diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2018-06-04 18:54:04 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-06-04 18:54:04 -0400 |
commit | 4057adafb395204af4ff93f3669ecb49eb45b3cf (patch) | |
tree | d6b0abf50ec5cd658fe958f90941c0192486549c | |
parent | 137f5ae4dae85011b13e3a7049414c4060ad94c0 (diff) | |
parent | 52f2b34f46223ca2789320fa10c13f6664c1b628 (diff) |
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull RCU updates from Ingo Molnar:
- updates to the handling of expedited grace periods
- updates to reduce lock contention in the rcu_node combining tree
[ These are in preparation for the consolidation of RCU-bh,
RCU-preempt, and RCU-sched into a single flavor, which was
requested by Linus in response to a security flaw whose root cause
included confusion between the multiple flavors of RCU ]
- torture-test updates that save their users some time and effort
- miscellaneous fixes
* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (44 commits)
rcu/x86: Provide early rcu_cpu_starting() callback
torture: Make kvm-find-errors.sh find build warnings
rcutorture: Abbreviate kvm.sh summary lines
rcutorture: Print end-of-test state in kvm.sh summary
rcutorture: Print end-of-test state
torture: Fold parse-torture.sh into parse-console.sh
torture: Add a script to edit output from failed runs
rcu: Update list of rcu_future_grace_period() trace events
rcu: Drop early GP request check from rcu_gp_kthread()
rcu: Simplify and inline cpu_needs_another_gp()
rcu: The rcu_gp_cleanup() function does not need cpu_needs_another_gp()
rcu: Make rcu_start_this_gp() check for out-of-range requests
rcu: Add funnel locking to rcu_start_this_gp()
rcu: Make rcu_start_future_gp() caller select grace period
rcu: Inline rcu_start_gp_advanced() into rcu_start_future_gp()
rcu: Clear request other than RCU_GP_FLAG_INIT at GP end
rcu: Cleanup, don't put ->completed into an int
rcu: Switch __rcu_process_callbacks() to rcu_accelerate_cbs()
rcu: Avoid __call_rcu_core() root rcu_node ->lock acquisition
rcu: Make rcu_migrate_callbacks wake GP kthread when needed
...
31 files changed, 631 insertions, 629 deletions
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index a27fbfb0efb8..65eb856526b7 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt | |||
@@ -1,3 +1,5 @@ | |||
1 | What is RCU? -- "Read, Copy, Update" | ||
2 | |||
1 | Please note that the "What is RCU?" LWN series is an excellent place | 3 | Please note that the "What is RCU?" LWN series is an excellent place |
2 | to start learning about RCU: | 4 | to start learning about RCU: |
3 | 5 | ||
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c index 7468de429087..3ea0047beb40 100644 --- a/arch/x86/kernel/cpu/mtrr/main.c +++ b/arch/x86/kernel/cpu/mtrr/main.c | |||
@@ -46,6 +46,7 @@ | |||
46 | #include <linux/pci.h> | 46 | #include <linux/pci.h> |
47 | #include <linux/smp.h> | 47 | #include <linux/smp.h> |
48 | #include <linux/syscore_ops.h> | 48 | #include <linux/syscore_ops.h> |
49 | #include <linux/rcupdate.h> | ||
49 | 50 | ||
50 | #include <asm/cpufeature.h> | 51 | #include <asm/cpufeature.h> |
51 | #include <asm/e820/api.h> | 52 | #include <asm/e820/api.h> |
@@ -793,6 +794,9 @@ void mtrr_ap_init(void) | |||
793 | 794 | ||
794 | if (!use_intel() || mtrr_aps_delayed_init) | 795 | if (!use_intel() || mtrr_aps_delayed_init) |
795 | return; | 796 | return; |
797 | |||
798 | rcu_cpu_starting(smp_processor_id()); | ||
799 | |||
796 | /* | 800 | /* |
797 | * Ideally we should hold mtrr_mutex here to avoid mtrr entries | 801 | * Ideally we should hold mtrr_mutex here to avoid mtrr entries |
798 | * changed, but this routine will be called in cpu boot time, | 802 | * changed, but this routine will be called in cpu boot time, |
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 04a20da76786..c8b30067b6ae 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c | |||
@@ -357,7 +357,7 @@ static void nvme_free_ns_head(struct kref *ref) | |||
357 | nvme_mpath_remove_disk(head); | 357 | nvme_mpath_remove_disk(head); |
358 | ida_simple_remove(&head->subsys->ns_ida, head->instance); | 358 | ida_simple_remove(&head->subsys->ns_ida, head->instance); |
359 | list_del_init(&head->entry); | 359 | list_del_init(&head->entry); |
360 | cleanup_srcu_struct(&head->srcu); | 360 | cleanup_srcu_struct_quiesced(&head->srcu); |
361 | nvme_put_subsystem(head->subsys); | 361 | nvme_put_subsystem(head->subsys); |
362 | kfree(head); | 362 | kfree(head); |
363 | } | 363 | } |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 36360d07f25b..e679b175b411 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -108,7 +108,6 @@ void rcu_sched_qs(void); | |||
108 | void rcu_bh_qs(void); | 108 | void rcu_bh_qs(void); |
109 | void rcu_check_callbacks(int user); | 109 | void rcu_check_callbacks(int user); |
110 | void rcu_report_dead(unsigned int cpu); | 110 | void rcu_report_dead(unsigned int cpu); |
111 | void rcu_cpu_starting(unsigned int cpu); | ||
112 | void rcutree_migrate_callbacks(int cpu); | 111 | void rcutree_migrate_callbacks(int cpu); |
113 | 112 | ||
114 | #ifdef CONFIG_RCU_STALL_COMMON | 113 | #ifdef CONFIG_RCU_STALL_COMMON |
@@ -188,13 +187,13 @@ static inline void exit_tasks_rcu_finish(void) { } | |||
188 | #endif /* #else #ifdef CONFIG_TASKS_RCU */ | 187 | #endif /* #else #ifdef CONFIG_TASKS_RCU */ |
189 | 188 | ||
190 | /** | 189 | /** |
191 | * cond_resched_rcu_qs - Report potential quiescent states to RCU | 190 | * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU |
192 | * | 191 | * |
193 | * This macro resembles cond_resched(), except that it is defined to | 192 | * This macro resembles cond_resched(), except that it is defined to |
194 | * report potential quiescent states to RCU-tasks even if the cond_resched() | 193 | * report potential quiescent states to RCU-tasks even if the cond_resched() |
195 | * machinery were to be shut off, as some advocate for PREEMPT kernels. | 194 | * machinery were to be shut off, as some advocate for PREEMPT kernels. |
196 | */ | 195 | */ |
197 | #define cond_resched_rcu_qs() \ | 196 | #define cond_resched_tasks_rcu_qs() \ |
198 | do { \ | 197 | do { \ |
199 | if (!cond_resched()) \ | 198 | if (!cond_resched()) \ |
200 | rcu_note_voluntary_context_switch_lite(current); \ | 199 | rcu_note_voluntary_context_switch_lite(current); \ |
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index ce9beec35e34..7b3c82e8a625 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h | |||
@@ -132,5 +132,6 @@ static inline void rcu_all_qs(void) { barrier(); } | |||
132 | #define rcutree_offline_cpu NULL | 132 | #define rcutree_offline_cpu NULL |
133 | #define rcutree_dead_cpu NULL | 133 | #define rcutree_dead_cpu NULL |
134 | #define rcutree_dying_cpu NULL | 134 | #define rcutree_dying_cpu NULL |
135 | static inline void rcu_cpu_starting(unsigned int cpu) { } | ||
135 | 136 | ||
136 | #endif /* __LINUX_RCUTINY_H */ | 137 | #endif /* __LINUX_RCUTINY_H */ |
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index fd996cdf1833..914655848ef6 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h | |||
@@ -74,6 +74,7 @@ static inline void synchronize_rcu_bh_expedited(void) | |||
74 | void rcu_barrier(void); | 74 | void rcu_barrier(void); |
75 | void rcu_barrier_bh(void); | 75 | void rcu_barrier_bh(void); |
76 | void rcu_barrier_sched(void); | 76 | void rcu_barrier_sched(void); |
77 | bool rcu_eqs_special_set(int cpu); | ||
77 | unsigned long get_state_synchronize_rcu(void); | 78 | unsigned long get_state_synchronize_rcu(void); |
78 | void cond_synchronize_rcu(unsigned long oldstate); | 79 | void cond_synchronize_rcu(unsigned long oldstate); |
79 | unsigned long get_state_synchronize_sched(void); | 80 | unsigned long get_state_synchronize_sched(void); |
@@ -100,5 +101,6 @@ int rcutree_online_cpu(unsigned int cpu); | |||
100 | int rcutree_offline_cpu(unsigned int cpu); | 101 | int rcutree_offline_cpu(unsigned int cpu); |
101 | int rcutree_dead_cpu(unsigned int cpu); | 102 | int rcutree_dead_cpu(unsigned int cpu); |
102 | int rcutree_dying_cpu(unsigned int cpu); | 103 | int rcutree_dying_cpu(unsigned int cpu); |
104 | void rcu_cpu_starting(unsigned int cpu); | ||
103 | 105 | ||
104 | #endif /* __LINUX_RCUTREE_H */ | 106 | #endif /* __LINUX_RCUTREE_H */ |
diff --git a/include/linux/sched.h b/include/linux/sched.h index ca3f3eae8980..5a0c10b45273 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -1661,7 +1661,6 @@ static inline int test_tsk_need_resched(struct task_struct *tsk) | |||
1661 | * explicit rescheduling in places that are safe. The return | 1661 | * explicit rescheduling in places that are safe. The return |
1662 | * value indicates whether a reschedule was done in fact. | 1662 | * value indicates whether a reschedule was done in fact. |
1663 | * cond_resched_lock() will drop the spinlock before scheduling, | 1663 | * cond_resched_lock() will drop the spinlock before scheduling, |
1664 | * cond_resched_softirq() will enable bhs before scheduling. | ||
1665 | */ | 1664 | */ |
1666 | #ifndef CONFIG_PREEMPT | 1665 | #ifndef CONFIG_PREEMPT |
1667 | extern int _cond_resched(void); | 1666 | extern int _cond_resched(void); |
@@ -1681,13 +1680,6 @@ extern int __cond_resched_lock(spinlock_t *lock); | |||
1681 | __cond_resched_lock(lock); \ | 1680 | __cond_resched_lock(lock); \ |
1682 | }) | 1681 | }) |
1683 | 1682 | ||
1684 | extern int __cond_resched_softirq(void); | ||
1685 | |||
1686 | #define cond_resched_softirq() ({ \ | ||
1687 | ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ | ||
1688 | __cond_resched_softirq(); \ | ||
1689 | }) | ||
1690 | |||
1691 | static inline void cond_resched_rcu(void) | 1683 | static inline void cond_resched_rcu(void) |
1692 | { | 1684 | { |
1693 | #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) | 1685 | #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) |
diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 33c1c698df09..91494d7e8e41 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h | |||
@@ -69,11 +69,45 @@ struct srcu_struct { }; | |||
69 | 69 | ||
70 | void call_srcu(struct srcu_struct *sp, struct rcu_head *head, | 70 | void call_srcu(struct srcu_struct *sp, struct rcu_head *head, |
71 | void (*func)(struct rcu_head *head)); | 71 | void (*func)(struct rcu_head *head)); |
72 | void cleanup_srcu_struct(struct srcu_struct *sp); | 72 | void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced); |
73 | int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); | 73 | int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); |
74 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); | 74 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); |
75 | void synchronize_srcu(struct srcu_struct *sp); | 75 | void synchronize_srcu(struct srcu_struct *sp); |
76 | 76 | ||
77 | /** | ||
78 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | ||
79 | * @sp: structure to clean up. | ||
80 | * | ||
81 | * Must invoke this after you are finished using a given srcu_struct that | ||
82 | * was initialized via init_srcu_struct(), else you leak memory. | ||
83 | */ | ||
84 | static inline void cleanup_srcu_struct(struct srcu_struct *sp) | ||
85 | { | ||
86 | _cleanup_srcu_struct(sp, false); | ||
87 | } | ||
88 | |||
89 | /** | ||
90 | * cleanup_srcu_struct_quiesced - deconstruct a quiesced sleep-RCU structure | ||
91 | * @sp: structure to clean up. | ||
92 | * | ||
93 | * Must invoke this after you are finished using a given srcu_struct that | ||
94 | * was initialized via init_srcu_struct(), else you leak memory. Also, | ||
95 | * all grace-period processing must have completed. | ||
96 | * | ||
97 | * "Completed" means that the last synchronize_srcu() and | ||
98 | * synchronize_srcu_expedited() calls must have returned before the call | ||
99 | * to cleanup_srcu_struct_quiesced(). It also means that the callback | ||
100 | * from the last call_srcu() must have been invoked before the call to | ||
101 | * cleanup_srcu_struct_quiesced(), but you can use srcu_barrier() to help | ||
102 | * with this last. Violating these rules will get you a WARN_ON() splat | ||
103 | * (with high probability, anyway), and will also cause the srcu_struct | ||
104 | * to be leaked. | ||
105 | */ | ||
106 | static inline void cleanup_srcu_struct_quiesced(struct srcu_struct *sp) | ||
107 | { | ||
108 | _cleanup_srcu_struct(sp, true); | ||
109 | } | ||
110 | |||
77 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 111 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
78 | 112 | ||
79 | /** | 113 | /** |
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index d8c33298c153..5936aac357ab 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h | |||
@@ -84,20 +84,21 @@ TRACE_EVENT(rcu_grace_period, | |||
84 | ); | 84 | ); |
85 | 85 | ||
86 | /* | 86 | /* |
87 | * Tracepoint for future grace-period events, including those for no-callbacks | 87 | * Tracepoint for future grace-period events. The caller should pull |
88 | * CPUs. The caller should pull the data from the rcu_node structure, | 88 | * the data from the rcu_node structure, other than rcuname, which comes |
89 | * other than rcuname, which comes from the rcu_state structure, and event, | 89 | * from the rcu_state structure, and event, which is one of the following: |
90 | * which is one of the following: | ||
91 | * | 90 | * |
92 | * "Startleaf": Request a nocb grace period based on leaf-node data. | 91 | * "Startleaf": Request a grace period based on leaf-node data. |
92 | * "Prestarted": Someone beat us to the request | ||
93 | * "Startedleaf": Leaf-node start proved sufficient. | 93 | * "Startedleaf": Leaf-node start proved sufficient. |
94 | * "Startedleafroot": Leaf-node start proved sufficient after checking root. | 94 | * "Startedleafroot": Leaf-node start proved sufficient after checking root. |
95 | * "Startedroot": Requested a nocb grace period based on root-node data. | 95 | * "Startedroot": Requested a nocb grace period based on root-node data. |
96 | * "NoGPkthread": The RCU grace-period kthread has not yet started. | ||
96 | * "StartWait": Start waiting for the requested grace period. | 97 | * "StartWait": Start waiting for the requested grace period. |
97 | * "ResumeWait": Resume waiting after signal. | 98 | * "ResumeWait": Resume waiting after signal. |
98 | * "EndWait": Complete wait. | 99 | * "EndWait": Complete wait. |
99 | * "Cleanup": Clean up rcu_node structure after previous GP. | 100 | * "Cleanup": Clean up rcu_node structure after previous GP. |
100 | * "CleanupMore": Clean up, and another no-CB GP is needed. | 101 | * "CleanupMore": Clean up, and another GP is needed. |
101 | */ | 102 | */ |
102 | TRACE_EVENT(rcu_future_grace_period, | 103 | TRACE_EVENT(rcu_future_grace_period, |
103 | 104 | ||
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 7a693e31184a..40cea6735c2d 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h | |||
@@ -270,6 +270,12 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) | |||
270 | } | 270 | } |
271 | } | 271 | } |
272 | 272 | ||
273 | /* Returns first leaf rcu_node of the specified RCU flavor. */ | ||
274 | #define rcu_first_leaf_node(rsp) ((rsp)->level[rcu_num_lvls - 1]) | ||
275 | |||
276 | /* Is this rcu_node a leaf? */ | ||
277 | #define rcu_is_leaf_node(rnp) ((rnp)->level == rcu_num_lvls - 1) | ||
278 | |||
273 | /* | 279 | /* |
274 | * Do a full breadth-first scan of the rcu_node structures for the | 280 | * Do a full breadth-first scan of the rcu_node structures for the |
275 | * specified rcu_state structure. | 281 | * specified rcu_state structure. |
@@ -284,8 +290,7 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) | |||
284 | * rcu_node tree with but one rcu_node structure, this loop is a no-op. | 290 | * rcu_node tree with but one rcu_node structure, this loop is a no-op. |
285 | */ | 291 | */ |
286 | #define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \ | 292 | #define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \ |
287 | for ((rnp) = &(rsp)->node[0]; \ | 293 | for ((rnp) = &(rsp)->node[0]; !rcu_is_leaf_node(rsp, rnp); (rnp)++) |
288 | (rnp) < (rsp)->level[rcu_num_lvls - 1]; (rnp)++) | ||
289 | 294 | ||
290 | /* | 295 | /* |
291 | * Scan the leaves of the rcu_node hierarchy for the specified rcu_state | 296 | * Scan the leaves of the rcu_node hierarchy for the specified rcu_state |
@@ -294,7 +299,7 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt) | |||
294 | * It is still a leaf node, even if it is also the root node. | 299 | * It is still a leaf node, even if it is also the root node. |
295 | */ | 300 | */ |
296 | #define rcu_for_each_leaf_node(rsp, rnp) \ | 301 | #define rcu_for_each_leaf_node(rsp, rnp) \ |
297 | for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \ | 302 | for ((rnp) = rcu_first_leaf_node(rsp); \ |
298 | (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++) | 303 | (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++) |
299 | 304 | ||
300 | /* | 305 | /* |
@@ -486,6 +491,7 @@ void rcu_force_quiescent_state(void); | |||
486 | void rcu_bh_force_quiescent_state(void); | 491 | void rcu_bh_force_quiescent_state(void); |
487 | void rcu_sched_force_quiescent_state(void); | 492 | void rcu_sched_force_quiescent_state(void); |
488 | extern struct workqueue_struct *rcu_gp_wq; | 493 | extern struct workqueue_struct *rcu_gp_wq; |
494 | extern struct workqueue_struct *rcu_par_gp_wq; | ||
489 | #endif /* #else #ifdef CONFIG_TINY_RCU */ | 495 | #endif /* #else #ifdef CONFIG_TINY_RCU */ |
490 | 496 | ||
491 | #ifdef CONFIG_RCU_NOCB_CPU | 497 | #ifdef CONFIG_RCU_NOCB_CPU |
diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 88cba7c2956c..5aff271adf1e 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c | |||
@@ -404,24 +404,6 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) | |||
404 | } | 404 | } |
405 | 405 | ||
406 | /* | 406 | /* |
407 | * Scan the specified rcu_segcblist structure for callbacks that need | ||
408 | * a grace period later than the one specified by "seq". We don't look | ||
409 | * at the RCU_DONE_TAIL or RCU_NEXT_TAIL segments because they don't | ||
410 | * have a grace-period sequence number. | ||
411 | */ | ||
412 | bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp, | ||
413 | unsigned long seq) | ||
414 | { | ||
415 | int i; | ||
416 | |||
417 | for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) | ||
418 | if (rsclp->tails[i - 1] != rsclp->tails[i] && | ||
419 | ULONG_CMP_LT(seq, rsclp->gp_seq[i])) | ||
420 | return true; | ||
421 | return false; | ||
422 | } | ||
423 | |||
424 | /* | ||
425 | * Merge the source rcu_segcblist structure into the destination | 407 | * Merge the source rcu_segcblist structure into the destination |
426 | * rcu_segcblist structure, then initialize the source. Any pending | 408 | * rcu_segcblist structure, then initialize the source. Any pending |
427 | * callbacks from the source get to start over. It is best to | 409 | * callbacks from the source get to start over. It is best to |
diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 581c12b63544..948470cef385 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h | |||
@@ -134,7 +134,5 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, | |||
134 | struct rcu_cblist *rclp); | 134 | struct rcu_cblist *rclp); |
135 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); | 135 | void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); |
136 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); | 136 | bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); |
137 | bool rcu_segcblist_future_gp_needed(struct rcu_segcblist *rsclp, | ||
138 | unsigned long seq); | ||
139 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, | 137 | void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, |
140 | struct rcu_segcblist *src_rsclp); | 138 | struct rcu_segcblist *src_rsclp); |
diff --git a/kernel/rcu/rcuperf.c b/kernel/rcu/rcuperf.c index 777e7a6a0292..e232846516b3 100644 --- a/kernel/rcu/rcuperf.c +++ b/kernel/rcu/rcuperf.c | |||
@@ -369,7 +369,7 @@ static bool __maybe_unused torturing_tasks(void) | |||
369 | */ | 369 | */ |
370 | static void rcu_perf_wait_shutdown(void) | 370 | static void rcu_perf_wait_shutdown(void) |
371 | { | 371 | { |
372 | cond_resched_rcu_qs(); | 372 | cond_resched_tasks_rcu_qs(); |
373 | if (atomic_read(&n_rcu_perf_writer_finished) < nrealwriters) | 373 | if (atomic_read(&n_rcu_perf_writer_finished) < nrealwriters) |
374 | return; | 374 | return; |
375 | while (!torture_must_stop()) | 375 | while (!torture_must_stop()) |
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 680c96d8c00f..e628fcfd1bde 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c | |||
@@ -593,7 +593,12 @@ static void srcu_torture_init(void) | |||
593 | 593 | ||
594 | static void srcu_torture_cleanup(void) | 594 | static void srcu_torture_cleanup(void) |
595 | { | 595 | { |
596 | cleanup_srcu_struct(&srcu_ctld); | 596 | static DEFINE_TORTURE_RANDOM(rand); |
597 | |||
598 | if (torture_random(&rand) & 0x800) | ||
599 | cleanup_srcu_struct(&srcu_ctld); | ||
600 | else | ||
601 | cleanup_srcu_struct_quiesced(&srcu_ctld); | ||
597 | srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ | 602 | srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ |
598 | } | 603 | } |
599 | 604 | ||
@@ -1609,6 +1614,9 @@ static enum cpuhp_state rcutor_hp; | |||
1609 | static void | 1614 | static void |
1610 | rcu_torture_cleanup(void) | 1615 | rcu_torture_cleanup(void) |
1611 | { | 1616 | { |
1617 | int flags = 0; | ||
1618 | unsigned long gpnum = 0; | ||
1619 | unsigned long completed = 0; | ||
1612 | int i; | 1620 | int i; |
1613 | 1621 | ||
1614 | rcutorture_record_test_transition(); | 1622 | rcutorture_record_test_transition(); |
@@ -1639,6 +1647,11 @@ rcu_torture_cleanup(void) | |||
1639 | fakewriter_tasks = NULL; | 1647 | fakewriter_tasks = NULL; |
1640 | } | 1648 | } |
1641 | 1649 | ||
1650 | rcutorture_get_gp_data(cur_ops->ttype, &flags, &gpnum, &completed); | ||
1651 | srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, | ||
1652 | &flags, &gpnum, &completed); | ||
1653 | pr_alert("%s: End-test grace-period state: g%lu c%lu f%#x\n", | ||
1654 | cur_ops->name, gpnum, completed, flags); | ||
1642 | torture_stop_kthread(rcu_torture_stats, stats_task); | 1655 | torture_stop_kthread(rcu_torture_stats, stats_task); |
1643 | torture_stop_kthread(rcu_torture_fqs, fqs_task); | 1656 | torture_stop_kthread(rcu_torture_fqs, fqs_task); |
1644 | for (i = 0; i < ncbflooders; i++) | 1657 | for (i = 0; i < ncbflooders; i++) |
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 76ac5f50b2c7..622792abe41a 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c | |||
@@ -86,16 +86,19 @@ EXPORT_SYMBOL_GPL(init_srcu_struct); | |||
86 | * Must invoke this after you are finished using a given srcu_struct that | 86 | * Must invoke this after you are finished using a given srcu_struct that |
87 | * was initialized via init_srcu_struct(), else you leak memory. | 87 | * was initialized via init_srcu_struct(), else you leak memory. |
88 | */ | 88 | */ |
89 | void cleanup_srcu_struct(struct srcu_struct *sp) | 89 | void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced) |
90 | { | 90 | { |
91 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); | 91 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); |
92 | flush_work(&sp->srcu_work); | 92 | if (quiesced) |
93 | WARN_ON(work_pending(&sp->srcu_work)); | ||
94 | else | ||
95 | flush_work(&sp->srcu_work); | ||
93 | WARN_ON(sp->srcu_gp_running); | 96 | WARN_ON(sp->srcu_gp_running); |
94 | WARN_ON(sp->srcu_gp_waiting); | 97 | WARN_ON(sp->srcu_gp_waiting); |
95 | WARN_ON(sp->srcu_cb_head); | 98 | WARN_ON(sp->srcu_cb_head); |
96 | WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail); | 99 | WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail); |
97 | } | 100 | } |
98 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | 101 | EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); |
99 | 102 | ||
100 | /* | 103 | /* |
101 | * Removes the count for the old reader from the appropriate element of | 104 | * Removes the count for the old reader from the appropriate element of |
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index fb560fca9ef4..b4123d7a2cec 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c | |||
@@ -366,24 +366,28 @@ static unsigned long srcu_get_delay(struct srcu_struct *sp) | |||
366 | return SRCU_INTERVAL; | 366 | return SRCU_INTERVAL; |
367 | } | 367 | } |
368 | 368 | ||
369 | /** | 369 | /* Helper for cleanup_srcu_struct() and cleanup_srcu_struct_quiesced(). */ |
370 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | 370 | void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced) |
371 | * @sp: structure to clean up. | ||
372 | * | ||
373 | * Must invoke this after you are finished using a given srcu_struct that | ||
374 | * was initialized via init_srcu_struct(), else you leak memory. | ||
375 | */ | ||
376 | void cleanup_srcu_struct(struct srcu_struct *sp) | ||
377 | { | 371 | { |
378 | int cpu; | 372 | int cpu; |
379 | 373 | ||
380 | if (WARN_ON(!srcu_get_delay(sp))) | 374 | if (WARN_ON(!srcu_get_delay(sp))) |
381 | return; /* Leakage unless caller handles error. */ | 375 | return; /* Just leak it! */ |
382 | if (WARN_ON(srcu_readers_active(sp))) | 376 | if (WARN_ON(srcu_readers_active(sp))) |
383 | return; /* Leakage unless caller handles error. */ | 377 | return; /* Just leak it! */ |
384 | flush_delayed_work(&sp->work); | 378 | if (quiesced) { |
379 | if (WARN_ON(delayed_work_pending(&sp->work))) | ||
380 | return; /* Just leak it! */ | ||
381 | } else { | ||
382 | flush_delayed_work(&sp->work); | ||
383 | } | ||
385 | for_each_possible_cpu(cpu) | 384 | for_each_possible_cpu(cpu) |
386 | flush_delayed_work(&per_cpu_ptr(sp->sda, cpu)->work); | 385 | if (quiesced) { |
386 | if (WARN_ON(delayed_work_pending(&per_cpu_ptr(sp->sda, cpu)->work))) | ||
387 | return; /* Just leak it! */ | ||
388 | } else { | ||
389 | flush_delayed_work(&per_cpu_ptr(sp->sda, cpu)->work); | ||
390 | } | ||
387 | if (WARN_ON(rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) != SRCU_STATE_IDLE) || | 391 | if (WARN_ON(rcu_seq_state(READ_ONCE(sp->srcu_gp_seq)) != SRCU_STATE_IDLE) || |
388 | WARN_ON(srcu_readers_active(sp))) { | 392 | WARN_ON(srcu_readers_active(sp))) { |
389 | pr_info("%s: Active srcu_struct %p state: %d\n", __func__, sp, rcu_seq_state(READ_ONCE(sp->srcu_gp_seq))); | 393 | pr_info("%s: Active srcu_struct %p state: %d\n", __func__, sp, rcu_seq_state(READ_ONCE(sp->srcu_gp_seq))); |
@@ -392,7 +396,7 @@ void cleanup_srcu_struct(struct srcu_struct *sp) | |||
392 | free_percpu(sp->sda); | 396 | free_percpu(sp->sda); |
393 | sp->sda = NULL; | 397 | sp->sda = NULL; |
394 | } | 398 | } |
395 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | 399 | EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); |
396 | 400 | ||
397 | /* | 401 | /* |
398 | * Counts the new reader in the appropriate per-CPU element of the | 402 | * Counts the new reader in the appropriate per-CPU element of the |
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 2a734692a581..aa7cade1b9f3 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
@@ -524,8 +524,6 @@ module_param(rcu_kick_kthreads, bool, 0644); | |||
524 | static ulong jiffies_till_sched_qs = HZ / 10; | 524 | static ulong jiffies_till_sched_qs = HZ / 10; |
525 | module_param(jiffies_till_sched_qs, ulong, 0444); | 525 | module_param(jiffies_till_sched_qs, ulong, 0444); |
526 | 526 | ||
527 | static bool rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, | ||
528 | struct rcu_data *rdp); | ||
529 | static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)); | 527 | static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)); |
530 | static void force_quiescent_state(struct rcu_state *rsp); | 528 | static void force_quiescent_state(struct rcu_state *rsp); |
531 | static int rcu_pending(void); | 529 | static int rcu_pending(void); |
@@ -711,44 +709,6 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) | |||
711 | } | 709 | } |
712 | 710 | ||
713 | /* | 711 | /* |
714 | * Is there any need for future grace periods? | ||
715 | * Interrupts must be disabled. If the caller does not hold the root | ||
716 | * rnp_node structure's ->lock, the results are advisory only. | ||
717 | */ | ||
718 | static int rcu_future_needs_gp(struct rcu_state *rsp) | ||
719 | { | ||
720 | struct rcu_node *rnp = rcu_get_root(rsp); | ||
721 | int idx = (READ_ONCE(rnp->completed) + 1) & 0x1; | ||
722 | int *fp = &rnp->need_future_gp[idx]; | ||
723 | |||
724 | lockdep_assert_irqs_disabled(); | ||
725 | return READ_ONCE(*fp); | ||
726 | } | ||
727 | |||
728 | /* | ||
729 | * Does the current CPU require a not-yet-started grace period? | ||
730 | * The caller must have disabled interrupts to prevent races with | ||
731 | * normal callback registry. | ||
732 | */ | ||
733 | static bool | ||
734 | cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) | ||
735 | { | ||
736 | lockdep_assert_irqs_disabled(); | ||
737 | if (rcu_gp_in_progress(rsp)) | ||
738 | return false; /* No, a grace period is already in progress. */ | ||
739 | if (rcu_future_needs_gp(rsp)) | ||
740 | return true; /* Yes, a no-CBs CPU needs one. */ | ||
741 | if (!rcu_segcblist_is_enabled(&rdp->cblist)) | ||
742 | return false; /* No, this is a no-CBs (or offline) CPU. */ | ||
743 | if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) | ||
744 | return true; /* Yes, CPU has newly registered callbacks. */ | ||
745 | if (rcu_segcblist_future_gp_needed(&rdp->cblist, | ||
746 | READ_ONCE(rsp->completed))) | ||
747 | return true; /* Yes, CBs for future grace period. */ | ||
748 | return false; /* No grace period needed. */ | ||
749 | } | ||
750 | |||
751 | /* | ||
752 | * Enter an RCU extended quiescent state, which can be either the | 712 | * Enter an RCU extended quiescent state, which can be either the |
753 | * idle loop or adaptive-tickless usermode execution. | 713 | * idle loop or adaptive-tickless usermode execution. |
754 | * | 714 | * |
@@ -1234,10 +1194,10 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) | |||
1234 | } | 1194 | } |
1235 | 1195 | ||
1236 | /* | 1196 | /* |
1237 | * Has this CPU encountered a cond_resched_rcu_qs() since the | 1197 | * Has this CPU encountered a cond_resched() since the beginning |
1238 | * beginning of the grace period? For this to be the case, | 1198 | * of the grace period? For this to be the case, the CPU has to |
1239 | * the CPU has to have noticed the current grace period. This | 1199 | * have noticed the current grace period. This might not be the |
1240 | * might not be the case for nohz_full CPUs looping in the kernel. | 1200 | * case for nohz_full CPUs looping in the kernel. |
1241 | */ | 1201 | */ |
1242 | jtsq = jiffies_till_sched_qs; | 1202 | jtsq = jiffies_till_sched_qs; |
1243 | ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu); | 1203 | ruqp = per_cpu_ptr(&rcu_dynticks.rcu_urgent_qs, rdp->cpu); |
@@ -1642,18 +1602,30 @@ static unsigned long rcu_cbs_completed(struct rcu_state *rsp, | |||
1642 | return rnp->completed + 1; | 1602 | return rnp->completed + 1; |
1643 | 1603 | ||
1644 | /* | 1604 | /* |
1605 | * If the current rcu_node structure believes that RCU is | ||
1606 | * idle, and if the rcu_state structure does not yet reflect | ||
1607 | * the start of a new grace period, then the next grace period | ||
1608 | * will suffice. The memory barrier is needed to accurately | ||
1609 | * sample the rsp->gpnum, and pairs with the second lock | ||
1610 | * acquisition in rcu_gp_init(), which is augmented with | ||
1611 | * smp_mb__after_unlock_lock() for this purpose. | ||
1612 | */ | ||
1613 | if (rnp->gpnum == rnp->completed) { | ||
1614 | smp_mb(); /* See above block comment. */ | ||
1615 | if (READ_ONCE(rsp->gpnum) == rnp->completed) | ||
1616 | return rnp->completed + 1; | ||
1617 | } | ||
1618 | |||
1619 | /* | ||
1645 | * Otherwise, wait for a possible partial grace period and | 1620 | * Otherwise, wait for a possible partial grace period and |
1646 | * then the subsequent full grace period. | 1621 | * then the subsequent full grace period. |
1647 | */ | 1622 | */ |
1648 | return rnp->completed + 2; | 1623 | return rnp->completed + 2; |
1649 | } | 1624 | } |
1650 | 1625 | ||
1651 | /* | 1626 | /* Trace-event wrapper function for trace_rcu_future_grace_period. */ |
1652 | * Trace-event helper function for rcu_start_future_gp() and | 1627 | static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, |
1653 | * rcu_nocb_wait_gp(). | 1628 | unsigned long c, const char *s) |
1654 | */ | ||
1655 | static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, | ||
1656 | unsigned long c, const char *s) | ||
1657 | { | 1629 | { |
1658 | trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, | 1630 | trace_rcu_future_grace_period(rdp->rsp->name, rnp->gpnum, |
1659 | rnp->completed, c, rnp->level, | 1631 | rnp->completed, c, rnp->level, |
@@ -1661,96 +1633,67 @@ static void trace_rcu_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, | |||
1661 | } | 1633 | } |
1662 | 1634 | ||
1663 | /* | 1635 | /* |
1664 | * Start some future grace period, as needed to handle newly arrived | 1636 | * Start the specified grace period, as needed to handle newly arrived |
1665 | * callbacks. The required future grace periods are recorded in each | 1637 | * callbacks. The required future grace periods are recorded in each |
1666 | * rcu_node structure's ->need_future_gp field. Returns true if there | 1638 | * rcu_node structure's ->need_future_gp[] field. Returns true if there |
1667 | * is reason to awaken the grace-period kthread. | 1639 | * is reason to awaken the grace-period kthread. |
1668 | * | 1640 | * |
1669 | * The caller must hold the specified rcu_node structure's ->lock. | 1641 | * The caller must hold the specified rcu_node structure's ->lock, which |
1642 | * is why the caller is responsible for waking the grace-period kthread. | ||
1670 | */ | 1643 | */ |
1671 | static bool __maybe_unused | 1644 | static bool rcu_start_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, |
1672 | rcu_start_future_gp(struct rcu_node *rnp, struct rcu_data *rdp, | 1645 | unsigned long c) |
1673 | unsigned long *c_out) | ||
1674 | { | 1646 | { |
1675 | unsigned long c; | ||
1676 | bool ret = false; | 1647 | bool ret = false; |
1677 | struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); | 1648 | struct rcu_state *rsp = rdp->rsp; |
1678 | 1649 | struct rcu_node *rnp_root; | |
1679 | raw_lockdep_assert_held_rcu_node(rnp); | ||
1680 | |||
1681 | /* | ||
1682 | * Pick up grace-period number for new callbacks. If this | ||
1683 | * grace period is already marked as needed, return to the caller. | ||
1684 | */ | ||
1685 | c = rcu_cbs_completed(rdp->rsp, rnp); | ||
1686 | trace_rcu_future_gp(rnp, rdp, c, TPS("Startleaf")); | ||
1687 | if (rnp->need_future_gp[c & 0x1]) { | ||
1688 | trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartleaf")); | ||
1689 | goto out; | ||
1690 | } | ||
1691 | 1650 | ||
1692 | /* | 1651 | /* |
1693 | * If either this rcu_node structure or the root rcu_node structure | 1652 | * Use funnel locking to either acquire the root rcu_node |
1694 | * believe that a grace period is in progress, then we must wait | 1653 | * structure's lock or bail out if the need for this grace period |
1695 | * for the one following, which is in "c". Because our request | 1654 | * has already been recorded -- or has already started. If there |
1696 | * will be noticed at the end of the current grace period, we don't | 1655 | * is already a grace period in progress in a non-leaf node, no |
1697 | * need to explicitly start one. We only do the lockless check | 1656 | * recording is needed because the end of the grace period will |
1698 | * of rnp_root's fields if the current rcu_node structure thinks | 1657 | * scan the leaf rcu_node structures. Note that rnp->lock must |
1699 | * there is no grace period in flight, and because we hold rnp->lock, | 1658 | * not be released. |
1700 | * the only possible change is when rnp_root's two fields are | ||
1701 | * equal, in which case rnp_root->gpnum might be concurrently | ||
1702 | * incremented. But that is OK, as it will just result in our | ||
1703 | * doing some extra useless work. | ||
1704 | */ | 1659 | */ |
1705 | if (rnp->gpnum != rnp->completed || | 1660 | raw_lockdep_assert_held_rcu_node(rnp); |
1706 | READ_ONCE(rnp_root->gpnum) != READ_ONCE(rnp_root->completed)) { | 1661 | trace_rcu_this_gp(rnp, rdp, c, TPS("Startleaf")); |
1707 | rnp->need_future_gp[c & 0x1]++; | 1662 | for (rnp_root = rnp; 1; rnp_root = rnp_root->parent) { |
1708 | trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleaf")); | 1663 | if (rnp_root != rnp) |
1709 | goto out; | 1664 | raw_spin_lock_rcu_node(rnp_root); |
1665 | WARN_ON_ONCE(ULONG_CMP_LT(rnp_root->gpnum + | ||
1666 | need_future_gp_mask(), c)); | ||
1667 | if (need_future_gp_element(rnp_root, c) || | ||
1668 | ULONG_CMP_GE(rnp_root->gpnum, c) || | ||
1669 | (rnp != rnp_root && | ||
1670 | rnp_root->gpnum != rnp_root->completed)) { | ||
1671 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Prestarted")); | ||
1672 | goto unlock_out; | ||
1673 | } | ||
1674 | need_future_gp_element(rnp_root, c) = true; | ||
1675 | if (rnp_root != rnp && rnp_root->parent != NULL) | ||
1676 | raw_spin_unlock_rcu_node(rnp_root); | ||
1677 | if (!rnp_root->parent) | ||
1678 | break; /* At root, and perhaps also leaf. */ | ||
1710 | } | 1679 | } |
1711 | 1680 | ||
1712 | /* | 1681 | /* If GP already in progress, just leave, otherwise start one. */ |
1713 | * There might be no grace period in progress. If we don't already | 1682 | if (rnp_root->gpnum != rnp_root->completed) { |
1714 | * hold it, acquire the root rcu_node structure's lock in order to | 1683 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedleafroot")); |
1715 | * start one (if needed). | ||
1716 | */ | ||
1717 | if (rnp != rnp_root) | ||
1718 | raw_spin_lock_rcu_node(rnp_root); | ||
1719 | |||
1720 | /* | ||
1721 | * Get a new grace-period number. If there really is no grace | ||
1722 | * period in progress, it will be smaller than the one we obtained | ||
1723 | * earlier. Adjust callbacks as needed. | ||
1724 | */ | ||
1725 | c = rcu_cbs_completed(rdp->rsp, rnp_root); | ||
1726 | if (!rcu_is_nocb_cpu(rdp->cpu)) | ||
1727 | (void)rcu_segcblist_accelerate(&rdp->cblist, c); | ||
1728 | |||
1729 | /* | ||
1730 | * If the needed for the required grace period is already | ||
1731 | * recorded, trace and leave. | ||
1732 | */ | ||
1733 | if (rnp_root->need_future_gp[c & 0x1]) { | ||
1734 | trace_rcu_future_gp(rnp, rdp, c, TPS("Prestartedroot")); | ||
1735 | goto unlock_out; | 1684 | goto unlock_out; |
1736 | } | 1685 | } |
1737 | 1686 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("Startedroot")); | |
1738 | /* Record the need for the future grace period. */ | 1687 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags | RCU_GP_FLAG_INIT); |
1739 | rnp_root->need_future_gp[c & 0x1]++; | 1688 | if (!rsp->gp_kthread) { |
1740 | 1689 | trace_rcu_this_gp(rnp_root, rdp, c, TPS("NoGPkthread")); | |
1741 | /* If a grace period is not already in progress, start one. */ | 1690 | goto unlock_out; |
1742 | if (rnp_root->gpnum != rnp_root->completed) { | ||
1743 | trace_rcu_future_gp(rnp, rdp, c, TPS("Startedleafroot")); | ||
1744 | } else { | ||
1745 | trace_rcu_future_gp(rnp, rdp, c, TPS("Startedroot")); | ||
1746 | ret = rcu_start_gp_advanced(rdp->rsp, rnp_root, rdp); | ||
1747 | } | 1691 | } |
1692 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), TPS("newreq")); | ||
1693 | ret = true; /* Caller must wake GP kthread. */ | ||
1748 | unlock_out: | 1694 | unlock_out: |
1749 | if (rnp != rnp_root) | 1695 | if (rnp != rnp_root) |
1750 | raw_spin_unlock_rcu_node(rnp_root); | 1696 | raw_spin_unlock_rcu_node(rnp_root); |
1751 | out: | ||
1752 | if (c_out != NULL) | ||
1753 | *c_out = c; | ||
1754 | return ret; | 1697 | return ret; |
1755 | } | 1698 | } |
1756 | 1699 | ||
@@ -1758,16 +1701,16 @@ out: | |||
1758 | * Clean up any old requests for the just-ended grace period. Also return | 1701 | * Clean up any old requests for the just-ended grace period. Also return |
1759 | * whether any additional grace periods have been requested. | 1702 | * whether any additional grace periods have been requested. |
1760 | */ | 1703 | */ |
1761 | static int rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) | 1704 | static bool rcu_future_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) |
1762 | { | 1705 | { |
1763 | int c = rnp->completed; | 1706 | unsigned long c = rnp->completed; |
1764 | int needmore; | 1707 | bool needmore; |
1765 | struct rcu_data *rdp = this_cpu_ptr(rsp->rda); | 1708 | struct rcu_data *rdp = this_cpu_ptr(rsp->rda); |
1766 | 1709 | ||
1767 | rnp->need_future_gp[c & 0x1] = 0; | 1710 | need_future_gp_element(rnp, c) = false; |
1768 | needmore = rnp->need_future_gp[(c + 1) & 0x1]; | 1711 | needmore = need_any_future_gp(rnp); |
1769 | trace_rcu_future_gp(rnp, rdp, c, | 1712 | trace_rcu_this_gp(rnp, rdp, c, |
1770 | needmore ? TPS("CleanupMore") : TPS("Cleanup")); | 1713 | needmore ? TPS("CleanupMore") : TPS("Cleanup")); |
1771 | return needmore; | 1714 | return needmore; |
1772 | } | 1715 | } |
1773 | 1716 | ||
@@ -1802,6 +1745,7 @@ static void rcu_gp_kthread_wake(struct rcu_state *rsp) | |||
1802 | static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, | 1745 | static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, |
1803 | struct rcu_data *rdp) | 1746 | struct rcu_data *rdp) |
1804 | { | 1747 | { |
1748 | unsigned long c; | ||
1805 | bool ret = false; | 1749 | bool ret = false; |
1806 | 1750 | ||
1807 | raw_lockdep_assert_held_rcu_node(rnp); | 1751 | raw_lockdep_assert_held_rcu_node(rnp); |
@@ -1820,8 +1764,9 @@ static bool rcu_accelerate_cbs(struct rcu_state *rsp, struct rcu_node *rnp, | |||
1820 | * accelerating callback invocation to an earlier grace-period | 1764 | * accelerating callback invocation to an earlier grace-period |
1821 | * number. | 1765 | * number. |
1822 | */ | 1766 | */ |
1823 | if (rcu_segcblist_accelerate(&rdp->cblist, rcu_cbs_completed(rsp, rnp))) | 1767 | c = rcu_cbs_completed(rsp, rnp); |
1824 | ret = rcu_start_future_gp(rnp, rdp, NULL); | 1768 | if (rcu_segcblist_accelerate(&rdp->cblist, c)) |
1769 | ret = rcu_start_this_gp(rnp, rdp, c); | ||
1825 | 1770 | ||
1826 | /* Trace depending on how much we were able to accelerate. */ | 1771 | /* Trace depending on how much we were able to accelerate. */ |
1827 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) | 1772 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) |
@@ -2049,7 +1994,7 @@ static bool rcu_gp_init(struct rcu_state *rsp) | |||
2049 | rnp->level, rnp->grplo, | 1994 | rnp->level, rnp->grplo, |
2050 | rnp->grphi, rnp->qsmask); | 1995 | rnp->grphi, rnp->qsmask); |
2051 | raw_spin_unlock_irq_rcu_node(rnp); | 1996 | raw_spin_unlock_irq_rcu_node(rnp); |
2052 | cond_resched_rcu_qs(); | 1997 | cond_resched_tasks_rcu_qs(); |
2053 | WRITE_ONCE(rsp->gp_activity, jiffies); | 1998 | WRITE_ONCE(rsp->gp_activity, jiffies); |
2054 | } | 1999 | } |
2055 | 2000 | ||
@@ -2108,7 +2053,6 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) | |||
2108 | { | 2053 | { |
2109 | unsigned long gp_duration; | 2054 | unsigned long gp_duration; |
2110 | bool needgp = false; | 2055 | bool needgp = false; |
2111 | int nocb = 0; | ||
2112 | struct rcu_data *rdp; | 2056 | struct rcu_data *rdp; |
2113 | struct rcu_node *rnp = rcu_get_root(rsp); | 2057 | struct rcu_node *rnp = rcu_get_root(rsp); |
2114 | struct swait_queue_head *sq; | 2058 | struct swait_queue_head *sq; |
@@ -2147,31 +2091,35 @@ static void rcu_gp_cleanup(struct rcu_state *rsp) | |||
2147 | if (rnp == rdp->mynode) | 2091 | if (rnp == rdp->mynode) |
2148 | needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; | 2092 | needgp = __note_gp_changes(rsp, rnp, rdp) || needgp; |
2149 | /* smp_mb() provided by prior unlock-lock pair. */ | 2093 | /* smp_mb() provided by prior unlock-lock pair. */ |
2150 | nocb += rcu_future_gp_cleanup(rsp, rnp); | 2094 | needgp = rcu_future_gp_cleanup(rsp, rnp) || needgp; |
2151 | sq = rcu_nocb_gp_get(rnp); | 2095 | sq = rcu_nocb_gp_get(rnp); |
2152 | raw_spin_unlock_irq_rcu_node(rnp); | 2096 | raw_spin_unlock_irq_rcu_node(rnp); |
2153 | rcu_nocb_gp_cleanup(sq); | 2097 | rcu_nocb_gp_cleanup(sq); |
2154 | cond_resched_rcu_qs(); | 2098 | cond_resched_tasks_rcu_qs(); |
2155 | WRITE_ONCE(rsp->gp_activity, jiffies); | 2099 | WRITE_ONCE(rsp->gp_activity, jiffies); |
2156 | rcu_gp_slow(rsp, gp_cleanup_delay); | 2100 | rcu_gp_slow(rsp, gp_cleanup_delay); |
2157 | } | 2101 | } |
2158 | rnp = rcu_get_root(rsp); | 2102 | rnp = rcu_get_root(rsp); |
2159 | raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */ | 2103 | raw_spin_lock_irq_rcu_node(rnp); /* Order GP before ->completed update. */ |
2160 | rcu_nocb_gp_set(rnp, nocb); | ||
2161 | 2104 | ||
2162 | /* Declare grace period done. */ | 2105 | /* Declare grace period done. */ |
2163 | WRITE_ONCE(rsp->completed, rsp->gpnum); | 2106 | WRITE_ONCE(rsp->completed, rsp->gpnum); |
2164 | trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); | 2107 | trace_rcu_grace_period(rsp->name, rsp->completed, TPS("end")); |
2165 | rsp->gp_state = RCU_GP_IDLE; | 2108 | rsp->gp_state = RCU_GP_IDLE; |
2109 | /* Check for GP requests since above loop. */ | ||
2166 | rdp = this_cpu_ptr(rsp->rda); | 2110 | rdp = this_cpu_ptr(rsp->rda); |
2111 | if (need_any_future_gp(rnp)) { | ||
2112 | trace_rcu_this_gp(rnp, rdp, rsp->completed - 1, | ||
2113 | TPS("CleanupMore")); | ||
2114 | needgp = true; | ||
2115 | } | ||
2167 | /* Advance CBs to reduce false positives below. */ | 2116 | /* Advance CBs to reduce false positives below. */ |
2168 | needgp = rcu_advance_cbs(rsp, rnp, rdp) || needgp; | 2117 | if (!rcu_accelerate_cbs(rsp, rnp, rdp) && needgp) { |
2169 | if (needgp || cpu_needs_another_gp(rsp, rdp)) { | ||
2170 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); | 2118 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); |
2171 | trace_rcu_grace_period(rsp->name, | 2119 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), |
2172 | READ_ONCE(rsp->gpnum), | ||
2173 | TPS("newreq")); | 2120 | TPS("newreq")); |
2174 | } | 2121 | } |
2122 | WRITE_ONCE(rsp->gp_flags, rsp->gp_flags & RCU_GP_FLAG_INIT); | ||
2175 | raw_spin_unlock_irq_rcu_node(rnp); | 2123 | raw_spin_unlock_irq_rcu_node(rnp); |
2176 | } | 2124 | } |
2177 | 2125 | ||
@@ -2202,7 +2150,7 @@ static int __noreturn rcu_gp_kthread(void *arg) | |||
2202 | /* Locking provides needed memory barrier. */ | 2150 | /* Locking provides needed memory barrier. */ |
2203 | if (rcu_gp_init(rsp)) | 2151 | if (rcu_gp_init(rsp)) |
2204 | break; | 2152 | break; |
2205 | cond_resched_rcu_qs(); | 2153 | cond_resched_tasks_rcu_qs(); |
2206 | WRITE_ONCE(rsp->gp_activity, jiffies); | 2154 | WRITE_ONCE(rsp->gp_activity, jiffies); |
2207 | WARN_ON(signal_pending(current)); | 2155 | WARN_ON(signal_pending(current)); |
2208 | trace_rcu_grace_period(rsp->name, | 2156 | trace_rcu_grace_period(rsp->name, |
@@ -2247,7 +2195,7 @@ static int __noreturn rcu_gp_kthread(void *arg) | |||
2247 | trace_rcu_grace_period(rsp->name, | 2195 | trace_rcu_grace_period(rsp->name, |
2248 | READ_ONCE(rsp->gpnum), | 2196 | READ_ONCE(rsp->gpnum), |
2249 | TPS("fqsend")); | 2197 | TPS("fqsend")); |
2250 | cond_resched_rcu_qs(); | 2198 | cond_resched_tasks_rcu_qs(); |
2251 | WRITE_ONCE(rsp->gp_activity, jiffies); | 2199 | WRITE_ONCE(rsp->gp_activity, jiffies); |
2252 | ret = 0; /* Force full wait till next FQS. */ | 2200 | ret = 0; /* Force full wait till next FQS. */ |
2253 | j = jiffies_till_next_fqs; | 2201 | j = jiffies_till_next_fqs; |
@@ -2260,7 +2208,7 @@ static int __noreturn rcu_gp_kthread(void *arg) | |||
2260 | } | 2208 | } |
2261 | } else { | 2209 | } else { |
2262 | /* Deal with stray signal. */ | 2210 | /* Deal with stray signal. */ |
2263 | cond_resched_rcu_qs(); | 2211 | cond_resched_tasks_rcu_qs(); |
2264 | WRITE_ONCE(rsp->gp_activity, jiffies); | 2212 | WRITE_ONCE(rsp->gp_activity, jiffies); |
2265 | WARN_ON(signal_pending(current)); | 2213 | WARN_ON(signal_pending(current)); |
2266 | trace_rcu_grace_period(rsp->name, | 2214 | trace_rcu_grace_period(rsp->name, |
@@ -2283,71 +2231,6 @@ static int __noreturn rcu_gp_kthread(void *arg) | |||
2283 | } | 2231 | } |
2284 | 2232 | ||
2285 | /* | 2233 | /* |
2286 | * Start a new RCU grace period if warranted, re-initializing the hierarchy | ||
2287 | * in preparation for detecting the next grace period. The caller must hold | ||
2288 | * the root node's ->lock and hard irqs must be disabled. | ||
2289 | * | ||
2290 | * Note that it is legal for a dying CPU (which is marked as offline) to | ||
2291 | * invoke this function. This can happen when the dying CPU reports its | ||
2292 | * quiescent state. | ||
2293 | * | ||
2294 | * Returns true if the grace-period kthread must be awakened. | ||
2295 | */ | ||
2296 | static bool | ||
2297 | rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, | ||
2298 | struct rcu_data *rdp) | ||
2299 | { | ||
2300 | raw_lockdep_assert_held_rcu_node(rnp); | ||
2301 | if (!rsp->gp_kthread || !cpu_needs_another_gp(rsp, rdp)) { | ||
2302 | /* | ||
2303 | * Either we have not yet spawned the grace-period | ||
2304 | * task, this CPU does not need another grace period, | ||
2305 | * or a grace period is already in progress. | ||
2306 | * Either way, don't start a new grace period. | ||
2307 | */ | ||
2308 | return false; | ||
2309 | } | ||
2310 | WRITE_ONCE(rsp->gp_flags, RCU_GP_FLAG_INIT); | ||
2311 | trace_rcu_grace_period(rsp->name, READ_ONCE(rsp->gpnum), | ||
2312 | TPS("newreq")); | ||
2313 | |||
2314 | /* | ||
2315 | * We can't do wakeups while holding the rnp->lock, as that | ||
2316 | * could cause possible deadlocks with the rq->lock. Defer | ||
2317 | * the wakeup to our caller. | ||
2318 | */ | ||
2319 | return true; | ||
2320 | } | ||
2321 | |||
2322 | /* | ||
2323 | * Similar to rcu_start_gp_advanced(), but also advance the calling CPU's | ||
2324 | * callbacks. Note that rcu_start_gp_advanced() cannot do this because it | ||
2325 | * is invoked indirectly from rcu_advance_cbs(), which would result in | ||
2326 | * endless recursion -- or would do so if it wasn't for the self-deadlock | ||
2327 | * that is encountered beforehand. | ||
2328 | * | ||
2329 | * Returns true if the grace-period kthread needs to be awakened. | ||
2330 | */ | ||
2331 | static bool rcu_start_gp(struct rcu_state *rsp) | ||
2332 | { | ||
2333 | struct rcu_data *rdp = this_cpu_ptr(rsp->rda); | ||
2334 | struct rcu_node *rnp = rcu_get_root(rsp); | ||
2335 | bool ret = false; | ||
2336 | |||
2337 | /* | ||
2338 | * If there is no grace period in progress right now, any | ||
2339 | * callbacks we have up to this point will be satisfied by the | ||
2340 | * next grace period. Also, advancing the callbacks reduces the | ||
2341 | * probability of false positives from cpu_needs_another_gp() | ||
2342 | * resulting in pointless grace periods. So, advance callbacks | ||
2343 | * then start the grace period! | ||
2344 | */ | ||
2345 | ret = rcu_advance_cbs(rsp, rnp, rdp) || ret; | ||
2346 | ret = rcu_start_gp_advanced(rsp, rnp, rdp) || ret; | ||
2347 | return ret; | ||
2348 | } | ||
2349 | |||
2350 | /* | ||
2351 | * Report a full set of quiescent states to the specified rcu_state data | 2234 | * Report a full set of quiescent states to the specified rcu_state data |
2352 | * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period | 2235 | * structure. Invoke rcu_gp_kthread_wake() to awaken the grace-period |
2353 | * kthread if another grace period is required. Whether we wake | 2236 | * kthread if another grace period is required. Whether we wake |
@@ -2398,7 +2281,7 @@ rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, | |||
2398 | return; | 2281 | return; |
2399 | } | 2282 | } |
2400 | WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ | 2283 | WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */ |
2401 | WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1 && | 2284 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp) && |
2402 | rcu_preempt_blocked_readers_cgp(rnp)); | 2285 | rcu_preempt_blocked_readers_cgp(rnp)); |
2403 | rnp->qsmask &= ~mask; | 2286 | rnp->qsmask &= ~mask; |
2404 | trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum, | 2287 | trace_rcu_quiescent_state_report(rsp->name, rnp->gpnum, |
@@ -2782,7 +2665,7 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *rsp)) | |||
2782 | struct rcu_node *rnp; | 2665 | struct rcu_node *rnp; |
2783 | 2666 | ||
2784 | rcu_for_each_leaf_node(rsp, rnp) { | 2667 | rcu_for_each_leaf_node(rsp, rnp) { |
2785 | cond_resched_rcu_qs(); | 2668 | cond_resched_tasks_rcu_qs(); |
2786 | mask = 0; | 2669 | mask = 0; |
2787 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | 2670 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
2788 | if (rnp->qsmask == 0) { | 2671 | if (rnp->qsmask == 0) { |
@@ -2874,22 +2757,27 @@ __rcu_process_callbacks(struct rcu_state *rsp) | |||
2874 | unsigned long flags; | 2757 | unsigned long flags; |
2875 | bool needwake; | 2758 | bool needwake; |
2876 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); | 2759 | struct rcu_data *rdp = raw_cpu_ptr(rsp->rda); |
2760 | struct rcu_node *rnp; | ||
2877 | 2761 | ||
2878 | WARN_ON_ONCE(!rdp->beenonline); | 2762 | WARN_ON_ONCE(!rdp->beenonline); |
2879 | 2763 | ||
2880 | /* Update RCU state based on any recent quiescent states. */ | 2764 | /* Update RCU state based on any recent quiescent states. */ |
2881 | rcu_check_quiescent_state(rsp, rdp); | 2765 | rcu_check_quiescent_state(rsp, rdp); |
2882 | 2766 | ||
2883 | /* Does this CPU require a not-yet-started grace period? */ | 2767 | /* No grace period and unregistered callbacks? */ |
2884 | local_irq_save(flags); | 2768 | if (!rcu_gp_in_progress(rsp) && |
2885 | if (cpu_needs_another_gp(rsp, rdp)) { | 2769 | rcu_segcblist_is_enabled(&rdp->cblist)) { |
2886 | raw_spin_lock_rcu_node(rcu_get_root(rsp)); /* irqs disabled. */ | 2770 | local_irq_save(flags); |
2887 | needwake = rcu_start_gp(rsp); | 2771 | if (rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) { |
2888 | raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(rsp), flags); | 2772 | local_irq_restore(flags); |
2889 | if (needwake) | 2773 | } else { |
2890 | rcu_gp_kthread_wake(rsp); | 2774 | rnp = rdp->mynode; |
2891 | } else { | 2775 | raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ |
2892 | local_irq_restore(flags); | 2776 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
2777 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
2778 | if (needwake) | ||
2779 | rcu_gp_kthread_wake(rsp); | ||
2780 | } | ||
2893 | } | 2781 | } |
2894 | 2782 | ||
2895 | /* If there are callbacks ready, invoke them. */ | 2783 | /* If there are callbacks ready, invoke them. */ |
@@ -2973,11 +2861,11 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp, | |||
2973 | 2861 | ||
2974 | /* Start a new grace period if one not already started. */ | 2862 | /* Start a new grace period if one not already started. */ |
2975 | if (!rcu_gp_in_progress(rsp)) { | 2863 | if (!rcu_gp_in_progress(rsp)) { |
2976 | struct rcu_node *rnp_root = rcu_get_root(rsp); | 2864 | struct rcu_node *rnp = rdp->mynode; |
2977 | 2865 | ||
2978 | raw_spin_lock_rcu_node(rnp_root); | 2866 | raw_spin_lock_rcu_node(rnp); |
2979 | needwake = rcu_start_gp(rsp); | 2867 | needwake = rcu_accelerate_cbs(rsp, rnp, rdp); |
2980 | raw_spin_unlock_rcu_node(rnp_root); | 2868 | raw_spin_unlock_rcu_node(rnp); |
2981 | if (needwake) | 2869 | if (needwake) |
2982 | rcu_gp_kthread_wake(rsp); | 2870 | rcu_gp_kthread_wake(rsp); |
2983 | } else { | 2871 | } else { |
@@ -3368,7 +3256,9 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) | |||
3368 | return 1; | 3256 | return 1; |
3369 | 3257 | ||
3370 | /* Has RCU gone idle with this CPU needing another grace period? */ | 3258 | /* Has RCU gone idle with this CPU needing another grace period? */ |
3371 | if (cpu_needs_another_gp(rsp, rdp)) | 3259 | if (!rcu_gp_in_progress(rsp) && |
3260 | rcu_segcblist_is_enabled(&rdp->cblist) && | ||
3261 | !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) | ||
3372 | return 1; | 3262 | return 1; |
3373 | 3263 | ||
3374 | /* Has another RCU grace period completed? */ | 3264 | /* Has another RCU grace period completed? */ |
@@ -3775,6 +3665,8 @@ int rcutree_dead_cpu(unsigned int cpu) | |||
3775 | return 0; | 3665 | return 0; |
3776 | } | 3666 | } |
3777 | 3667 | ||
3668 | static DEFINE_PER_CPU(int, rcu_cpu_started); | ||
3669 | |||
3778 | /* | 3670 | /* |
3779 | * Mark the specified CPU as being online so that subsequent grace periods | 3671 | * Mark the specified CPU as being online so that subsequent grace periods |
3780 | * (both expedited and normal) will wait on it. Note that this means that | 3672 | * (both expedited and normal) will wait on it. Note that this means that |
@@ -3796,6 +3688,11 @@ void rcu_cpu_starting(unsigned int cpu) | |||
3796 | struct rcu_node *rnp; | 3688 | struct rcu_node *rnp; |
3797 | struct rcu_state *rsp; | 3689 | struct rcu_state *rsp; |
3798 | 3690 | ||
3691 | if (per_cpu(rcu_cpu_started, cpu)) | ||
3692 | return; | ||
3693 | |||
3694 | per_cpu(rcu_cpu_started, cpu) = 1; | ||
3695 | |||
3799 | for_each_rcu_flavor(rsp) { | 3696 | for_each_rcu_flavor(rsp) { |
3800 | rdp = per_cpu_ptr(rsp->rda, cpu); | 3697 | rdp = per_cpu_ptr(rsp->rda, cpu); |
3801 | rnp = rdp->mynode; | 3698 | rnp = rdp->mynode; |
@@ -3852,6 +3749,8 @@ void rcu_report_dead(unsigned int cpu) | |||
3852 | preempt_enable(); | 3749 | preempt_enable(); |
3853 | for_each_rcu_flavor(rsp) | 3750 | for_each_rcu_flavor(rsp) |
3854 | rcu_cleanup_dying_idle_cpu(cpu, rsp); | 3751 | rcu_cleanup_dying_idle_cpu(cpu, rsp); |
3752 | |||
3753 | per_cpu(rcu_cpu_started, cpu) = 0; | ||
3855 | } | 3754 | } |
3856 | 3755 | ||
3857 | /* Migrate the dead CPU's callbacks to the current CPU. */ | 3756 | /* Migrate the dead CPU's callbacks to the current CPU. */ |
@@ -3861,6 +3760,7 @@ static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp) | |||
3861 | struct rcu_data *my_rdp; | 3760 | struct rcu_data *my_rdp; |
3862 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | 3761 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
3863 | struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); | 3762 | struct rcu_node *rnp_root = rcu_get_root(rdp->rsp); |
3763 | bool needwake; | ||
3864 | 3764 | ||
3865 | if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist)) | 3765 | if (rcu_is_nocb_cpu(cpu) || rcu_segcblist_empty(&rdp->cblist)) |
3866 | return; /* No callbacks to migrate. */ | 3766 | return; /* No callbacks to migrate. */ |
@@ -3872,12 +3772,15 @@ static void rcu_migrate_callbacks(int cpu, struct rcu_state *rsp) | |||
3872 | return; | 3772 | return; |
3873 | } | 3773 | } |
3874 | raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ | 3774 | raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */ |
3875 | rcu_advance_cbs(rsp, rnp_root, rdp); /* Leverage recent GPs. */ | 3775 | /* Leverage recent GPs and set GP for new callbacks. */ |
3876 | rcu_advance_cbs(rsp, rnp_root, my_rdp); /* Assign GP to pending CBs. */ | 3776 | needwake = rcu_advance_cbs(rsp, rnp_root, rdp) || |
3777 | rcu_advance_cbs(rsp, rnp_root, my_rdp); | ||
3877 | rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); | 3778 | rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); |
3878 | WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != | 3779 | WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != |
3879 | !rcu_segcblist_n_cbs(&my_rdp->cblist)); | 3780 | !rcu_segcblist_n_cbs(&my_rdp->cblist)); |
3880 | raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags); | 3781 | raw_spin_unlock_irqrestore_rcu_node(rnp_root, flags); |
3782 | if (needwake) | ||
3783 | rcu_gp_kthread_wake(rsp); | ||
3881 | WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || | 3784 | WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || |
3882 | !rcu_segcblist_empty(&rdp->cblist), | 3785 | !rcu_segcblist_empty(&rdp->cblist), |
3883 | "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n", | 3786 | "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n", |
@@ -4056,7 +3959,7 @@ static void __init rcu_init_one(struct rcu_state *rsp) | |||
4056 | 3959 | ||
4057 | init_swait_queue_head(&rsp->gp_wq); | 3960 | init_swait_queue_head(&rsp->gp_wq); |
4058 | init_swait_queue_head(&rsp->expedited_wq); | 3961 | init_swait_queue_head(&rsp->expedited_wq); |
4059 | rnp = rsp->level[rcu_num_lvls - 1]; | 3962 | rnp = rcu_first_leaf_node(rsp); |
4060 | for_each_possible_cpu(i) { | 3963 | for_each_possible_cpu(i) { |
4061 | while (i > rnp->grphi) | 3964 | while (i > rnp->grphi) |
4062 | rnp++; | 3965 | rnp++; |
@@ -4168,6 +4071,7 @@ static void __init rcu_dump_rcu_node_tree(struct rcu_state *rsp) | |||
4168 | } | 4071 | } |
4169 | 4072 | ||
4170 | struct workqueue_struct *rcu_gp_wq; | 4073 | struct workqueue_struct *rcu_gp_wq; |
4074 | struct workqueue_struct *rcu_par_gp_wq; | ||
4171 | 4075 | ||
4172 | void __init rcu_init(void) | 4076 | void __init rcu_init(void) |
4173 | { | 4077 | { |
@@ -4199,6 +4103,8 @@ void __init rcu_init(void) | |||
4199 | /* Create workqueue for expedited GPs and for Tree SRCU. */ | 4103 | /* Create workqueue for expedited GPs and for Tree SRCU. */ |
4200 | rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0); | 4104 | rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0); |
4201 | WARN_ON(!rcu_gp_wq); | 4105 | WARN_ON(!rcu_gp_wq); |
4106 | rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0); | ||
4107 | WARN_ON(!rcu_par_gp_wq); | ||
4202 | } | 4108 | } |
4203 | 4109 | ||
4204 | #include "tree_exp.h" | 4110 | #include "tree_exp.h" |
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index f491ab4f2e8e..78e051dffc5b 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
@@ -58,6 +58,14 @@ struct rcu_dynticks { | |||
58 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ | 58 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ |
59 | }; | 59 | }; |
60 | 60 | ||
61 | /* Communicate arguments to a workqueue handler. */ | ||
62 | struct rcu_exp_work { | ||
63 | smp_call_func_t rew_func; | ||
64 | struct rcu_state *rew_rsp; | ||
65 | unsigned long rew_s; | ||
66 | struct work_struct rew_work; | ||
67 | }; | ||
68 | |||
61 | /* RCU's kthread states for tracing. */ | 69 | /* RCU's kthread states for tracing. */ |
62 | #define RCU_KTHREAD_STOPPED 0 | 70 | #define RCU_KTHREAD_STOPPED 0 |
63 | #define RCU_KTHREAD_RUNNING 1 | 71 | #define RCU_KTHREAD_RUNNING 1 |
@@ -150,15 +158,32 @@ struct rcu_node { | |||
150 | struct swait_queue_head nocb_gp_wq[2]; | 158 | struct swait_queue_head nocb_gp_wq[2]; |
151 | /* Place for rcu_nocb_kthread() to wait GP. */ | 159 | /* Place for rcu_nocb_kthread() to wait GP. */ |
152 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ | 160 | #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ |
153 | int need_future_gp[2]; | 161 | u8 need_future_gp[4]; /* Counts of upcoming GP requests. */ |
154 | /* Counts of upcoming no-CB GP requests. */ | ||
155 | raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; | 162 | raw_spinlock_t fqslock ____cacheline_internodealigned_in_smp; |
156 | 163 | ||
157 | spinlock_t exp_lock ____cacheline_internodealigned_in_smp; | 164 | spinlock_t exp_lock ____cacheline_internodealigned_in_smp; |
158 | unsigned long exp_seq_rq; | 165 | unsigned long exp_seq_rq; |
159 | wait_queue_head_t exp_wq[4]; | 166 | wait_queue_head_t exp_wq[4]; |
167 | struct rcu_exp_work rew; | ||
168 | bool exp_need_flush; /* Need to flush workitem? */ | ||
160 | } ____cacheline_internodealigned_in_smp; | 169 | } ____cacheline_internodealigned_in_smp; |
161 | 170 | ||
171 | /* Accessors for ->need_future_gp[] array. */ | ||
172 | #define need_future_gp_mask() \ | ||
173 | (ARRAY_SIZE(((struct rcu_node *)NULL)->need_future_gp) - 1) | ||
174 | #define need_future_gp_element(rnp, c) \ | ||
175 | ((rnp)->need_future_gp[(c) & need_future_gp_mask()]) | ||
176 | #define need_any_future_gp(rnp) \ | ||
177 | ({ \ | ||
178 | int __i; \ | ||
179 | bool __nonzero = false; \ | ||
180 | \ | ||
181 | for (__i = 0; __i < ARRAY_SIZE((rnp)->need_future_gp); __i++) \ | ||
182 | __nonzero = __nonzero || \ | ||
183 | READ_ONCE((rnp)->need_future_gp[__i]); \ | ||
184 | __nonzero; \ | ||
185 | }) | ||
186 | |||
162 | /* | 187 | /* |
163 | * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and | 188 | * Bitmasks in an rcu_node cover the interval [grplo, grphi] of CPU IDs, and |
164 | * are indexed relative to this interval rather than the global CPU ID space. | 189 | * are indexed relative to this interval rather than the global CPU ID space. |
@@ -224,10 +249,6 @@ struct rcu_data { | |||
224 | #ifdef CONFIG_RCU_FAST_NO_HZ | 249 | #ifdef CONFIG_RCU_FAST_NO_HZ |
225 | struct rcu_head oom_head; | 250 | struct rcu_head oom_head; |
226 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ | 251 | #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */ |
227 | atomic_long_t exp_workdone0; /* # done by workqueue. */ | ||
228 | atomic_long_t exp_workdone1; /* # done by others #1. */ | ||
229 | atomic_long_t exp_workdone2; /* # done by others #2. */ | ||
230 | atomic_long_t exp_workdone3; /* # done by others #3. */ | ||
231 | int exp_dynticks_snap; /* Double-check need for IPI. */ | 252 | int exp_dynticks_snap; /* Double-check need for IPI. */ |
232 | 253 | ||
233 | /* 6) Callback offloading. */ | 254 | /* 6) Callback offloading. */ |
@@ -408,7 +429,6 @@ extern struct rcu_state rcu_preempt_state; | |||
408 | #endif /* #ifdef CONFIG_PREEMPT_RCU */ | 429 | #endif /* #ifdef CONFIG_PREEMPT_RCU */ |
409 | 430 | ||
410 | int rcu_dynticks_snap(struct rcu_dynticks *rdtp); | 431 | int rcu_dynticks_snap(struct rcu_dynticks *rdtp); |
411 | bool rcu_eqs_special_set(int cpu); | ||
412 | 432 | ||
413 | #ifdef CONFIG_RCU_BOOST | 433 | #ifdef CONFIG_RCU_BOOST |
414 | DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); | 434 | DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); |
@@ -438,7 +458,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); | |||
438 | static void invoke_rcu_callbacks_kthread(void); | 458 | static void invoke_rcu_callbacks_kthread(void); |
439 | static bool rcu_is_callbacks_kthread(void); | 459 | static bool rcu_is_callbacks_kthread(void); |
440 | #ifdef CONFIG_RCU_BOOST | 460 | #ifdef CONFIG_RCU_BOOST |
441 | static void rcu_preempt_do_callbacks(void); | ||
442 | static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, | 461 | static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, |
443 | struct rcu_node *rnp); | 462 | struct rcu_node *rnp); |
444 | #endif /* #ifdef CONFIG_RCU_BOOST */ | 463 | #endif /* #ifdef CONFIG_RCU_BOOST */ |
@@ -454,7 +473,6 @@ static void print_cpu_stall_info_end(void); | |||
454 | static void zero_cpu_stall_ticks(struct rcu_data *rdp); | 473 | static void zero_cpu_stall_ticks(struct rcu_data *rdp); |
455 | static void increment_cpu_stall_ticks(void); | 474 | static void increment_cpu_stall_ticks(void); |
456 | static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu); | 475 | static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu); |
457 | static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq); | ||
458 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); | 476 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); |
459 | static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); | 477 | static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); |
460 | static void rcu_init_one_nocb(struct rcu_node *rnp); | 478 | static void rcu_init_one_nocb(struct rcu_node *rnp); |
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index f72eefab8543..d40708e8c5d6 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h | |||
@@ -20,6 +20,8 @@ | |||
20 | * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 20 | * Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> |
21 | */ | 21 | */ |
22 | 22 | ||
23 | #include <linux/lockdep.h> | ||
24 | |||
23 | /* | 25 | /* |
24 | * Record the start of an expedited grace period. | 26 | * Record the start of an expedited grace period. |
25 | */ | 27 | */ |
@@ -154,15 +156,35 @@ static void __maybe_unused sync_exp_reset_tree(struct rcu_state *rsp) | |||
154 | * for the current expedited grace period. Works only for preemptible | 156 | * for the current expedited grace period. Works only for preemptible |
155 | * RCU -- other RCU implementation use other means. | 157 | * RCU -- other RCU implementation use other means. |
156 | * | 158 | * |
157 | * Caller must hold the rcu_state's exp_mutex. | 159 | * Caller must hold the specificed rcu_node structure's ->lock |
158 | */ | 160 | */ |
159 | static bool sync_rcu_preempt_exp_done(struct rcu_node *rnp) | 161 | static bool sync_rcu_preempt_exp_done(struct rcu_node *rnp) |
160 | { | 162 | { |
163 | raw_lockdep_assert_held_rcu_node(rnp); | ||
164 | |||
161 | return rnp->exp_tasks == NULL && | 165 | return rnp->exp_tasks == NULL && |
162 | READ_ONCE(rnp->expmask) == 0; | 166 | READ_ONCE(rnp->expmask) == 0; |
163 | } | 167 | } |
164 | 168 | ||
165 | /* | 169 | /* |
170 | * Like sync_rcu_preempt_exp_done(), but this function assumes the caller | ||
171 | * doesn't hold the rcu_node's ->lock, and will acquire and release the lock | ||
172 | * itself | ||
173 | */ | ||
174 | static bool sync_rcu_preempt_exp_done_unlocked(struct rcu_node *rnp) | ||
175 | { | ||
176 | unsigned long flags; | ||
177 | bool ret; | ||
178 | |||
179 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | ||
180 | ret = sync_rcu_preempt_exp_done(rnp); | ||
181 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
182 | |||
183 | return ret; | ||
184 | } | ||
185 | |||
186 | |||
187 | /* | ||
166 | * Report the exit from RCU read-side critical section for the last task | 188 | * Report the exit from RCU read-side critical section for the last task |
167 | * that queued itself during or before the current expedited preemptible-RCU | 189 | * that queued itself during or before the current expedited preemptible-RCU |
168 | * grace period. This event is reported either to the rcu_node structure on | 190 | * grace period. This event is reported either to the rcu_node structure on |
@@ -170,8 +192,7 @@ static bool sync_rcu_preempt_exp_done(struct rcu_node *rnp) | |||
170 | * recursively up the tree. (Calm down, calm down, we do the recursion | 192 | * recursively up the tree. (Calm down, calm down, we do the recursion |
171 | * iteratively!) | 193 | * iteratively!) |
172 | * | 194 | * |
173 | * Caller must hold the rcu_state's exp_mutex and the specified rcu_node | 195 | * Caller must hold the specified rcu_node structure's ->lock. |
174 | * structure's ->lock. | ||
175 | */ | 196 | */ |
176 | static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, | 197 | static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, |
177 | bool wake, unsigned long flags) | 198 | bool wake, unsigned long flags) |
@@ -207,8 +228,6 @@ static void __rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, | |||
207 | /* | 228 | /* |
208 | * Report expedited quiescent state for specified node. This is a | 229 | * Report expedited quiescent state for specified node. This is a |
209 | * lock-acquisition wrapper function for __rcu_report_exp_rnp(). | 230 | * lock-acquisition wrapper function for __rcu_report_exp_rnp(). |
210 | * | ||
211 | * Caller must hold the rcu_state's exp_mutex. | ||
212 | */ | 231 | */ |
213 | static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp, | 232 | static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp, |
214 | struct rcu_node *rnp, bool wake) | 233 | struct rcu_node *rnp, bool wake) |
@@ -221,8 +240,7 @@ static void __maybe_unused rcu_report_exp_rnp(struct rcu_state *rsp, | |||
221 | 240 | ||
222 | /* | 241 | /* |
223 | * Report expedited quiescent state for multiple CPUs, all covered by the | 242 | * Report expedited quiescent state for multiple CPUs, all covered by the |
224 | * specified leaf rcu_node structure. Caller must hold the rcu_state's | 243 | * specified leaf rcu_node structure. |
225 | * exp_mutex. | ||
226 | */ | 244 | */ |
227 | static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp, | 245 | static void rcu_report_exp_cpu_mult(struct rcu_state *rsp, struct rcu_node *rnp, |
228 | unsigned long mask, bool wake) | 246 | unsigned long mask, bool wake) |
@@ -248,14 +266,12 @@ static void rcu_report_exp_rdp(struct rcu_state *rsp, struct rcu_data *rdp, | |||
248 | } | 266 | } |
249 | 267 | ||
250 | /* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */ | 268 | /* Common code for synchronize_{rcu,sched}_expedited() work-done checking. */ |
251 | static bool sync_exp_work_done(struct rcu_state *rsp, atomic_long_t *stat, | 269 | static bool sync_exp_work_done(struct rcu_state *rsp, unsigned long s) |
252 | unsigned long s) | ||
253 | { | 270 | { |
254 | if (rcu_exp_gp_seq_done(rsp, s)) { | 271 | if (rcu_exp_gp_seq_done(rsp, s)) { |
255 | trace_rcu_exp_grace_period(rsp->name, s, TPS("done")); | 272 | trace_rcu_exp_grace_period(rsp->name, s, TPS("done")); |
256 | /* Ensure test happens before caller kfree(). */ | 273 | /* Ensure test happens before caller kfree(). */ |
257 | smp_mb__before_atomic(); /* ^^^ */ | 274 | smp_mb__before_atomic(); /* ^^^ */ |
258 | atomic_long_inc(stat); | ||
259 | return true; | 275 | return true; |
260 | } | 276 | } |
261 | return false; | 277 | return false; |
@@ -289,7 +305,7 @@ static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s) | |||
289 | * promoting locality and is not strictly needed for correctness. | 305 | * promoting locality and is not strictly needed for correctness. |
290 | */ | 306 | */ |
291 | for (; rnp != NULL; rnp = rnp->parent) { | 307 | for (; rnp != NULL; rnp = rnp->parent) { |
292 | if (sync_exp_work_done(rsp, &rdp->exp_workdone1, s)) | 308 | if (sync_exp_work_done(rsp, s)) |
293 | return true; | 309 | return true; |
294 | 310 | ||
295 | /* Work not done, either wait here or go up. */ | 311 | /* Work not done, either wait here or go up. */ |
@@ -302,8 +318,7 @@ static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s) | |||
302 | rnp->grplo, rnp->grphi, | 318 | rnp->grplo, rnp->grphi, |
303 | TPS("wait")); | 319 | TPS("wait")); |
304 | wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], | 320 | wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], |
305 | sync_exp_work_done(rsp, | 321 | sync_exp_work_done(rsp, s)); |
306 | &rdp->exp_workdone2, s)); | ||
307 | return true; | 322 | return true; |
308 | } | 323 | } |
309 | rnp->exp_seq_rq = s; /* Followers can wait on us. */ | 324 | rnp->exp_seq_rq = s; /* Followers can wait on us. */ |
@@ -313,7 +328,7 @@ static bool exp_funnel_lock(struct rcu_state *rsp, unsigned long s) | |||
313 | } | 328 | } |
314 | mutex_lock(&rsp->exp_mutex); | 329 | mutex_lock(&rsp->exp_mutex); |
315 | fastpath: | 330 | fastpath: |
316 | if (sync_exp_work_done(rsp, &rdp->exp_workdone3, s)) { | 331 | if (sync_exp_work_done(rsp, s)) { |
317 | mutex_unlock(&rsp->exp_mutex); | 332 | mutex_unlock(&rsp->exp_mutex); |
318 | return true; | 333 | return true; |
319 | } | 334 | } |
@@ -362,93 +377,129 @@ static void sync_sched_exp_online_cleanup(int cpu) | |||
362 | } | 377 | } |
363 | 378 | ||
364 | /* | 379 | /* |
365 | * Select the nodes that the upcoming expedited grace period needs | 380 | * Select the CPUs within the specified rcu_node that the upcoming |
366 | * to wait for. | 381 | * expedited grace period needs to wait for. |
367 | */ | 382 | */ |
368 | static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, | 383 | static void sync_rcu_exp_select_node_cpus(struct work_struct *wp) |
369 | smp_call_func_t func) | ||
370 | { | 384 | { |
371 | int cpu; | 385 | int cpu; |
372 | unsigned long flags; | 386 | unsigned long flags; |
387 | smp_call_func_t func; | ||
373 | unsigned long mask_ofl_test; | 388 | unsigned long mask_ofl_test; |
374 | unsigned long mask_ofl_ipi; | 389 | unsigned long mask_ofl_ipi; |
375 | int ret; | 390 | int ret; |
376 | struct rcu_node *rnp; | 391 | struct rcu_exp_work *rewp = |
377 | 392 | container_of(wp, struct rcu_exp_work, rew_work); | |
378 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("reset")); | 393 | struct rcu_node *rnp = container_of(rewp, struct rcu_node, rew); |
379 | sync_exp_reset_tree(rsp); | 394 | struct rcu_state *rsp = rewp->rew_rsp; |
380 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("select")); | ||
381 | rcu_for_each_leaf_node(rsp, rnp) { | ||
382 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | ||
383 | 395 | ||
384 | /* Each pass checks a CPU for identity, offline, and idle. */ | 396 | func = rewp->rew_func; |
385 | mask_ofl_test = 0; | 397 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
386 | for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { | ||
387 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); | ||
388 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | ||
389 | struct rcu_dynticks *rdtp = per_cpu_ptr(&rcu_dynticks, cpu); | ||
390 | int snap; | ||
391 | 398 | ||
392 | if (raw_smp_processor_id() == cpu || | 399 | /* Each pass checks a CPU for identity, offline, and idle. */ |
393 | !(rnp->qsmaskinitnext & mask)) { | 400 | mask_ofl_test = 0; |
401 | for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { | ||
402 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); | ||
403 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | ||
404 | struct rcu_dynticks *rdtp = per_cpu_ptr(&rcu_dynticks, cpu); | ||
405 | int snap; | ||
406 | |||
407 | if (raw_smp_processor_id() == cpu || | ||
408 | !(rnp->qsmaskinitnext & mask)) { | ||
409 | mask_ofl_test |= mask; | ||
410 | } else { | ||
411 | snap = rcu_dynticks_snap(rdtp); | ||
412 | if (rcu_dynticks_in_eqs(snap)) | ||
394 | mask_ofl_test |= mask; | 413 | mask_ofl_test |= mask; |
395 | } else { | 414 | else |
396 | snap = rcu_dynticks_snap(rdtp); | 415 | rdp->exp_dynticks_snap = snap; |
397 | if (rcu_dynticks_in_eqs(snap)) | ||
398 | mask_ofl_test |= mask; | ||
399 | else | ||
400 | rdp->exp_dynticks_snap = snap; | ||
401 | } | ||
402 | } | 416 | } |
403 | mask_ofl_ipi = rnp->expmask & ~mask_ofl_test; | 417 | } |
404 | 418 | mask_ofl_ipi = rnp->expmask & ~mask_ofl_test; | |
405 | /* | ||
406 | * Need to wait for any blocked tasks as well. Note that | ||
407 | * additional blocking tasks will also block the expedited | ||
408 | * GP until such time as the ->expmask bits are cleared. | ||
409 | */ | ||
410 | if (rcu_preempt_has_tasks(rnp)) | ||
411 | rnp->exp_tasks = rnp->blkd_tasks.next; | ||
412 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
413 | 419 | ||
414 | /* IPI the remaining CPUs for expedited quiescent state. */ | 420 | /* |
415 | for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { | 421 | * Need to wait for any blocked tasks as well. Note that |
416 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); | 422 | * additional blocking tasks will also block the expedited GP |
417 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | 423 | * until such time as the ->expmask bits are cleared. |
424 | */ | ||
425 | if (rcu_preempt_has_tasks(rnp)) | ||
426 | rnp->exp_tasks = rnp->blkd_tasks.next; | ||
427 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
418 | 428 | ||
419 | if (!(mask_ofl_ipi & mask)) | 429 | /* IPI the remaining CPUs for expedited quiescent state. */ |
420 | continue; | 430 | for_each_leaf_node_cpu_mask(rnp, cpu, rnp->expmask) { |
431 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); | ||
432 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | ||
433 | |||
434 | if (!(mask_ofl_ipi & mask)) | ||
435 | continue; | ||
421 | retry_ipi: | 436 | retry_ipi: |
422 | if (rcu_dynticks_in_eqs_since(rdp->dynticks, | 437 | if (rcu_dynticks_in_eqs_since(rdp->dynticks, |
423 | rdp->exp_dynticks_snap)) { | 438 | rdp->exp_dynticks_snap)) { |
424 | mask_ofl_test |= mask; | 439 | mask_ofl_test |= mask; |
425 | continue; | 440 | continue; |
426 | } | 441 | } |
427 | ret = smp_call_function_single(cpu, func, rsp, 0); | 442 | ret = smp_call_function_single(cpu, func, rsp, 0); |
428 | if (!ret) { | 443 | if (!ret) { |
429 | mask_ofl_ipi &= ~mask; | 444 | mask_ofl_ipi &= ~mask; |
430 | continue; | 445 | continue; |
431 | } | 446 | } |
432 | /* Failed, raced with CPU hotplug operation. */ | 447 | /* Failed, raced with CPU hotplug operation. */ |
433 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | 448 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
434 | if ((rnp->qsmaskinitnext & mask) && | 449 | if ((rnp->qsmaskinitnext & mask) && |
435 | (rnp->expmask & mask)) { | 450 | (rnp->expmask & mask)) { |
436 | /* Online, so delay for a bit and try again. */ | 451 | /* Online, so delay for a bit and try again. */ |
437 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
438 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("selectofl")); | ||
439 | schedule_timeout_uninterruptible(1); | ||
440 | goto retry_ipi; | ||
441 | } | ||
442 | /* CPU really is offline, so we can ignore it. */ | ||
443 | if (!(rnp->expmask & mask)) | ||
444 | mask_ofl_ipi &= ~mask; | ||
445 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | 452 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
453 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("selectofl")); | ||
454 | schedule_timeout_uninterruptible(1); | ||
455 | goto retry_ipi; | ||
456 | } | ||
457 | /* CPU really is offline, so we can ignore it. */ | ||
458 | if (!(rnp->expmask & mask)) | ||
459 | mask_ofl_ipi &= ~mask; | ||
460 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | ||
461 | } | ||
462 | /* Report quiescent states for those that went offline. */ | ||
463 | mask_ofl_test |= mask_ofl_ipi; | ||
464 | if (mask_ofl_test) | ||
465 | rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false); | ||
466 | } | ||
467 | |||
468 | /* | ||
469 | * Select the nodes that the upcoming expedited grace period needs | ||
470 | * to wait for. | ||
471 | */ | ||
472 | static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, | ||
473 | smp_call_func_t func) | ||
474 | { | ||
475 | struct rcu_node *rnp; | ||
476 | |||
477 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("reset")); | ||
478 | sync_exp_reset_tree(rsp); | ||
479 | trace_rcu_exp_grace_period(rsp->name, rcu_exp_gp_seq_endval(rsp), TPS("select")); | ||
480 | |||
481 | /* Schedule work for each leaf rcu_node structure. */ | ||
482 | rcu_for_each_leaf_node(rsp, rnp) { | ||
483 | rnp->exp_need_flush = false; | ||
484 | if (!READ_ONCE(rnp->expmask)) | ||
485 | continue; /* Avoid early boot non-existent wq. */ | ||
486 | rnp->rew.rew_func = func; | ||
487 | rnp->rew.rew_rsp = rsp; | ||
488 | if (!READ_ONCE(rcu_par_gp_wq) || | ||
489 | rcu_scheduler_active != RCU_SCHEDULER_RUNNING) { | ||
490 | /* No workqueues yet. */ | ||
491 | sync_rcu_exp_select_node_cpus(&rnp->rew.rew_work); | ||
492 | continue; | ||
446 | } | 493 | } |
447 | /* Report quiescent states for those that went offline. */ | 494 | INIT_WORK(&rnp->rew.rew_work, sync_rcu_exp_select_node_cpus); |
448 | mask_ofl_test |= mask_ofl_ipi; | 495 | queue_work_on(rnp->grplo, rcu_par_gp_wq, &rnp->rew.rew_work); |
449 | if (mask_ofl_test) | 496 | rnp->exp_need_flush = true; |
450 | rcu_report_exp_cpu_mult(rsp, rnp, mask_ofl_test, false); | ||
451 | } | 497 | } |
498 | |||
499 | /* Wait for workqueue jobs (if any) to complete. */ | ||
500 | rcu_for_each_leaf_node(rsp, rnp) | ||
501 | if (rnp->exp_need_flush) | ||
502 | flush_work(&rnp->rew.rew_work); | ||
452 | } | 503 | } |
453 | 504 | ||
454 | static void synchronize_sched_expedited_wait(struct rcu_state *rsp) | 505 | static void synchronize_sched_expedited_wait(struct rcu_state *rsp) |
@@ -469,9 +520,9 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) | |||
469 | for (;;) { | 520 | for (;;) { |
470 | ret = swait_event_timeout( | 521 | ret = swait_event_timeout( |
471 | rsp->expedited_wq, | 522 | rsp->expedited_wq, |
472 | sync_rcu_preempt_exp_done(rnp_root), | 523 | sync_rcu_preempt_exp_done_unlocked(rnp_root), |
473 | jiffies_stall); | 524 | jiffies_stall); |
474 | if (ret > 0 || sync_rcu_preempt_exp_done(rnp_root)) | 525 | if (ret > 0 || sync_rcu_preempt_exp_done_unlocked(rnp_root)) |
475 | return; | 526 | return; |
476 | WARN_ON(ret < 0); /* workqueues should not be signaled. */ | 527 | WARN_ON(ret < 0); /* workqueues should not be signaled. */ |
477 | if (rcu_cpu_stall_suppress) | 528 | if (rcu_cpu_stall_suppress) |
@@ -504,7 +555,7 @@ static void synchronize_sched_expedited_wait(struct rcu_state *rsp) | |||
504 | rcu_for_each_node_breadth_first(rsp, rnp) { | 555 | rcu_for_each_node_breadth_first(rsp, rnp) { |
505 | if (rnp == rnp_root) | 556 | if (rnp == rnp_root) |
506 | continue; /* printed unconditionally */ | 557 | continue; /* printed unconditionally */ |
507 | if (sync_rcu_preempt_exp_done(rnp)) | 558 | if (sync_rcu_preempt_exp_done_unlocked(rnp)) |
508 | continue; | 559 | continue; |
509 | pr_cont(" l=%u:%d-%d:%#lx/%c", | 560 | pr_cont(" l=%u:%d-%d:%#lx/%c", |
510 | rnp->level, rnp->grplo, rnp->grphi, | 561 | rnp->level, rnp->grplo, rnp->grphi, |
@@ -560,14 +611,6 @@ static void rcu_exp_wait_wake(struct rcu_state *rsp, unsigned long s) | |||
560 | mutex_unlock(&rsp->exp_wake_mutex); | 611 | mutex_unlock(&rsp->exp_wake_mutex); |
561 | } | 612 | } |
562 | 613 | ||
563 | /* Let the workqueue handler know what it is supposed to do. */ | ||
564 | struct rcu_exp_work { | ||
565 | smp_call_func_t rew_func; | ||
566 | struct rcu_state *rew_rsp; | ||
567 | unsigned long rew_s; | ||
568 | struct work_struct rew_work; | ||
569 | }; | ||
570 | |||
571 | /* | 614 | /* |
572 | * Common code to drive an expedited grace period forward, used by | 615 | * Common code to drive an expedited grace period forward, used by |
573 | * workqueues and mid-boot-time tasks. | 616 | * workqueues and mid-boot-time tasks. |
@@ -633,7 +676,7 @@ static void _synchronize_rcu_expedited(struct rcu_state *rsp, | |||
633 | rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); | 676 | rdp = per_cpu_ptr(rsp->rda, raw_smp_processor_id()); |
634 | rnp = rcu_get_root(rsp); | 677 | rnp = rcu_get_root(rsp); |
635 | wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], | 678 | wait_event(rnp->exp_wq[rcu_seq_ctr(s) & 0x3], |
636 | sync_exp_work_done(rsp, &rdp->exp_workdone0, s)); | 679 | sync_exp_work_done(rsp, s)); |
637 | smp_mb(); /* Workqueue actions happen before return. */ | 680 | smp_mb(); /* Workqueue actions happen before return. */ |
638 | 681 | ||
639 | /* Let the next expedited grace period start. */ | 682 | /* Let the next expedited grace period start. */ |
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 84fbee4686d3..7fd12039e512 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -182,7 +182,7 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) | |||
182 | 182 | ||
183 | raw_lockdep_assert_held_rcu_node(rnp); | 183 | raw_lockdep_assert_held_rcu_node(rnp); |
184 | WARN_ON_ONCE(rdp->mynode != rnp); | 184 | WARN_ON_ONCE(rdp->mynode != rnp); |
185 | WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1); | 185 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); |
186 | 186 | ||
187 | /* | 187 | /* |
188 | * Decide where to queue the newly blocked task. In theory, | 188 | * Decide where to queue the newly blocked task. In theory, |
@@ -384,6 +384,50 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp) | |||
384 | } | 384 | } |
385 | 385 | ||
386 | /* | 386 | /* |
387 | * Preemptible RCU implementation for rcu_read_lock(). | ||
388 | * Just increment ->rcu_read_lock_nesting, shared state will be updated | ||
389 | * if we block. | ||
390 | */ | ||
391 | void __rcu_read_lock(void) | ||
392 | { | ||
393 | current->rcu_read_lock_nesting++; | ||
394 | barrier(); /* critical section after entry code. */ | ||
395 | } | ||
396 | EXPORT_SYMBOL_GPL(__rcu_read_lock); | ||
397 | |||
398 | /* | ||
399 | * Preemptible RCU implementation for rcu_read_unlock(). | ||
400 | * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost | ||
401 | * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then | ||
402 | * invoke rcu_read_unlock_special() to clean up after a context switch | ||
403 | * in an RCU read-side critical section and other special cases. | ||
404 | */ | ||
405 | void __rcu_read_unlock(void) | ||
406 | { | ||
407 | struct task_struct *t = current; | ||
408 | |||
409 | if (t->rcu_read_lock_nesting != 1) { | ||
410 | --t->rcu_read_lock_nesting; | ||
411 | } else { | ||
412 | barrier(); /* critical section before exit code. */ | ||
413 | t->rcu_read_lock_nesting = INT_MIN; | ||
414 | barrier(); /* assign before ->rcu_read_unlock_special load */ | ||
415 | if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s))) | ||
416 | rcu_read_unlock_special(t); | ||
417 | barrier(); /* ->rcu_read_unlock_special load before assign */ | ||
418 | t->rcu_read_lock_nesting = 0; | ||
419 | } | ||
420 | #ifdef CONFIG_PROVE_LOCKING | ||
421 | { | ||
422 | int rrln = READ_ONCE(t->rcu_read_lock_nesting); | ||
423 | |||
424 | WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2); | ||
425 | } | ||
426 | #endif /* #ifdef CONFIG_PROVE_LOCKING */ | ||
427 | } | ||
428 | EXPORT_SYMBOL_GPL(__rcu_read_unlock); | ||
429 | |||
430 | /* | ||
387 | * Advance a ->blkd_tasks-list pointer to the next entry, instead | 431 | * Advance a ->blkd_tasks-list pointer to the next entry, instead |
388 | * returning NULL if at the end of the list. | 432 | * returning NULL if at the end of the list. |
389 | */ | 433 | */ |
@@ -489,7 +533,7 @@ void rcu_read_unlock_special(struct task_struct *t) | |||
489 | rnp = t->rcu_blocked_node; | 533 | rnp = t->rcu_blocked_node; |
490 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ | 534 | raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ |
491 | WARN_ON_ONCE(rnp != t->rcu_blocked_node); | 535 | WARN_ON_ONCE(rnp != t->rcu_blocked_node); |
492 | WARN_ON_ONCE(rnp->level != rcu_num_lvls - 1); | 536 | WARN_ON_ONCE(!rcu_is_leaf_node(rnp)); |
493 | empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); | 537 | empty_norm = !rcu_preempt_blocked_readers_cgp(rnp); |
494 | empty_exp = sync_rcu_preempt_exp_done(rnp); | 538 | empty_exp = sync_rcu_preempt_exp_done(rnp); |
495 | smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ | 539 | smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ |
@@ -685,15 +729,6 @@ static void rcu_preempt_check_callbacks(void) | |||
685 | t->rcu_read_unlock_special.b.need_qs = true; | 729 | t->rcu_read_unlock_special.b.need_qs = true; |
686 | } | 730 | } |
687 | 731 | ||
688 | #ifdef CONFIG_RCU_BOOST | ||
689 | |||
690 | static void rcu_preempt_do_callbacks(void) | ||
691 | { | ||
692 | rcu_do_batch(rcu_state_p, this_cpu_ptr(rcu_data_p)); | ||
693 | } | ||
694 | |||
695 | #endif /* #ifdef CONFIG_RCU_BOOST */ | ||
696 | |||
697 | /** | 732 | /** |
698 | * call_rcu() - Queue an RCU callback for invocation after a grace period. | 733 | * call_rcu() - Queue an RCU callback for invocation after a grace period. |
699 | * @head: structure to be used for queueing the RCU updates. | 734 | * @head: structure to be used for queueing the RCU updates. |
@@ -1140,7 +1175,7 @@ static void rcu_kthread_do_work(void) | |||
1140 | { | 1175 | { |
1141 | rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); | 1176 | rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); |
1142 | rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); | 1177 | rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); |
1143 | rcu_preempt_do_callbacks(); | 1178 | rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data)); |
1144 | } | 1179 | } |
1145 | 1180 | ||
1146 | static void rcu_cpu_kthread_setup(unsigned int cpu) | 1181 | static void rcu_cpu_kthread_setup(unsigned int cpu) |
@@ -1607,7 +1642,7 @@ static int rcu_oom_notify(struct notifier_block *self, | |||
1607 | 1642 | ||
1608 | for_each_online_cpu(cpu) { | 1643 | for_each_online_cpu(cpu) { |
1609 | smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1); | 1644 | smp_call_function_single(cpu, rcu_oom_notify_cpu, NULL, 1); |
1610 | cond_resched_rcu_qs(); | 1645 | cond_resched_tasks_rcu_qs(); |
1611 | } | 1646 | } |
1612 | 1647 | ||
1613 | /* Unconditionally decrement: no need to wake ourselves up. */ | 1648 | /* Unconditionally decrement: no need to wake ourselves up. */ |
@@ -1780,19 +1815,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) | |||
1780 | swake_up_all(sq); | 1815 | swake_up_all(sq); |
1781 | } | 1816 | } |
1782 | 1817 | ||
1783 | /* | ||
1784 | * Set the root rcu_node structure's ->need_future_gp field | ||
1785 | * based on the sum of those of all rcu_node structures. This does | ||
1786 | * double-count the root rcu_node structure's requests, but this | ||
1787 | * is necessary to handle the possibility of a rcu_nocb_kthread() | ||
1788 | * having awakened during the time that the rcu_node structures | ||
1789 | * were being updated for the end of the previous grace period. | ||
1790 | */ | ||
1791 | static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq) | ||
1792 | { | ||
1793 | rnp->need_future_gp[(rnp->completed + 1) & 0x1] += nrq; | ||
1794 | } | ||
1795 | |||
1796 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) | 1818 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) |
1797 | { | 1819 | { |
1798 | return &rnp->nocb_gp_wq[rnp->completed & 0x1]; | 1820 | return &rnp->nocb_gp_wq[rnp->completed & 0x1]; |
@@ -1966,7 +1988,7 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp, | |||
1966 | trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, | 1988 | trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, |
1967 | TPS("WakeOvf")); | 1989 | TPS("WakeOvf")); |
1968 | } else { | 1990 | } else { |
1969 | wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE, | 1991 | wake_nocb_leader_defer(rdp, RCU_NOCB_WAKE_FORCE, |
1970 | TPS("WakeOvfIsDeferred")); | 1992 | TPS("WakeOvfIsDeferred")); |
1971 | } | 1993 | } |
1972 | rdp->qlen_last_fqs_check = LONG_MAX / 2; | 1994 | rdp->qlen_last_fqs_check = LONG_MAX / 2; |
@@ -2048,7 +2070,8 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp) | |||
2048 | struct rcu_node *rnp = rdp->mynode; | 2070 | struct rcu_node *rnp = rdp->mynode; |
2049 | 2071 | ||
2050 | raw_spin_lock_irqsave_rcu_node(rnp, flags); | 2072 | raw_spin_lock_irqsave_rcu_node(rnp, flags); |
2051 | needwake = rcu_start_future_gp(rnp, rdp, &c); | 2073 | c = rcu_cbs_completed(rdp->rsp, rnp); |
2074 | needwake = rcu_start_this_gp(rnp, rdp, c); | ||
2052 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); | 2075 | raw_spin_unlock_irqrestore_rcu_node(rnp, flags); |
2053 | if (needwake) | 2076 | if (needwake) |
2054 | rcu_gp_kthread_wake(rdp->rsp); | 2077 | rcu_gp_kthread_wake(rdp->rsp); |
@@ -2057,7 +2080,7 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp) | |||
2057 | * Wait for the grace period. Do so interruptibly to avoid messing | 2080 | * Wait for the grace period. Do so interruptibly to avoid messing |
2058 | * up the load average. | 2081 | * up the load average. |
2059 | */ | 2082 | */ |
2060 | trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait")); | 2083 | trace_rcu_this_gp(rnp, rdp, c, TPS("StartWait")); |
2061 | for (;;) { | 2084 | for (;;) { |
2062 | swait_event_interruptible( | 2085 | swait_event_interruptible( |
2063 | rnp->nocb_gp_wq[c & 0x1], | 2086 | rnp->nocb_gp_wq[c & 0x1], |
@@ -2065,9 +2088,9 @@ static void rcu_nocb_wait_gp(struct rcu_data *rdp) | |||
2065 | if (likely(d)) | 2088 | if (likely(d)) |
2066 | break; | 2089 | break; |
2067 | WARN_ON(signal_pending(current)); | 2090 | WARN_ON(signal_pending(current)); |
2068 | trace_rcu_future_gp(rnp, rdp, c, TPS("ResumeWait")); | 2091 | trace_rcu_this_gp(rnp, rdp, c, TPS("ResumeWait")); |
2069 | } | 2092 | } |
2070 | trace_rcu_future_gp(rnp, rdp, c, TPS("EndWait")); | 2093 | trace_rcu_this_gp(rnp, rdp, c, TPS("EndWait")); |
2071 | smp_mb(); /* Ensure that CB invocation happens after GP end. */ | 2094 | smp_mb(); /* Ensure that CB invocation happens after GP end. */ |
2072 | } | 2095 | } |
2073 | 2096 | ||
@@ -2236,7 +2259,7 @@ static int rcu_nocb_kthread(void *arg) | |||
2236 | cl++; | 2259 | cl++; |
2237 | c++; | 2260 | c++; |
2238 | local_bh_enable(); | 2261 | local_bh_enable(); |
2239 | cond_resched_rcu_qs(); | 2262 | cond_resched_tasks_rcu_qs(); |
2240 | list = next; | 2263 | list = next; |
2241 | } | 2264 | } |
2242 | trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1); | 2265 | trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1); |
@@ -2292,7 +2315,7 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp) | |||
2292 | void __init rcu_init_nohz(void) | 2315 | void __init rcu_init_nohz(void) |
2293 | { | 2316 | { |
2294 | int cpu; | 2317 | int cpu; |
2295 | bool need_rcu_nocb_mask = true; | 2318 | bool need_rcu_nocb_mask = false; |
2296 | struct rcu_state *rsp; | 2319 | struct rcu_state *rsp; |
2297 | 2320 | ||
2298 | #if defined(CONFIG_NO_HZ_FULL) | 2321 | #if defined(CONFIG_NO_HZ_FULL) |
@@ -2315,7 +2338,7 @@ void __init rcu_init_nohz(void) | |||
2315 | #endif /* #if defined(CONFIG_NO_HZ_FULL) */ | 2338 | #endif /* #if defined(CONFIG_NO_HZ_FULL) */ |
2316 | 2339 | ||
2317 | if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { | 2340 | if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) { |
2318 | pr_info("\tNote: kernel parameter 'rcu_nocbs=' contains nonexistent CPUs.\n"); | 2341 | pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n"); |
2319 | cpumask_and(rcu_nocb_mask, cpu_possible_mask, | 2342 | cpumask_and(rcu_nocb_mask, cpu_possible_mask, |
2320 | rcu_nocb_mask); | 2343 | rcu_nocb_mask); |
2321 | } | 2344 | } |
@@ -2495,10 +2518,6 @@ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) | |||
2495 | { | 2518 | { |
2496 | } | 2519 | } |
2497 | 2520 | ||
2498 | static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq) | ||
2499 | { | ||
2500 | } | ||
2501 | |||
2502 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) | 2521 | static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) |
2503 | { | 2522 | { |
2504 | return NULL; | 2523 | return NULL; |
@@ -2587,8 +2606,7 @@ static bool rcu_nohz_full_cpu(struct rcu_state *rsp) | |||
2587 | } | 2606 | } |
2588 | 2607 | ||
2589 | /* | 2608 | /* |
2590 | * Bind the grace-period kthread for the sysidle flavor of RCU to the | 2609 | * Bind the RCU grace-period kthreads to the housekeeping CPU. |
2591 | * timekeeping CPU. | ||
2592 | */ | 2610 | */ |
2593 | static void rcu_bind_gp_kthread(void) | 2611 | static void rcu_bind_gp_kthread(void) |
2594 | { | 2612 | { |
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 68fa19a5e7bd..4c230a60ece4 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c | |||
@@ -226,54 +226,6 @@ core_initcall(rcu_set_runtime_mode); | |||
226 | 226 | ||
227 | #endif /* #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU) */ | 227 | #endif /* #if !defined(CONFIG_TINY_RCU) || defined(CONFIG_SRCU) */ |
228 | 228 | ||
229 | #ifdef CONFIG_PREEMPT_RCU | ||
230 | |||
231 | /* | ||
232 | * Preemptible RCU implementation for rcu_read_lock(). | ||
233 | * Just increment ->rcu_read_lock_nesting, shared state will be updated | ||
234 | * if we block. | ||
235 | */ | ||
236 | void __rcu_read_lock(void) | ||
237 | { | ||
238 | current->rcu_read_lock_nesting++; | ||
239 | barrier(); /* critical section after entry code. */ | ||
240 | } | ||
241 | EXPORT_SYMBOL_GPL(__rcu_read_lock); | ||
242 | |||
243 | /* | ||
244 | * Preemptible RCU implementation for rcu_read_unlock(). | ||
245 | * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost | ||
246 | * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then | ||
247 | * invoke rcu_read_unlock_special() to clean up after a context switch | ||
248 | * in an RCU read-side critical section and other special cases. | ||
249 | */ | ||
250 | void __rcu_read_unlock(void) | ||
251 | { | ||
252 | struct task_struct *t = current; | ||
253 | |||
254 | if (t->rcu_read_lock_nesting != 1) { | ||
255 | --t->rcu_read_lock_nesting; | ||
256 | } else { | ||
257 | barrier(); /* critical section before exit code. */ | ||
258 | t->rcu_read_lock_nesting = INT_MIN; | ||
259 | barrier(); /* assign before ->rcu_read_unlock_special load */ | ||
260 | if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s))) | ||
261 | rcu_read_unlock_special(t); | ||
262 | barrier(); /* ->rcu_read_unlock_special load before assign */ | ||
263 | t->rcu_read_lock_nesting = 0; | ||
264 | } | ||
265 | #ifdef CONFIG_PROVE_LOCKING | ||
266 | { | ||
267 | int rrln = READ_ONCE(t->rcu_read_lock_nesting); | ||
268 | |||
269 | WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2); | ||
270 | } | ||
271 | #endif /* #ifdef CONFIG_PROVE_LOCKING */ | ||
272 | } | ||
273 | EXPORT_SYMBOL_GPL(__rcu_read_unlock); | ||
274 | |||
275 | #endif /* #ifdef CONFIG_PREEMPT_RCU */ | ||
276 | |||
277 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 229 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
278 | static struct lock_class_key rcu_lock_key; | 230 | static struct lock_class_key rcu_lock_key; |
279 | struct lockdep_map rcu_lock_map = | 231 | struct lockdep_map rcu_lock_map = |
@@ -624,7 +576,7 @@ EXPORT_SYMBOL_GPL(call_rcu_tasks); | |||
624 | * grace period has elapsed, in other words after all currently | 576 | * grace period has elapsed, in other words after all currently |
625 | * executing rcu-tasks read-side critical sections have elapsed. These | 577 | * executing rcu-tasks read-side critical sections have elapsed. These |
626 | * read-side critical sections are delimited by calls to schedule(), | 578 | * read-side critical sections are delimited by calls to schedule(), |
627 | * cond_resched_rcu_qs(), idle execution, userspace execution, calls | 579 | * cond_resched_tasks_rcu_qs(), idle execution, userspace execution, calls |
628 | * to synchronize_rcu_tasks(), and (in theory, anyway) cond_resched(). | 580 | * to synchronize_rcu_tasks(), and (in theory, anyway) cond_resched(). |
629 | * | 581 | * |
630 | * This is a very specialized primitive, intended only for a few uses in | 582 | * This is a very specialized primitive, intended only for a few uses in |
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 211890edf37e..e27034bd954e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
@@ -5025,20 +5025,6 @@ int __cond_resched_lock(spinlock_t *lock) | |||
5025 | } | 5025 | } |
5026 | EXPORT_SYMBOL(__cond_resched_lock); | 5026 | EXPORT_SYMBOL(__cond_resched_lock); |
5027 | 5027 | ||
5028 | int __sched __cond_resched_softirq(void) | ||
5029 | { | ||
5030 | BUG_ON(!in_softirq()); | ||
5031 | |||
5032 | if (should_resched(SOFTIRQ_DISABLE_OFFSET)) { | ||
5033 | local_bh_enable(); | ||
5034 | preempt_schedule_common(); | ||
5035 | local_bh_disable(); | ||
5036 | return 1; | ||
5037 | } | ||
5038 | return 0; | ||
5039 | } | ||
5040 | EXPORT_SYMBOL(__cond_resched_softirq); | ||
5041 | |||
5042 | /** | 5028 | /** |
5043 | * yield - yield the current processor to other threads. | 5029 | * yield - yield the current processor to other threads. |
5044 | * | 5030 | * |
diff --git a/kernel/softirq.c b/kernel/softirq.c index 177de3640c78..03981f1c39ea 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
@@ -145,8 +145,7 @@ static void __local_bh_enable(unsigned int cnt) | |||
145 | } | 145 | } |
146 | 146 | ||
147 | /* | 147 | /* |
148 | * Special-case - softirqs can safely be enabled in | 148 | * Special-case - softirqs can safely be enabled by __do_softirq(), |
149 | * cond_resched_softirq(), or by __do_softirq(), | ||
150 | * without processing still-pending softirqs: | 149 | * without processing still-pending softirqs: |
151 | */ | 150 | */ |
152 | void _local_bh_enable(void) | 151 | void _local_bh_enable(void) |
diff --git a/kernel/torture.c b/kernel/torture.c index 37b94012a3f8..3de1efbecd6a 100644 --- a/kernel/torture.c +++ b/kernel/torture.c | |||
@@ -574,7 +574,7 @@ void stutter_wait(const char *title) | |||
574 | { | 574 | { |
575 | int spt; | 575 | int spt; |
576 | 576 | ||
577 | cond_resched_rcu_qs(); | 577 | cond_resched_tasks_rcu_qs(); |
578 | spt = READ_ONCE(stutter_pause_test); | 578 | spt = READ_ONCE(stutter_pause_test); |
579 | for (; spt; spt = READ_ONCE(stutter_pause_test)) { | 579 | for (; spt; spt = READ_ONCE(stutter_pause_test)) { |
580 | if (spt == 1) { | 580 | if (spt == 1) { |
diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index 22fee766081b..80e0b2aca703 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c | |||
@@ -159,13 +159,13 @@ static int benchmark_event_kthread(void *arg) | |||
159 | * wants to run, schedule in, but if the CPU is idle, | 159 | * wants to run, schedule in, but if the CPU is idle, |
160 | * we'll keep burning cycles. | 160 | * we'll keep burning cycles. |
161 | * | 161 | * |
162 | * Note the _rcu_qs() version of cond_resched() will | 162 | * Note the tasks_rcu_qs() version of cond_resched() will |
163 | * notify synchronize_rcu_tasks() that this thread has | 163 | * notify synchronize_rcu_tasks() that this thread has |
164 | * passed a quiescent state for rcu_tasks. Otherwise | 164 | * passed a quiescent state for rcu_tasks. Otherwise |
165 | * this thread will never voluntarily schedule which would | 165 | * this thread will never voluntarily schedule which would |
166 | * block synchronize_rcu_tasks() indefinitely. | 166 | * block synchronize_rcu_tasks() indefinitely. |
167 | */ | 167 | */ |
168 | cond_resched(); | 168 | cond_resched_tasks_rcu_qs(); |
169 | } | 169 | } |
170 | 170 | ||
171 | return 0; | 171 | return 0; |
diff --git a/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh new file mode 100755 index 000000000000..98f650c9bf54 --- /dev/null +++ b/tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh | |||
@@ -0,0 +1,56 @@ | |||
1 | #!/bin/sh | ||
2 | # | ||
3 | # Invoke a text editor on all console.log files for all runs with diagnostics, | ||
4 | # that is, on all such files having a console.log.diags counterpart. | ||
5 | # Note that both console.log.diags and console.log are passed to the | ||
6 | # editor (currently defaulting to "vi"), allowing the user to get an | ||
7 | # idea of what to search for in the console.log file. | ||
8 | # | ||
9 | # Usage: kvm-find-errors.sh directory | ||
10 | # | ||
11 | # The "directory" above should end with the date/time directory, for example, | ||
12 | # "tools/testing/selftests/rcutorture/res/2018.02.25-14:27:27". | ||
13 | |||
14 | rundir="${1}" | ||
15 | if test -z "$rundir" -o ! -d "$rundir" | ||
16 | then | ||
17 | echo Usage: $0 directory | ||
18 | fi | ||
19 | editor=${EDITOR-vi} | ||
20 | |||
21 | # Find builds with errors | ||
22 | files= | ||
23 | for i in ${rundir}/*/Make.out | ||
24 | do | ||
25 | if egrep -q "error:|warning:" < $i | ||
26 | then | ||
27 | egrep "error:|warning:" < $i > $i.diags | ||
28 | files="$files $i.diags $i" | ||
29 | fi | ||
30 | done | ||
31 | if test -n "$files" | ||
32 | then | ||
33 | $editor $files | ||
34 | else | ||
35 | echo No build errors. | ||
36 | fi | ||
37 | if grep -q -e "--buildonly" < ${rundir}/log | ||
38 | then | ||
39 | echo Build-only run, no console logs to check. | ||
40 | fi | ||
41 | |||
42 | # Find console logs with errors | ||
43 | files= | ||
44 | for i in ${rundir}/*/console.log | ||
45 | do | ||
46 | if test -r $i.diags | ||
47 | then | ||
48 | files="$files $i.diags $i" | ||
49 | fi | ||
50 | done | ||
51 | if test -n "$files" | ||
52 | then | ||
53 | $editor $files | ||
54 | else | ||
55 | echo No errors in console logs. | ||
56 | fi | ||
diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcu.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcu.sh index c2e1bb6d0cba..477ecb1293ab 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcu.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck-rcu.sh | |||
@@ -34,11 +34,15 @@ fi | |||
34 | 34 | ||
35 | configfile=`echo $i | sed -e 's/^.*\///'` | 35 | configfile=`echo $i | sed -e 's/^.*\///'` |
36 | ngps=`grep ver: $i/console.log 2> /dev/null | tail -1 | sed -e 's/^.* ver: //' -e 's/ .*$//'` | 36 | ngps=`grep ver: $i/console.log 2> /dev/null | tail -1 | sed -e 's/^.* ver: //' -e 's/ .*$//'` |
37 | stopstate="`grep 'End-test grace-period state: g' $i/console.log 2> /dev/null | | ||
38 | tail -1 | sed -e 's/^\[[ 0-9.]*] //' | | ||
39 | awk '{ print \"[\" $1 \" \" $5 \" \" $6 \" \" $7 \"]\"; }' | | ||
40 | tr -d '\012\015'`" | ||
37 | if test -z "$ngps" | 41 | if test -z "$ngps" |
38 | then | 42 | then |
39 | echo "$configfile -------" | 43 | echo "$configfile ------- " $stopstate |
40 | else | 44 | else |
41 | title="$configfile ------- $ngps grace periods" | 45 | title="$configfile ------- $ngps GPs" |
42 | dur=`sed -e 's/^.* rcutorture.shutdown_secs=//' -e 's/ .*$//' < $i/qemu-cmd 2> /dev/null` | 46 | dur=`sed -e 's/^.* rcutorture.shutdown_secs=//' -e 's/ .*$//' < $i/qemu-cmd 2> /dev/null` |
43 | if test -z "$dur" | 47 | if test -z "$dur" |
44 | then | 48 | then |
@@ -46,9 +50,9 @@ else | |||
46 | else | 50 | else |
47 | ngpsps=`awk -v ngps=$ngps -v dur=$dur ' | 51 | ngpsps=`awk -v ngps=$ngps -v dur=$dur ' |
48 | BEGIN { print ngps / dur }' < /dev/null` | 52 | BEGIN { print ngps / dur }' < /dev/null` |
49 | title="$title ($ngpsps per second)" | 53 | title="$title ($ngpsps/s)" |
50 | fi | 54 | fi |
51 | echo $title | 55 | echo $title $stopstate |
52 | nclosecalls=`grep --binary-files=text 'torture: Reader Batch' $i/console.log | tail -1 | awk '{for (i=NF-8;i<=NF;i++) sum+=$i; } END {print sum}'` | 56 | nclosecalls=`grep --binary-files=text 'torture: Reader Batch' $i/console.log | tail -1 | awk '{for (i=NF-8;i<=NF;i++) sum+=$i; } END {print sum}'` |
53 | if test -z "$nclosecalls" | 57 | if test -z "$nclosecalls" |
54 | then | 58 | then |
diff --git a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh index f7e988f369dd..c27e97824163 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-recheck.sh | |||
@@ -48,10 +48,6 @@ do | |||
48 | cat $i/Make.oldconfig.err | 48 | cat $i/Make.oldconfig.err |
49 | fi | 49 | fi |
50 | parse-build.sh $i/Make.out $configfile | 50 | parse-build.sh $i/Make.out $configfile |
51 | if test "$TORTURE_SUITE" != rcuperf | ||
52 | then | ||
53 | parse-torture.sh $i/console.log $configfile | ||
54 | fi | ||
55 | parse-console.sh $i/console.log $configfile | 51 | parse-console.sh $i/console.log $configfile |
56 | if test -r $i/Warnings | 52 | if test -r $i/Warnings |
57 | then | 53 | then |
diff --git a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh index 5f8fbb0d7c17..c5b0f94341d9 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-test-1-run.sh | |||
@@ -267,5 +267,4 @@ then | |||
267 | echo Unknown PID, cannot kill qemu command | 267 | echo Unknown PID, cannot kill qemu command |
268 | fi | 268 | fi |
269 | 269 | ||
270 | parse-torture.sh $resdir/console.log $title | ||
271 | parse-console.sh $resdir/console.log $title | 270 | parse-console.sh $resdir/console.log $title |
diff --git a/tools/testing/selftests/rcutorture/bin/parse-console.sh b/tools/testing/selftests/rcutorture/bin/parse-console.sh index 08aa7d50ae0e..17293436f551 100755 --- a/tools/testing/selftests/rcutorture/bin/parse-console.sh +++ b/tools/testing/selftests/rcutorture/bin/parse-console.sh | |||
@@ -24,57 +24,146 @@ | |||
24 | # | 24 | # |
25 | # Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 25 | # Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> |
26 | 26 | ||
27 | T=${TMPDIR-/tmp}/parse-console.sh.$$ | ||
27 | file="$1" | 28 | file="$1" |
28 | title="$2" | 29 | title="$2" |
29 | 30 | ||
31 | trap 'rm -f $T.seq $T.diags' 0 | ||
32 | |||
30 | . functions.sh | 33 | . functions.sh |
31 | 34 | ||
35 | # Check for presence and readability of console output file | ||
36 | if test -f "$file" -a -r "$file" | ||
37 | then | ||
38 | : | ||
39 | else | ||
40 | echo $title unreadable console output file: $file | ||
41 | exit 1 | ||
42 | fi | ||
32 | if grep -Pq '\x00' < $file | 43 | if grep -Pq '\x00' < $file |
33 | then | 44 | then |
34 | print_warning Console output contains nul bytes, old qemu still running? | 45 | print_warning Console output contains nul bytes, old qemu still running? |
35 | fi | 46 | fi |
36 | egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:|detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state|rcu_.*kthread starved for' < $file | grep -v 'ODEBUG: ' | grep -v 'Warning: unable to open an initial console' > $1.diags | 47 | cat /dev/null > $file.diags |
37 | if test -s $1.diags | 48 | |
49 | # Check for proper termination, except that rcuperf runs don't indicate this. | ||
50 | if test "$TORTURE_SUITE" != rcuperf | ||
38 | then | 51 | then |
39 | print_warning Assertion failure in $file $title | 52 | # check for abject failure |
40 | # cat $1.diags | 53 | |
54 | if grep -q FAILURE $file || grep -q -e '-torture.*!!!' $file | ||
55 | then | ||
56 | nerrs=`grep --binary-files=text '!!!' $file | | ||
57 | tail -1 | | ||
58 | awk ' | ||
59 | { | ||
60 | for (i=NF-8;i<=NF;i++) | ||
61 | sum+=$i; | ||
62 | } | ||
63 | END { print sum }'` | ||
64 | print_bug $title FAILURE, $nerrs instances | ||
65 | exit | ||
66 | fi | ||
67 | |||
68 | grep --binary-files=text 'torture:.*ver:' $file | | ||
69 | egrep --binary-files=text -v '\(null\)|rtc: 000000000* ' | | ||
70 | sed -e 's/^(initramfs)[^]]*] //' -e 's/^\[[^]]*] //' | | ||
71 | awk ' | ||
72 | BEGIN { | ||
73 | ver = 0; | ||
74 | badseq = 0; | ||
75 | } | ||
76 | |||
77 | { | ||
78 | if (!badseq && ($5 + 0 != $5 || $5 <= ver)) { | ||
79 | badseqno1 = ver; | ||
80 | badseqno2 = $5; | ||
81 | badseqnr = NR; | ||
82 | badseq = 1; | ||
83 | } | ||
84 | ver = $5 | ||
85 | } | ||
86 | |||
87 | END { | ||
88 | if (badseq) { | ||
89 | if (badseqno1 == badseqno2 && badseqno2 == ver) | ||
90 | print "GP HANG at " ver " torture stat " badseqnr; | ||
91 | else | ||
92 | print "BAD SEQ " badseqno1 ":" badseqno2 " last:" ver " version " badseqnr; | ||
93 | } | ||
94 | }' > $T.seq | ||
95 | |||
96 | if grep -q SUCCESS $file | ||
97 | then | ||
98 | if test -s $T.seq | ||
99 | then | ||
100 | print_warning $title `cat $T.seq` | ||
101 | echo " " $file | ||
102 | exit 2 | ||
103 | fi | ||
104 | else | ||
105 | if grep -q "_HOTPLUG:" $file | ||
106 | then | ||
107 | print_warning HOTPLUG FAILURES $title `cat $T.seq` | ||
108 | echo " " $file | ||
109 | exit 3 | ||
110 | fi | ||
111 | echo $title no success message, `grep --binary-files=text 'ver:' $file | wc -l` successful version messages | ||
112 | if test -s $T.seq | ||
113 | then | ||
114 | print_warning $title `cat $T.seq` | ||
115 | fi | ||
116 | exit 2 | ||
117 | fi | ||
118 | fi | tee -a $file.diags | ||
119 | |||
120 | egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:|detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state|rcu_.*kthread starved for' < $file | | ||
121 | grep -v 'ODEBUG: ' | | ||
122 | grep -v 'Warning: unable to open an initial console' > $T.diags | ||
123 | if test -s $T.diags | ||
124 | then | ||
125 | print_warning "Assertion failure in $file $title" | ||
126 | # cat $T.diags | ||
41 | summary="" | 127 | summary="" |
42 | n_badness=`grep -c Badness $1` | 128 | n_badness=`grep -c Badness $file` |
43 | if test "$n_badness" -ne 0 | 129 | if test "$n_badness" -ne 0 |
44 | then | 130 | then |
45 | summary="$summary Badness: $n_badness" | 131 | summary="$summary Badness: $n_badness" |
46 | fi | 132 | fi |
47 | n_warn=`grep -v 'Warning: unable to open an initial console' $1 | egrep -c 'WARNING:|Warn'` | 133 | n_warn=`grep -v 'Warning: unable to open an initial console' $file | egrep -c 'WARNING:|Warn'` |
48 | if test "$n_warn" -ne 0 | 134 | if test "$n_warn" -ne 0 |
49 | then | 135 | then |
50 | summary="$summary Warnings: $n_warn" | 136 | summary="$summary Warnings: $n_warn" |
51 | fi | 137 | fi |
52 | n_bugs=`egrep -c 'BUG|Oops:' $1` | 138 | n_bugs=`egrep -c 'BUG|Oops:' $file` |
53 | if test "$n_bugs" -ne 0 | 139 | if test "$n_bugs" -ne 0 |
54 | then | 140 | then |
55 | summary="$summary Bugs: $n_bugs" | 141 | summary="$summary Bugs: $n_bugs" |
56 | fi | 142 | fi |
57 | n_calltrace=`grep -c 'Call Trace:' $1` | 143 | n_calltrace=`grep -c 'Call Trace:' $file` |
58 | if test "$n_calltrace" -ne 0 | 144 | if test "$n_calltrace" -ne 0 |
59 | then | 145 | then |
60 | summary="$summary Call Traces: $n_calltrace" | 146 | summary="$summary Call Traces: $n_calltrace" |
61 | fi | 147 | fi |
62 | n_lockdep=`grep -c =========== $1` | 148 | n_lockdep=`grep -c =========== $file` |
63 | if test "$n_badness" -ne 0 | 149 | if test "$n_badness" -ne 0 |
64 | then | 150 | then |
65 | summary="$summary lockdep: $n_badness" | 151 | summary="$summary lockdep: $n_badness" |
66 | fi | 152 | fi |
67 | n_stalls=`egrep -c 'detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state' $1` | 153 | n_stalls=`egrep -c 'detected stalls on CPUs/tasks:|self-detected stall on CPU|Stall ended before state dump start|\?\?\? Writer stall state' $file` |
68 | if test "$n_stalls" -ne 0 | 154 | if test "$n_stalls" -ne 0 |
69 | then | 155 | then |
70 | summary="$summary Stalls: $n_stalls" | 156 | summary="$summary Stalls: $n_stalls" |
71 | fi | 157 | fi |
72 | n_starves=`grep -c 'rcu_.*kthread starved for' $1` | 158 | n_starves=`grep -c 'rcu_.*kthread starved for' $file` |
73 | if test "$n_starves" -ne 0 | 159 | if test "$n_starves" -ne 0 |
74 | then | 160 | then |
75 | summary="$summary Starves: $n_starves" | 161 | summary="$summary Starves: $n_starves" |
76 | fi | 162 | fi |
77 | print_warning Summary: $summary | 163 | print_warning Summary: $summary |
78 | else | 164 | cat $T.diags >> $file.diags |
79 | rm $1.diags | 165 | fi |
166 | if ! test -s $file.diags | ||
167 | then | ||
168 | rm -f $file.diags | ||
80 | fi | 169 | fi |
diff --git a/tools/testing/selftests/rcutorture/bin/parse-torture.sh b/tools/testing/selftests/rcutorture/bin/parse-torture.sh deleted file mode 100755 index 5987e50cfeb4..000000000000 --- a/tools/testing/selftests/rcutorture/bin/parse-torture.sh +++ /dev/null | |||
@@ -1,105 +0,0 @@ | |||
1 | #!/bin/bash | ||
2 | # | ||
3 | # Check the console output from a torture run for goodness. | ||
4 | # The "file" is a pathname on the local system, and "title" is | ||
5 | # a text string for error-message purposes. | ||
6 | # | ||
7 | # The file must contain torture output, but can be interspersed | ||
8 | # with other dmesg text, as in console-log output. | ||
9 | # | ||
10 | # Usage: parse-torture.sh file title | ||
11 | # | ||
12 | # This program is free software; you can redistribute it and/or modify | ||
13 | # it under the terms of the GNU General Public License as published by | ||
14 | # the Free Software Foundation; either version 2 of the License, or | ||
15 | # (at your option) any later version. | ||
16 | # | ||
17 | # This program is distributed in the hope that it will be useful, | ||
18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
20 | # GNU General Public License for more details. | ||
21 | # | ||
22 | # You should have received a copy of the GNU General Public License | ||
23 | # along with this program; if not, you can access it online at | ||
24 | # http://www.gnu.org/licenses/gpl-2.0.html. | ||
25 | # | ||
26 | # Copyright (C) IBM Corporation, 2011 | ||
27 | # | ||
28 | # Authors: Paul E. McKenney <paulmck@linux.vnet.ibm.com> | ||
29 | |||
30 | T=${TMPDIR-/tmp}/parse-torture.sh.$$ | ||
31 | file="$1" | ||
32 | title="$2" | ||
33 | |||
34 | trap 'rm -f $T.seq' 0 | ||
35 | |||
36 | . functions.sh | ||
37 | |||
38 | # check for presence of torture output file. | ||
39 | |||
40 | if test -f "$file" -a -r "$file" | ||
41 | then | ||
42 | : | ||
43 | else | ||
44 | echo $title unreadable torture output file: $file | ||
45 | exit 1 | ||
46 | fi | ||
47 | |||
48 | # check for abject failure | ||
49 | |||
50 | if grep -q FAILURE $file || grep -q -e '-torture.*!!!' $file | ||
51 | then | ||
52 | nerrs=`grep --binary-files=text '!!!' $file | tail -1 | awk '{for (i=NF-8;i<=NF;i++) sum+=$i; } END {print sum}'` | ||
53 | print_bug $title FAILURE, $nerrs instances | ||
54 | echo " " $url | ||
55 | exit | ||
56 | fi | ||
57 | |||
58 | grep --binary-files=text 'torture:.*ver:' $file | egrep --binary-files=text -v '\(null\)|rtc: 000000000* ' | sed -e 's/^(initramfs)[^]]*] //' -e 's/^\[[^]]*] //' | | ||
59 | awk ' | ||
60 | BEGIN { | ||
61 | ver = 0; | ||
62 | badseq = 0; | ||
63 | } | ||
64 | |||
65 | { | ||
66 | if (!badseq && ($5 + 0 != $5 || $5 <= ver)) { | ||
67 | badseqno1 = ver; | ||
68 | badseqno2 = $5; | ||
69 | badseqnr = NR; | ||
70 | badseq = 1; | ||
71 | } | ||
72 | ver = $5 | ||
73 | } | ||
74 | |||
75 | END { | ||
76 | if (badseq) { | ||
77 | if (badseqno1 == badseqno2 && badseqno2 == ver) | ||
78 | print "GP HANG at " ver " torture stat " badseqnr; | ||
79 | else | ||
80 | print "BAD SEQ " badseqno1 ":" badseqno2 " last:" ver " version " badseqnr; | ||
81 | } | ||
82 | }' > $T.seq | ||
83 | |||
84 | if grep -q SUCCESS $file | ||
85 | then | ||
86 | if test -s $T.seq | ||
87 | then | ||
88 | print_warning $title $title `cat $T.seq` | ||
89 | echo " " $file | ||
90 | exit 2 | ||
91 | fi | ||
92 | else | ||
93 | if grep -q "_HOTPLUG:" $file | ||
94 | then | ||
95 | print_warning HOTPLUG FAILURES $title `cat $T.seq` | ||
96 | echo " " $file | ||
97 | exit 3 | ||
98 | fi | ||
99 | echo $title no success message, `grep --binary-files=text 'ver:' $file | wc -l` successful version messages | ||
100 | if test -s $T.seq | ||
101 | then | ||
102 | print_warning $title `cat $T.seq` | ||
103 | fi | ||
104 | exit 2 | ||
105 | fi | ||