summaryrefslogtreecommitdiffstats
path: root/kernel/rcu/srcutiny.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-12-26 16:07:19 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2018-12-26 16:07:19 -0500
commit792bf4d871dea8b69be2aaabdd320d7c6ed15985 (patch)
tree8cec3755ff6df5f82b12420fb6ad6a4d531ebfd1 /kernel/rcu/srcutiny.c
parenteed9688f8513189295887e5a27ec7f576754b60e (diff)
parent4bbfd7467cfc7d42e18d3008fa6a28ffd56e901a (diff)
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull RCU updates from Ingo Molnar: "The biggest RCU changes in this cycle were: - Convert RCU's BUG_ON() and similar calls to WARN_ON() and similar. - Replace calls of RCU-bh and RCU-sched update-side functions to their vanilla RCU counterparts. This series is a step towards complete removal of the RCU-bh and RCU-sched update-side functions. ( Note that some of these conversions are going upstream via their respective maintainers. ) - Documentation updates, including a number of flavor-consolidation updates from Joel Fernandes. - Miscellaneous fixes. - Automate generation of the initrd filesystem used for rcutorture testing. - Convert spin_is_locked() assertions to instead use lockdep. ( Note that some of these conversions are going upstream via their respective maintainers. ) - SRCU updates, especially including a fix from Dennis Krein for a bag-on-head-class bug. - RCU torture-test updates" * 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (112 commits) rcutorture: Don't do busted forward-progress testing rcutorture: Use 100ms buckets for forward-progress callback histograms rcutorture: Recover from OOM during forward-progress tests rcutorture: Print forward-progress test age upon failure rcutorture: Print time since GP end upon forward-progress failure rcutorture: Print histogram of CB invocation at OOM time rcutorture: Print GP age upon forward-progress failure rcu: Print per-CPU callback counts for forward-progress failures rcu: Account for nocb-CPU callback counts in RCU CPU stall warnings rcutorture: Dump grace-period diagnostics upon forward-progress OOM rcutorture: Prepare for asynchronous access to rcu_fwd_startat torture: Remove unnecessary "ret" variables rcutorture: Affinity forward-progress test to avoid housekeeping CPUs rcutorture: Break up too-long rcu_torture_fwd_prog() function rcutorture: Remove cbflood facility torture: Bring any extra CPUs online during kernel startup rcutorture: Add call_rcu() flooding forward-progress tests rcutorture/formal: Replace synchronize_sched() with synchronize_rcu() tools/kernel.h: Replace synchronize_sched() with synchronize_rcu() net/decnet: Replace rcu_barrier_bh() with rcu_barrier() ...
Diffstat (limited to 'kernel/rcu/srcutiny.c')
-rw-r--r--kernel/rcu/srcutiny.c120
1 files changed, 60 insertions, 60 deletions
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index b46e6683f8c9..32dfd6522548 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -37,30 +37,30 @@ int rcu_scheduler_active __read_mostly;
37static LIST_HEAD(srcu_boot_list); 37static LIST_HEAD(srcu_boot_list);
38static bool srcu_init_done; 38static bool srcu_init_done;
39 39
40static int init_srcu_struct_fields(struct srcu_struct *sp) 40static int init_srcu_struct_fields(struct srcu_struct *ssp)
41{ 41{
42 sp->srcu_lock_nesting[0] = 0; 42 ssp->srcu_lock_nesting[0] = 0;
43 sp->srcu_lock_nesting[1] = 0; 43 ssp->srcu_lock_nesting[1] = 0;
44 init_swait_queue_head(&sp->srcu_wq); 44 init_swait_queue_head(&ssp->srcu_wq);
45 sp->srcu_cb_head = NULL; 45 ssp->srcu_cb_head = NULL;
46 sp->srcu_cb_tail = &sp->srcu_cb_head; 46 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
47 sp->srcu_gp_running = false; 47 ssp->srcu_gp_running = false;
48 sp->srcu_gp_waiting = false; 48 ssp->srcu_gp_waiting = false;
49 sp->srcu_idx = 0; 49 ssp->srcu_idx = 0;
50 INIT_WORK(&sp->srcu_work, srcu_drive_gp); 50 INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
51 INIT_LIST_HEAD(&sp->srcu_work.entry); 51 INIT_LIST_HEAD(&ssp->srcu_work.entry);
52 return 0; 52 return 0;
53} 53}
54 54
55#ifdef CONFIG_DEBUG_LOCK_ALLOC 55#ifdef CONFIG_DEBUG_LOCK_ALLOC
56 56
57int __init_srcu_struct(struct srcu_struct *sp, const char *name, 57int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
58 struct lock_class_key *key) 58 struct lock_class_key *key)
59{ 59{
60 /* Don't re-initialize a lock while it is held. */ 60 /* Don't re-initialize a lock while it is held. */
61 debug_check_no_locks_freed((void *)sp, sizeof(*sp)); 61 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));
62 lockdep_init_map(&sp->dep_map, name, key, 0); 62 lockdep_init_map(&ssp->dep_map, name, key, 0);
63 return init_srcu_struct_fields(sp); 63 return init_srcu_struct_fields(ssp);
64} 64}
65EXPORT_SYMBOL_GPL(__init_srcu_struct); 65EXPORT_SYMBOL_GPL(__init_srcu_struct);
66 66
@@ -68,15 +68,15 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct);
68 68
69/* 69/*
70 * init_srcu_struct - initialize a sleep-RCU structure 70 * init_srcu_struct - initialize a sleep-RCU structure
71 * @sp: structure to initialize. 71 * @ssp: structure to initialize.
72 * 72 *
73 * Must invoke this on a given srcu_struct before passing that srcu_struct 73 * Must invoke this on a given srcu_struct before passing that srcu_struct
74 * to any other function. Each srcu_struct represents a separate domain 74 * to any other function. Each srcu_struct represents a separate domain
75 * of SRCU protection. 75 * of SRCU protection.
76 */ 76 */
77int init_srcu_struct(struct srcu_struct *sp) 77int init_srcu_struct(struct srcu_struct *ssp)
78{ 78{
79 return init_srcu_struct_fields(sp); 79 return init_srcu_struct_fields(ssp);
80} 80}
81EXPORT_SYMBOL_GPL(init_srcu_struct); 81EXPORT_SYMBOL_GPL(init_srcu_struct);
82 82
@@ -84,22 +84,22 @@ EXPORT_SYMBOL_GPL(init_srcu_struct);
84 84
85/* 85/*
86 * cleanup_srcu_struct - deconstruct a sleep-RCU structure 86 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
87 * @sp: structure to clean up. 87 * @ssp: structure to clean up.
88 * 88 *
89 * Must invoke this after you are finished using a given srcu_struct that 89 * Must invoke this after you are finished using a given srcu_struct that
90 * was initialized via init_srcu_struct(), else you leak memory. 90 * was initialized via init_srcu_struct(), else you leak memory.
91 */ 91 */
92void _cleanup_srcu_struct(struct srcu_struct *sp, bool quiesced) 92void _cleanup_srcu_struct(struct srcu_struct *ssp, bool quiesced)
93{ 93{
94 WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); 94 WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);
95 if (quiesced) 95 if (quiesced)
96 WARN_ON(work_pending(&sp->srcu_work)); 96 WARN_ON(work_pending(&ssp->srcu_work));
97 else 97 else
98 flush_work(&sp->srcu_work); 98 flush_work(&ssp->srcu_work);
99 WARN_ON(sp->srcu_gp_running); 99 WARN_ON(ssp->srcu_gp_running);
100 WARN_ON(sp->srcu_gp_waiting); 100 WARN_ON(ssp->srcu_gp_waiting);
101 WARN_ON(sp->srcu_cb_head); 101 WARN_ON(ssp->srcu_cb_head);
102 WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail); 102 WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
103} 103}
104EXPORT_SYMBOL_GPL(_cleanup_srcu_struct); 104EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);
105 105
@@ -107,13 +107,13 @@ EXPORT_SYMBOL_GPL(_cleanup_srcu_struct);
107 * Removes the count for the old reader from the appropriate element of 107 * Removes the count for the old reader from the appropriate element of
108 * the srcu_struct. 108 * the srcu_struct.
109 */ 109 */
110void __srcu_read_unlock(struct srcu_struct *sp, int idx) 110void __srcu_read_unlock(struct srcu_struct *ssp, int idx)
111{ 111{
112 int newval = sp->srcu_lock_nesting[idx] - 1; 112 int newval = ssp->srcu_lock_nesting[idx] - 1;
113 113
114 WRITE_ONCE(sp->srcu_lock_nesting[idx], newval); 114 WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
115 if (!newval && READ_ONCE(sp->srcu_gp_waiting)) 115 if (!newval && READ_ONCE(ssp->srcu_gp_waiting))
116 swake_up_one(&sp->srcu_wq); 116 swake_up_one(&ssp->srcu_wq);
117} 117}
118EXPORT_SYMBOL_GPL(__srcu_read_unlock); 118EXPORT_SYMBOL_GPL(__srcu_read_unlock);
119 119
@@ -127,24 +127,24 @@ void srcu_drive_gp(struct work_struct *wp)
127 int idx; 127 int idx;
128 struct rcu_head *lh; 128 struct rcu_head *lh;
129 struct rcu_head *rhp; 129 struct rcu_head *rhp;
130 struct srcu_struct *sp; 130 struct srcu_struct *ssp;
131 131
132 sp = container_of(wp, struct srcu_struct, srcu_work); 132 ssp = container_of(wp, struct srcu_struct, srcu_work);
133 if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head)) 133 if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head))
134 return; /* Already running or nothing to do. */ 134 return; /* Already running or nothing to do. */
135 135
136 /* Remove recently arrived callbacks and wait for readers. */ 136 /* Remove recently arrived callbacks and wait for readers. */
137 WRITE_ONCE(sp->srcu_gp_running, true); 137 WRITE_ONCE(ssp->srcu_gp_running, true);
138 local_irq_disable(); 138 local_irq_disable();
139 lh = sp->srcu_cb_head; 139 lh = ssp->srcu_cb_head;
140 sp->srcu_cb_head = NULL; 140 ssp->srcu_cb_head = NULL;
141 sp->srcu_cb_tail = &sp->srcu_cb_head; 141 ssp->srcu_cb_tail = &ssp->srcu_cb_head;
142 local_irq_enable(); 142 local_irq_enable();
143 idx = sp->srcu_idx; 143 idx = ssp->srcu_idx;
144 WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx); 144 WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx);
145 WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ 145 WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */
146 swait_event_exclusive(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx])); 146 swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
147 WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ 147 WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
148 148
149 /* Invoke the callbacks we removed above. */ 149 /* Invoke the callbacks we removed above. */
150 while (lh) { 150 while (lh) {
@@ -161,9 +161,9 @@ void srcu_drive_gp(struct work_struct *wp)
161 * at interrupt level, but the ->srcu_gp_running checks will 161 * at interrupt level, but the ->srcu_gp_running checks will
162 * straighten that out. 162 * straighten that out.
163 */ 163 */
164 WRITE_ONCE(sp->srcu_gp_running, false); 164 WRITE_ONCE(ssp->srcu_gp_running, false);
165 if (READ_ONCE(sp->srcu_cb_head)) 165 if (READ_ONCE(ssp->srcu_cb_head))
166 schedule_work(&sp->srcu_work); 166 schedule_work(&ssp->srcu_work);
167} 167}
168EXPORT_SYMBOL_GPL(srcu_drive_gp); 168EXPORT_SYMBOL_GPL(srcu_drive_gp);
169 169
@@ -171,7 +171,7 @@ EXPORT_SYMBOL_GPL(srcu_drive_gp);
171 * Enqueue an SRCU callback on the specified srcu_struct structure, 171 * Enqueue an SRCU callback on the specified srcu_struct structure,
172 * initiating grace-period processing if it is not already running. 172 * initiating grace-period processing if it is not already running.
173 */ 173 */
174void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, 174void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
175 rcu_callback_t func) 175 rcu_callback_t func)
176{ 176{
177 unsigned long flags; 177 unsigned long flags;
@@ -179,14 +179,14 @@ void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
179 rhp->func = func; 179 rhp->func = func;
180 rhp->next = NULL; 180 rhp->next = NULL;
181 local_irq_save(flags); 181 local_irq_save(flags);
182 *sp->srcu_cb_tail = rhp; 182 *ssp->srcu_cb_tail = rhp;
183 sp->srcu_cb_tail = &rhp->next; 183 ssp->srcu_cb_tail = &rhp->next;
184 local_irq_restore(flags); 184 local_irq_restore(flags);
185 if (!READ_ONCE(sp->srcu_gp_running)) { 185 if (!READ_ONCE(ssp->srcu_gp_running)) {
186 if (likely(srcu_init_done)) 186 if (likely(srcu_init_done))
187 schedule_work(&sp->srcu_work); 187 schedule_work(&ssp->srcu_work);
188 else if (list_empty(&sp->srcu_work.entry)) 188 else if (list_empty(&ssp->srcu_work.entry))
189 list_add(&sp->srcu_work.entry, &srcu_boot_list); 189 list_add(&ssp->srcu_work.entry, &srcu_boot_list);
190 } 190 }
191} 191}
192EXPORT_SYMBOL_GPL(call_srcu); 192EXPORT_SYMBOL_GPL(call_srcu);
@@ -194,13 +194,13 @@ EXPORT_SYMBOL_GPL(call_srcu);
194/* 194/*
195 * synchronize_srcu - wait for prior SRCU read-side critical-section completion 195 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
196 */ 196 */
197void synchronize_srcu(struct srcu_struct *sp) 197void synchronize_srcu(struct srcu_struct *ssp)
198{ 198{
199 struct rcu_synchronize rs; 199 struct rcu_synchronize rs;
200 200
201 init_rcu_head_on_stack(&rs.head); 201 init_rcu_head_on_stack(&rs.head);
202 init_completion(&rs.completion); 202 init_completion(&rs.completion);
203 call_srcu(sp, &rs.head, wakeme_after_rcu); 203 call_srcu(ssp, &rs.head, wakeme_after_rcu);
204 wait_for_completion(&rs.completion); 204 wait_for_completion(&rs.completion);
205 destroy_rcu_head_on_stack(&rs.head); 205 destroy_rcu_head_on_stack(&rs.head);
206} 206}
@@ -219,13 +219,13 @@ void __init rcu_scheduler_starting(void)
219 */ 219 */
220void __init srcu_init(void) 220void __init srcu_init(void)
221{ 221{
222 struct srcu_struct *sp; 222 struct srcu_struct *ssp;
223 223
224 srcu_init_done = true; 224 srcu_init_done = true;
225 while (!list_empty(&srcu_boot_list)) { 225 while (!list_empty(&srcu_boot_list)) {
226 sp = list_first_entry(&srcu_boot_list, 226 ssp = list_first_entry(&srcu_boot_list,
227 struct srcu_struct, srcu_work.entry); 227 struct srcu_struct, srcu_work.entry);
228 list_del_init(&sp->srcu_work.entry); 228 list_del_init(&ssp->srcu_work.entry);
229 schedule_work(&sp->srcu_work); 229 schedule_work(&ssp->srcu_work);
230 } 230 }
231} 231}