diff options
Diffstat (limited to 'kernel/rcutiny.c')
-rw-r--r-- | kernel/rcutiny.c | 294 |
1 files changed, 294 insertions, 0 deletions
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c new file mode 100644 index 000000000000..0b54efde66ac --- /dev/null +++ b/kernel/rcutiny.c | |||
@@ -0,0 +1,294 @@ | |||
1 | /* | ||
2 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License as published by | ||
6 | * the Free Software Foundation; either version 2 of the License, or | ||
7 | * (at your option) any later version. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, | ||
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | * GNU General Public License for more details. | ||
13 | * | ||
14 | * You should have received a copy of the GNU General Public License | ||
15 | * along with this program; if not, write to the Free Software | ||
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
17 | * | ||
18 | * Copyright IBM Corporation, 2008 | ||
19 | * | ||
20 | * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> | ||
21 | * | ||
22 | * For detailed explanation of Read-Copy Update mechanism see - | ||
23 | * Documentation/RCU | ||
24 | */ | ||
25 | |||
26 | #include <linux/types.h> | ||
27 | #include <linux/kernel.h> | ||
28 | #include <linux/init.h> | ||
29 | #include <linux/rcupdate.h> | ||
30 | #include <linux/interrupt.h> | ||
31 | #include <linux/sched.h> | ||
32 | #include <linux/module.h> | ||
33 | #include <linux/completion.h> | ||
34 | #include <linux/moduleparam.h> | ||
35 | #include <linux/notifier.h> | ||
36 | #include <linux/cpu.h> | ||
37 | #include <linux/mutex.h> | ||
38 | #include <linux/time.h> | ||
39 | |||
40 | /* Global control variables for rcupdate callback mechanism. */ | ||
41 | struct rcu_ctrlblk { | ||
42 | struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ | ||
43 | struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ | ||
44 | struct rcu_head **curtail; /* ->next pointer of last CB. */ | ||
45 | }; | ||
46 | |||
47 | /* Definition for rcupdate control block. */ | ||
48 | static struct rcu_ctrlblk rcu_ctrlblk = { | ||
49 | .rcucblist = NULL, | ||
50 | .donetail = &rcu_ctrlblk.rcucblist, | ||
51 | .curtail = &rcu_ctrlblk.rcucblist, | ||
52 | }; | ||
53 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { | ||
54 | .rcucblist = NULL, | ||
55 | .donetail = &rcu_bh_ctrlblk.rcucblist, | ||
56 | .curtail = &rcu_bh_ctrlblk.rcucblist, | ||
57 | }; | ||
58 | |||
59 | #ifdef CONFIG_NO_HZ | ||
60 | |||
61 | static long rcu_dynticks_nesting = 1; | ||
62 | |||
63 | /* | ||
64 | * Enter dynticks-idle mode, which is an extended quiescent state | ||
65 | * if we have fully entered that mode (i.e., if the new value of | ||
66 | * dynticks_nesting is zero). | ||
67 | */ | ||
68 | void rcu_enter_nohz(void) | ||
69 | { | ||
70 | if (--rcu_dynticks_nesting == 0) | ||
71 | rcu_sched_qs(0); /* implies rcu_bh_qsctr_inc(0) */ | ||
72 | } | ||
73 | |||
74 | /* | ||
75 | * Exit dynticks-idle mode, so that we are no longer in an extended | ||
76 | * quiescent state. | ||
77 | */ | ||
78 | void rcu_exit_nohz(void) | ||
79 | { | ||
80 | rcu_dynticks_nesting++; | ||
81 | } | ||
82 | |||
83 | #endif /* #ifdef CONFIG_NO_HZ */ | ||
84 | |||
85 | /* | ||
86 | * Helper function for rcu_qsctr_inc() and rcu_bh_qsctr_inc(). | ||
87 | * Also disable irqs to avoid confusion due to interrupt handlers invoking | ||
88 | * call_rcu(). | ||
89 | */ | ||
90 | static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) | ||
91 | { | ||
92 | unsigned long flags; | ||
93 | |||
94 | local_irq_save(flags); | ||
95 | if (rcp->rcucblist != NULL && | ||
96 | rcp->donetail != rcp->curtail) { | ||
97 | rcp->donetail = rcp->curtail; | ||
98 | local_irq_restore(flags); | ||
99 | return 1; | ||
100 | } | ||
101 | local_irq_restore(flags); | ||
102 | return 0; | ||
103 | } | ||
104 | |||
105 | /* | ||
106 | * Record an rcu quiescent state. And an rcu_bh quiescent state while we | ||
107 | * are at it, given that any rcu quiescent state is also an rcu_bh | ||
108 | * quiescent state. Use "+" instead of "||" to defeat short circuiting. | ||
109 | */ | ||
110 | void rcu_sched_qs(int cpu) | ||
111 | { | ||
112 | if (rcu_qsctr_help(&rcu_ctrlblk) + rcu_qsctr_help(&rcu_bh_ctrlblk)) | ||
113 | raise_softirq(RCU_SOFTIRQ); | ||
114 | } | ||
115 | |||
116 | /* | ||
117 | * Record an rcu_bh quiescent state. | ||
118 | */ | ||
119 | void rcu_bh_qs(int cpu) | ||
120 | { | ||
121 | if (rcu_qsctr_help(&rcu_bh_ctrlblk)) | ||
122 | raise_softirq(RCU_SOFTIRQ); | ||
123 | } | ||
124 | |||
125 | /* | ||
126 | * Check to see if the scheduling-clock interrupt came from an extended | ||
127 | * quiescent state, and, if so, tell RCU about it. | ||
128 | */ | ||
129 | void rcu_check_callbacks(int cpu, int user) | ||
130 | { | ||
131 | if (user || | ||
132 | (idle_cpu(cpu) && | ||
133 | !in_softirq() && | ||
134 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) | ||
135 | rcu_sched_qs(cpu); | ||
136 | else if (!in_softirq()) | ||
137 | rcu_bh_qs(cpu); | ||
138 | } | ||
139 | |||
140 | /* | ||
141 | * Helper function for rcu_process_callbacks() that operates on the | ||
142 | * specified rcu_ctrlkblk structure. | ||
143 | */ | ||
144 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) | ||
145 | { | ||
146 | unsigned long flags; | ||
147 | struct rcu_head *next, *list; | ||
148 | |||
149 | /* If no RCU callbacks ready to invoke, just return. */ | ||
150 | if (&rcp->rcucblist == rcp->donetail) | ||
151 | return; | ||
152 | |||
153 | /* Move the ready-to-invoke callbacks to a local list. */ | ||
154 | local_irq_save(flags); | ||
155 | list = rcp->rcucblist; | ||
156 | rcp->rcucblist = *rcp->donetail; | ||
157 | *rcp->donetail = NULL; | ||
158 | if (rcp->curtail == rcp->donetail) | ||
159 | rcp->curtail = &rcp->rcucblist; | ||
160 | rcp->donetail = &rcp->rcucblist; | ||
161 | local_irq_restore(flags); | ||
162 | |||
163 | /* Invoke the callbacks on the local list. */ | ||
164 | while (list) { | ||
165 | next = list->next; | ||
166 | prefetch(next); | ||
167 | list->func(list); | ||
168 | list = next; | ||
169 | } | ||
170 | } | ||
171 | |||
172 | /* | ||
173 | * Invoke any callbacks whose grace period has completed. | ||
174 | */ | ||
175 | static void rcu_process_callbacks(struct softirq_action *unused) | ||
176 | { | ||
177 | __rcu_process_callbacks(&rcu_ctrlblk); | ||
178 | __rcu_process_callbacks(&rcu_bh_ctrlblk); | ||
179 | } | ||
180 | |||
181 | /* | ||
182 | * Null function to handle CPU being onlined. Longer term, we want to | ||
183 | * make TINY_RCU avoid using rcupdate.c, but later... | ||
184 | */ | ||
185 | int rcu_cpu_notify(struct notifier_block *self, | ||
186 | unsigned long action, void *hcpu) | ||
187 | { | ||
188 | return NOTIFY_OK; | ||
189 | } | ||
190 | |||
191 | /* | ||
192 | * Wait for a grace period to elapse. But it is illegal to invoke | ||
193 | * synchronize_sched() from within an RCU read-side critical section. | ||
194 | * Therefore, any legal call to synchronize_sched() is a quiescent | ||
195 | * state, and so on a UP system, synchronize_sched() need do nothing. | ||
196 | * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the | ||
197 | * benefits of doing might_sleep() to reduce latency.) | ||
198 | * | ||
199 | * Cool, huh? (Due to Josh Triplett.) | ||
200 | * | ||
201 | * But we want to make this a static inline later. | ||
202 | */ | ||
203 | void synchronize_sched(void) | ||
204 | { | ||
205 | cond_resched(); | ||
206 | } | ||
207 | EXPORT_SYMBOL_GPL(synchronize_sched); | ||
208 | |||
209 | void synchronize_rcu_bh(void) | ||
210 | { | ||
211 | synchronize_sched(); | ||
212 | } | ||
213 | EXPORT_SYMBOL_GPL(synchronize_rcu_bh); | ||
214 | |||
215 | /* | ||
216 | * Helper function for call_rcu() and call_rcu_bh(). | ||
217 | */ | ||
218 | static void __call_rcu(struct rcu_head *head, | ||
219 | void (*func)(struct rcu_head *rcu), | ||
220 | struct rcu_ctrlblk *rcp) | ||
221 | { | ||
222 | unsigned long flags; | ||
223 | |||
224 | head->func = func; | ||
225 | head->next = NULL; | ||
226 | local_irq_save(flags); | ||
227 | *rcp->curtail = head; | ||
228 | rcp->curtail = &head->next; | ||
229 | local_irq_restore(flags); | ||
230 | } | ||
231 | |||
232 | /* | ||
233 | * Post an RCU callback to be invoked after the end of an RCU grace | ||
234 | * period. But since we have but one CPU, that would be after any | ||
235 | * quiescent state. | ||
236 | */ | ||
237 | void call_rcu(struct rcu_head *head, | ||
238 | void (*func)(struct rcu_head *rcu)) | ||
239 | { | ||
240 | __call_rcu(head, func, &rcu_ctrlblk); | ||
241 | } | ||
242 | EXPORT_SYMBOL_GPL(call_rcu); | ||
243 | |||
244 | /* | ||
245 | * Post an RCU bottom-half callback to be invoked after any subsequent | ||
246 | * quiescent state. | ||
247 | */ | ||
248 | void call_rcu_bh(struct rcu_head *head, | ||
249 | void (*func)(struct rcu_head *rcu)) | ||
250 | { | ||
251 | __call_rcu(head, func, &rcu_bh_ctrlblk); | ||
252 | } | ||
253 | EXPORT_SYMBOL_GPL(call_rcu_bh); | ||
254 | |||
255 | void rcu_barrier(void) | ||
256 | { | ||
257 | struct rcu_synchronize rcu; | ||
258 | |||
259 | init_completion(&rcu.completion); | ||
260 | /* Will wake me after RCU finished. */ | ||
261 | call_rcu(&rcu.head, wakeme_after_rcu); | ||
262 | /* Wait for it. */ | ||
263 | wait_for_completion(&rcu.completion); | ||
264 | } | ||
265 | EXPORT_SYMBOL_GPL(rcu_barrier); | ||
266 | |||
267 | void rcu_barrier_bh(void) | ||
268 | { | ||
269 | struct rcu_synchronize rcu; | ||
270 | |||
271 | init_completion(&rcu.completion); | ||
272 | /* Will wake me after RCU finished. */ | ||
273 | call_rcu_bh(&rcu.head, wakeme_after_rcu); | ||
274 | /* Wait for it. */ | ||
275 | wait_for_completion(&rcu.completion); | ||
276 | } | ||
277 | EXPORT_SYMBOL_GPL(rcu_barrier_bh); | ||
278 | |||
279 | void rcu_barrier_sched(void) | ||
280 | { | ||
281 | struct rcu_synchronize rcu; | ||
282 | |||
283 | init_completion(&rcu.completion); | ||
284 | /* Will wake me after RCU finished. */ | ||
285 | call_rcu_sched(&rcu.head, wakeme_after_rcu); | ||
286 | /* Wait for it. */ | ||
287 | wait_for_completion(&rcu.completion); | ||
288 | } | ||
289 | EXPORT_SYMBOL_GPL(rcu_barrier_sched); | ||
290 | |||
291 | void __rcu_init(void) | ||
292 | { | ||
293 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | ||
294 | } | ||