diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2016-12-12 12:09:54 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2016-12-12 12:09:54 -0500 |
commit | 718c0ddd6aa911fd2a6fb1b6e050fbaee8060e61 (patch) | |
tree | 8f502a67605f864773d96dca7c8c6e1a9fa3f322 | |
parent | 8fa3b6f9392bf6d90cb7b908e07bd90166639f0a (diff) | |
parent | af91a81131aee3e233a977632a23b839857a327b (diff) |
Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull RCU updates from Ingo Molnar:
"The main RCU changes in this development cycle were:
- Miscellaneous fixes, including a change to call_rcu()'s rcu_head
alignment check.
- Security-motivated list consistency checks, which are disabled by
default behind DEBUG_LIST.
- Torture-test updates.
- Documentation updates, yet again just simple changes"
* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
torture: Prevent jitter from delaying build-only runs
torture: Remove obsolete files from rcutorture .gitignore
rcu: Don't kick unless grace period or request
rcu: Make expedited grace periods recheck dyntick idle state
torture: Trace long read-side delays
rcu: RCU_TRACE enables event tracing as well as debugfs
rcu: Remove obsolete comment from __call_rcu()
rcu: Remove obsolete rcu_check_callbacks() header comment
rcu: Tighten up __call_rcu() rcu_head alignment check
Documentation/RCU: Fix minor typo
documentation: Present updated RCU guarantee
bug: Avoid Kconfig warning for BUG_ON_DATA_CORRUPTION
lib/Kconfig.debug: Fix typo in select statement
lkdtm: Add tests for struct list corruption
bug: Provide toggle for BUG on data corruption
list: Split list_del() debug checking into separate function
rculist: Consolidate DEBUG_LIST for list_add_rcu()
list: Split list_add() debug checking into separate function
-rw-r--r-- | Documentation/RCU/Design/Requirements/Requirements.html | 25 | ||||
-rw-r--r-- | Documentation/RCU/whatisRCU.txt | 2 | ||||
-rw-r--r-- | drivers/misc/lkdtm.h | 2 | ||||
-rw-r--r-- | drivers/misc/lkdtm_bugs.c | 68 | ||||
-rw-r--r-- | drivers/misc/lkdtm_core.c | 2 | ||||
-rw-r--r-- | include/linux/bug.h | 17 | ||||
-rw-r--r-- | include/linux/list.h | 37 | ||||
-rw-r--r-- | include/linux/rculist.h | 8 | ||||
-rw-r--r-- | include/trace/events/rcu.h | 5 | ||||
-rw-r--r-- | kernel/rcu/rcutorture.c | 11 | ||||
-rw-r--r-- | kernel/rcu/tree.c | 17 | ||||
-rw-r--r-- | kernel/rcu/tree.h | 1 | ||||
-rw-r--r-- | kernel/rcu/tree_exp.h | 12 | ||||
-rw-r--r-- | lib/Kconfig.debug | 15 | ||||
-rw-r--r-- | lib/list_debug.c | 99 | ||||
-rw-r--r-- | tools/testing/selftests/rcutorture/.gitignore | 2 | ||||
-rwxr-xr-x | tools/testing/selftests/rcutorture/bin/kvm.sh | 5 |
17 files changed, 221 insertions, 107 deletions
diff --git a/Documentation/RCU/Design/Requirements/Requirements.html b/Documentation/RCU/Design/Requirements/Requirements.html index a4d3838130e4..39bcb74ea733 100644 --- a/Documentation/RCU/Design/Requirements/Requirements.html +++ b/Documentation/RCU/Design/Requirements/Requirements.html | |||
@@ -547,7 +547,7 @@ The <tt>rcu_access_pointer()</tt> on line 6 is similar to | |||
547 | It could reuse a value formerly fetched from this same pointer. | 547 | It could reuse a value formerly fetched from this same pointer. |
548 | It could also fetch the pointer from <tt>gp</tt> in a byte-at-a-time | 548 | It could also fetch the pointer from <tt>gp</tt> in a byte-at-a-time |
549 | manner, resulting in <i>load tearing</i>, in turn resulting a bytewise | 549 | manner, resulting in <i>load tearing</i>, in turn resulting a bytewise |
550 | mash-up of two distince pointer values. | 550 | mash-up of two distinct pointer values. |
551 | It might even use value-speculation optimizations, where it makes | 551 | It might even use value-speculation optimizations, where it makes |
552 | a wrong guess, but by the time it gets around to checking the | 552 | a wrong guess, but by the time it gets around to checking the |
553 | value, an update has changed the pointer to match the wrong guess. | 553 | value, an update has changed the pointer to match the wrong guess. |
@@ -659,6 +659,29 @@ systems with more than one CPU: | |||
659 | In other words, a given instance of <tt>synchronize_rcu()</tt> | 659 | In other words, a given instance of <tt>synchronize_rcu()</tt> |
660 | can avoid waiting on a given RCU read-side critical section only | 660 | can avoid waiting on a given RCU read-side critical section only |
661 | if it can prove that <tt>synchronize_rcu()</tt> started first. | 661 | if it can prove that <tt>synchronize_rcu()</tt> started first. |
662 | |||
663 | <p> | ||
664 | A related question is “When <tt>rcu_read_lock()</tt> | ||
665 | doesn't generate any code, why does it matter how it relates | ||
666 | to a grace period?” | ||
667 | The answer is that it is not the relationship of | ||
668 | <tt>rcu_read_lock()</tt> itself that is important, but rather | ||
669 | the relationship of the code within the enclosed RCU read-side | ||
670 | critical section to the code preceding and following the | ||
671 | grace period. | ||
672 | If we take this viewpoint, then a given RCU read-side critical | ||
673 | section begins before a given grace period when some access | ||
674 | preceding the grace period observes the effect of some access | ||
675 | within the critical section, in which case none of the accesses | ||
676 | within the critical section may observe the effects of any | ||
677 | access following the grace period. | ||
678 | |||
679 | <p> | ||
680 | As of late 2016, mathematical models of RCU take this | ||
681 | viewpoint, for example, see slides 62 and 63 | ||
682 | of the | ||
683 | <a href="http://www2.rdrop.com/users/paulmck/scalability/paper/LinuxMM.2016.10.04c.LCE.pdf">2016 LinuxCon EU</a> | ||
684 | presentation. | ||
662 | </font></td></tr> | 685 | </font></td></tr> |
663 | <tr><td> </td></tr> | 686 | <tr><td> </td></tr> |
664 | </table> | 687 | </table> |
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index 204422719197..5cbd8b2395b8 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt | |||
@@ -237,7 +237,7 @@ rcu_dereference() | |||
237 | 237 | ||
238 | The reader uses rcu_dereference() to fetch an RCU-protected | 238 | The reader uses rcu_dereference() to fetch an RCU-protected |
239 | pointer, which returns a value that may then be safely | 239 | pointer, which returns a value that may then be safely |
240 | dereferenced. Note that rcu_deference() does not actually | 240 | dereferenced. Note that rcu_dereference() does not actually |
241 | dereference the pointer, instead, it protects the pointer for | 241 | dereference the pointer, instead, it protects the pointer for |
242 | later dereferencing. It also executes any needed memory-barrier | 242 | later dereferencing. It also executes any needed memory-barrier |
243 | instructions for a given CPU architecture. Currently, only Alpha | 243 | instructions for a given CPU architecture. Currently, only Alpha |
diff --git a/drivers/misc/lkdtm.h b/drivers/misc/lkdtm.h index fdf954c2107f..cfa1039c62e7 100644 --- a/drivers/misc/lkdtm.h +++ b/drivers/misc/lkdtm.h | |||
@@ -21,6 +21,8 @@ void lkdtm_SPINLOCKUP(void); | |||
21 | void lkdtm_HUNG_TASK(void); | 21 | void lkdtm_HUNG_TASK(void); |
22 | void lkdtm_ATOMIC_UNDERFLOW(void); | 22 | void lkdtm_ATOMIC_UNDERFLOW(void); |
23 | void lkdtm_ATOMIC_OVERFLOW(void); | 23 | void lkdtm_ATOMIC_OVERFLOW(void); |
24 | void lkdtm_CORRUPT_LIST_ADD(void); | ||
25 | void lkdtm_CORRUPT_LIST_DEL(void); | ||
24 | 26 | ||
25 | /* lkdtm_heap.c */ | 27 | /* lkdtm_heap.c */ |
26 | void lkdtm_OVERWRITE_ALLOCATION(void); | 28 | void lkdtm_OVERWRITE_ALLOCATION(void); |
diff --git a/drivers/misc/lkdtm_bugs.c b/drivers/misc/lkdtm_bugs.c index 182ae1894b32..f336206d4b1f 100644 --- a/drivers/misc/lkdtm_bugs.c +++ b/drivers/misc/lkdtm_bugs.c | |||
@@ -5,8 +5,13 @@ | |||
5 | * test source files. | 5 | * test source files. |
6 | */ | 6 | */ |
7 | #include "lkdtm.h" | 7 | #include "lkdtm.h" |
8 | #include <linux/list.h> | ||
8 | #include <linux/sched.h> | 9 | #include <linux/sched.h> |
9 | 10 | ||
11 | struct lkdtm_list { | ||
12 | struct list_head node; | ||
13 | }; | ||
14 | |||
10 | /* | 15 | /* |
11 | * Make sure our attempts to over run the kernel stack doesn't trigger | 16 | * Make sure our attempts to over run the kernel stack doesn't trigger |
12 | * a compiler warning when CONFIG_FRAME_WARN is set. Then make sure we | 17 | * a compiler warning when CONFIG_FRAME_WARN is set. Then make sure we |
@@ -146,3 +151,66 @@ void lkdtm_ATOMIC_OVERFLOW(void) | |||
146 | pr_info("attempting bad atomic overflow\n"); | 151 | pr_info("attempting bad atomic overflow\n"); |
147 | atomic_inc(&over); | 152 | atomic_inc(&over); |
148 | } | 153 | } |
154 | |||
155 | void lkdtm_CORRUPT_LIST_ADD(void) | ||
156 | { | ||
157 | /* | ||
158 | * Initially, an empty list via LIST_HEAD: | ||
159 | * test_head.next = &test_head | ||
160 | * test_head.prev = &test_head | ||
161 | */ | ||
162 | LIST_HEAD(test_head); | ||
163 | struct lkdtm_list good, bad; | ||
164 | void *target[2] = { }; | ||
165 | void *redirection = ⌖ | ||
166 | |||
167 | pr_info("attempting good list addition\n"); | ||
168 | |||
169 | /* | ||
170 | * Adding to the list performs these actions: | ||
171 | * test_head.next->prev = &good.node | ||
172 | * good.node.next = test_head.next | ||
173 | * good.node.prev = test_head | ||
174 | * test_head.next = good.node | ||
175 | */ | ||
176 | list_add(&good.node, &test_head); | ||
177 | |||
178 | pr_info("attempting corrupted list addition\n"); | ||
179 | /* | ||
180 | * In simulating this "write what where" primitive, the "what" is | ||
181 | * the address of &bad.node, and the "where" is the address held | ||
182 | * by "redirection". | ||
183 | */ | ||
184 | test_head.next = redirection; | ||
185 | list_add(&bad.node, &test_head); | ||
186 | |||
187 | if (target[0] == NULL && target[1] == NULL) | ||
188 | pr_err("Overwrite did not happen, but no BUG?!\n"); | ||
189 | else | ||
190 | pr_err("list_add() corruption not detected!\n"); | ||
191 | } | ||
192 | |||
193 | void lkdtm_CORRUPT_LIST_DEL(void) | ||
194 | { | ||
195 | LIST_HEAD(test_head); | ||
196 | struct lkdtm_list item; | ||
197 | void *target[2] = { }; | ||
198 | void *redirection = ⌖ | ||
199 | |||
200 | list_add(&item.node, &test_head); | ||
201 | |||
202 | pr_info("attempting good list removal\n"); | ||
203 | list_del(&item.node); | ||
204 | |||
205 | pr_info("attempting corrupted list removal\n"); | ||
206 | list_add(&item.node, &test_head); | ||
207 | |||
208 | /* As with the list_add() test above, this corrupts "next". */ | ||
209 | item.node.next = redirection; | ||
210 | list_del(&item.node); | ||
211 | |||
212 | if (target[0] == NULL && target[1] == NULL) | ||
213 | pr_err("Overwrite did not happen, but no BUG?!\n"); | ||
214 | else | ||
215 | pr_err("list_del() corruption not detected!\n"); | ||
216 | } | ||
diff --git a/drivers/misc/lkdtm_core.c b/drivers/misc/lkdtm_core.c index f9154b8d67f6..7eeb71a75549 100644 --- a/drivers/misc/lkdtm_core.c +++ b/drivers/misc/lkdtm_core.c | |||
@@ -197,6 +197,8 @@ struct crashtype crashtypes[] = { | |||
197 | CRASHTYPE(EXCEPTION), | 197 | CRASHTYPE(EXCEPTION), |
198 | CRASHTYPE(LOOP), | 198 | CRASHTYPE(LOOP), |
199 | CRASHTYPE(OVERFLOW), | 199 | CRASHTYPE(OVERFLOW), |
200 | CRASHTYPE(CORRUPT_LIST_ADD), | ||
201 | CRASHTYPE(CORRUPT_LIST_DEL), | ||
200 | CRASHTYPE(CORRUPT_STACK), | 202 | CRASHTYPE(CORRUPT_STACK), |
201 | CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE), | 203 | CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE), |
202 | CRASHTYPE(OVERWRITE_ALLOCATION), | 204 | CRASHTYPE(OVERWRITE_ALLOCATION), |
diff --git a/include/linux/bug.h b/include/linux/bug.h index 292d6a10b0c2..baff2e8fc8a8 100644 --- a/include/linux/bug.h +++ b/include/linux/bug.h | |||
@@ -121,4 +121,21 @@ static inline enum bug_trap_type report_bug(unsigned long bug_addr, | |||
121 | } | 121 | } |
122 | 122 | ||
123 | #endif /* CONFIG_GENERIC_BUG */ | 123 | #endif /* CONFIG_GENERIC_BUG */ |
124 | |||
125 | /* | ||
126 | * Since detected data corruption should stop operation on the affected | ||
127 | * structures, this returns false if the corruption condition is found. | ||
128 | */ | ||
129 | #define CHECK_DATA_CORRUPTION(condition, fmt, ...) \ | ||
130 | do { \ | ||
131 | if (unlikely(condition)) { \ | ||
132 | if (IS_ENABLED(CONFIG_BUG_ON_DATA_CORRUPTION)) { \ | ||
133 | pr_err(fmt, ##__VA_ARGS__); \ | ||
134 | BUG(); \ | ||
135 | } else \ | ||
136 | WARN(1, fmt, ##__VA_ARGS__); \ | ||
137 | return false; \ | ||
138 | } \ | ||
139 | } while (0) | ||
140 | |||
124 | #endif /* _LINUX_BUG_H */ | 141 | #endif /* _LINUX_BUG_H */ |
diff --git a/include/linux/list.h b/include/linux/list.h index 5809e9a2de5b..d1039ecaf94f 100644 --- a/include/linux/list.h +++ b/include/linux/list.h | |||
@@ -28,27 +28,42 @@ static inline void INIT_LIST_HEAD(struct list_head *list) | |||
28 | list->prev = list; | 28 | list->prev = list; |
29 | } | 29 | } |
30 | 30 | ||
31 | #ifdef CONFIG_DEBUG_LIST | ||
32 | extern bool __list_add_valid(struct list_head *new, | ||
33 | struct list_head *prev, | ||
34 | struct list_head *next); | ||
35 | extern bool __list_del_entry_valid(struct list_head *entry); | ||
36 | #else | ||
37 | static inline bool __list_add_valid(struct list_head *new, | ||
38 | struct list_head *prev, | ||
39 | struct list_head *next) | ||
40 | { | ||
41 | return true; | ||
42 | } | ||
43 | static inline bool __list_del_entry_valid(struct list_head *entry) | ||
44 | { | ||
45 | return true; | ||
46 | } | ||
47 | #endif | ||
48 | |||
31 | /* | 49 | /* |
32 | * Insert a new entry between two known consecutive entries. | 50 | * Insert a new entry between two known consecutive entries. |
33 | * | 51 | * |
34 | * This is only for internal list manipulation where we know | 52 | * This is only for internal list manipulation where we know |
35 | * the prev/next entries already! | 53 | * the prev/next entries already! |
36 | */ | 54 | */ |
37 | #ifndef CONFIG_DEBUG_LIST | ||
38 | static inline void __list_add(struct list_head *new, | 55 | static inline void __list_add(struct list_head *new, |
39 | struct list_head *prev, | 56 | struct list_head *prev, |
40 | struct list_head *next) | 57 | struct list_head *next) |
41 | { | 58 | { |
59 | if (!__list_add_valid(new, prev, next)) | ||
60 | return; | ||
61 | |||
42 | next->prev = new; | 62 | next->prev = new; |
43 | new->next = next; | 63 | new->next = next; |
44 | new->prev = prev; | 64 | new->prev = prev; |
45 | WRITE_ONCE(prev->next, new); | 65 | WRITE_ONCE(prev->next, new); |
46 | } | 66 | } |
47 | #else | ||
48 | extern void __list_add(struct list_head *new, | ||
49 | struct list_head *prev, | ||
50 | struct list_head *next); | ||
51 | #endif | ||
52 | 67 | ||
53 | /** | 68 | /** |
54 | * list_add - add a new entry | 69 | * list_add - add a new entry |
@@ -96,22 +111,20 @@ static inline void __list_del(struct list_head * prev, struct list_head * next) | |||
96 | * Note: list_empty() on entry does not return true after this, the entry is | 111 | * Note: list_empty() on entry does not return true after this, the entry is |
97 | * in an undefined state. | 112 | * in an undefined state. |
98 | */ | 113 | */ |
99 | #ifndef CONFIG_DEBUG_LIST | ||
100 | static inline void __list_del_entry(struct list_head *entry) | 114 | static inline void __list_del_entry(struct list_head *entry) |
101 | { | 115 | { |
116 | if (!__list_del_entry_valid(entry)) | ||
117 | return; | ||
118 | |||
102 | __list_del(entry->prev, entry->next); | 119 | __list_del(entry->prev, entry->next); |
103 | } | 120 | } |
104 | 121 | ||
105 | static inline void list_del(struct list_head *entry) | 122 | static inline void list_del(struct list_head *entry) |
106 | { | 123 | { |
107 | __list_del(entry->prev, entry->next); | 124 | __list_del_entry(entry); |
108 | entry->next = LIST_POISON1; | 125 | entry->next = LIST_POISON1; |
109 | entry->prev = LIST_POISON2; | 126 | entry->prev = LIST_POISON2; |
110 | } | 127 | } |
111 | #else | ||
112 | extern void __list_del_entry(struct list_head *entry); | ||
113 | extern void list_del(struct list_head *entry); | ||
114 | #endif | ||
115 | 128 | ||
116 | /** | 129 | /** |
117 | * list_replace - replace old entry by new one | 130 | * list_replace - replace old entry by new one |
diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 8beb98dcf14f..4f7a9561b8c4 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h | |||
@@ -45,19 +45,17 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list) | |||
45 | * This is only for internal list manipulation where we know | 45 | * This is only for internal list manipulation where we know |
46 | * the prev/next entries already! | 46 | * the prev/next entries already! |
47 | */ | 47 | */ |
48 | #ifndef CONFIG_DEBUG_LIST | ||
49 | static inline void __list_add_rcu(struct list_head *new, | 48 | static inline void __list_add_rcu(struct list_head *new, |
50 | struct list_head *prev, struct list_head *next) | 49 | struct list_head *prev, struct list_head *next) |
51 | { | 50 | { |
51 | if (!__list_add_valid(new, prev, next)) | ||
52 | return; | ||
53 | |||
52 | new->next = next; | 54 | new->next = next; |
53 | new->prev = prev; | 55 | new->prev = prev; |
54 | rcu_assign_pointer(list_next_rcu(prev), new); | 56 | rcu_assign_pointer(list_next_rcu(prev), new); |
55 | next->prev = new; | 57 | next->prev = new; |
56 | } | 58 | } |
57 | #else | ||
58 | void __list_add_rcu(struct list_head *new, | ||
59 | struct list_head *prev, struct list_head *next); | ||
60 | #endif | ||
61 | 59 | ||
62 | /** | 60 | /** |
63 | * list_add_rcu - add a new entry to rcu-protected list | 61 | * list_add_rcu - add a new entry to rcu-protected list |
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index d3e756539d44..9d4f9b3a2b7b 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h | |||
@@ -698,7 +698,10 @@ TRACE_EVENT(rcu_batch_end, | |||
698 | /* | 698 | /* |
699 | * Tracepoint for rcutorture readers. The first argument is the name | 699 | * Tracepoint for rcutorture readers. The first argument is the name |
700 | * of the RCU flavor from rcutorture's viewpoint and the second argument | 700 | * of the RCU flavor from rcutorture's viewpoint and the second argument |
701 | * is the callback address. | 701 | * is the callback address. The third argument is the start time in |
702 | * seconds, and the last two arguments are the grace period numbers | ||
703 | * at the beginning and end of the read, respectively. Note that the | ||
704 | * callback address can be NULL. | ||
702 | */ | 705 | */ |
703 | TRACE_EVENT(rcu_torture_read, | 706 | TRACE_EVENT(rcu_torture_read, |
704 | 707 | ||
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index bf08fee53dc7..87c51225ceec 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c | |||
@@ -289,15 +289,24 @@ static int rcu_torture_read_lock(void) __acquires(RCU) | |||
289 | 289 | ||
290 | static void rcu_read_delay(struct torture_random_state *rrsp) | 290 | static void rcu_read_delay(struct torture_random_state *rrsp) |
291 | { | 291 | { |
292 | unsigned long started; | ||
293 | unsigned long completed; | ||
292 | const unsigned long shortdelay_us = 200; | 294 | const unsigned long shortdelay_us = 200; |
293 | const unsigned long longdelay_ms = 50; | 295 | const unsigned long longdelay_ms = 50; |
296 | unsigned long long ts; | ||
294 | 297 | ||
295 | /* We want a short delay sometimes to make a reader delay the grace | 298 | /* We want a short delay sometimes to make a reader delay the grace |
296 | * period, and we want a long delay occasionally to trigger | 299 | * period, and we want a long delay occasionally to trigger |
297 | * force_quiescent_state. */ | 300 | * force_quiescent_state. */ |
298 | 301 | ||
299 | if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) | 302 | if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { |
303 | started = cur_ops->completed(); | ||
304 | ts = rcu_trace_clock_local(); | ||
300 | mdelay(longdelay_ms); | 305 | mdelay(longdelay_ms); |
306 | completed = cur_ops->completed(); | ||
307 | do_trace_rcu_torture_read(cur_ops->name, NULL, ts, | ||
308 | started, completed); | ||
309 | } | ||
301 | if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) | 310 | if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) |
302 | udelay(shortdelay_us); | 311 | udelay(shortdelay_us); |
303 | #ifdef CONFIG_PREEMPT | 312 | #ifdef CONFIG_PREEMPT |
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 69a5611a7e7c..96c52e43f7ca 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
@@ -1304,7 +1304,8 @@ static void rcu_stall_kick_kthreads(struct rcu_state *rsp) | |||
1304 | if (!rcu_kick_kthreads) | 1304 | if (!rcu_kick_kthreads) |
1305 | return; | 1305 | return; |
1306 | j = READ_ONCE(rsp->jiffies_kick_kthreads); | 1306 | j = READ_ONCE(rsp->jiffies_kick_kthreads); |
1307 | if (time_after(jiffies, j) && rsp->gp_kthread) { | 1307 | if (time_after(jiffies, j) && rsp->gp_kthread && |
1308 | (rcu_gp_in_progress(rsp) || READ_ONCE(rsp->gp_flags))) { | ||
1308 | WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name); | 1309 | WARN_ONCE(1, "Kicking %s grace-period kthread\n", rsp->name); |
1309 | rcu_ftrace_dump(DUMP_ALL); | 1310 | rcu_ftrace_dump(DUMP_ALL); |
1310 | wake_up_process(rsp->gp_kthread); | 1311 | wake_up_process(rsp->gp_kthread); |
@@ -2828,8 +2829,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) | |||
2828 | * Also schedule RCU core processing. | 2829 | * Also schedule RCU core processing. |
2829 | * | 2830 | * |
2830 | * This function must be called from hardirq context. It is normally | 2831 | * This function must be called from hardirq context. It is normally |
2831 | * invoked from the scheduling-clock interrupt. If rcu_pending returns | 2832 | * invoked from the scheduling-clock interrupt. |
2832 | * false, there is no point in invoking rcu_check_callbacks(). | ||
2833 | */ | 2833 | */ |
2834 | void rcu_check_callbacks(int user) | 2834 | void rcu_check_callbacks(int user) |
2835 | { | 2835 | { |
@@ -3121,7 +3121,9 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, | |||
3121 | unsigned long flags; | 3121 | unsigned long flags; |
3122 | struct rcu_data *rdp; | 3122 | struct rcu_data *rdp; |
3123 | 3123 | ||
3124 | WARN_ON_ONCE((unsigned long)head & 0x1); /* Misaligned rcu_head! */ | 3124 | /* Misaligned rcu_head! */ |
3125 | WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); | ||
3126 | |||
3125 | if (debug_rcu_head_queue(head)) { | 3127 | if (debug_rcu_head_queue(head)) { |
3126 | /* Probable double call_rcu(), so leak the callback. */ | 3128 | /* Probable double call_rcu(), so leak the callback. */ |
3127 | WRITE_ONCE(head->func, rcu_leak_callback); | 3129 | WRITE_ONCE(head->func, rcu_leak_callback); |
@@ -3130,13 +3132,6 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func, | |||
3130 | } | 3132 | } |
3131 | head->func = func; | 3133 | head->func = func; |
3132 | head->next = NULL; | 3134 | head->next = NULL; |
3133 | |||
3134 | /* | ||
3135 | * Opportunistically note grace-period endings and beginnings. | ||
3136 | * Note that we might see a beginning right after we see an | ||
3137 | * end, but never vice versa, since this CPU has to pass through | ||
3138 | * a quiescent state betweentimes. | ||
3139 | */ | ||
3140 | local_irq_save(flags); | 3135 | local_irq_save(flags); |
3141 | rdp = this_cpu_ptr(rsp->rda); | 3136 | rdp = this_cpu_ptr(rsp->rda); |
3142 | 3137 | ||
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index e99a5234d9ed..fe98dd24adf8 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h | |||
@@ -404,6 +404,7 @@ struct rcu_data { | |||
404 | atomic_long_t exp_workdone1; /* # done by others #1. */ | 404 | atomic_long_t exp_workdone1; /* # done by others #1. */ |
405 | atomic_long_t exp_workdone2; /* # done by others #2. */ | 405 | atomic_long_t exp_workdone2; /* # done by others #2. */ |
406 | atomic_long_t exp_workdone3; /* # done by others #3. */ | 406 | atomic_long_t exp_workdone3; /* # done by others #3. */ |
407 | int exp_dynticks_snap; /* Double-check need for IPI. */ | ||
407 | 408 | ||
408 | /* 7) Callback offloading. */ | 409 | /* 7) Callback offloading. */ |
409 | #ifdef CONFIG_RCU_NOCB_CPU | 410 | #ifdef CONFIG_RCU_NOCB_CPU |
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 24343eb87b58..d3053e99fdb6 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h | |||
@@ -358,8 +358,10 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, | |||
358 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | 358 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); |
359 | struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | 359 | struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); |
360 | 360 | ||
361 | rdp->exp_dynticks_snap = | ||
362 | atomic_add_return(0, &rdtp->dynticks); | ||
361 | if (raw_smp_processor_id() == cpu || | 363 | if (raw_smp_processor_id() == cpu || |
362 | !(atomic_add_return(0, &rdtp->dynticks) & 0x1) || | 364 | !(rdp->exp_dynticks_snap & 0x1) || |
363 | !(rnp->qsmaskinitnext & rdp->grpmask)) | 365 | !(rnp->qsmaskinitnext & rdp->grpmask)) |
364 | mask_ofl_test |= rdp->grpmask; | 366 | mask_ofl_test |= rdp->grpmask; |
365 | } | 367 | } |
@@ -377,9 +379,17 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp, | |||
377 | /* IPI the remaining CPUs for expedited quiescent state. */ | 379 | /* IPI the remaining CPUs for expedited quiescent state. */ |
378 | for_each_leaf_node_possible_cpu(rnp, cpu) { | 380 | for_each_leaf_node_possible_cpu(rnp, cpu) { |
379 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); | 381 | unsigned long mask = leaf_node_cpu_bit(rnp, cpu); |
382 | struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); | ||
383 | struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); | ||
384 | |||
380 | if (!(mask_ofl_ipi & mask)) | 385 | if (!(mask_ofl_ipi & mask)) |
381 | continue; | 386 | continue; |
382 | retry_ipi: | 387 | retry_ipi: |
388 | if (atomic_add_return(0, &rdtp->dynticks) != | ||
389 | rdp->exp_dynticks_snap) { | ||
390 | mask_ofl_test |= mask; | ||
391 | continue; | ||
392 | } | ||
383 | ret = smp_call_function_single(cpu, func, rsp, 0); | 393 | ret = smp_call_function_single(cpu, func, rsp, 0); |
384 | if (!ret) { | 394 | if (!ret) { |
385 | mask_ofl_ipi &= ~mask; | 395 | mask_ofl_ipi &= ~mask; |
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a6c8db1d62f6..9bb7d825ba14 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug | |||
@@ -1218,7 +1218,7 @@ config DEBUG_BUGVERBOSE | |||
1218 | 1218 | ||
1219 | config DEBUG_LIST | 1219 | config DEBUG_LIST |
1220 | bool "Debug linked list manipulation" | 1220 | bool "Debug linked list manipulation" |
1221 | depends on DEBUG_KERNEL | 1221 | depends on DEBUG_KERNEL || BUG_ON_DATA_CORRUPTION |
1222 | help | 1222 | help |
1223 | Enable this to turn on extended checks in the linked-list | 1223 | Enable this to turn on extended checks in the linked-list |
1224 | walking routines. | 1224 | walking routines. |
@@ -1434,7 +1434,8 @@ config RCU_TRACE | |||
1434 | select TRACE_CLOCK | 1434 | select TRACE_CLOCK |
1435 | help | 1435 | help |
1436 | This option provides tracing in RCU which presents stats | 1436 | This option provides tracing in RCU which presents stats |
1437 | in debugfs for debugging RCU implementation. | 1437 | in debugfs for debugging RCU implementation. It also enables |
1438 | additional tracepoints for ftrace-style event tracing. | ||
1438 | 1439 | ||
1439 | Say Y here if you want to enable RCU tracing | 1440 | Say Y here if you want to enable RCU tracing |
1440 | Say N if you are unsure. | 1441 | Say N if you are unsure. |
@@ -1964,6 +1965,16 @@ config TEST_STATIC_KEYS | |||
1964 | 1965 | ||
1965 | If unsure, say N. | 1966 | If unsure, say N. |
1966 | 1967 | ||
1968 | config BUG_ON_DATA_CORRUPTION | ||
1969 | bool "Trigger a BUG when data corruption is detected" | ||
1970 | select DEBUG_LIST | ||
1971 | help | ||
1972 | Select this option if the kernel should BUG when it encounters | ||
1973 | data corruption in kernel memory structures when they get checked | ||
1974 | for validity. | ||
1975 | |||
1976 | If unsure, say N. | ||
1977 | |||
1967 | source "samples/Kconfig" | 1978 | source "samples/Kconfig" |
1968 | 1979 | ||
1969 | source "lib/Kconfig.kgdb" | 1980 | source "lib/Kconfig.kgdb" |
diff --git a/lib/list_debug.c b/lib/list_debug.c index 3859bf63561c..7f7bfa55eb6d 100644 --- a/lib/list_debug.c +++ b/lib/list_debug.c | |||
@@ -2,8 +2,7 @@ | |||
2 | * Copyright 2006, Red Hat, Inc., Dave Jones | 2 | * Copyright 2006, Red Hat, Inc., Dave Jones |
3 | * Released under the General Public License (GPL). | 3 | * Released under the General Public License (GPL). |
4 | * | 4 | * |
5 | * This file contains the linked list implementations for | 5 | * This file contains the linked list validation for DEBUG_LIST. |
6 | * DEBUG_LIST. | ||
7 | */ | 6 | */ |
8 | 7 | ||
9 | #include <linux/export.h> | 8 | #include <linux/export.h> |
@@ -13,88 +12,48 @@ | |||
13 | #include <linux/rculist.h> | 12 | #include <linux/rculist.h> |
14 | 13 | ||
15 | /* | 14 | /* |
16 | * Insert a new entry between two known consecutive entries. | 15 | * Check that the data structures for the list manipulations are reasonably |
17 | * | 16 | * valid. Failures here indicate memory corruption (and possibly an exploit |
18 | * This is only for internal list manipulation where we know | 17 | * attempt). |
19 | * the prev/next entries already! | ||
20 | */ | 18 | */ |
21 | 19 | ||
22 | void __list_add(struct list_head *new, | 20 | bool __list_add_valid(struct list_head *new, struct list_head *prev, |
23 | struct list_head *prev, | 21 | struct list_head *next) |
24 | struct list_head *next) | ||
25 | { | 22 | { |
26 | WARN(next->prev != prev, | 23 | CHECK_DATA_CORRUPTION(next->prev != prev, |
27 | "list_add corruption. next->prev should be " | 24 | "list_add corruption. next->prev should be prev (%p), but was %p. (next=%p).\n", |
28 | "prev (%p), but was %p. (next=%p).\n", | ||
29 | prev, next->prev, next); | 25 | prev, next->prev, next); |
30 | WARN(prev->next != next, | 26 | CHECK_DATA_CORRUPTION(prev->next != next, |
31 | "list_add corruption. prev->next should be " | 27 | "list_add corruption. prev->next should be next (%p), but was %p. (prev=%p).\n", |
32 | "next (%p), but was %p. (prev=%p).\n", | ||
33 | next, prev->next, prev); | 28 | next, prev->next, prev); |
34 | WARN(new == prev || new == next, | 29 | CHECK_DATA_CORRUPTION(new == prev || new == next, |
35 | "list_add double add: new=%p, prev=%p, next=%p.\n", | 30 | "list_add double add: new=%p, prev=%p, next=%p.\n", |
36 | new, prev, next); | 31 | new, prev, next); |
37 | next->prev = new; | 32 | |
38 | new->next = next; | 33 | return true; |
39 | new->prev = prev; | ||
40 | WRITE_ONCE(prev->next, new); | ||
41 | } | 34 | } |
42 | EXPORT_SYMBOL(__list_add); | 35 | EXPORT_SYMBOL(__list_add_valid); |
43 | 36 | ||
44 | void __list_del_entry(struct list_head *entry) | 37 | bool __list_del_entry_valid(struct list_head *entry) |
45 | { | 38 | { |
46 | struct list_head *prev, *next; | 39 | struct list_head *prev, *next; |
47 | 40 | ||
48 | prev = entry->prev; | 41 | prev = entry->prev; |
49 | next = entry->next; | 42 | next = entry->next; |
50 | 43 | ||
51 | if (WARN(next == LIST_POISON1, | 44 | CHECK_DATA_CORRUPTION(next == LIST_POISON1, |
52 | "list_del corruption, %p->next is LIST_POISON1 (%p)\n", | 45 | "list_del corruption, %p->next is LIST_POISON1 (%p)\n", |
53 | entry, LIST_POISON1) || | 46 | entry, LIST_POISON1); |
54 | WARN(prev == LIST_POISON2, | 47 | CHECK_DATA_CORRUPTION(prev == LIST_POISON2, |
55 | "list_del corruption, %p->prev is LIST_POISON2 (%p)\n", | 48 | "list_del corruption, %p->prev is LIST_POISON2 (%p)\n", |
56 | entry, LIST_POISON2) || | 49 | entry, LIST_POISON2); |
57 | WARN(prev->next != entry, | 50 | CHECK_DATA_CORRUPTION(prev->next != entry, |
58 | "list_del corruption. prev->next should be %p, " | 51 | "list_del corruption. prev->next should be %p, but was %p\n", |
59 | "but was %p\n", entry, prev->next) || | 52 | entry, prev->next); |
60 | WARN(next->prev != entry, | 53 | CHECK_DATA_CORRUPTION(next->prev != entry, |
61 | "list_del corruption. next->prev should be %p, " | 54 | "list_del corruption. next->prev should be %p, but was %p\n", |
62 | "but was %p\n", entry, next->prev)) | 55 | entry, next->prev); |
63 | return; | 56 | return true; |
64 | |||
65 | __list_del(prev, next); | ||
66 | } | ||
67 | EXPORT_SYMBOL(__list_del_entry); | ||
68 | 57 | ||
69 | /** | ||
70 | * list_del - deletes entry from list. | ||
71 | * @entry: the element to delete from the list. | ||
72 | * Note: list_empty on entry does not return true after this, the entry is | ||
73 | * in an undefined state. | ||
74 | */ | ||
75 | void list_del(struct list_head *entry) | ||
76 | { | ||
77 | __list_del_entry(entry); | ||
78 | entry->next = LIST_POISON1; | ||
79 | entry->prev = LIST_POISON2; | ||
80 | } | ||
81 | EXPORT_SYMBOL(list_del); | ||
82 | |||
83 | /* | ||
84 | * RCU variants. | ||
85 | */ | ||
86 | void __list_add_rcu(struct list_head *new, | ||
87 | struct list_head *prev, struct list_head *next) | ||
88 | { | ||
89 | WARN(next->prev != prev, | ||
90 | "list_add_rcu corruption. next->prev should be prev (%p), but was %p. (next=%p).\n", | ||
91 | prev, next->prev, next); | ||
92 | WARN(prev->next != next, | ||
93 | "list_add_rcu corruption. prev->next should be next (%p), but was %p. (prev=%p).\n", | ||
94 | next, prev->next, prev); | ||
95 | new->next = next; | ||
96 | new->prev = prev; | ||
97 | rcu_assign_pointer(list_next_rcu(prev), new); | ||
98 | next->prev = new; | ||
99 | } | 58 | } |
100 | EXPORT_SYMBOL(__list_add_rcu); | 59 | EXPORT_SYMBOL(__list_del_entry_valid); |
diff --git a/tools/testing/selftests/rcutorture/.gitignore b/tools/testing/selftests/rcutorture/.gitignore index 05838f6f2ebe..ccc240275d1c 100644 --- a/tools/testing/selftests/rcutorture/.gitignore +++ b/tools/testing/selftests/rcutorture/.gitignore | |||
@@ -1,6 +1,4 @@ | |||
1 | initrd | 1 | initrd |
2 | linux-2.6 | ||
3 | b[0-9]* | 2 | b[0-9]* |
4 | rcu-test-image | ||
5 | res | 3 | res |
6 | *.swp | 4 | *.swp |
diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index 0aed965f0062..3b3c1b693ee1 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh | |||
@@ -303,6 +303,7 @@ then | |||
303 | fi | 303 | fi |
304 | ___EOF___ | 304 | ___EOF___ |
305 | awk < $T/cfgcpu.pack \ | 305 | awk < $T/cfgcpu.pack \ |
306 | -v TORTURE_BUILDONLY="$TORTURE_BUILDONLY" \ | ||
306 | -v CONFIGDIR="$CONFIGFRAG/" \ | 307 | -v CONFIGDIR="$CONFIGFRAG/" \ |
307 | -v KVM="$KVM" \ | 308 | -v KVM="$KVM" \ |
308 | -v ncpus=$cpus \ | 309 | -v ncpus=$cpus \ |
@@ -375,6 +376,10 @@ function dump(first, pastlast, batchnum) | |||
375 | njitter = ncpus; | 376 | njitter = ncpus; |
376 | else | 377 | else |
377 | njitter = ja[1]; | 378 | njitter = ja[1]; |
379 | if (TORTURE_BUILDONLY && njitter != 0) { | ||
380 | njitter = 0; | ||
381 | print "echo Build-only run, so suppressing jitter >> " rd "/log" | ||
382 | } | ||
378 | for (j = 0; j < njitter; j++) | 383 | for (j = 0; j < njitter; j++) |
379 | print "jitter.sh " j " " dur " " ja[2] " " ja[3] "&" | 384 | print "jitter.sh " j " " dur " " ja[2] " " ja[3] "&" |
380 | print "wait" | 385 | print "wait" |