diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2010-03-13 17:43:01 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2010-03-13 17:43:01 -0500 |
commit | 4e3eaddd142e2142c048c5052a0a9d2604fccfc6 (patch) | |
tree | 5bc45a286502e54e790c54948f22364c5afd9d89 | |
parent | 8655e7e3ddec60603c4f6c14cdf642e2ba198df8 (diff) | |
parent | b97c4bc16734a2e597dac7f91ee9eb78f4aeef9a (diff) |
Merge branch 'core-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'core-fixes-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
locking: Make sparse work with inline spinlocks and rwlocks
x86/mce: Fix RCU lockdep splats
rcu: Increase RCU CPU stall timeouts if PROVE_RCU
ftrace: Replace read_barrier_depends() with rcu_dereference_raw()
rcu: Suppress RCU lockdep warnings during early boot
rcu, ftrace: Fix RCU lockdep splat in ftrace_perf_buf_prepare()
rcu: Suppress __mpol_dup() false positive from RCU lockdep
rcu: Make rcu_read_lock_sched_held() handle !PREEMPT
rcu: Add control variables to lockdep_rcu_dereference() diagnostics
rcu, cgroup: Relax the check in task_subsys_state() as early boot is now handled by lockdep-RCU
rcu: Use wrapper function instead of exporting tasklist_lock
sched, rcu: Fix rcu_dereference() for RCU-lockdep
rcu: Make task_subsys_state() RCU-lockdep checks handle boot-time use
rcu: Fix holdoff for accelerated GPs for last non-dynticked CPU
x86/gart: Unexport gart_iommu_aperture
Fix trivial conflicts in kernel/trace/ftrace.c
-rw-r--r-- | arch/x86/kernel/aperture_64.c | 1 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mcheck/mce.c | 11 | ||||
-rw-r--r-- | include/linux/cred.h | 2 | ||||
-rw-r--r-- | include/linux/rcupdate.h | 45 | ||||
-rw-r--r-- | include/linux/rwlock.h | 20 | ||||
-rw-r--r-- | include/linux/sched.h | 4 | ||||
-rw-r--r-- | include/linux/spinlock.h | 13 | ||||
-rw-r--r-- | include/trace/ftrace.h | 4 | ||||
-rw-r--r-- | kernel/exit.c | 2 | ||||
-rw-r--r-- | kernel/fork.c | 9 | ||||
-rw-r--r-- | kernel/lockdep.c | 1 | ||||
-rw-r--r-- | kernel/pid.c | 4 | ||||
-rw-r--r-- | kernel/rcutree.h | 21 | ||||
-rw-r--r-- | kernel/rcutree_plugin.h | 8 | ||||
-rw-r--r-- | kernel/sched_fair.c | 2 | ||||
-rw-r--r-- | kernel/trace/ftrace.c | 22 | ||||
-rw-r--r-- | kernel/trace/trace_event_profile.c | 4 | ||||
-rw-r--r-- | mm/mempolicy.c | 2 |
18 files changed, 120 insertions, 55 deletions
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index f147a95fd84a..3704997e8b25 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c | |||
@@ -31,7 +31,6 @@ | |||
31 | #include <asm/x86_init.h> | 31 | #include <asm/x86_init.h> |
32 | 32 | ||
33 | int gart_iommu_aperture; | 33 | int gart_iommu_aperture; |
34 | EXPORT_SYMBOL_GPL(gart_iommu_aperture); | ||
35 | int gart_iommu_aperture_disabled __initdata; | 34 | int gart_iommu_aperture_disabled __initdata; |
36 | int gart_iommu_aperture_allowed __initdata; | 35 | int gart_iommu_aperture_allowed __initdata; |
37 | 36 | ||
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 28cba46bf32c..bd58de4d7a29 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c | |||
@@ -46,6 +46,11 @@ | |||
46 | 46 | ||
47 | #include "mce-internal.h" | 47 | #include "mce-internal.h" |
48 | 48 | ||
49 | #define rcu_dereference_check_mce(p) \ | ||
50 | rcu_dereference_check((p), \ | ||
51 | rcu_read_lock_sched_held() || \ | ||
52 | lockdep_is_held(&mce_read_mutex)) | ||
53 | |||
49 | #define CREATE_TRACE_POINTS | 54 | #define CREATE_TRACE_POINTS |
50 | #include <trace/events/mce.h> | 55 | #include <trace/events/mce.h> |
51 | 56 | ||
@@ -158,7 +163,7 @@ void mce_log(struct mce *mce) | |||
158 | mce->finished = 0; | 163 | mce->finished = 0; |
159 | wmb(); | 164 | wmb(); |
160 | for (;;) { | 165 | for (;;) { |
161 | entry = rcu_dereference(mcelog.next); | 166 | entry = rcu_dereference_check_mce(mcelog.next); |
162 | for (;;) { | 167 | for (;;) { |
163 | /* | 168 | /* |
164 | * When the buffer fills up discard new entries. | 169 | * When the buffer fills up discard new entries. |
@@ -1500,7 +1505,7 @@ static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize, | |||
1500 | return -ENOMEM; | 1505 | return -ENOMEM; |
1501 | 1506 | ||
1502 | mutex_lock(&mce_read_mutex); | 1507 | mutex_lock(&mce_read_mutex); |
1503 | next = rcu_dereference(mcelog.next); | 1508 | next = rcu_dereference_check_mce(mcelog.next); |
1504 | 1509 | ||
1505 | /* Only supports full reads right now */ | 1510 | /* Only supports full reads right now */ |
1506 | if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) { | 1511 | if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) { |
@@ -1565,7 +1570,7 @@ timeout: | |||
1565 | static unsigned int mce_poll(struct file *file, poll_table *wait) | 1570 | static unsigned int mce_poll(struct file *file, poll_table *wait) |
1566 | { | 1571 | { |
1567 | poll_wait(file, &mce_wait, wait); | 1572 | poll_wait(file, &mce_wait, wait); |
1568 | if (rcu_dereference(mcelog.next)) | 1573 | if (rcu_dereference_check_mce(mcelog.next)) |
1569 | return POLLIN | POLLRDNORM; | 1574 | return POLLIN | POLLRDNORM; |
1570 | return 0; | 1575 | return 0; |
1571 | } | 1576 | } |
diff --git a/include/linux/cred.h b/include/linux/cred.h index 4db09f89b637..52507c3e1387 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h | |||
@@ -280,7 +280,7 @@ static inline void put_cred(const struct cred *_cred) | |||
280 | * task or by holding tasklist_lock to prevent it from being unlinked. | 280 | * task or by holding tasklist_lock to prevent it from being unlinked. |
281 | */ | 281 | */ |
282 | #define __task_cred(task) \ | 282 | #define __task_cred(task) \ |
283 | ((const struct cred *)(rcu_dereference_check((task)->real_cred, rcu_read_lock_held() || lockdep_is_held(&tasklist_lock)))) | 283 | ((const struct cred *)(rcu_dereference_check((task)->real_cred, rcu_read_lock_held() || lockdep_tasklist_lock_is_held()))) |
284 | 284 | ||
285 | /** | 285 | /** |
286 | * get_task_cred - Get another task's objective credentials | 286 | * get_task_cred - Get another task's objective credentials |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a005cac5e302..3024050c82a1 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -101,6 +101,11 @@ extern struct lockdep_map rcu_sched_lock_map; | |||
101 | # define rcu_read_release_sched() \ | 101 | # define rcu_read_release_sched() \ |
102 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) | 102 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) |
103 | 103 | ||
104 | static inline int debug_lockdep_rcu_enabled(void) | ||
105 | { | ||
106 | return likely(rcu_scheduler_active && debug_locks); | ||
107 | } | ||
108 | |||
104 | /** | 109 | /** |
105 | * rcu_read_lock_held - might we be in RCU read-side critical section? | 110 | * rcu_read_lock_held - might we be in RCU read-side critical section? |
106 | * | 111 | * |
@@ -108,12 +113,14 @@ extern struct lockdep_map rcu_sched_lock_map; | |||
108 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, | 113 | * an RCU read-side critical section. In absence of CONFIG_PROVE_LOCKING, |
109 | * this assumes we are in an RCU read-side critical section unless it can | 114 | * this assumes we are in an RCU read-side critical section unless it can |
110 | * prove otherwise. | 115 | * prove otherwise. |
116 | * | ||
117 | * Check rcu_scheduler_active to prevent false positives during boot. | ||
111 | */ | 118 | */ |
112 | static inline int rcu_read_lock_held(void) | 119 | static inline int rcu_read_lock_held(void) |
113 | { | 120 | { |
114 | if (debug_locks) | 121 | if (!debug_lockdep_rcu_enabled()) |
115 | return lock_is_held(&rcu_lock_map); | 122 | return 1; |
116 | return 1; | 123 | return lock_is_held(&rcu_lock_map); |
117 | } | 124 | } |
118 | 125 | ||
119 | /** | 126 | /** |
@@ -123,12 +130,14 @@ static inline int rcu_read_lock_held(void) | |||
123 | * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, | 130 | * an RCU-bh read-side critical section. In absence of CONFIG_PROVE_LOCKING, |
124 | * this assumes we are in an RCU-bh read-side critical section unless it can | 131 | * this assumes we are in an RCU-bh read-side critical section unless it can |
125 | * prove otherwise. | 132 | * prove otherwise. |
133 | * | ||
134 | * Check rcu_scheduler_active to prevent false positives during boot. | ||
126 | */ | 135 | */ |
127 | static inline int rcu_read_lock_bh_held(void) | 136 | static inline int rcu_read_lock_bh_held(void) |
128 | { | 137 | { |
129 | if (debug_locks) | 138 | if (!debug_lockdep_rcu_enabled()) |
130 | return lock_is_held(&rcu_bh_lock_map); | 139 | return 1; |
131 | return 1; | 140 | return lock_is_held(&rcu_bh_lock_map); |
132 | } | 141 | } |
133 | 142 | ||
134 | /** | 143 | /** |
@@ -139,15 +148,26 @@ static inline int rcu_read_lock_bh_held(void) | |||
139 | * this assumes we are in an RCU-sched read-side critical section unless it | 148 | * this assumes we are in an RCU-sched read-side critical section unless it |
140 | * can prove otherwise. Note that disabling of preemption (including | 149 | * can prove otherwise. Note that disabling of preemption (including |
141 | * disabling irqs) counts as an RCU-sched read-side critical section. | 150 | * disabling irqs) counts as an RCU-sched read-side critical section. |
151 | * | ||
152 | * Check rcu_scheduler_active to prevent false positives during boot. | ||
142 | */ | 153 | */ |
154 | #ifdef CONFIG_PREEMPT | ||
143 | static inline int rcu_read_lock_sched_held(void) | 155 | static inline int rcu_read_lock_sched_held(void) |
144 | { | 156 | { |
145 | int lockdep_opinion = 0; | 157 | int lockdep_opinion = 0; |
146 | 158 | ||
159 | if (!debug_lockdep_rcu_enabled()) | ||
160 | return 1; | ||
147 | if (debug_locks) | 161 | if (debug_locks) |
148 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | 162 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
149 | return lockdep_opinion || preempt_count() != 0 || !rcu_scheduler_active; | 163 | return lockdep_opinion || preempt_count() != 0; |
164 | } | ||
165 | #else /* #ifdef CONFIG_PREEMPT */ | ||
166 | static inline int rcu_read_lock_sched_held(void) | ||
167 | { | ||
168 | return 1; | ||
150 | } | 169 | } |
170 | #endif /* #else #ifdef CONFIG_PREEMPT */ | ||
151 | 171 | ||
152 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 172 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
153 | 173 | ||
@@ -168,10 +188,17 @@ static inline int rcu_read_lock_bh_held(void) | |||
168 | return 1; | 188 | return 1; |
169 | } | 189 | } |
170 | 190 | ||
191 | #ifdef CONFIG_PREEMPT | ||
171 | static inline int rcu_read_lock_sched_held(void) | 192 | static inline int rcu_read_lock_sched_held(void) |
172 | { | 193 | { |
173 | return preempt_count() != 0 || !rcu_scheduler_active; | 194 | return !rcu_scheduler_active || preempt_count() != 0; |
195 | } | ||
196 | #else /* #ifdef CONFIG_PREEMPT */ | ||
197 | static inline int rcu_read_lock_sched_held(void) | ||
198 | { | ||
199 | return 1; | ||
174 | } | 200 | } |
201 | #endif /* #else #ifdef CONFIG_PREEMPT */ | ||
175 | 202 | ||
176 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 203 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
177 | 204 | ||
@@ -188,7 +215,7 @@ static inline int rcu_read_lock_sched_held(void) | |||
188 | */ | 215 | */ |
189 | #define rcu_dereference_check(p, c) \ | 216 | #define rcu_dereference_check(p, c) \ |
190 | ({ \ | 217 | ({ \ |
191 | if (debug_locks && !(c)) \ | 218 | if (debug_lockdep_rcu_enabled() && !(c)) \ |
192 | lockdep_rcu_dereference(__FILE__, __LINE__); \ | 219 | lockdep_rcu_dereference(__FILE__, __LINE__); \ |
193 | rcu_dereference_raw(p); \ | 220 | rcu_dereference_raw(p); \ |
194 | }) | 221 | }) |
diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 71e0b00b6f2c..bc2994ed66e1 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h | |||
@@ -29,25 +29,25 @@ do { \ | |||
29 | #endif | 29 | #endif |
30 | 30 | ||
31 | #ifdef CONFIG_DEBUG_SPINLOCK | 31 | #ifdef CONFIG_DEBUG_SPINLOCK |
32 | extern void do_raw_read_lock(rwlock_t *lock); | 32 | extern void do_raw_read_lock(rwlock_t *lock) __acquires(lock); |
33 | #define do_raw_read_lock_flags(lock, flags) do_raw_read_lock(lock) | 33 | #define do_raw_read_lock_flags(lock, flags) do_raw_read_lock(lock) |
34 | extern int do_raw_read_trylock(rwlock_t *lock); | 34 | extern int do_raw_read_trylock(rwlock_t *lock); |
35 | extern void do_raw_read_unlock(rwlock_t *lock); | 35 | extern void do_raw_read_unlock(rwlock_t *lock) __releases(lock); |
36 | extern void do_raw_write_lock(rwlock_t *lock); | 36 | extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock); |
37 | #define do_raw_write_lock_flags(lock, flags) do_raw_write_lock(lock) | 37 | #define do_raw_write_lock_flags(lock, flags) do_raw_write_lock(lock) |
38 | extern int do_raw_write_trylock(rwlock_t *lock); | 38 | extern int do_raw_write_trylock(rwlock_t *lock); |
39 | extern void do_raw_write_unlock(rwlock_t *lock); | 39 | extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock); |
40 | #else | 40 | #else |
41 | # define do_raw_read_lock(rwlock) arch_read_lock(&(rwlock)->raw_lock) | 41 | # define do_raw_read_lock(rwlock) do {__acquire(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) |
42 | # define do_raw_read_lock_flags(lock, flags) \ | 42 | # define do_raw_read_lock_flags(lock, flags) \ |
43 | arch_read_lock_flags(&(lock)->raw_lock, *(flags)) | 43 | do {__acquire(lock); arch_read_lock_flags(&(lock)->raw_lock, *(flags)); } while (0) |
44 | # define do_raw_read_trylock(rwlock) arch_read_trylock(&(rwlock)->raw_lock) | 44 | # define do_raw_read_trylock(rwlock) arch_read_trylock(&(rwlock)->raw_lock) |
45 | # define do_raw_read_unlock(rwlock) arch_read_unlock(&(rwlock)->raw_lock) | 45 | # define do_raw_read_unlock(rwlock) do {arch_read_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) |
46 | # define do_raw_write_lock(rwlock) arch_write_lock(&(rwlock)->raw_lock) | 46 | # define do_raw_write_lock(rwlock) do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0) |
47 | # define do_raw_write_lock_flags(lock, flags) \ | 47 | # define do_raw_write_lock_flags(lock, flags) \ |
48 | arch_write_lock_flags(&(lock)->raw_lock, *(flags)) | 48 | do {__acquire(lock); arch_write_lock_flags(&(lock)->raw_lock, *(flags)); } while (0) |
49 | # define do_raw_write_trylock(rwlock) arch_write_trylock(&(rwlock)->raw_lock) | 49 | # define do_raw_write_trylock(rwlock) arch_write_trylock(&(rwlock)->raw_lock) |
50 | # define do_raw_write_unlock(rwlock) arch_write_unlock(&(rwlock)->raw_lock) | 50 | # define do_raw_write_unlock(rwlock) do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) |
51 | #endif | 51 | #endif |
52 | 52 | ||
53 | #define read_can_lock(rwlock) arch_read_can_lock(&(rwlock)->raw_lock) | 53 | #define read_can_lock(rwlock) arch_read_can_lock(&(rwlock)->raw_lock) |
diff --git a/include/linux/sched.h b/include/linux/sched.h index 8d70ff802da2..dad7f668ebf7 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -258,6 +258,10 @@ extern spinlock_t mmlist_lock; | |||
258 | 258 | ||
259 | struct task_struct; | 259 | struct task_struct; |
260 | 260 | ||
261 | #ifdef CONFIG_PROVE_RCU | ||
262 | extern int lockdep_tasklist_lock_is_held(void); | ||
263 | #endif /* #ifdef CONFIG_PROVE_RCU */ | ||
264 | |||
261 | extern void sched_init(void); | 265 | extern void sched_init(void); |
262 | extern void sched_init_smp(void); | 266 | extern void sched_init_smp(void); |
263 | extern asmlinkage void schedule_tail(struct task_struct *prev); | 267 | extern asmlinkage void schedule_tail(struct task_struct *prev); |
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 86088213334a..89fac6a3f78b 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h | |||
@@ -128,19 +128,21 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } | |||
128 | #define raw_spin_unlock_wait(lock) arch_spin_unlock_wait(&(lock)->raw_lock) | 128 | #define raw_spin_unlock_wait(lock) arch_spin_unlock_wait(&(lock)->raw_lock) |
129 | 129 | ||
130 | #ifdef CONFIG_DEBUG_SPINLOCK | 130 | #ifdef CONFIG_DEBUG_SPINLOCK |
131 | extern void do_raw_spin_lock(raw_spinlock_t *lock); | 131 | extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock); |
132 | #define do_raw_spin_lock_flags(lock, flags) do_raw_spin_lock(lock) | 132 | #define do_raw_spin_lock_flags(lock, flags) do_raw_spin_lock(lock) |
133 | extern int do_raw_spin_trylock(raw_spinlock_t *lock); | 133 | extern int do_raw_spin_trylock(raw_spinlock_t *lock); |
134 | extern void do_raw_spin_unlock(raw_spinlock_t *lock); | 134 | extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); |
135 | #else | 135 | #else |
136 | static inline void do_raw_spin_lock(raw_spinlock_t *lock) | 136 | static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) |
137 | { | 137 | { |
138 | __acquire(lock); | ||
138 | arch_spin_lock(&lock->raw_lock); | 139 | arch_spin_lock(&lock->raw_lock); |
139 | } | 140 | } |
140 | 141 | ||
141 | static inline void | 142 | static inline void |
142 | do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) | 143 | do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lock) |
143 | { | 144 | { |
145 | __acquire(lock); | ||
144 | arch_spin_lock_flags(&lock->raw_lock, *flags); | 146 | arch_spin_lock_flags(&lock->raw_lock, *flags); |
145 | } | 147 | } |
146 | 148 | ||
@@ -149,9 +151,10 @@ static inline int do_raw_spin_trylock(raw_spinlock_t *lock) | |||
149 | return arch_spin_trylock(&(lock)->raw_lock); | 151 | return arch_spin_trylock(&(lock)->raw_lock); |
150 | } | 152 | } |
151 | 153 | ||
152 | static inline void do_raw_spin_unlock(raw_spinlock_t *lock) | 154 | static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) |
153 | { | 155 | { |
154 | arch_spin_unlock(&lock->raw_lock); | 156 | arch_spin_unlock(&lock->raw_lock); |
157 | __release(lock); | ||
155 | } | 158 | } |
156 | #endif | 159 | #endif |
157 | 160 | ||
diff --git a/include/trace/ftrace.h b/include/trace/ftrace.h index 0804cd594803..601ad7744247 100644 --- a/include/trace/ftrace.h +++ b/include/trace/ftrace.h | |||
@@ -699,9 +699,9 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ | |||
699 | * __cpu = smp_processor_id(); | 699 | * __cpu = smp_processor_id(); |
700 | * | 700 | * |
701 | * if (in_nmi()) | 701 | * if (in_nmi()) |
702 | * trace_buf = rcu_dereference(perf_trace_buf_nmi); | 702 | * trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); |
703 | * else | 703 | * else |
704 | * trace_buf = rcu_dereference(perf_trace_buf); | 704 | * trace_buf = rcu_dereference_sched(perf_trace_buf); |
705 | * | 705 | * |
706 | * if (!trace_buf) | 706 | * if (!trace_buf) |
707 | * goto end; | 707 | * goto end; |
diff --git a/kernel/exit.c b/kernel/exit.c index ce1e48c2d93d..cce59cb5ee6a 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -87,7 +87,7 @@ static void __exit_signal(struct task_struct *tsk) | |||
87 | 87 | ||
88 | sighand = rcu_dereference_check(tsk->sighand, | 88 | sighand = rcu_dereference_check(tsk->sighand, |
89 | rcu_read_lock_held() || | 89 | rcu_read_lock_held() || |
90 | lockdep_is_held(&tasklist_lock)); | 90 | lockdep_tasklist_lock_is_held()); |
91 | spin_lock(&sighand->siglock); | 91 | spin_lock(&sighand->siglock); |
92 | 92 | ||
93 | posix_cpu_timers_exit(tsk); | 93 | posix_cpu_timers_exit(tsk); |
diff --git a/kernel/fork.c b/kernel/fork.c index 1beb6c303c41..4799c5f0e6d0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -86,7 +86,14 @@ int max_threads; /* tunable limit on nr_threads */ | |||
86 | DEFINE_PER_CPU(unsigned long, process_counts) = 0; | 86 | DEFINE_PER_CPU(unsigned long, process_counts) = 0; |
87 | 87 | ||
88 | __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ | 88 | __cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ |
89 | EXPORT_SYMBOL_GPL(tasklist_lock); | 89 | |
90 | #ifdef CONFIG_PROVE_RCU | ||
91 | int lockdep_tasklist_lock_is_held(void) | ||
92 | { | ||
93 | return lockdep_is_held(&tasklist_lock); | ||
94 | } | ||
95 | EXPORT_SYMBOL_GPL(lockdep_tasklist_lock_is_held); | ||
96 | #endif /* #ifdef CONFIG_PROVE_RCU */ | ||
90 | 97 | ||
91 | int nr_processes(void) | 98 | int nr_processes(void) |
92 | { | 99 | { |
diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 0c30d0455de1..681bc2e1e187 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c | |||
@@ -3822,6 +3822,7 @@ void lockdep_rcu_dereference(const char *file, const int line) | |||
3822 | printk("%s:%d invoked rcu_dereference_check() without protection!\n", | 3822 | printk("%s:%d invoked rcu_dereference_check() without protection!\n", |
3823 | file, line); | 3823 | file, line); |
3824 | printk("\nother info that might help us debug this:\n\n"); | 3824 | printk("\nother info that might help us debug this:\n\n"); |
3825 | printk("\nrcu_scheduler_active = %d, debug_locks = %d\n", rcu_scheduler_active, debug_locks); | ||
3825 | lockdep_print_held_locks(curr); | 3826 | lockdep_print_held_locks(curr); |
3826 | printk("\nstack backtrace:\n"); | 3827 | printk("\nstack backtrace:\n"); |
3827 | dump_stack(); | 3828 | dump_stack(); |
diff --git a/kernel/pid.c b/kernel/pid.c index 86b296943e5f..aebb30d9c233 100644 --- a/kernel/pid.c +++ b/kernel/pid.c | |||
@@ -367,7 +367,9 @@ struct task_struct *pid_task(struct pid *pid, enum pid_type type) | |||
367 | struct task_struct *result = NULL; | 367 | struct task_struct *result = NULL; |
368 | if (pid) { | 368 | if (pid) { |
369 | struct hlist_node *first; | 369 | struct hlist_node *first; |
370 | first = rcu_dereference_check(pid->tasks[type].first, rcu_read_lock_held() || lockdep_is_held(&tasklist_lock)); | 370 | first = rcu_dereference_check(pid->tasks[type].first, |
371 | rcu_read_lock_held() || | ||
372 | lockdep_tasklist_lock_is_held()); | ||
371 | if (first) | 373 | if (first) |
372 | result = hlist_entry(first, struct task_struct, pids[(type)].node); | 374 | result = hlist_entry(first, struct task_struct, pids[(type)].node); |
373 | } | 375 | } |
diff --git a/kernel/rcutree.h b/kernel/rcutree.h index 1439eb504c22..4a525a30e08e 100644 --- a/kernel/rcutree.h +++ b/kernel/rcutree.h | |||
@@ -246,12 +246,21 @@ struct rcu_data { | |||
246 | 246 | ||
247 | #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ | 247 | #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ |
248 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | 248 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR |
249 | #define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ) /* for rsp->jiffies_stall */ | 249 | |
250 | #define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ) /* for rsp->jiffies_stall */ | 250 | #ifdef CONFIG_PROVE_RCU |
251 | #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */ | 251 | #define RCU_STALL_DELAY_DELTA (5 * HZ) |
252 | /* to take at least one */ | 252 | #else |
253 | /* scheduling clock irq */ | 253 | #define RCU_STALL_DELAY_DELTA 0 |
254 | /* before ratting on them. */ | 254 | #endif |
255 | |||
256 | #define RCU_SECONDS_TILL_STALL_CHECK (10 * HZ + RCU_STALL_DELAY_DELTA) | ||
257 | /* for rsp->jiffies_stall */ | ||
258 | #define RCU_SECONDS_TILL_STALL_RECHECK (30 * HZ + RCU_STALL_DELAY_DELTA) | ||
259 | /* for rsp->jiffies_stall */ | ||
260 | #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */ | ||
261 | /* to take at least one */ | ||
262 | /* scheduling clock irq */ | ||
263 | /* before ratting on them. */ | ||
255 | 264 | ||
256 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | 265 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ |
257 | 266 | ||
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index 464ad2cdee00..79b53bda8943 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h | |||
@@ -1010,6 +1010,10 @@ int rcu_needs_cpu(int cpu) | |||
1010 | int c = 0; | 1010 | int c = 0; |
1011 | int thatcpu; | 1011 | int thatcpu; |
1012 | 1012 | ||
1013 | /* Check for being in the holdoff period. */ | ||
1014 | if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) | ||
1015 | return rcu_needs_cpu_quick_check(cpu); | ||
1016 | |||
1013 | /* Don't bother unless we are the last non-dyntick-idle CPU. */ | 1017 | /* Don't bother unless we are the last non-dyntick-idle CPU. */ |
1014 | for_each_cpu_not(thatcpu, nohz_cpu_mask) | 1018 | for_each_cpu_not(thatcpu, nohz_cpu_mask) |
1015 | if (thatcpu != cpu) { | 1019 | if (thatcpu != cpu) { |
@@ -1041,10 +1045,8 @@ int rcu_needs_cpu(int cpu) | |||
1041 | } | 1045 | } |
1042 | 1046 | ||
1043 | /* If RCU callbacks are still pending, RCU still needs this CPU. */ | 1047 | /* If RCU callbacks are still pending, RCU still needs this CPU. */ |
1044 | if (c) { | 1048 | if (c) |
1045 | raise_softirq(RCU_SOFTIRQ); | 1049 | raise_softirq(RCU_SOFTIRQ); |
1046 | per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; | ||
1047 | } | ||
1048 | return c; | 1050 | return c; |
1049 | } | 1051 | } |
1050 | 1052 | ||
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 3e1fd96c6cf9..5a5ea2cd924f 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
@@ -3476,7 +3476,7 @@ static void run_rebalance_domains(struct softirq_action *h) | |||
3476 | 3476 | ||
3477 | static inline int on_null_domain(int cpu) | 3477 | static inline int on_null_domain(int cpu) |
3478 | { | 3478 | { |
3479 | return !rcu_dereference(cpu_rq(cpu)->sd); | 3479 | return !rcu_dereference_sched(cpu_rq(cpu)->sd); |
3480 | } | 3480 | } |
3481 | 3481 | ||
3482 | /* | 3482 | /* |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index bb53edbb5c8c..d9062f5cc0c0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/ctype.h> | 27 | #include <linux/ctype.h> |
28 | #include <linux/list.h> | 28 | #include <linux/list.h> |
29 | #include <linux/hash.h> | 29 | #include <linux/hash.h> |
30 | #include <linux/rcupdate.h> | ||
30 | 31 | ||
31 | #include <trace/events/sched.h> | 32 | #include <trace/events/sched.h> |
32 | 33 | ||
@@ -84,18 +85,22 @@ ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | |||
84 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; | 85 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
85 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; | 86 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
86 | 87 | ||
88 | /* | ||
89 | * Traverse the ftrace_list, invoking all entries. The reason that we | ||
90 | * can use rcu_dereference_raw() is that elements removed from this list | ||
91 | * are simply leaked, so there is no need to interact with a grace-period | ||
92 | * mechanism. The rcu_dereference_raw() calls are needed to handle | ||
93 | * concurrent insertions into the ftrace_list. | ||
94 | * | ||
95 | * Silly Alpha and silly pointer-speculation compiler optimizations! | ||
96 | */ | ||
87 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) | 97 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) |
88 | { | 98 | { |
89 | struct ftrace_ops *op = ftrace_list; | 99 | struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ |
90 | |||
91 | /* in case someone actually ports this to alpha! */ | ||
92 | read_barrier_depends(); | ||
93 | 100 | ||
94 | while (op != &ftrace_list_end) { | 101 | while (op != &ftrace_list_end) { |
95 | /* silly alpha */ | ||
96 | read_barrier_depends(); | ||
97 | op->func(ip, parent_ip); | 102 | op->func(ip, parent_ip); |
98 | op = op->next; | 103 | op = rcu_dereference_raw(op->next); /*see above*/ |
99 | }; | 104 | }; |
100 | } | 105 | } |
101 | 106 | ||
@@ -150,8 +155,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops) | |||
150 | * the ops->next pointer is valid before another CPU sees | 155 | * the ops->next pointer is valid before another CPU sees |
151 | * the ops pointer included into the ftrace_list. | 156 | * the ops pointer included into the ftrace_list. |
152 | */ | 157 | */ |
153 | smp_wmb(); | 158 | rcu_assign_pointer(ftrace_list, ops); |
154 | ftrace_list = ops; | ||
155 | 159 | ||
156 | if (ftrace_enabled) { | 160 | if (ftrace_enabled) { |
157 | ftrace_func_t func; | 161 | ftrace_func_t func; |
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c index f0d693005075..c1cc3ab633de 100644 --- a/kernel/trace/trace_event_profile.c +++ b/kernel/trace/trace_event_profile.c | |||
@@ -138,9 +138,9 @@ __kprobes void *ftrace_perf_buf_prepare(int size, unsigned short type, | |||
138 | cpu = smp_processor_id(); | 138 | cpu = smp_processor_id(); |
139 | 139 | ||
140 | if (in_nmi()) | 140 | if (in_nmi()) |
141 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | 141 | trace_buf = rcu_dereference_sched(perf_trace_buf_nmi); |
142 | else | 142 | else |
143 | trace_buf = rcu_dereference(perf_trace_buf); | 143 | trace_buf = rcu_dereference_sched(perf_trace_buf); |
144 | 144 | ||
145 | if (!trace_buf) | 145 | if (!trace_buf) |
146 | goto err; | 146 | goto err; |
diff --git a/mm/mempolicy.c b/mm/mempolicy.c index bda230e52acd..643f66e10187 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c | |||
@@ -1756,10 +1756,12 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) | |||
1756 | 1756 | ||
1757 | if (!new) | 1757 | if (!new) |
1758 | return ERR_PTR(-ENOMEM); | 1758 | return ERR_PTR(-ENOMEM); |
1759 | rcu_read_lock(); | ||
1759 | if (current_cpuset_is_being_rebound()) { | 1760 | if (current_cpuset_is_being_rebound()) { |
1760 | nodemask_t mems = cpuset_mems_allowed(current); | 1761 | nodemask_t mems = cpuset_mems_allowed(current); |
1761 | mpol_rebind_policy(old, &mems); | 1762 | mpol_rebind_policy(old, &mems); |
1762 | } | 1763 | } |
1764 | rcu_read_unlock(); | ||
1763 | *new = *old; | 1765 | *new = *old; |
1764 | atomic_set(&new->refcnt, 1); | 1766 | atomic_set(&new->refcnt, 1); |
1765 | return new; | 1767 | return new; |