diff options
| -rw-r--r-- | fs/autofs4/autofs_i.h | 1 | ||||
| -rw-r--r-- | include/linux/dcache.h | 1 | ||||
| -rw-r--r-- | include/linux/rcupdate.h | 40 | ||||
| -rw-r--r-- | include/linux/rcupdate_wait.h | 50 | ||||
| -rw-r--r-- | include/linux/rcutiny.h | 11 | ||||
| -rw-r--r-- | kernel/rcu/srcu.c | 2 | ||||
| -rw-r--r-- | kernel/rcu/tiny.c | 14 | ||||
| -rw-r--r-- | kernel/rcu/tree.c | 2 | ||||
| -rw-r--r-- | kernel/rcu/update.c | 1 | ||||
| -rw-r--r-- | kernel/sched/core.c | 1 |
10 files changed, 71 insertions, 52 deletions
diff --git a/fs/autofs4/autofs_i.h b/fs/autofs4/autofs_i.h index c885daae68c8..beef981aa54f 100644 --- a/fs/autofs4/autofs_i.h +++ b/fs/autofs4/autofs_i.h | |||
| @@ -14,6 +14,7 @@ | |||
| 14 | #include <linux/mutex.h> | 14 | #include <linux/mutex.h> |
| 15 | #include <linux/spinlock.h> | 15 | #include <linux/spinlock.h> |
| 16 | #include <linux/list.h> | 16 | #include <linux/list.h> |
| 17 | #include <linux/completion.h> | ||
| 17 | 18 | ||
| 18 | /* This is the range of ioctl() numbers we claim as ours */ | 19 | /* This is the range of ioctl() numbers we claim as ours */ |
| 19 | #define AUTOFS_IOC_FIRST AUTOFS_IOC_READY | 20 | #define AUTOFS_IOC_FIRST AUTOFS_IOC_READY |
diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 591b6c16f9c1..d2e38dc6172c 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h | |||
| @@ -11,6 +11,7 @@ | |||
| 11 | #include <linux/rcupdate.h> | 11 | #include <linux/rcupdate.h> |
| 12 | #include <linux/lockref.h> | 12 | #include <linux/lockref.h> |
| 13 | #include <linux/stringhash.h> | 13 | #include <linux/stringhash.h> |
| 14 | #include <linux/wait.h> | ||
| 14 | 15 | ||
| 15 | struct path; | 16 | struct path; |
| 16 | struct vfsmount; | 17 | struct vfsmount; |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 6ade6a52d9d4..de88b33c0974 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
| @@ -40,7 +40,6 @@ | |||
| 40 | #include <linux/cpumask.h> | 40 | #include <linux/cpumask.h> |
| 41 | #include <linux/seqlock.h> | 41 | #include <linux/seqlock.h> |
| 42 | #include <linux/lockdep.h> | 42 | #include <linux/lockdep.h> |
| 43 | #include <linux/completion.h> | ||
| 44 | #include <linux/debugobjects.h> | 43 | #include <linux/debugobjects.h> |
| 45 | #include <linux/bug.h> | 44 | #include <linux/bug.h> |
| 46 | #include <linux/compiler.h> | 45 | #include <linux/compiler.h> |
| @@ -226,45 +225,6 @@ void call_rcu_sched(struct rcu_head *head, | |||
| 226 | 225 | ||
| 227 | void synchronize_sched(void); | 226 | void synchronize_sched(void); |
| 228 | 227 | ||
| 229 | /* | ||
| 230 | * Structure allowing asynchronous waiting on RCU. | ||
| 231 | */ | ||
| 232 | struct rcu_synchronize { | ||
| 233 | struct rcu_head head; | ||
| 234 | struct completion completion; | ||
| 235 | }; | ||
| 236 | void wakeme_after_rcu(struct rcu_head *head); | ||
| 237 | |||
| 238 | void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array, | ||
| 239 | struct rcu_synchronize *rs_array); | ||
| 240 | |||
| 241 | #define _wait_rcu_gp(checktiny, ...) \ | ||
| 242 | do { \ | ||
| 243 | call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ | ||
| 244 | struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ | ||
| 245 | __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \ | ||
| 246 | __crcu_array, __rs_array); \ | ||
| 247 | } while (0) | ||
| 248 | |||
| 249 | #define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__) | ||
| 250 | |||
| 251 | /** | ||
| 252 | * synchronize_rcu_mult - Wait concurrently for multiple grace periods | ||
| 253 | * @...: List of call_rcu() functions for the flavors to wait on. | ||
| 254 | * | ||
| 255 | * This macro waits concurrently for multiple flavors of RCU grace periods. | ||
| 256 | * For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait | ||
| 257 | * on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU | ||
| 258 | * domain requires you to write a wrapper function for that SRCU domain's | ||
| 259 | * call_srcu() function, supplying the corresponding srcu_struct. | ||
| 260 | * | ||
| 261 | * If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU | ||
| 262 | * or RCU-bh, given that anywhere synchronize_rcu_mult() can be called | ||
| 263 | * is automatically a grace period. | ||
| 264 | */ | ||
| 265 | #define synchronize_rcu_mult(...) \ | ||
| 266 | _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) | ||
| 267 | |||
| 268 | /** | 228 | /** |
| 269 | * call_rcu_tasks() - Queue an RCU for invocation task-based grace period | 229 | * call_rcu_tasks() - Queue an RCU for invocation task-based grace period |
| 270 | * @head: structure to be used for queueing the RCU updates. | 230 | * @head: structure to be used for queueing the RCU updates. |
diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h new file mode 100644 index 000000000000..e774b4f5f220 --- /dev/null +++ b/include/linux/rcupdate_wait.h | |||
| @@ -0,0 +1,50 @@ | |||
| 1 | #ifndef _LINUX_SCHED_RCUPDATE_WAIT_H | ||
| 2 | #define _LINUX_SCHED_RCUPDATE_WAIT_H | ||
| 3 | |||
| 4 | /* | ||
| 5 | * RCU synchronization types and methods: | ||
| 6 | */ | ||
| 7 | |||
| 8 | #include <linux/rcupdate.h> | ||
| 9 | #include <linux/completion.h> | ||
| 10 | |||
| 11 | /* | ||
| 12 | * Structure allowing asynchronous waiting on RCU. | ||
| 13 | */ | ||
| 14 | struct rcu_synchronize { | ||
| 15 | struct rcu_head head; | ||
| 16 | struct completion completion; | ||
| 17 | }; | ||
| 18 | void wakeme_after_rcu(struct rcu_head *head); | ||
| 19 | |||
| 20 | void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array, | ||
| 21 | struct rcu_synchronize *rs_array); | ||
| 22 | |||
| 23 | #define _wait_rcu_gp(checktiny, ...) \ | ||
| 24 | do { \ | ||
| 25 | call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ | ||
| 26 | struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ | ||
| 27 | __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \ | ||
| 28 | __crcu_array, __rs_array); \ | ||
| 29 | } while (0) | ||
| 30 | |||
| 31 | #define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__) | ||
| 32 | |||
| 33 | /** | ||
| 34 | * synchronize_rcu_mult - Wait concurrently for multiple grace periods | ||
| 35 | * @...: List of call_rcu() functions for the flavors to wait on. | ||
| 36 | * | ||
| 37 | * This macro waits concurrently for multiple flavors of RCU grace periods. | ||
| 38 | * For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait | ||
| 39 | * on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU | ||
| 40 | * domain requires you to write a wrapper function for that SRCU domain's | ||
| 41 | * call_srcu() function, supplying the corresponding srcu_struct. | ||
| 42 | * | ||
| 43 | * If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU | ||
| 44 | * or RCU-bh, given that anywhere synchronize_rcu_mult() can be called | ||
| 45 | * is automatically a grace period. | ||
| 46 | */ | ||
| 47 | #define synchronize_rcu_mult(...) \ | ||
| 48 | _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) | ||
| 49 | |||
| 50 | #endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */ | ||
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 4f9b2fa2173d..b452953e21c8 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h | |||
| @@ -53,15 +53,8 @@ static inline void cond_synchronize_sched(unsigned long oldstate) | |||
| 53 | might_sleep(); | 53 | might_sleep(); |
| 54 | } | 54 | } |
| 55 | 55 | ||
| 56 | static inline void rcu_barrier_bh(void) | 56 | extern void rcu_barrier_bh(void); |
| 57 | { | 57 | extern void rcu_barrier_sched(void); |
| 58 | wait_rcu_gp(call_rcu_bh); | ||
| 59 | } | ||
| 60 | |||
| 61 | static inline void rcu_barrier_sched(void) | ||
| 62 | { | ||
| 63 | wait_rcu_gp(call_rcu_sched); | ||
| 64 | } | ||
| 65 | 58 | ||
| 66 | static inline void synchronize_rcu_expedited(void) | 59 | static inline void synchronize_rcu_expedited(void) |
| 67 | { | 60 | { |
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c index e773129c8b08..ef3bcfb15b39 100644 --- a/kernel/rcu/srcu.c +++ b/kernel/rcu/srcu.c | |||
| @@ -30,7 +30,7 @@ | |||
| 30 | #include <linux/mutex.h> | 30 | #include <linux/mutex.h> |
| 31 | #include <linux/percpu.h> | 31 | #include <linux/percpu.h> |
| 32 | #include <linux/preempt.h> | 32 | #include <linux/preempt.h> |
| 33 | #include <linux/rcupdate.h> | 33 | #include <linux/rcupdate_wait.h> |
| 34 | #include <linux/sched.h> | 34 | #include <linux/sched.h> |
| 35 | #include <linux/smp.h> | 35 | #include <linux/smp.h> |
| 36 | #include <linux/delay.h> | 36 | #include <linux/delay.h> |
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index fa6a48d3917b..6ad330dbbae2 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c | |||
| @@ -25,7 +25,7 @@ | |||
| 25 | #include <linux/completion.h> | 25 | #include <linux/completion.h> |
| 26 | #include <linux/interrupt.h> | 26 | #include <linux/interrupt.h> |
| 27 | #include <linux/notifier.h> | 27 | #include <linux/notifier.h> |
| 28 | #include <linux/rcupdate.h> | 28 | #include <linux/rcupdate_wait.h> |
| 29 | #include <linux/kernel.h> | 29 | #include <linux/kernel.h> |
| 30 | #include <linux/export.h> | 30 | #include <linux/export.h> |
| 31 | #include <linux/mutex.h> | 31 | #include <linux/mutex.h> |
| @@ -47,6 +47,18 @@ static void __call_rcu(struct rcu_head *head, | |||
| 47 | 47 | ||
| 48 | #include "tiny_plugin.h" | 48 | #include "tiny_plugin.h" |
| 49 | 49 | ||
| 50 | void rcu_barrier_bh(void) | ||
| 51 | { | ||
| 52 | wait_rcu_gp(call_rcu_bh); | ||
| 53 | } | ||
| 54 | EXPORT_SYMBOL(rcu_barrier_bh); | ||
| 55 | |||
| 56 | void rcu_barrier_sched(void) | ||
| 57 | { | ||
| 58 | wait_rcu_gp(call_rcu_sched); | ||
| 59 | } | ||
| 60 | EXPORT_SYMBOL(rcu_barrier_sched); | ||
| 61 | |||
| 50 | #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) | 62 | #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) |
| 51 | 63 | ||
| 52 | /* | 64 | /* |
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index d80e0d2f68c6..cb62ce23ffc7 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c | |||
| @@ -32,7 +32,7 @@ | |||
| 32 | #include <linux/init.h> | 32 | #include <linux/init.h> |
| 33 | #include <linux/spinlock.h> | 33 | #include <linux/spinlock.h> |
| 34 | #include <linux/smp.h> | 34 | #include <linux/smp.h> |
| 35 | #include <linux/rcupdate.h> | 35 | #include <linux/rcupdate_wait.h> |
| 36 | #include <linux/interrupt.h> | 36 | #include <linux/interrupt.h> |
| 37 | #include <linux/sched.h> | 37 | #include <linux/sched.h> |
| 38 | #include <linux/nmi.h> | 38 | #include <linux/nmi.h> |
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 9e03db9ea9c0..a0e90e0afc75 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c | |||
| @@ -49,6 +49,7 @@ | |||
| 49 | #include <linux/moduleparam.h> | 49 | #include <linux/moduleparam.h> |
| 50 | #include <linux/kthread.h> | 50 | #include <linux/kthread.h> |
| 51 | #include <linux/tick.h> | 51 | #include <linux/tick.h> |
| 52 | #include <linux/rcupdate_wait.h> | ||
| 52 | 53 | ||
| 53 | #define CREATE_TRACE_POINTS | 54 | #define CREATE_TRACE_POINTS |
| 54 | 55 | ||
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2d51ec65dc73..1bd317db9810 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
| @@ -10,6 +10,7 @@ | |||
| 10 | #include <linux/delayacct.h> | 10 | #include <linux/delayacct.h> |
| 11 | #include <linux/init_task.h> | 11 | #include <linux/init_task.h> |
| 12 | #include <linux/context_tracking.h> | 12 | #include <linux/context_tracking.h> |
| 13 | #include <linux/rcupdate_wait.h> | ||
| 13 | 14 | ||
| 14 | #include <linux/blkdev.h> | 15 | #include <linux/blkdev.h> |
| 15 | #include <linux/kprobes.h> | 16 | #include <linux/kprobes.h> |
