diff options
Diffstat (limited to 'include/linux')
-rw-r--r-- | include/linux/cpu.h | 1 | ||||
-rw-r--r-- | include/linux/hardirq.h | 21 | ||||
-rw-r--r-- | include/linux/rcupdate.h | 115 | ||||
-rw-r--r-- | include/linux/sched.h | 8 | ||||
-rw-r--r-- | include/linux/srcu.h | 87 | ||||
-rw-r--r-- | include/linux/tick.h | 11 |
6 files changed, 163 insertions, 80 deletions
diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 6cb60fd2ea8..305c263021e 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h | |||
@@ -27,6 +27,7 @@ struct cpu { | |||
27 | 27 | ||
28 | extern int register_cpu(struct cpu *cpu, int num); | 28 | extern int register_cpu(struct cpu *cpu, int num); |
29 | extern struct sys_device *get_cpu_sysdev(unsigned cpu); | 29 | extern struct sys_device *get_cpu_sysdev(unsigned cpu); |
30 | extern bool cpu_is_hotpluggable(unsigned cpu); | ||
30 | 31 | ||
31 | extern int cpu_add_sysdev_attr(struct sysdev_attribute *attr); | 32 | extern int cpu_add_sysdev_attr(struct sysdev_attribute *attr); |
32 | extern void cpu_remove_sysdev_attr(struct sysdev_attribute *attr); | 33 | extern void cpu_remove_sysdev_attr(struct sysdev_attribute *attr); |
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index f743883f769..bb7f3097185 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h | |||
@@ -139,20 +139,7 @@ static inline void account_system_vtime(struct task_struct *tsk) | |||
139 | extern void account_system_vtime(struct task_struct *tsk); | 139 | extern void account_system_vtime(struct task_struct *tsk); |
140 | #endif | 140 | #endif |
141 | 141 | ||
142 | #if defined(CONFIG_NO_HZ) | ||
143 | #if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU) | 142 | #if defined(CONFIG_TINY_RCU) || defined(CONFIG_TINY_PREEMPT_RCU) |
144 | extern void rcu_enter_nohz(void); | ||
145 | extern void rcu_exit_nohz(void); | ||
146 | |||
147 | static inline void rcu_irq_enter(void) | ||
148 | { | ||
149 | rcu_exit_nohz(); | ||
150 | } | ||
151 | |||
152 | static inline void rcu_irq_exit(void) | ||
153 | { | ||
154 | rcu_enter_nohz(); | ||
155 | } | ||
156 | 143 | ||
157 | static inline void rcu_nmi_enter(void) | 144 | static inline void rcu_nmi_enter(void) |
158 | { | 145 | { |
@@ -163,17 +150,9 @@ static inline void rcu_nmi_exit(void) | |||
163 | } | 150 | } |
164 | 151 | ||
165 | #else | 152 | #else |
166 | extern void rcu_irq_enter(void); | ||
167 | extern void rcu_irq_exit(void); | ||
168 | extern void rcu_nmi_enter(void); | 153 | extern void rcu_nmi_enter(void); |
169 | extern void rcu_nmi_exit(void); | 154 | extern void rcu_nmi_exit(void); |
170 | #endif | 155 | #endif |
171 | #else | ||
172 | # define rcu_irq_enter() do { } while (0) | ||
173 | # define rcu_irq_exit() do { } while (0) | ||
174 | # define rcu_nmi_enter() do { } while (0) | ||
175 | # define rcu_nmi_exit() do { } while (0) | ||
176 | #endif /* #if defined(CONFIG_NO_HZ) */ | ||
177 | 156 | ||
178 | /* | 157 | /* |
179 | * It is safe to do non-atomic ops on ->hardirq_context, | 158 | * It is safe to do non-atomic ops on ->hardirq_context, |
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 2cf4226ade7..81c04f4348e 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -51,6 +51,8 @@ extern int rcutorture_runnable; /* for sysctl */ | |||
51 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) | 51 | #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) |
52 | extern void rcutorture_record_test_transition(void); | 52 | extern void rcutorture_record_test_transition(void); |
53 | extern void rcutorture_record_progress(unsigned long vernum); | 53 | extern void rcutorture_record_progress(unsigned long vernum); |
54 | extern void do_trace_rcu_torture_read(char *rcutorturename, | ||
55 | struct rcu_head *rhp); | ||
54 | #else | 56 | #else |
55 | static inline void rcutorture_record_test_transition(void) | 57 | static inline void rcutorture_record_test_transition(void) |
56 | { | 58 | { |
@@ -58,6 +60,12 @@ static inline void rcutorture_record_test_transition(void) | |||
58 | static inline void rcutorture_record_progress(unsigned long vernum) | 60 | static inline void rcutorture_record_progress(unsigned long vernum) |
59 | { | 61 | { |
60 | } | 62 | } |
63 | #ifdef CONFIG_RCU_TRACE | ||
64 | extern void do_trace_rcu_torture_read(char *rcutorturename, | ||
65 | struct rcu_head *rhp); | ||
66 | #else | ||
67 | #define do_trace_rcu_torture_read(rcutorturename, rhp) do { } while (0) | ||
68 | #endif | ||
61 | #endif | 69 | #endif |
62 | 70 | ||
63 | #define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b)) | 71 | #define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b)) |
@@ -177,23 +185,10 @@ extern void rcu_sched_qs(int cpu); | |||
177 | extern void rcu_bh_qs(int cpu); | 185 | extern void rcu_bh_qs(int cpu); |
178 | extern void rcu_check_callbacks(int cpu, int user); | 186 | extern void rcu_check_callbacks(int cpu, int user); |
179 | struct notifier_block; | 187 | struct notifier_block; |
180 | 188 | extern void rcu_idle_enter(void); | |
181 | #ifdef CONFIG_NO_HZ | 189 | extern void rcu_idle_exit(void); |
182 | 190 | extern void rcu_irq_enter(void); | |
183 | extern void rcu_enter_nohz(void); | 191 | extern void rcu_irq_exit(void); |
184 | extern void rcu_exit_nohz(void); | ||
185 | |||
186 | #else /* #ifdef CONFIG_NO_HZ */ | ||
187 | |||
188 | static inline void rcu_enter_nohz(void) | ||
189 | { | ||
190 | } | ||
191 | |||
192 | static inline void rcu_exit_nohz(void) | ||
193 | { | ||
194 | } | ||
195 | |||
196 | #endif /* #else #ifdef CONFIG_NO_HZ */ | ||
197 | 192 | ||
198 | /* | 193 | /* |
199 | * Infrastructure to implement the synchronize_() primitives in | 194 | * Infrastructure to implement the synchronize_() primitives in |
@@ -233,22 +228,30 @@ static inline void destroy_rcu_head_on_stack(struct rcu_head *head) | |||
233 | 228 | ||
234 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 229 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
235 | 230 | ||
236 | extern struct lockdep_map rcu_lock_map; | 231 | #ifdef CONFIG_PROVE_RCU |
237 | # define rcu_read_acquire() \ | 232 | extern int rcu_is_cpu_idle(void); |
238 | lock_acquire(&rcu_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | 233 | #else /* !CONFIG_PROVE_RCU */ |
239 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) | 234 | static inline int rcu_is_cpu_idle(void) |
235 | { | ||
236 | return 0; | ||
237 | } | ||
238 | #endif /* else !CONFIG_PROVE_RCU */ | ||
240 | 239 | ||
241 | extern struct lockdep_map rcu_bh_lock_map; | 240 | static inline void rcu_lock_acquire(struct lockdep_map *map) |
242 | # define rcu_read_acquire_bh() \ | 241 | { |
243 | lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | 242 | WARN_ON_ONCE(rcu_is_cpu_idle()); |
244 | # define rcu_read_release_bh() lock_release(&rcu_bh_lock_map, 1, _THIS_IP_) | 243 | lock_acquire(map, 0, 0, 2, 1, NULL, _THIS_IP_); |
244 | } | ||
245 | 245 | ||
246 | extern struct lockdep_map rcu_sched_lock_map; | 246 | static inline void rcu_lock_release(struct lockdep_map *map) |
247 | # define rcu_read_acquire_sched() \ | 247 | { |
248 | lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, NULL, _THIS_IP_) | 248 | WARN_ON_ONCE(rcu_is_cpu_idle()); |
249 | # define rcu_read_release_sched() \ | 249 | lock_release(map, 1, _THIS_IP_); |
250 | lock_release(&rcu_sched_lock_map, 1, _THIS_IP_) | 250 | } |
251 | 251 | ||
252 | extern struct lockdep_map rcu_lock_map; | ||
253 | extern struct lockdep_map rcu_bh_lock_map; | ||
254 | extern struct lockdep_map rcu_sched_lock_map; | ||
252 | extern int debug_lockdep_rcu_enabled(void); | 255 | extern int debug_lockdep_rcu_enabled(void); |
253 | 256 | ||
254 | /** | 257 | /** |
@@ -262,11 +265,18 @@ extern int debug_lockdep_rcu_enabled(void); | |||
262 | * | 265 | * |
263 | * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot | 266 | * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot |
264 | * and while lockdep is disabled. | 267 | * and while lockdep is disabled. |
268 | * | ||
269 | * Note that rcu_read_lock() and the matching rcu_read_unlock() must | ||
270 | * occur in the same context, for example, it is illegal to invoke | ||
271 | * rcu_read_unlock() in process context if the matching rcu_read_lock() | ||
272 | * was invoked from within an irq handler. | ||
265 | */ | 273 | */ |
266 | static inline int rcu_read_lock_held(void) | 274 | static inline int rcu_read_lock_held(void) |
267 | { | 275 | { |
268 | if (!debug_lockdep_rcu_enabled()) | 276 | if (!debug_lockdep_rcu_enabled()) |
269 | return 1; | 277 | return 1; |
278 | if (rcu_is_cpu_idle()) | ||
279 | return 0; | ||
270 | return lock_is_held(&rcu_lock_map); | 280 | return lock_is_held(&rcu_lock_map); |
271 | } | 281 | } |
272 | 282 | ||
@@ -290,6 +300,19 @@ extern int rcu_read_lock_bh_held(void); | |||
290 | * | 300 | * |
291 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot | 301 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot |
292 | * and while lockdep is disabled. | 302 | * and while lockdep is disabled. |
303 | * | ||
304 | * Note that if the CPU is in the idle loop from an RCU point of | ||
305 | * view (ie: that we are in the section between rcu_idle_enter() and | ||
306 | * rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU | ||
307 | * did an rcu_read_lock(). The reason for this is that RCU ignores CPUs | ||
308 | * that are in such a section, considering these as in extended quiescent | ||
309 | * state, so such a CPU is effectively never in an RCU read-side critical | ||
310 | * section regardless of what RCU primitives it invokes. This state of | ||
311 | * affairs is required --- we need to keep an RCU-free window in idle | ||
312 | * where the CPU may possibly enter into low power mode. This way we can | ||
313 | * notice an extended quiescent state to other CPUs that started a grace | ||
314 | * period. Otherwise we would delay any grace period as long as we run in | ||
315 | * the idle task. | ||
293 | */ | 316 | */ |
294 | #ifdef CONFIG_PREEMPT_COUNT | 317 | #ifdef CONFIG_PREEMPT_COUNT |
295 | static inline int rcu_read_lock_sched_held(void) | 318 | static inline int rcu_read_lock_sched_held(void) |
@@ -298,6 +321,8 @@ static inline int rcu_read_lock_sched_held(void) | |||
298 | 321 | ||
299 | if (!debug_lockdep_rcu_enabled()) | 322 | if (!debug_lockdep_rcu_enabled()) |
300 | return 1; | 323 | return 1; |
324 | if (rcu_is_cpu_idle()) | ||
325 | return 0; | ||
301 | if (debug_locks) | 326 | if (debug_locks) |
302 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | 327 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
303 | return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); | 328 | return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); |
@@ -311,12 +336,8 @@ static inline int rcu_read_lock_sched_held(void) | |||
311 | 336 | ||
312 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 337 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
313 | 338 | ||
314 | # define rcu_read_acquire() do { } while (0) | 339 | # define rcu_lock_acquire(a) do { } while (0) |
315 | # define rcu_read_release() do { } while (0) | 340 | # define rcu_lock_release(a) do { } while (0) |
316 | # define rcu_read_acquire_bh() do { } while (0) | ||
317 | # define rcu_read_release_bh() do { } while (0) | ||
318 | # define rcu_read_acquire_sched() do { } while (0) | ||
319 | # define rcu_read_release_sched() do { } while (0) | ||
320 | 341 | ||
321 | static inline int rcu_read_lock_held(void) | 342 | static inline int rcu_read_lock_held(void) |
322 | { | 343 | { |
@@ -637,7 +658,7 @@ static inline void rcu_read_lock(void) | |||
637 | { | 658 | { |
638 | __rcu_read_lock(); | 659 | __rcu_read_lock(); |
639 | __acquire(RCU); | 660 | __acquire(RCU); |
640 | rcu_read_acquire(); | 661 | rcu_lock_acquire(&rcu_lock_map); |
641 | } | 662 | } |
642 | 663 | ||
643 | /* | 664 | /* |
@@ -657,7 +678,7 @@ static inline void rcu_read_lock(void) | |||
657 | */ | 678 | */ |
658 | static inline void rcu_read_unlock(void) | 679 | static inline void rcu_read_unlock(void) |
659 | { | 680 | { |
660 | rcu_read_release(); | 681 | rcu_lock_release(&rcu_lock_map); |
661 | __release(RCU); | 682 | __release(RCU); |
662 | __rcu_read_unlock(); | 683 | __rcu_read_unlock(); |
663 | } | 684 | } |
@@ -673,12 +694,17 @@ static inline void rcu_read_unlock(void) | |||
673 | * critical sections in interrupt context can use just rcu_read_lock(), | 694 | * critical sections in interrupt context can use just rcu_read_lock(), |
674 | * though this should at least be commented to avoid confusing people | 695 | * though this should at least be commented to avoid confusing people |
675 | * reading the code. | 696 | * reading the code. |
697 | * | ||
698 | * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh() | ||
699 | * must occur in the same context, for example, it is illegal to invoke | ||
700 | * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh() | ||
701 | * was invoked from some other task. | ||
676 | */ | 702 | */ |
677 | static inline void rcu_read_lock_bh(void) | 703 | static inline void rcu_read_lock_bh(void) |
678 | { | 704 | { |
679 | local_bh_disable(); | 705 | local_bh_disable(); |
680 | __acquire(RCU_BH); | 706 | __acquire(RCU_BH); |
681 | rcu_read_acquire_bh(); | 707 | rcu_lock_acquire(&rcu_bh_lock_map); |
682 | } | 708 | } |
683 | 709 | ||
684 | /* | 710 | /* |
@@ -688,7 +714,7 @@ static inline void rcu_read_lock_bh(void) | |||
688 | */ | 714 | */ |
689 | static inline void rcu_read_unlock_bh(void) | 715 | static inline void rcu_read_unlock_bh(void) |
690 | { | 716 | { |
691 | rcu_read_release_bh(); | 717 | rcu_lock_release(&rcu_bh_lock_map); |
692 | __release(RCU_BH); | 718 | __release(RCU_BH); |
693 | local_bh_enable(); | 719 | local_bh_enable(); |
694 | } | 720 | } |
@@ -700,12 +726,17 @@ static inline void rcu_read_unlock_bh(void) | |||
700 | * are being done using call_rcu_sched() or synchronize_rcu_sched(). | 726 | * are being done using call_rcu_sched() or synchronize_rcu_sched(). |
701 | * Read-side critical sections can also be introduced by anything that | 727 | * Read-side critical sections can also be introduced by anything that |
702 | * disables preemption, including local_irq_disable() and friends. | 728 | * disables preemption, including local_irq_disable() and friends. |
729 | * | ||
730 | * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched() | ||
731 | * must occur in the same context, for example, it is illegal to invoke | ||
732 | * rcu_read_unlock_sched() from process context if the matching | ||
733 | * rcu_read_lock_sched() was invoked from an NMI handler. | ||
703 | */ | 734 | */ |
704 | static inline void rcu_read_lock_sched(void) | 735 | static inline void rcu_read_lock_sched(void) |
705 | { | 736 | { |
706 | preempt_disable(); | 737 | preempt_disable(); |
707 | __acquire(RCU_SCHED); | 738 | __acquire(RCU_SCHED); |
708 | rcu_read_acquire_sched(); | 739 | rcu_lock_acquire(&rcu_sched_lock_map); |
709 | } | 740 | } |
710 | 741 | ||
711 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ | 742 | /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ |
@@ -722,7 +753,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void) | |||
722 | */ | 753 | */ |
723 | static inline void rcu_read_unlock_sched(void) | 754 | static inline void rcu_read_unlock_sched(void) |
724 | { | 755 | { |
725 | rcu_read_release_sched(); | 756 | rcu_lock_release(&rcu_sched_lock_map); |
726 | __release(RCU_SCHED); | 757 | __release(RCU_SCHED); |
727 | preempt_enable(); | 758 | preempt_enable(); |
728 | } | 759 | } |
diff --git a/include/linux/sched.h b/include/linux/sched.h index 1c4f3e9b9bc..4a7e4d333a2 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -2070,6 +2070,14 @@ extern int sched_setscheduler(struct task_struct *, int, | |||
2070 | extern int sched_setscheduler_nocheck(struct task_struct *, int, | 2070 | extern int sched_setscheduler_nocheck(struct task_struct *, int, |
2071 | const struct sched_param *); | 2071 | const struct sched_param *); |
2072 | extern struct task_struct *idle_task(int cpu); | 2072 | extern struct task_struct *idle_task(int cpu); |
2073 | /** | ||
2074 | * is_idle_task - is the specified task an idle task? | ||
2075 | * @tsk: the task in question. | ||
2076 | */ | ||
2077 | static inline bool is_idle_task(struct task_struct *p) | ||
2078 | { | ||
2079 | return p->pid == 0; | ||
2080 | } | ||
2073 | extern struct task_struct *curr_task(int cpu); | 2081 | extern struct task_struct *curr_task(int cpu); |
2074 | extern void set_curr_task(int cpu, struct task_struct *p); | 2082 | extern void set_curr_task(int cpu, struct task_struct *p); |
2075 | 2083 | ||
diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 58971e891f4..e1b005918bb 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h | |||
@@ -28,6 +28,7 @@ | |||
28 | #define _LINUX_SRCU_H | 28 | #define _LINUX_SRCU_H |
29 | 29 | ||
30 | #include <linux/mutex.h> | 30 | #include <linux/mutex.h> |
31 | #include <linux/rcupdate.h> | ||
31 | 32 | ||
32 | struct srcu_struct_array { | 33 | struct srcu_struct_array { |
33 | int c[2]; | 34 | int c[2]; |
@@ -60,18 +61,10 @@ int __init_srcu_struct(struct srcu_struct *sp, const char *name, | |||
60 | __init_srcu_struct((sp), #sp, &__srcu_key); \ | 61 | __init_srcu_struct((sp), #sp, &__srcu_key); \ |
61 | }) | 62 | }) |
62 | 63 | ||
63 | # define srcu_read_acquire(sp) \ | ||
64 | lock_acquire(&(sp)->dep_map, 0, 0, 2, 1, NULL, _THIS_IP_) | ||
65 | # define srcu_read_release(sp) \ | ||
66 | lock_release(&(sp)->dep_map, 1, _THIS_IP_) | ||
67 | |||
68 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 64 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
69 | 65 | ||
70 | int init_srcu_struct(struct srcu_struct *sp); | 66 | int init_srcu_struct(struct srcu_struct *sp); |
71 | 67 | ||
72 | # define srcu_read_acquire(sp) do { } while (0) | ||
73 | # define srcu_read_release(sp) do { } while (0) | ||
74 | |||
75 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 68 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
76 | 69 | ||
77 | void cleanup_srcu_struct(struct srcu_struct *sp); | 70 | void cleanup_srcu_struct(struct srcu_struct *sp); |
@@ -90,12 +83,32 @@ long srcu_batches_completed(struct srcu_struct *sp); | |||
90 | * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, | 83 | * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, |
91 | * this assumes we are in an SRCU read-side critical section unless it can | 84 | * this assumes we are in an SRCU read-side critical section unless it can |
92 | * prove otherwise. | 85 | * prove otherwise. |
86 | * | ||
87 | * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot | ||
88 | * and while lockdep is disabled. | ||
89 | * | ||
90 | * Note that if the CPU is in the idle loop from an RCU point of view | ||
91 | * (ie: that we are in the section between rcu_idle_enter() and | ||
92 | * rcu_idle_exit()) then srcu_read_lock_held() returns false even if | ||
93 | * the CPU did an srcu_read_lock(). The reason for this is that RCU | ||
94 | * ignores CPUs that are in such a section, considering these as in | ||
95 | * extended quiescent state, so such a CPU is effectively never in an | ||
96 | * RCU read-side critical section regardless of what RCU primitives it | ||
97 | * invokes. This state of affairs is required --- we need to keep an | ||
98 | * RCU-free window in idle where the CPU may possibly enter into low | ||
99 | * power mode. This way we can notice an extended quiescent state to | ||
100 | * other CPUs that started a grace period. Otherwise we would delay any | ||
101 | * grace period as long as we run in the idle task. | ||
93 | */ | 102 | */ |
94 | static inline int srcu_read_lock_held(struct srcu_struct *sp) | 103 | static inline int srcu_read_lock_held(struct srcu_struct *sp) |
95 | { | 104 | { |
96 | if (debug_locks) | 105 | if (rcu_is_cpu_idle()) |
97 | return lock_is_held(&sp->dep_map); | 106 | return 0; |
98 | return 1; | 107 | |
108 | if (!debug_lockdep_rcu_enabled()) | ||
109 | return 1; | ||
110 | |||
111 | return lock_is_held(&sp->dep_map); | ||
99 | } | 112 | } |
100 | 113 | ||
101 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 114 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
@@ -145,12 +158,17 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp) | |||
145 | * one way to indirectly wait on an SRCU grace period is to acquire | 158 | * one way to indirectly wait on an SRCU grace period is to acquire |
146 | * a mutex that is held elsewhere while calling synchronize_srcu() or | 159 | * a mutex that is held elsewhere while calling synchronize_srcu() or |
147 | * synchronize_srcu_expedited(). | 160 | * synchronize_srcu_expedited(). |
161 | * | ||
162 | * Note that srcu_read_lock() and the matching srcu_read_unlock() must | ||
163 | * occur in the same context, for example, it is illegal to invoke | ||
164 | * srcu_read_unlock() in an irq handler if the matching srcu_read_lock() | ||
165 | * was invoked in process context. | ||
148 | */ | 166 | */ |
149 | static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) | 167 | static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) |
150 | { | 168 | { |
151 | int retval = __srcu_read_lock(sp); | 169 | int retval = __srcu_read_lock(sp); |
152 | 170 | ||
153 | srcu_read_acquire(sp); | 171 | rcu_lock_acquire(&(sp)->dep_map); |
154 | return retval; | 172 | return retval; |
155 | } | 173 | } |
156 | 174 | ||
@@ -164,8 +182,51 @@ static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) | |||
164 | static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) | 182 | static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) |
165 | __releases(sp) | 183 | __releases(sp) |
166 | { | 184 | { |
167 | srcu_read_release(sp); | 185 | rcu_lock_release(&(sp)->dep_map); |
186 | __srcu_read_unlock(sp, idx); | ||
187 | } | ||
188 | |||
189 | /** | ||
190 | * srcu_read_lock_raw - register a new reader for an SRCU-protected structure. | ||
191 | * @sp: srcu_struct in which to register the new reader. | ||
192 | * | ||
193 | * Enter an SRCU read-side critical section. Similar to srcu_read_lock(), | ||
194 | * but avoids the RCU-lockdep checking. This means that it is legal to | ||
195 | * use srcu_read_lock_raw() in one context, for example, in an exception | ||
196 | * handler, and then have the matching srcu_read_unlock_raw() in another | ||
197 | * context, for example in the task that took the exception. | ||
198 | * | ||
199 | * However, the entire SRCU read-side critical section must reside within a | ||
200 | * single task. For example, beware of using srcu_read_lock_raw() in | ||
201 | * a device interrupt handler and srcu_read_unlock() in the interrupted | ||
202 | * task: This will not work if interrupts are threaded. | ||
203 | */ | ||
204 | static inline int srcu_read_lock_raw(struct srcu_struct *sp) | ||
205 | { | ||
206 | unsigned long flags; | ||
207 | int ret; | ||
208 | |||
209 | local_irq_save(flags); | ||
210 | ret = __srcu_read_lock(sp); | ||
211 | local_irq_restore(flags); | ||
212 | return ret; | ||
213 | } | ||
214 | |||
215 | /** | ||
216 | * srcu_read_unlock_raw - unregister reader from an SRCU-protected structure. | ||
217 | * @sp: srcu_struct in which to unregister the old reader. | ||
218 | * @idx: return value from corresponding srcu_read_lock_raw(). | ||
219 | * | ||
220 | * Exit an SRCU read-side critical section without lockdep-RCU checking. | ||
221 | * See srcu_read_lock_raw() for more details. | ||
222 | */ | ||
223 | static inline void srcu_read_unlock_raw(struct srcu_struct *sp, int idx) | ||
224 | { | ||
225 | unsigned long flags; | ||
226 | |||
227 | local_irq_save(flags); | ||
168 | __srcu_read_unlock(sp, idx); | 228 | __srcu_read_unlock(sp, idx); |
229 | local_irq_restore(flags); | ||
169 | } | 230 | } |
170 | 231 | ||
171 | #endif | 232 | #endif |
diff --git a/include/linux/tick.h b/include/linux/tick.h index b232ccc0ee2..ab8be90b5cc 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h | |||
@@ -7,6 +7,7 @@ | |||
7 | #define _LINUX_TICK_H | 7 | #define _LINUX_TICK_H |
8 | 8 | ||
9 | #include <linux/clockchips.h> | 9 | #include <linux/clockchips.h> |
10 | #include <linux/irqflags.h> | ||
10 | 11 | ||
11 | #ifdef CONFIG_GENERIC_CLOCKEVENTS | 12 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
12 | 13 | ||
@@ -121,14 +122,16 @@ static inline int tick_oneshot_mode_active(void) { return 0; } | |||
121 | #endif /* !CONFIG_GENERIC_CLOCKEVENTS */ | 122 | #endif /* !CONFIG_GENERIC_CLOCKEVENTS */ |
122 | 123 | ||
123 | # ifdef CONFIG_NO_HZ | 124 | # ifdef CONFIG_NO_HZ |
124 | extern void tick_nohz_stop_sched_tick(int inidle); | 125 | extern void tick_nohz_idle_enter(void); |
125 | extern void tick_nohz_restart_sched_tick(void); | 126 | extern void tick_nohz_idle_exit(void); |
127 | extern void tick_nohz_irq_exit(void); | ||
126 | extern ktime_t tick_nohz_get_sleep_length(void); | 128 | extern ktime_t tick_nohz_get_sleep_length(void); |
127 | extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); | 129 | extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); |
128 | extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); | 130 | extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); |
129 | # else | 131 | # else |
130 | static inline void tick_nohz_stop_sched_tick(int inidle) { } | 132 | static inline void tick_nohz_idle_enter(void) { } |
131 | static inline void tick_nohz_restart_sched_tick(void) { } | 133 | static inline void tick_nohz_idle_exit(void) { } |
134 | |||
132 | static inline ktime_t tick_nohz_get_sleep_length(void) | 135 | static inline ktime_t tick_nohz_get_sleep_length(void) |
133 | { | 136 | { |
134 | ktime_t len = { .tv64 = NSEC_PER_SEC/HZ }; | 137 | ktime_t len = { .tv64 = NSEC_PER_SEC/HZ }; |