diff options
-rw-r--r-- | include/linux/bit_spinlock.h | 2 | ||||
-rw-r--r-- | include/linux/hardirq.h | 4 | ||||
-rw-r--r-- | include/linux/pagemap.h | 4 | ||||
-rw-r--r-- | include/linux/preempt.h | 26 | ||||
-rw-r--r-- | include/linux/rcupdate.h | 12 | ||||
-rw-r--r-- | include/linux/sched.h | 2 | ||||
-rw-r--r-- | kernel/Kconfig.preempt | 3 | ||||
-rw-r--r-- | kernel/sched.c | 2 |
8 files changed, 33 insertions, 22 deletions
diff --git a/include/linux/bit_spinlock.h b/include/linux/bit_spinlock.h index b4326bfa684f..564d997e2168 100644 --- a/include/linux/bit_spinlock.h +++ b/include/linux/bit_spinlock.h | |||
@@ -88,7 +88,7 @@ static inline int bit_spin_is_locked(int bitnum, unsigned long *addr) | |||
88 | { | 88 | { |
89 | #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) | 89 | #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) |
90 | return test_bit(bitnum, addr); | 90 | return test_bit(bitnum, addr); |
91 | #elif defined CONFIG_PREEMPT | 91 | #elif defined CONFIG_PREEMPT_COUNT |
92 | return preempt_count(); | 92 | return preempt_count(); |
93 | #else | 93 | #else |
94 | return 1; | 94 | return 1; |
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index ba362171e8ae..f743883f769e 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h | |||
@@ -93,7 +93,7 @@ | |||
93 | */ | 93 | */ |
94 | #define in_nmi() (preempt_count() & NMI_MASK) | 94 | #define in_nmi() (preempt_count() & NMI_MASK) |
95 | 95 | ||
96 | #if defined(CONFIG_PREEMPT) | 96 | #if defined(CONFIG_PREEMPT_COUNT) |
97 | # define PREEMPT_CHECK_OFFSET 1 | 97 | # define PREEMPT_CHECK_OFFSET 1 |
98 | #else | 98 | #else |
99 | # define PREEMPT_CHECK_OFFSET 0 | 99 | # define PREEMPT_CHECK_OFFSET 0 |
@@ -115,7 +115,7 @@ | |||
115 | #define in_atomic_preempt_off() \ | 115 | #define in_atomic_preempt_off() \ |
116 | ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) | 116 | ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) |
117 | 117 | ||
118 | #ifdef CONFIG_PREEMPT | 118 | #ifdef CONFIG_PREEMPT_COUNT |
119 | # define preemptible() (preempt_count() == 0 && !irqs_disabled()) | 119 | # define preemptible() (preempt_count() == 0 && !irqs_disabled()) |
120 | # define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1) | 120 | # define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1) |
121 | #else | 121 | #else |
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 716875e53520..8e38d4c140ff 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h | |||
@@ -134,7 +134,7 @@ static inline int page_cache_get_speculative(struct page *page) | |||
134 | VM_BUG_ON(in_interrupt()); | 134 | VM_BUG_ON(in_interrupt()); |
135 | 135 | ||
136 | #if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) | 136 | #if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) |
137 | # ifdef CONFIG_PREEMPT | 137 | # ifdef CONFIG_PREEMPT_COUNT |
138 | VM_BUG_ON(!in_atomic()); | 138 | VM_BUG_ON(!in_atomic()); |
139 | # endif | 139 | # endif |
140 | /* | 140 | /* |
@@ -172,7 +172,7 @@ static inline int page_cache_add_speculative(struct page *page, int count) | |||
172 | VM_BUG_ON(in_interrupt()); | 172 | VM_BUG_ON(in_interrupt()); |
173 | 173 | ||
174 | #if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) | 174 | #if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) |
175 | # ifdef CONFIG_PREEMPT | 175 | # ifdef CONFIG_PREEMPT_COUNT |
176 | VM_BUG_ON(!in_atomic()); | 176 | VM_BUG_ON(!in_atomic()); |
177 | # endif | 177 | # endif |
178 | VM_BUG_ON(page_count(page) == 0); | 178 | VM_BUG_ON(page_count(page) == 0); |
diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 2e681d9555bd..58969b2a8a82 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h | |||
@@ -27,6 +27,21 @@ | |||
27 | 27 | ||
28 | asmlinkage void preempt_schedule(void); | 28 | asmlinkage void preempt_schedule(void); |
29 | 29 | ||
30 | #define preempt_check_resched() \ | ||
31 | do { \ | ||
32 | if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ | ||
33 | preempt_schedule(); \ | ||
34 | } while (0) | ||
35 | |||
36 | #else /* !CONFIG_PREEMPT */ | ||
37 | |||
38 | #define preempt_check_resched() do { } while (0) | ||
39 | |||
40 | #endif /* CONFIG_PREEMPT */ | ||
41 | |||
42 | |||
43 | #ifdef CONFIG_PREEMPT_COUNT | ||
44 | |||
30 | #define preempt_disable() \ | 45 | #define preempt_disable() \ |
31 | do { \ | 46 | do { \ |
32 | inc_preempt_count(); \ | 47 | inc_preempt_count(); \ |
@@ -39,12 +54,6 @@ do { \ | |||
39 | dec_preempt_count(); \ | 54 | dec_preempt_count(); \ |
40 | } while (0) | 55 | } while (0) |
41 | 56 | ||
42 | #define preempt_check_resched() \ | ||
43 | do { \ | ||
44 | if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ | ||
45 | preempt_schedule(); \ | ||
46 | } while (0) | ||
47 | |||
48 | #define preempt_enable() \ | 57 | #define preempt_enable() \ |
49 | do { \ | 58 | do { \ |
50 | preempt_enable_no_resched(); \ | 59 | preempt_enable_no_resched(); \ |
@@ -80,18 +89,17 @@ do { \ | |||
80 | preempt_check_resched(); \ | 89 | preempt_check_resched(); \ |
81 | } while (0) | 90 | } while (0) |
82 | 91 | ||
83 | #else | 92 | #else /* !CONFIG_PREEMPT_COUNT */ |
84 | 93 | ||
85 | #define preempt_disable() do { } while (0) | 94 | #define preempt_disable() do { } while (0) |
86 | #define preempt_enable_no_resched() do { } while (0) | 95 | #define preempt_enable_no_resched() do { } while (0) |
87 | #define preempt_enable() do { } while (0) | 96 | #define preempt_enable() do { } while (0) |
88 | #define preempt_check_resched() do { } while (0) | ||
89 | 97 | ||
90 | #define preempt_disable_notrace() do { } while (0) | 98 | #define preempt_disable_notrace() do { } while (0) |
91 | #define preempt_enable_no_resched_notrace() do { } while (0) | 99 | #define preempt_enable_no_resched_notrace() do { } while (0) |
92 | #define preempt_enable_notrace() do { } while (0) | 100 | #define preempt_enable_notrace() do { } while (0) |
93 | 101 | ||
94 | #endif | 102 | #endif /* CONFIG_PREEMPT_COUNT */ |
95 | 103 | ||
96 | #ifdef CONFIG_PREEMPT_NOTIFIERS | 104 | #ifdef CONFIG_PREEMPT_NOTIFIERS |
97 | 105 | ||
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 99f9aa7c2804..8f4f881a0ad8 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h | |||
@@ -239,7 +239,7 @@ extern int rcu_read_lock_bh_held(void); | |||
239 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot | 239 | * Check debug_lockdep_rcu_enabled() to prevent false positives during boot |
240 | * and while lockdep is disabled. | 240 | * and while lockdep is disabled. |
241 | */ | 241 | */ |
242 | #ifdef CONFIG_PREEMPT | 242 | #ifdef CONFIG_PREEMPT_COUNT |
243 | static inline int rcu_read_lock_sched_held(void) | 243 | static inline int rcu_read_lock_sched_held(void) |
244 | { | 244 | { |
245 | int lockdep_opinion = 0; | 245 | int lockdep_opinion = 0; |
@@ -250,12 +250,12 @@ static inline int rcu_read_lock_sched_held(void) | |||
250 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); | 250 | lockdep_opinion = lock_is_held(&rcu_sched_lock_map); |
251 | return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); | 251 | return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); |
252 | } | 252 | } |
253 | #else /* #ifdef CONFIG_PREEMPT */ | 253 | #else /* #ifdef CONFIG_PREEMPT_COUNT */ |
254 | static inline int rcu_read_lock_sched_held(void) | 254 | static inline int rcu_read_lock_sched_held(void) |
255 | { | 255 | { |
256 | return 1; | 256 | return 1; |
257 | } | 257 | } |
258 | #endif /* #else #ifdef CONFIG_PREEMPT */ | 258 | #endif /* #else #ifdef CONFIG_PREEMPT_COUNT */ |
259 | 259 | ||
260 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 260 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
261 | 261 | ||
@@ -276,17 +276,17 @@ static inline int rcu_read_lock_bh_held(void) | |||
276 | return 1; | 276 | return 1; |
277 | } | 277 | } |
278 | 278 | ||
279 | #ifdef CONFIG_PREEMPT | 279 | #ifdef CONFIG_PREEMPT_COUNT |
280 | static inline int rcu_read_lock_sched_held(void) | 280 | static inline int rcu_read_lock_sched_held(void) |
281 | { | 281 | { |
282 | return preempt_count() != 0 || irqs_disabled(); | 282 | return preempt_count() != 0 || irqs_disabled(); |
283 | } | 283 | } |
284 | #else /* #ifdef CONFIG_PREEMPT */ | 284 | #else /* #ifdef CONFIG_PREEMPT_COUNT */ |
285 | static inline int rcu_read_lock_sched_held(void) | 285 | static inline int rcu_read_lock_sched_held(void) |
286 | { | 286 | { |
287 | return 1; | 287 | return 1; |
288 | } | 288 | } |
289 | #endif /* #else #ifdef CONFIG_PREEMPT */ | 289 | #endif /* #else #ifdef CONFIG_PREEMPT_COUNT */ |
290 | 290 | ||
291 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | 291 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
292 | 292 | ||
diff --git a/include/linux/sched.h b/include/linux/sched.h index 483c1ed5bc4d..4ecd5cbe7e24 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -2502,7 +2502,7 @@ extern int _cond_resched(void); | |||
2502 | 2502 | ||
2503 | extern int __cond_resched_lock(spinlock_t *lock); | 2503 | extern int __cond_resched_lock(spinlock_t *lock); |
2504 | 2504 | ||
2505 | #ifdef CONFIG_PREEMPT | 2505 | #ifdef CONFIG_PREEMPT_COUNT |
2506 | #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET | 2506 | #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET |
2507 | #else | 2507 | #else |
2508 | #define PREEMPT_LOCK_OFFSET 0 | 2508 | #define PREEMPT_LOCK_OFFSET 0 |
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index bf987b95b356..24e7cb0ba26a 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt | |||
@@ -35,6 +35,7 @@ config PREEMPT_VOLUNTARY | |||
35 | 35 | ||
36 | config PREEMPT | 36 | config PREEMPT |
37 | bool "Preemptible Kernel (Low-Latency Desktop)" | 37 | bool "Preemptible Kernel (Low-Latency Desktop)" |
38 | select PREEMPT_COUNT | ||
38 | help | 39 | help |
39 | This option reduces the latency of the kernel by making | 40 | This option reduces the latency of the kernel by making |
40 | all kernel code (that is not executing in a critical section) | 41 | all kernel code (that is not executing in a critical section) |
@@ -52,3 +53,5 @@ config PREEMPT | |||
52 | 53 | ||
53 | endchoice | 54 | endchoice |
54 | 55 | ||
56 | config PREEMPT_COUNT | ||
57 | bool \ No newline at end of file | ||
diff --git a/kernel/sched.c b/kernel/sched.c index 01d9536aaa8e..90ad7cf2b290 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -2843,7 +2843,7 @@ void sched_fork(struct task_struct *p) | |||
2843 | #if defined(CONFIG_SMP) | 2843 | #if defined(CONFIG_SMP) |
2844 | p->on_cpu = 0; | 2844 | p->on_cpu = 0; |
2845 | #endif | 2845 | #endif |
2846 | #ifdef CONFIG_PREEMPT | 2846 | #ifdef CONFIG_PREEMPT_COUNT |
2847 | /* Want to start with kernel preemption disabled. */ | 2847 | /* Want to start with kernel preemption disabled. */ |
2848 | task_thread_info(p)->preempt_count = 1; | 2848 | task_thread_info(p)->preempt_count = 1; |
2849 | #endif | 2849 | #endif |