aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/bit_spinlock.h2
-rw-r--r--include/linux/hardirq.h4
-rw-r--r--include/linux/pagemap.h4
-rw-r--r--include/linux/preempt.h26
-rw-r--r--include/linux/rcupdate.h12
-rw-r--r--include/linux/sched.h2
-rw-r--r--kernel/Kconfig.preempt3
-rw-r--r--kernel/sched.c2
8 files changed, 33 insertions, 22 deletions
diff --git a/include/linux/bit_spinlock.h b/include/linux/bit_spinlock.h
index b4326bfa684..564d997e216 100644
--- a/include/linux/bit_spinlock.h
+++ b/include/linux/bit_spinlock.h
@@ -88,7 +88,7 @@ static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
88{ 88{
89#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) 89#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
90 return test_bit(bitnum, addr); 90 return test_bit(bitnum, addr);
91#elif defined CONFIG_PREEMPT 91#elif defined CONFIG_PREEMPT_COUNT
92 return preempt_count(); 92 return preempt_count();
93#else 93#else
94 return 1; 94 return 1;
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index ba362171e8a..f743883f769 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -93,7 +93,7 @@
93 */ 93 */
94#define in_nmi() (preempt_count() & NMI_MASK) 94#define in_nmi() (preempt_count() & NMI_MASK)
95 95
96#if defined(CONFIG_PREEMPT) 96#if defined(CONFIG_PREEMPT_COUNT)
97# define PREEMPT_CHECK_OFFSET 1 97# define PREEMPT_CHECK_OFFSET 1
98#else 98#else
99# define PREEMPT_CHECK_OFFSET 0 99# define PREEMPT_CHECK_OFFSET 0
@@ -115,7 +115,7 @@
115#define in_atomic_preempt_off() \ 115#define in_atomic_preempt_off() \
116 ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET) 116 ((preempt_count() & ~PREEMPT_ACTIVE) != PREEMPT_CHECK_OFFSET)
117 117
118#ifdef CONFIG_PREEMPT 118#ifdef CONFIG_PREEMPT_COUNT
119# define preemptible() (preempt_count() == 0 && !irqs_disabled()) 119# define preemptible() (preempt_count() == 0 && !irqs_disabled())
120# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1) 120# define IRQ_EXIT_OFFSET (HARDIRQ_OFFSET-1)
121#else 121#else
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 716875e5352..8e38d4c140f 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -134,7 +134,7 @@ static inline int page_cache_get_speculative(struct page *page)
134 VM_BUG_ON(in_interrupt()); 134 VM_BUG_ON(in_interrupt());
135 135
136#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) 136#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU)
137# ifdef CONFIG_PREEMPT 137# ifdef CONFIG_PREEMPT_COUNT
138 VM_BUG_ON(!in_atomic()); 138 VM_BUG_ON(!in_atomic());
139# endif 139# endif
140 /* 140 /*
@@ -172,7 +172,7 @@ static inline int page_cache_add_speculative(struct page *page, int count)
172 VM_BUG_ON(in_interrupt()); 172 VM_BUG_ON(in_interrupt());
173 173
174#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU) 174#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU)
175# ifdef CONFIG_PREEMPT 175# ifdef CONFIG_PREEMPT_COUNT
176 VM_BUG_ON(!in_atomic()); 176 VM_BUG_ON(!in_atomic());
177# endif 177# endif
178 VM_BUG_ON(page_count(page) == 0); 178 VM_BUG_ON(page_count(page) == 0);
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 2e681d9555b..58969b2a8a8 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -27,6 +27,21 @@
27 27
28asmlinkage void preempt_schedule(void); 28asmlinkage void preempt_schedule(void);
29 29
30#define preempt_check_resched() \
31do { \
32 if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
33 preempt_schedule(); \
34} while (0)
35
36#else /* !CONFIG_PREEMPT */
37
38#define preempt_check_resched() do { } while (0)
39
40#endif /* CONFIG_PREEMPT */
41
42
43#ifdef CONFIG_PREEMPT_COUNT
44
30#define preempt_disable() \ 45#define preempt_disable() \
31do { \ 46do { \
32 inc_preempt_count(); \ 47 inc_preempt_count(); \
@@ -39,12 +54,6 @@ do { \
39 dec_preempt_count(); \ 54 dec_preempt_count(); \
40} while (0) 55} while (0)
41 56
42#define preempt_check_resched() \
43do { \
44 if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
45 preempt_schedule(); \
46} while (0)
47
48#define preempt_enable() \ 57#define preempt_enable() \
49do { \ 58do { \
50 preempt_enable_no_resched(); \ 59 preempt_enable_no_resched(); \
@@ -80,18 +89,17 @@ do { \
80 preempt_check_resched(); \ 89 preempt_check_resched(); \
81} while (0) 90} while (0)
82 91
83#else 92#else /* !CONFIG_PREEMPT_COUNT */
84 93
85#define preempt_disable() do { } while (0) 94#define preempt_disable() do { } while (0)
86#define preempt_enable_no_resched() do { } while (0) 95#define preempt_enable_no_resched() do { } while (0)
87#define preempt_enable() do { } while (0) 96#define preempt_enable() do { } while (0)
88#define preempt_check_resched() do { } while (0)
89 97
90#define preempt_disable_notrace() do { } while (0) 98#define preempt_disable_notrace() do { } while (0)
91#define preempt_enable_no_resched_notrace() do { } while (0) 99#define preempt_enable_no_resched_notrace() do { } while (0)
92#define preempt_enable_notrace() do { } while (0) 100#define preempt_enable_notrace() do { } while (0)
93 101
94#endif 102#endif /* CONFIG_PREEMPT_COUNT */
95 103
96#ifdef CONFIG_PREEMPT_NOTIFIERS 104#ifdef CONFIG_PREEMPT_NOTIFIERS
97 105
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 99f9aa7c280..8f4f881a0ad 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -239,7 +239,7 @@ extern int rcu_read_lock_bh_held(void);
239 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot 239 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot
240 * and while lockdep is disabled. 240 * and while lockdep is disabled.
241 */ 241 */
242#ifdef CONFIG_PREEMPT 242#ifdef CONFIG_PREEMPT_COUNT
243static inline int rcu_read_lock_sched_held(void) 243static inline int rcu_read_lock_sched_held(void)
244{ 244{
245 int lockdep_opinion = 0; 245 int lockdep_opinion = 0;
@@ -250,12 +250,12 @@ static inline int rcu_read_lock_sched_held(void)
250 lockdep_opinion = lock_is_held(&rcu_sched_lock_map); 250 lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
251 return lockdep_opinion || preempt_count() != 0 || irqs_disabled(); 251 return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
252} 252}
253#else /* #ifdef CONFIG_PREEMPT */ 253#else /* #ifdef CONFIG_PREEMPT_COUNT */
254static inline int rcu_read_lock_sched_held(void) 254static inline int rcu_read_lock_sched_held(void)
255{ 255{
256 return 1; 256 return 1;
257} 257}
258#endif /* #else #ifdef CONFIG_PREEMPT */ 258#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
259 259
260#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 260#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
261 261
@@ -276,17 +276,17 @@ static inline int rcu_read_lock_bh_held(void)
276 return 1; 276 return 1;
277} 277}
278 278
279#ifdef CONFIG_PREEMPT 279#ifdef CONFIG_PREEMPT_COUNT
280static inline int rcu_read_lock_sched_held(void) 280static inline int rcu_read_lock_sched_held(void)
281{ 281{
282 return preempt_count() != 0 || irqs_disabled(); 282 return preempt_count() != 0 || irqs_disabled();
283} 283}
284#else /* #ifdef CONFIG_PREEMPT */ 284#else /* #ifdef CONFIG_PREEMPT_COUNT */
285static inline int rcu_read_lock_sched_held(void) 285static inline int rcu_read_lock_sched_held(void)
286{ 286{
287 return 1; 287 return 1;
288} 288}
289#endif /* #else #ifdef CONFIG_PREEMPT */ 289#endif /* #else #ifdef CONFIG_PREEMPT_COUNT */
290 290
291#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 291#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
292 292
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 483c1ed5bc4..4ecd5cbe7e2 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2502,7 +2502,7 @@ extern int _cond_resched(void);
2502 2502
2503extern int __cond_resched_lock(spinlock_t *lock); 2503extern int __cond_resched_lock(spinlock_t *lock);
2504 2504
2505#ifdef CONFIG_PREEMPT 2505#ifdef CONFIG_PREEMPT_COUNT
2506#define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET 2506#define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET
2507#else 2507#else
2508#define PREEMPT_LOCK_OFFSET 0 2508#define PREEMPT_LOCK_OFFSET 0
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index bf987b95b35..24e7cb0ba26 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -35,6 +35,7 @@ config PREEMPT_VOLUNTARY
35 35
36config PREEMPT 36config PREEMPT
37 bool "Preemptible Kernel (Low-Latency Desktop)" 37 bool "Preemptible Kernel (Low-Latency Desktop)"
38 select PREEMPT_COUNT
38 help 39 help
39 This option reduces the latency of the kernel by making 40 This option reduces the latency of the kernel by making
40 all kernel code (that is not executing in a critical section) 41 all kernel code (that is not executing in a critical section)
@@ -52,3 +53,5 @@ config PREEMPT
52 53
53endchoice 54endchoice
54 55
56config PREEMPT_COUNT
57 bool \ No newline at end of file
diff --git a/kernel/sched.c b/kernel/sched.c
index 01d9536aaa8..90ad7cf2b29 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2843,7 +2843,7 @@ void sched_fork(struct task_struct *p)
2843#if defined(CONFIG_SMP) 2843#if defined(CONFIG_SMP)
2844 p->on_cpu = 0; 2844 p->on_cpu = 0;
2845#endif 2845#endif
2846#ifdef CONFIG_PREEMPT 2846#ifdef CONFIG_PREEMPT_COUNT
2847 /* Want to start with kernel preemption disabled. */ 2847 /* Want to start with kernel preemption disabled. */
2848 task_thread_info(p)->preempt_count = 1; 2848 task_thread_info(p)->preempt_count = 1;
2849#endif 2849#endif