aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNick Piggin <npiggin@suse.de>2008-01-30 07:31:20 -0500
committerIngo Molnar <mingo@elte.hu>2008-01-30 07:31:20 -0500
commit95c354fe9f7d6decc08a92aa26eb233ecc2155bf (patch)
treeec9267032ea875e84216cfb20acb2cfc7c62149f
parenta95d67f87e1a5f1b4429be3ba3bf7b4051657908 (diff)
spinlock: lockbreak cleanup
The break_lock data structure and code for spinlocks is quite nasty. Not only does it double the size of a spinlock but it changes locking to a potentially less optimal trylock. Put all of that under CONFIG_GENERIC_LOCKBREAK, and introduce a __raw_spin_is_contended that uses the lock data itself to determine whether there are waiters on the lock, to be used if CONFIG_GENERIC_LOCKBREAK is not set. Rename need_lockbreak to spin_needbreak, make it use spin_is_contended to decouple it from the spinlock implementation, and make it typesafe (rwlocks do not have any need_lockbreak sites -- why do they even get bloated up with that break_lock then?). Signed-off-by: Nick Piggin <npiggin@suse.de> Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r--arch/arm/Kconfig5
-rw-r--r--arch/ia64/Kconfig5
-rw-r--r--arch/m32r/Kconfig5
-rw-r--r--arch/mips/Kconfig5
-rw-r--r--arch/parisc/Kconfig5
-rw-r--r--arch/powerpc/Kconfig5
-rw-r--r--arch/sparc64/Kconfig5
-rw-r--r--arch/x86/Kconfig4
-rw-r--r--fs/jbd/checkpoint.c3
-rw-r--r--fs/jbd/commit.c2
-rw-r--r--fs/jbd2/checkpoint.c3
-rw-r--r--fs/jbd2/commit.c2
-rw-r--r--include/linux/sched.h21
-rw-r--r--include/linux/spinlock.h6
-rw-r--r--include/linux/spinlock_types.h4
-rw-r--r--include/linux/spinlock_up.h2
-rw-r--r--kernel/sched.c16
-rw-r--r--kernel/spinlock.c3
-rw-r--r--mm/memory.c8
19 files changed, 72 insertions, 37 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index de211ac3853e..77201d3f7479 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -91,6 +91,11 @@ config GENERIC_IRQ_PROBE
91 bool 91 bool
92 default y 92 default y
93 93
94config GENERIC_LOCKBREAK
95 bool
96 default y
97 depends on SMP && PREEMPT
98
94config RWSEM_GENERIC_SPINLOCK 99config RWSEM_GENERIC_SPINLOCK
95 bool 100 bool
96 default y 101 default y
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index bef47725d4ad..4a81b7fb191a 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -42,6 +42,11 @@ config MMU
42config SWIOTLB 42config SWIOTLB
43 bool 43 bool
44 44
45config GENERIC_LOCKBREAK
46 bool
47 default y
48 depends on SMP && PREEMPT
49
45config RWSEM_XCHGADD_ALGORITHM 50config RWSEM_XCHGADD_ALGORITHM
46 bool 51 bool
47 default y 52 default y
diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig
index ab9a264cb194..f7237c5f531e 100644
--- a/arch/m32r/Kconfig
+++ b/arch/m32r/Kconfig
@@ -235,6 +235,11 @@ config IRAM_SIZE
235# Define implied options from the CPU selection here 235# Define implied options from the CPU selection here
236# 236#
237 237
238config GENERIC_LOCKBREAK
239 bool
240 default y
241 depends on SMP && PREEMPT
242
238config RWSEM_GENERIC_SPINLOCK 243config RWSEM_GENERIC_SPINLOCK
239 bool 244 bool
240 depends on M32R 245 depends on M32R
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 6b0f85f02c79..4fad0a34b997 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -694,6 +694,11 @@ source "arch/mips/vr41xx/Kconfig"
694 694
695endmenu 695endmenu
696 696
697config GENERIC_LOCKBREAK
698 bool
699 default y
700 depends on SMP && PREEMPT
701
697config RWSEM_GENERIC_SPINLOCK 702config RWSEM_GENERIC_SPINLOCK
698 bool 703 bool
699 default y 704 default y
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index b8ef1787a191..2b649c46631c 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -19,6 +19,11 @@ config MMU
19config STACK_GROWSUP 19config STACK_GROWSUP
20 def_bool y 20 def_bool y
21 21
22config GENERIC_LOCKBREAK
23 bool
24 default y
25 depends on SMP && PREEMPT
26
22config RWSEM_GENERIC_SPINLOCK 27config RWSEM_GENERIC_SPINLOCK
23 def_bool y 28 def_bool y
24 29
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 232c298c933f..c17a194beb0e 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -53,6 +53,11 @@ config RWSEM_XCHGADD_ALGORITHM
53 bool 53 bool
54 default y 54 default y
55 55
56config GENERIC_LOCKBREAK
57 bool
58 default y
59 depends on SMP && PREEMPT
60
56config ARCH_HAS_ILOG2_U32 61config ARCH_HAS_ILOG2_U32
57 bool 62 bool
58 default y 63 default y
diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig
index 10b212a1f9f5..1e25bce0366d 100644
--- a/arch/sparc64/Kconfig
+++ b/arch/sparc64/Kconfig
@@ -200,6 +200,11 @@ config US2E_FREQ
200 If in doubt, say N. 200 If in doubt, say N.
201 201
202# Global things across all Sun machines. 202# Global things across all Sun machines.
203config GENERIC_LOCKBREAK
204 bool
205 default y
206 depends on SMP && PREEMPT
207
203config RWSEM_GENERIC_SPINLOCK 208config RWSEM_GENERIC_SPINLOCK
204 bool 209 bool
205 210
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 23936301db56..db434f8171d3 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -19,6 +19,10 @@ config X86_64
19config X86 19config X86
20 def_bool y 20 def_bool y
21 21
22config GENERIC_LOCKBREAK
23 def_bool y
24 depends on SMP && PREEMPT
25
22config GENERIC_TIME 26config GENERIC_TIME
23 def_bool y 27 def_bool y
24 28
diff --git a/fs/jbd/checkpoint.c b/fs/jbd/checkpoint.c
index 0f69c416eebc..a5432bbbfb88 100644
--- a/fs/jbd/checkpoint.c
+++ b/fs/jbd/checkpoint.c
@@ -347,7 +347,8 @@ restart:
347 break; 347 break;
348 } 348 }
349 retry = __process_buffer(journal, jh, bhs,&batch_count); 349 retry = __process_buffer(journal, jh, bhs,&batch_count);
350 if (!retry && lock_need_resched(&journal->j_list_lock)){ 350 if (!retry && (need_resched() ||
351 spin_needbreak(&journal->j_list_lock))) {
351 spin_unlock(&journal->j_list_lock); 352 spin_unlock(&journal->j_list_lock);
352 retry = 1; 353 retry = 1;
353 break; 354 break;
diff --git a/fs/jbd/commit.c b/fs/jbd/commit.c
index 610264b99a8e..31853eb65b4c 100644
--- a/fs/jbd/commit.c
+++ b/fs/jbd/commit.c
@@ -265,7 +265,7 @@ write_out_data:
265 put_bh(bh); 265 put_bh(bh);
266 } 266 }
267 267
268 if (lock_need_resched(&journal->j_list_lock)) { 268 if (need_resched() || spin_needbreak(&journal->j_list_lock)) {
269 spin_unlock(&journal->j_list_lock); 269 spin_unlock(&journal->j_list_lock);
270 goto write_out_data; 270 goto write_out_data;
271 } 271 }
diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c
index 1b7f282c1ae9..6914598022ce 100644
--- a/fs/jbd2/checkpoint.c
+++ b/fs/jbd2/checkpoint.c
@@ -353,7 +353,8 @@ restart:
353 } 353 }
354 retry = __process_buffer(journal, jh, bhs, &batch_count, 354 retry = __process_buffer(journal, jh, bhs, &batch_count,
355 transaction); 355 transaction);
356 if (!retry && lock_need_resched(&journal->j_list_lock)){ 356 if (!retry && (need_resched() ||
357 spin_needbreak(&journal->j_list_lock))) {
357 spin_unlock(&journal->j_list_lock); 358 spin_unlock(&journal->j_list_lock);
358 retry = 1; 359 retry = 1;
359 break; 360 break;
diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c
index da8d0eb3b7b9..4f302d279279 100644
--- a/fs/jbd2/commit.c
+++ b/fs/jbd2/commit.c
@@ -341,7 +341,7 @@ write_out_data:
341 put_bh(bh); 341 put_bh(bh);
342 } 342 }
343 343
344 if (lock_need_resched(&journal->j_list_lock)) { 344 if (need_resched() || spin_needbreak(&journal->j_list_lock)) {
345 spin_unlock(&journal->j_list_lock); 345 spin_unlock(&journal->j_list_lock);
346 goto write_out_data; 346 goto write_out_data;
347 } 347 }
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 2d0546e884ea..9d4797609aa5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1922,23 +1922,16 @@ extern int cond_resched_softirq(void);
1922 1922
1923/* 1923/*
1924 * Does a critical section need to be broken due to another 1924 * Does a critical section need to be broken due to another
1925 * task waiting?: 1925 * task waiting?: (technically does not depend on CONFIG_PREEMPT,
1926 * but a general need for low latency)
1926 */ 1927 */
1927#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP) 1928static inline int spin_needbreak(spinlock_t *lock)
1928# define need_lockbreak(lock) ((lock)->break_lock)
1929#else
1930# define need_lockbreak(lock) 0
1931#endif
1932
1933/*
1934 * Does a critical section need to be broken due to another
1935 * task waiting or preemption being signalled:
1936 */
1937static inline int lock_need_resched(spinlock_t *lock)
1938{ 1929{
1939 if (need_lockbreak(lock) || need_resched()) 1930#ifdef CONFIG_PREEMPT
1940 return 1; 1931 return spin_is_contended(lock);
1932#else
1941 return 0; 1933 return 0;
1934#endif
1942} 1935}
1943 1936
1944/* 1937/*
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index c376f3b36c89..124449733c55 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -120,6 +120,12 @@ do { \
120 120
121#define spin_is_locked(lock) __raw_spin_is_locked(&(lock)->raw_lock) 121#define spin_is_locked(lock) __raw_spin_is_locked(&(lock)->raw_lock)
122 122
123#ifdef CONFIG_GENERIC_LOCKBREAK
124#define spin_is_contended(lock) ((lock)->break_lock)
125#else
126#define spin_is_contended(lock) __raw_spin_is_contended(&(lock)->raw_lock)
127#endif
128
123/** 129/**
124 * spin_unlock_wait - wait until the spinlock gets unlocked 130 * spin_unlock_wait - wait until the spinlock gets unlocked
125 * @lock: the spinlock in question. 131 * @lock: the spinlock in question.
diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h
index f6a3a951b79e..68d88f71f1a2 100644
--- a/include/linux/spinlock_types.h
+++ b/include/linux/spinlock_types.h
@@ -19,7 +19,7 @@
19 19
20typedef struct { 20typedef struct {
21 raw_spinlock_t raw_lock; 21 raw_spinlock_t raw_lock;
22#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP) 22#ifdef CONFIG_GENERIC_LOCKBREAK
23 unsigned int break_lock; 23 unsigned int break_lock;
24#endif 24#endif
25#ifdef CONFIG_DEBUG_SPINLOCK 25#ifdef CONFIG_DEBUG_SPINLOCK
@@ -35,7 +35,7 @@ typedef struct {
35 35
36typedef struct { 36typedef struct {
37 raw_rwlock_t raw_lock; 37 raw_rwlock_t raw_lock;
38#if defined(CONFIG_PREEMPT) && defined(CONFIG_SMP) 38#ifdef CONFIG_GENERIC_LOCKBREAK
39 unsigned int break_lock; 39 unsigned int break_lock;
40#endif 40#endif
41#ifdef CONFIG_DEBUG_SPINLOCK 41#ifdef CONFIG_DEBUG_SPINLOCK
diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h
index ea54c4c9a4ec..938234c4a996 100644
--- a/include/linux/spinlock_up.h
+++ b/include/linux/spinlock_up.h
@@ -64,6 +64,8 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock)
64# define __raw_spin_trylock(lock) ({ (void)(lock); 1; }) 64# define __raw_spin_trylock(lock) ({ (void)(lock); 1; })
65#endif /* DEBUG_SPINLOCK */ 65#endif /* DEBUG_SPINLOCK */
66 66
67#define __raw_spin_is_contended(lock) (((void)(lock), 0))
68
67#define __raw_read_can_lock(lock) (((void)(lock), 1)) 69#define __raw_read_can_lock(lock) (((void)(lock), 1))
68#define __raw_write_can_lock(lock) (((void)(lock), 1)) 70#define __raw_write_can_lock(lock) (((void)(lock), 1))
69 71
diff --git a/kernel/sched.c b/kernel/sched.c
index 524285e46fa7..ba4c88088f62 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -4945,19 +4945,15 @@ EXPORT_SYMBOL(_cond_resched);
4945 */ 4945 */
4946int cond_resched_lock(spinlock_t *lock) 4946int cond_resched_lock(spinlock_t *lock)
4947{ 4947{
4948 int resched = need_resched() && system_state == SYSTEM_RUNNING;
4948 int ret = 0; 4949 int ret = 0;
4949 4950
4950 if (need_lockbreak(lock)) { 4951 if (spin_needbreak(lock) || resched) {
4951 spin_unlock(lock); 4952 spin_unlock(lock);
4952 cpu_relax(); 4953 if (resched && need_resched())
4953 ret = 1; 4954 __cond_resched();
4954 spin_lock(lock); 4955 else
4955 } 4956 cpu_relax();
4956 if (need_resched() && system_state == SYSTEM_RUNNING) {
4957 spin_release(&lock->dep_map, 1, _THIS_IP_);
4958 _raw_spin_unlock(lock);
4959 preempt_enable_no_resched();
4960 __cond_resched();
4961 ret = 1; 4957 ret = 1;
4962 spin_lock(lock); 4958 spin_lock(lock);
4963 } 4959 }
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index cd72424c2662..ae28c8245123 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -65,8 +65,7 @@ EXPORT_SYMBOL(_write_trylock);
65 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are 65 * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66 * not re-enabled during lock-acquire (which the preempt-spin-ops do): 66 * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67 */ 67 */
68#if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \ 68#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC)
69 defined(CONFIG_DEBUG_LOCK_ALLOC)
70 69
71void __lockfunc _read_lock(rwlock_t *lock) 70void __lockfunc _read_lock(rwlock_t *lock)
72{ 71{
diff --git a/mm/memory.c b/mm/memory.c
index 4b0144b24c12..673ebbf499c7 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -513,8 +513,7 @@ again:
513 if (progress >= 32) { 513 if (progress >= 32) {
514 progress = 0; 514 progress = 0;
515 if (need_resched() || 515 if (need_resched() ||
516 need_lockbreak(src_ptl) || 516 spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
517 need_lockbreak(dst_ptl))
518 break; 517 break;
519 } 518 }
520 if (pte_none(*src_pte)) { 519 if (pte_none(*src_pte)) {
@@ -853,7 +852,7 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
853 tlb_finish_mmu(*tlbp, tlb_start, start); 852 tlb_finish_mmu(*tlbp, tlb_start, start);
854 853
855 if (need_resched() || 854 if (need_resched() ||
856 (i_mmap_lock && need_lockbreak(i_mmap_lock))) { 855 (i_mmap_lock && spin_needbreak(i_mmap_lock))) {
857 if (i_mmap_lock) { 856 if (i_mmap_lock) {
858 *tlbp = NULL; 857 *tlbp = NULL;
859 goto out; 858 goto out;
@@ -1768,8 +1767,7 @@ again:
1768 1767
1769 restart_addr = zap_page_range(vma, start_addr, 1768 restart_addr = zap_page_range(vma, start_addr,
1770 end_addr - start_addr, details); 1769 end_addr - start_addr, details);
1771 need_break = need_resched() || 1770 need_break = need_resched() || spin_needbreak(details->i_mmap_lock);
1772 need_lockbreak(details->i_mmap_lock);
1773 1771
1774 if (restart_addr >= end_addr) { 1772 if (restart_addr >= end_addr) {
1775 /* We have now completed this vma: mark it so */ 1773 /* We have now completed this vma: mark it so */