aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-08-04 19:09:06 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-08-04 19:09:06 -0400
commit8efb90cf1e80129fad197b916714e1d01ee183d2 (patch)
tree8eacf9e5f34cc7a9e2b553974b9f9181d61be294
parent5bda4f638f36ef4c4e3b1397b02affc3db94356e (diff)
parent3a6bfbc91df04b081a44d419e0260bad54abddf7 (diff)
Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking updates from Ingo Molnar: "The main changes in this cycle are: - big rtmutex and futex cleanup and robustification from Thomas Gleixner - mutex optimizations and refinements from Jason Low - arch_mutex_cpu_relax() removal and related cleanups - smaller lockdep tweaks" * 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (23 commits) arch, locking: Ciao arch_mutex_cpu_relax() locking/lockdep: Only ask for /proc/lock_stat output when available locking/mutexes: Optimize mutex trylock slowpath locking/mutexes: Try to acquire mutex only if it is unlocked locking/mutexes: Delete the MUTEX_SHOW_NO_WAITER macro locking/mutexes: Correct documentation on mutex optimistic spinning rtmutex: Make the rtmutex tester depend on BROKEN futex: Simplify futex_lock_pi_atomic() and make it more robust futex: Split out the first waiter attachment from lookup_pi_state() futex: Split out the waiter check from lookup_pi_state() futex: Use futex_top_waiter() in lookup_pi_state() futex: Make unlock_pi more robust rtmutex: Avoid pointless requeueing in the deadlock detection chain walk rtmutex: Cleanup deadlock detector debug logic rtmutex: Confine deadlock logic to futex rtmutex: Simplify remove_waiter() rtmutex: Document pi chain walk rtmutex: Clarify the boost/deboost part rtmutex: No need to keep task ref for lock owner check rtmutex: Simplify and document try_to_take_rtmutex() ...
Diffstat
-rw-r--r--arch/alpha/include/asm/processor.h1
-rw-r--r--arch/arc/include/asm/processor.h2
-rw-r--r--arch/arm/include/asm/processor.h2
-rw-r--r--arch/arm64/include/asm/processor.h1
-rw-r--r--arch/avr32/include/asm/processor.h1
-rw-r--r--arch/blackfin/include/asm/processor.h2
-rw-r--r--arch/c6x/include/asm/processor.h1
-rw-r--r--arch/cris/include/asm/processor.h1
-rw-r--r--arch/hexagon/include/asm/processor.h1
-rw-r--r--arch/ia64/include/asm/processor.h1
-rw-r--r--arch/m32r/include/asm/processor.h1
-rw-r--r--arch/m68k/include/asm/processor.h1
-rw-r--r--arch/metag/include/asm/processor.h1
-rw-r--r--arch/microblaze/include/asm/processor.h1
-rw-r--r--arch/mips/include/asm/processor.h1
-rw-r--r--arch/mn10300/include/asm/processor.h2
-rw-r--r--arch/openrisc/include/asm/processor.h1
-rw-r--r--arch/parisc/include/asm/processor.h1
-rw-r--r--arch/powerpc/include/asm/processor.h2
-rw-r--r--arch/s390/include/asm/processor.h2
-rw-r--r--arch/score/include/asm/processor.h1
-rw-r--r--arch/sh/include/asm/processor.h1
-rw-r--r--arch/sparc/include/asm/processor_32.h2
-rw-r--r--arch/sparc/include/asm/processor_64.h1
-rw-r--r--arch/tile/include/asm/processor.h2
-rw-r--r--arch/unicore32/include/asm/processor.h1
-rw-r--r--arch/x86/include/asm/barrier.h2
-rw-r--r--arch/x86/include/asm/processor.h2
-rw-r--r--arch/x86/include/asm/qrwlock.h2
-rw-r--r--arch/x86/um/asm/processor.h3
-rw-r--r--arch/xtensa/include/asm/processor.h1
-rw-r--r--include/linux/mutex.h4
-rw-r--r--include/linux/rtmutex.h6
-rw-r--r--include/linux/seqlock.h2
-rw-r--r--kernel/futex.c402
-rw-r--r--kernel/locking/lockdep.c2
-rw-r--r--kernel/locking/mcs_spinlock.c8
-rw-r--r--kernel/locking/mcs_spinlock.h4
-rw-r--r--kernel/locking/mutex.c39
-rw-r--r--kernel/locking/qrwlock.c9
-rw-r--r--kernel/locking/rtmutex-debug.c5
-rw-r--r--kernel/locking/rtmutex-debug.h7
-rw-r--r--kernel/locking/rtmutex.c562
-rw-r--r--kernel/locking/rtmutex.h7
-rw-r--r--kernel/locking/rtmutex_common.h22
-rw-r--r--kernel/locking/rwsem-xadd.c4
-rw-r--r--lib/Kconfig.debug2
-rw-r--r--lib/lockref.c3
48 files changed, 707 insertions, 425 deletions
diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h
index 6cb7fe85c4b5..b4cf03690394 100644
--- a/arch/alpha/include/asm/processor.h
+++ b/arch/alpha/include/asm/processor.h
@@ -57,6 +57,7 @@ unsigned long get_wchan(struct task_struct *p);
57 ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp) 57 ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp)
58 58
59#define cpu_relax() barrier() 59#define cpu_relax() barrier()
60#define cpu_relax_lowlatency() cpu_relax()
60 61
61#define ARCH_HAS_PREFETCH 62#define ARCH_HAS_PREFETCH
62#define ARCH_HAS_PREFETCHW 63#define ARCH_HAS_PREFETCHW
diff --git a/arch/arc/include/asm/processor.h b/arch/arc/include/asm/processor.h
index d99f9b37cd15..82588f3ba77f 100644
--- a/arch/arc/include/asm/processor.h
+++ b/arch/arc/include/asm/processor.h
@@ -62,6 +62,8 @@ unsigned long thread_saved_pc(struct task_struct *t);
62#define cpu_relax() do { } while (0) 62#define cpu_relax() do { } while (0)
63#endif 63#endif
64 64
65#define cpu_relax_lowlatency() cpu_relax()
66
65#define copy_segments(tsk, mm) do { } while (0) 67#define copy_segments(tsk, mm) do { } while (0)
66#define release_segments(mm) do { } while (0) 68#define release_segments(mm) do { } while (0)
67 69
diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h
index c3d5fc124a05..8a1e8e995dae 100644
--- a/arch/arm/include/asm/processor.h
+++ b/arch/arm/include/asm/processor.h
@@ -82,6 +82,8 @@ unsigned long get_wchan(struct task_struct *p);
82#define cpu_relax() barrier() 82#define cpu_relax() barrier()
83#endif 83#endif
84 84
85#define cpu_relax_lowlatency() cpu_relax()
86
85#define task_pt_regs(p) \ 87#define task_pt_regs(p) \
86 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) 88 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
87 89
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 77712454486b..3df21feeabdd 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -129,6 +129,7 @@ extern void release_thread(struct task_struct *);
129unsigned long get_wchan(struct task_struct *p); 129unsigned long get_wchan(struct task_struct *p);
130 130
131#define cpu_relax() barrier() 131#define cpu_relax() barrier()
132#define cpu_relax_lowlatency() cpu_relax()
132 133
133/* Thread switching */ 134/* Thread switching */
134extern struct task_struct *cpu_switch_to(struct task_struct *prev, 135extern struct task_struct *cpu_switch_to(struct task_struct *prev,
diff --git a/arch/avr32/include/asm/processor.h b/arch/avr32/include/asm/processor.h
index 972adcc1e8f4..941593c7d9f3 100644
--- a/arch/avr32/include/asm/processor.h
+++ b/arch/avr32/include/asm/processor.h
@@ -92,6 +92,7 @@ extern struct avr32_cpuinfo boot_cpu_data;
92#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3)) 92#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3))
93 93
94#define cpu_relax() barrier() 94#define cpu_relax() barrier()
95#define cpu_relax_lowlatency() cpu_relax()
95#define cpu_sync_pipeline() asm volatile("sub pc, -2" : : : "memory") 96#define cpu_sync_pipeline() asm volatile("sub pc, -2" : : : "memory")
96 97
97struct cpu_context { 98struct cpu_context {
diff --git a/arch/blackfin/include/asm/processor.h b/arch/blackfin/include/asm/processor.h
index d0e72e9475a6..7acd46653df3 100644
--- a/arch/blackfin/include/asm/processor.h
+++ b/arch/blackfin/include/asm/processor.h
@@ -99,7 +99,7 @@ unsigned long get_wchan(struct task_struct *p);
99#define KSTK_ESP(tsk) ((tsk) == current ? rdusp() : (tsk)->thread.usp) 99#define KSTK_ESP(tsk) ((tsk) == current ? rdusp() : (tsk)->thread.usp)
100 100
101#define cpu_relax() smp_mb() 101#define cpu_relax() smp_mb()
102 102#define cpu_relax_lowlatency() cpu_relax()
103 103
104/* Get the Silicon Revision of the chip */ 104/* Get the Silicon Revision of the chip */
105static inline uint32_t __pure bfin_revid(void) 105static inline uint32_t __pure bfin_revid(void)
diff --git a/arch/c6x/include/asm/processor.h b/arch/c6x/include/asm/processor.h
index b9eb3da7f278..f2ef31be2f8b 100644
--- a/arch/c6x/include/asm/processor.h
+++ b/arch/c6x/include/asm/processor.h
@@ -121,6 +121,7 @@ extern unsigned long get_wchan(struct task_struct *p);
121#define KSTK_ESP(task) (task_pt_regs(task)->sp) 121#define KSTK_ESP(task) (task_pt_regs(task)->sp)
122 122
123#define cpu_relax() do { } while (0) 123#define cpu_relax() do { } while (0)
124#define cpu_relax_lowlatency() cpu_relax()
124 125
125extern const struct seq_operations cpuinfo_op; 126extern const struct seq_operations cpuinfo_op;
126 127
diff --git a/arch/cris/include/asm/processor.h b/arch/cris/include/asm/processor.h
index 15b815df29c1..862126b58116 100644
--- a/arch/cris/include/asm/processor.h
+++ b/arch/cris/include/asm/processor.h
@@ -63,6 +63,7 @@ static inline void release_thread(struct task_struct *dead_task)
63#define init_stack (init_thread_union.stack) 63#define init_stack (init_thread_union.stack)
64 64
65#define cpu_relax() barrier() 65#define cpu_relax() barrier()
66#define cpu_relax_lowlatency() cpu_relax()
66 67
67void default_idle(void); 68void default_idle(void);
68 69
diff --git a/arch/hexagon/include/asm/processor.h b/arch/hexagon/include/asm/processor.h
index 45a825402f63..d8501137c8d0 100644
--- a/arch/hexagon/include/asm/processor.h
+++ b/arch/hexagon/include/asm/processor.h
@@ -56,6 +56,7 @@ struct thread_struct {
56} 56}
57 57
58#define cpu_relax() __vmyield() 58#define cpu_relax() __vmyield()
59#define cpu_relax_lowlatency() cpu_relax()
59 60
60/* 61/*
61 * Decides where the kernel will search for a free chunk of vm space during 62 * Decides where the kernel will search for a free chunk of vm space during
diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h
index efd1b927ccb7..c7367130ab14 100644
--- a/arch/ia64/include/asm/processor.h
+++ b/arch/ia64/include/asm/processor.h
@@ -548,6 +548,7 @@ ia64_eoi (void)
548} 548}
549 549
550#define cpu_relax() ia64_hint(ia64_hint_pause) 550#define cpu_relax() ia64_hint(ia64_hint_pause)
551#define cpu_relax_lowlatency() cpu_relax()
551 552
552static inline int 553static inline int
553ia64_get_irr(unsigned int vector) 554ia64_get_irr(unsigned int vector)
diff --git a/arch/m32r/include/asm/processor.h b/arch/m32r/include/asm/processor.h
index 5767367550c6..9f8fd9bef70f 100644
--- a/arch/m32r/include/asm/processor.h
+++ b/arch/m32r/include/asm/processor.h
@@ -133,5 +133,6 @@ unsigned long get_wchan(struct task_struct *p);
133#define KSTK_ESP(tsk) ((tsk)->thread.sp) 133#define KSTK_ESP(tsk) ((tsk)->thread.sp)
134 134
135#define cpu_relax() barrier() 135#define cpu_relax() barrier()
136#define cpu_relax_lowlatency() cpu_relax()
136 137
137#endif /* _ASM_M32R_PROCESSOR_H */ 138#endif /* _ASM_M32R_PROCESSOR_H */
diff --git a/arch/m68k/include/asm/processor.h b/arch/m68k/include/asm/processor.h
index b0768a657920..20dda1d4b860 100644
--- a/arch/m68k/include/asm/processor.h
+++ b/arch/m68k/include/asm/processor.h
@@ -176,5 +176,6 @@ unsigned long get_wchan(struct task_struct *p);
176#define task_pt_regs(tsk) ((struct pt_regs *) ((tsk)->thread.esp0)) 176#define task_pt_regs(tsk) ((struct pt_regs *) ((tsk)->thread.esp0))
177 177
178#define cpu_relax() barrier() 178#define cpu_relax() barrier()
179#define cpu_relax_lowlatency() cpu_relax()
179 180
180#endif 181#endif
diff --git a/arch/metag/include/asm/processor.h b/arch/metag/include/asm/processor.h
index a8a37477c66e..881071c07942 100644
--- a/arch/metag/include/asm/processor.h
+++ b/arch/metag/include/asm/processor.h
@@ -155,6 +155,7 @@ unsigned long get_wchan(struct task_struct *p);
155#define user_stack_pointer(regs) ((regs)->ctx.AX[0].U0) 155#define user_stack_pointer(regs) ((regs)->ctx.AX[0].U0)
156 156
157#define cpu_relax() barrier() 157#define cpu_relax() barrier()
158#define cpu_relax_lowlatency() cpu_relax()
158 159
159extern void setup_priv(void); 160extern void setup_priv(void);
160 161
diff --git a/arch/microblaze/include/asm/processor.h b/arch/microblaze/include/asm/processor.h
index 9d31b057c355..497a988d79c2 100644
--- a/arch/microblaze/include/asm/processor.h
+++ b/arch/microblaze/include/asm/processor.h
@@ -22,6 +22,7 @@
22extern const struct seq_operations cpuinfo_op; 22extern const struct seq_operations cpuinfo_op;
23 23
24# define cpu_relax() barrier() 24# define cpu_relax() barrier()
25# define cpu_relax_lowlatency() cpu_relax()
25 26
26#define task_pt_regs(tsk) \ 27#define task_pt_regs(tsk) \
27 (((struct pt_regs *)(THREAD_SIZE + task_stack_page(tsk))) - 1) 28 (((struct pt_regs *)(THREAD_SIZE + task_stack_page(tsk))) - 1)
diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h
index ad70cba8daff..d5098bc554f4 100644
--- a/arch/mips/include/asm/processor.h
+++ b/arch/mips/include/asm/processor.h
@@ -367,6 +367,7 @@ unsigned long get_wchan(struct task_struct *p);
367#define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status) 367#define KSTK_STATUS(tsk) (task_pt_regs(tsk)->cp0_status)
368 368
369#define cpu_relax() barrier() 369#define cpu_relax() barrier()
370#define cpu_relax_lowlatency() cpu_relax()
370 371
371/* 372/*
372 * Return_address is a replacement for __builtin_return_address(count) 373 * Return_address is a replacement for __builtin_return_address(count)
diff --git a/arch/mn10300/include/asm/processor.h b/arch/mn10300/include/asm/processor.h
index 8b80b19d0c8a..769d5ed8e992 100644
--- a/arch/mn10300/include/asm/processor.h
+++ b/arch/mn10300/include/asm/processor.h
@@ -68,7 +68,9 @@ extern struct mn10300_cpuinfo cpu_data[];
68extern void identify_cpu(struct mn10300_cpuinfo *); 68extern void identify_cpu(struct mn10300_cpuinfo *);
69extern void print_cpu_info(struct mn10300_cpuinfo *); 69extern void print_cpu_info(struct mn10300_cpuinfo *);
70extern void dodgy_tsc(void); 70extern void dodgy_tsc(void);
71
71#define cpu_relax() barrier() 72#define cpu_relax() barrier()
73#define cpu_relax_lowlatency() cpu_relax()
72 74
73/* 75/*
74 * User space process size: 1.75GB (default). 76 * User space process size: 1.75GB (default).
diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h
index cab746fa9e87..4d235e3d2534 100644
--- a/arch/openrisc/include/asm/processor.h
+++ b/arch/openrisc/include/asm/processor.h
@@ -101,6 +101,7 @@ extern unsigned long thread_saved_pc(struct task_struct *t);
101#define init_stack (init_thread_union.stack) 101#define init_stack (init_thread_union.stack)
102 102
103#define cpu_relax() barrier() 103#define cpu_relax() barrier()
104#define cpu_relax_lowlatency() cpu_relax()
104 105
105#endif /* __ASSEMBLY__ */ 106#endif /* __ASSEMBLY__ */
106#endif /* __ASM_OPENRISC_PROCESSOR_H */ 107#endif /* __ASM_OPENRISC_PROCESSOR_H */
diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h
index d951c9681ab3..689a8ade3606 100644
--- a/arch/parisc/include/asm/processor.h
+++ b/arch/parisc/include/asm/processor.h
@@ -338,6 +338,7 @@ extern unsigned long get_wchan(struct task_struct *p);
338#define KSTK_ESP(tsk) ((tsk)->thread.regs.gr[30]) 338#define KSTK_ESP(tsk) ((tsk)->thread.regs.gr[30])
339 339
340#define cpu_relax() barrier() 340#define cpu_relax() barrier()
341#define cpu_relax_lowlatency() cpu_relax()
341 342
342/* Used as a macro to identify the combined VIPT/PIPT cached 343/* Used as a macro to identify the combined VIPT/PIPT cached
343 * CPUs which require a guarantee of coherency (no inequivalent 344 * CPUs which require a guarantee of coherency (no inequivalent
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 6d59072e13a7..dda7ac4c80bd 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -400,6 +400,8 @@ static inline unsigned long __pack_fe01(unsigned int fpmode)
400#define cpu_relax() barrier() 400#define cpu_relax() barrier()
401#endif 401#endif
402 402
403#define cpu_relax_lowlatency() cpu_relax()
404
403/* Check that a certain kernel stack pointer is valid in task_struct p */ 405/* Check that a certain kernel stack pointer is valid in task_struct p */
404int validate_sp(unsigned long sp, struct task_struct *p, 406int validate_sp(unsigned long sp, struct task_struct *p,
405 unsigned long nbytes); 407 unsigned long nbytes);
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 6f02d452bbee..e568fc8a7250 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -217,7 +217,7 @@ static inline void cpu_relax(void)
217 barrier(); 217 barrier();
218} 218}
219 219
220#define arch_mutex_cpu_relax() barrier() 220#define cpu_relax_lowlatency() barrier()
221 221
222static inline void psw_set_key(unsigned int key) 222static inline void psw_set_key(unsigned int key)
223{ 223{
diff --git a/arch/score/include/asm/processor.h b/arch/score/include/asm/processor.h
index d9a922d8711b..851f441991d2 100644
--- a/arch/score/include/asm/processor.h
+++ b/arch/score/include/asm/processor.h
@@ -24,6 +24,7 @@ extern unsigned long get_wchan(struct task_struct *p);
24#define current_text_addr() ({ __label__ _l; _l: &&_l; }) 24#define current_text_addr() ({ __label__ _l; _l: &&_l; })
25 25
26#define cpu_relax() barrier() 26#define cpu_relax() barrier()
27#define cpu_relax_lowlatency() cpu_relax()
27#define release_thread(thread) do {} while (0) 28#define release_thread(thread) do {} while (0)
28 29
29/* 30/*
diff --git a/arch/sh/include/asm/processor.h b/arch/sh/include/asm/processor.h
index 5448f9bbf4ab..1506897648aa 100644
--- a/arch/sh/include/asm/processor.h
+++ b/arch/sh/include/asm/processor.h
@@ -97,6 +97,7 @@ extern struct sh_cpuinfo cpu_data[];
97 97
98#define cpu_sleep() __asm__ __volatile__ ("sleep" : : : "memory") 98#define cpu_sleep() __asm__ __volatile__ ("sleep" : : : "memory")
99#define cpu_relax() barrier() 99#define cpu_relax() barrier()
100#define cpu_relax_lowlatency() cpu_relax()
100 101
101void default_idle(void); 102void default_idle(void);
102void stop_this_cpu(void *); 103void stop_this_cpu(void *);
diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h
index a564817bbc2e..812fd08f3e62 100644
--- a/arch/sparc/include/asm/processor_32.h
+++ b/arch/sparc/include/asm/processor_32.h
@@ -119,6 +119,8 @@ extern struct task_struct *last_task_used_math;
119int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); 119int do_mathemu(struct pt_regs *regs, struct task_struct *fpt);
120 120
121#define cpu_relax() barrier() 121#define cpu_relax() barrier()
122#define cpu_relax_lowlatency() cpu_relax()
123
122extern void (*sparc_idle)(void); 124extern void (*sparc_idle)(void);
123 125
124#endif 126#endif
diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h
index 7028fe1a7c04..6924bdefe148 100644
--- a/arch/sparc/include/asm/processor_64.h
+++ b/arch/sparc/include/asm/processor_64.h
@@ -216,6 +216,7 @@ unsigned long get_wchan(struct task_struct *task);
216 "nop\n\t" \ 216 "nop\n\t" \
217 ".previous" \ 217 ".previous" \
218 ::: "memory") 218 ::: "memory")
219#define cpu_relax_lowlatency() cpu_relax()
219 220
220/* Prefetch support. This is tuned for UltraSPARC-III and later. 221/* Prefetch support. This is tuned for UltraSPARC-III and later.
221 * UltraSPARC-I will treat these as nops, and UltraSPARC-II has 222 * UltraSPARC-I will treat these as nops, and UltraSPARC-II has
diff --git a/arch/tile/include/asm/processor.h b/arch/tile/include/asm/processor.h
index 42323636c459..dd4f9f17e30a 100644
--- a/arch/tile/include/asm/processor.h
+++ b/arch/tile/include/asm/processor.h
@@ -266,6 +266,8 @@ static inline void cpu_relax(void)
266 barrier(); 266 barrier();
267} 267}
268 268
269#define cpu_relax_lowlatency() cpu_relax()
270
269/* Info on this processor (see fs/proc/cpuinfo.c) */ 271/* Info on this processor (see fs/proc/cpuinfo.c) */
270struct seq_operations; 272struct seq_operations;
271extern const struct seq_operations cpuinfo_op; 273extern const struct seq_operations cpuinfo_op;
diff --git a/arch/unicore32/include/asm/processor.h b/arch/unicore32/include/asm/processor.h
index 4eaa42167667..8d21b7adf26b 100644
--- a/arch/unicore32/include/asm/processor.h
+++ b/arch/unicore32/include/asm/processor.h
@@ -71,6 +71,7 @@ extern void release_thread(struct task_struct *);
71unsigned long get_wchan(struct task_struct *p); 71unsigned long get_wchan(struct task_struct *p);
72 72
73#define cpu_relax() barrier() 73#define cpu_relax() barrier()
74#define cpu_relax_lowlatency() cpu_relax()
74 75
75#define task_pt_regs(p) \ 76#define task_pt_regs(p) \
76 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1) 77 ((struct pt_regs *)(THREAD_START_SP + task_stack_page(p)) - 1)
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 5c7198cca5ed..0f4460b5636d 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -99,7 +99,7 @@
99#if defined(CONFIG_X86_PPRO_FENCE) 99#if defined(CONFIG_X86_PPRO_FENCE)
100 100
101/* 101/*
102 * For either of these options x86 doesn't have a strong TSO memory 102 * For this option x86 doesn't have a strong TSO memory
103 * model and we should fall back to full barriers. 103 * model and we should fall back to full barriers.
104 */ 104 */
105 105
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index a4ea02351f4d..32cc237f8e20 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -696,6 +696,8 @@ static inline void cpu_relax(void)
696 rep_nop(); 696 rep_nop();
697} 697}
698 698
699#define cpu_relax_lowlatency() cpu_relax()
700
699/* Stop speculative execution and prefetching of modified code. */ 701/* Stop speculative execution and prefetching of modified code. */
700static inline void sync_core(void) 702static inline void sync_core(void)
701{ 703{
diff --git a/arch/x86/include/asm/qrwlock.h b/arch/x86/include/asm/qrwlock.h
index 70f46f07f94e..ae0e241e228b 100644
--- a/arch/x86/include/asm/qrwlock.h
+++ b/arch/x86/include/asm/qrwlock.h
@@ -3,7 +3,7 @@
3 3
4#include <asm-generic/qrwlock_types.h> 4#include <asm-generic/qrwlock_types.h>
5 5
6#if !defined(CONFIG_X86_OOSTORE) && !defined(CONFIG_X86_PPRO_FENCE) 6#ifndef CONFIG_X86_PPRO_FENCE
7#define queue_write_unlock queue_write_unlock 7#define queue_write_unlock queue_write_unlock
8static inline void queue_write_unlock(struct qrwlock *lock) 8static inline void queue_write_unlock(struct qrwlock *lock)
9{ 9{
diff --git a/arch/x86/um/asm/processor.h b/arch/x86/um/asm/processor.h
index 04f82e020f2b..2a206d2b14ab 100644
--- a/arch/x86/um/asm/processor.h
+++ b/arch/x86/um/asm/processor.h
@@ -25,7 +25,8 @@ static inline void rep_nop(void)
25 __asm__ __volatile__("rep;nop": : :"memory"); 25 __asm__ __volatile__("rep;nop": : :"memory");
26} 26}
27 27
28#define cpu_relax() rep_nop() 28#define cpu_relax() rep_nop()
29#define cpu_relax_lowlatency() cpu_relax()
29 30
30#include <asm/processor-generic.h> 31#include <asm/processor-generic.h>
31 32
diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h
index abb59708a3b7..b61bdf0eea25 100644
--- a/arch/xtensa/include/asm/processor.h
+++ b/arch/xtensa/include/asm/processor.h
@@ -182,6 +182,7 @@ extern unsigned long get_wchan(struct task_struct *p);
182#define KSTK_ESP(tsk) (task_pt_regs(tsk)->areg[1]) 182#define KSTK_ESP(tsk) (task_pt_regs(tsk)->areg[1])
183 183
184#define cpu_relax() barrier() 184#define cpu_relax() barrier()
185#define cpu_relax_lowlatency() cpu_relax()
185 186
186/* Special register access. */ 187/* Special register access. */
187 188
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index 42aa9b9ecd5f..8d5535c58cc2 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -176,8 +176,4 @@ extern void mutex_unlock(struct mutex *lock);
176 176
177extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); 177extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
178 178
179#ifndef arch_mutex_cpu_relax
180# define arch_mutex_cpu_relax() cpu_relax()
181#endif
182
183#endif /* __LINUX_MUTEX_H */ 179#endif /* __LINUX_MUTEX_H */
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index 3aed8d737e1a..1abba5ce2a2f 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -90,11 +90,9 @@ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name);
90extern void rt_mutex_destroy(struct rt_mutex *lock); 90extern void rt_mutex_destroy(struct rt_mutex *lock);
91 91
92extern void rt_mutex_lock(struct rt_mutex *lock); 92extern void rt_mutex_lock(struct rt_mutex *lock);
93extern int rt_mutex_lock_interruptible(struct rt_mutex *lock, 93extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
94 int detect_deadlock);
95extern int rt_mutex_timed_lock(struct rt_mutex *lock, 94extern int rt_mutex_timed_lock(struct rt_mutex *lock,
96 struct hrtimer_sleeper *timeout, 95 struct hrtimer_sleeper *timeout);
97 int detect_deadlock);
98 96
99extern int rt_mutex_trylock(struct rt_mutex *lock); 97extern int rt_mutex_trylock(struct rt_mutex *lock);
100 98
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 535f158977b9..8cf350325dc6 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -164,8 +164,6 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)
164static inline unsigned raw_seqcount_begin(const seqcount_t *s) 164static inline unsigned raw_seqcount_begin(const seqcount_t *s)
165{ 165{
166 unsigned ret = ACCESS_ONCE(s->sequence); 166 unsigned ret = ACCESS_ONCE(s->sequence);
167
168 seqcount_lockdep_reader_access(s);
169 smp_rmb(); 167 smp_rmb();
170 return ret & ~1; 168 return ret & ~1;
171} 169}
diff --git a/kernel/futex.c b/kernel/futex.c
index b632b5f3f094..d3a9d946d0b7 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -792,94 +792,91 @@ void exit_pi_state_list(struct task_struct *curr)
792 * [10] There is no transient state which leaves owner and user space 792 * [10] There is no transient state which leaves owner and user space
793 * TID out of sync. 793 * TID out of sync.
794 */ 794 */
795static int 795
796lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, 796/*
797 union futex_key *key, struct futex_pi_state **ps) 797 * Validate that the existing waiter has a pi_state and sanity check
798 * the pi_state against the user space value. If correct, attach to
799 * it.
800 */
801static int attach_to_pi_state(u32 uval, struct futex_pi_state *pi_state,
802 struct futex_pi_state **ps)
798{ 803{
799 struct futex_pi_state *pi_state = NULL;
800 struct futex_q *this, *next;
801 struct task_struct *p;
802 pid_t pid = uval & FUTEX_TID_MASK; 804 pid_t pid = uval & FUTEX_TID_MASK;
803 805
804 plist_for_each_entry_safe(this, next, &hb->chain, list) { 806 /*
805 if (match_futex(&this->key, key)) { 807 * Userspace might have messed up non-PI and PI futexes [3]
806 /* 808 */
807 * Sanity check the waiter before increasing 809 if (unlikely(!pi_state))
808 * the refcount and attaching to it. 810 return -EINVAL;
809 */
810 pi_state = this->pi_state;
811 /*
812 * Userspace might have messed up non-PI and
813 * PI futexes [3]
814 */
815 if (unlikely(!pi_state))
816 return -EINVAL;
817 811
818 WARN_ON(!atomic_read(&pi_state->refcount)); 812 WARN_ON(!atomic_read(&pi_state->refcount));
819 813
814 /*
815 * Handle the owner died case:
816 */
817 if (uval & FUTEX_OWNER_DIED) {
818 /*
819 * exit_pi_state_list sets owner to NULL and wakes the
820 * topmost waiter. The task which acquires the
821 * pi_state->rt_mutex will fixup owner.
822 */
823 if (!pi_state->owner) {
820 /* 824 /*
821 * Handle the owner died case: 825 * No pi state owner, but the user space TID
826 * is not 0. Inconsistent state. [5]
822 */ 827 */
823 if (uval & FUTEX_OWNER_DIED) { 828 if (pid)
824 /* 829 return -EINVAL;
825 * exit_pi_state_list sets owner to NULL and
826 * wakes the topmost waiter. The task which
827 * acquires the pi_state->rt_mutex will fixup
828 * owner.
829 */
830 if (!pi_state->owner) {
831 /*
832 * No pi state owner, but the user
833 * space TID is not 0. Inconsistent
834 * state. [5]
835 */
836 if (pid)
837 return -EINVAL;
838 /*
839 * Take a ref on the state and
840 * return. [4]
841 */
842 goto out_state;
843 }
844
845 /*
846 * If TID is 0, then either the dying owner
847 * has not yet executed exit_pi_state_list()
848 * or some waiter acquired the rtmutex in the
849 * pi state, but did not yet fixup the TID in
850 * user space.
851 *
852 * Take a ref on the state and return. [6]
853 */
854 if (!pid)
855 goto out_state;
856 } else {
857 /*
858 * If the owner died bit is not set,
859 * then the pi_state must have an
860 * owner. [7]
861 */
862 if (!pi_state->owner)
863 return -EINVAL;
864 }
865
866 /* 830 /*
867 * Bail out if user space manipulated the 831 * Take a ref on the state and return success. [4]
868 * futex value. If pi state exists then the
869 * owner TID must be the same as the user
870 * space TID. [9/10]
871 */ 832 */
872 if (pid != task_pid_vnr(pi_state->owner)) 833 goto out_state;
873 return -EINVAL;
874
875 out_state:
876 atomic_inc(&pi_state->refcount);
877 *ps = pi_state;
878 return 0;
879 } 834 }
835
836 /*
837 * If TID is 0, then either the dying owner has not
838 * yet executed exit_pi_state_list() or some waiter
839 * acquired the rtmutex in the pi state, but did not
840 * yet fixup the TID in user space.
841 *
842 * Take a ref on the state and return success. [6]
843 */
844 if (!pid)
845 goto out_state;
846 } else {
847 /*
848 * If the owner died bit is not set, then the pi_state
849 * must have an owner. [7]
850 */
851 if (!pi_state->owner)
852 return -EINVAL;
880 } 853 }
881 854
882 /* 855 /*
856 * Bail out if user space manipulated the futex value. If pi
857 * state exists then the owner TID must be the same as the
858 * user space TID. [9/10]
859 */
860 if (pid != task_pid_vnr(pi_state->owner))
861 return -EINVAL;
862out_state:
863 atomic_inc(&pi_state->refcount);
864 *ps = pi_state;
865 return 0;
866}
867
868/*
869 * Lookup the task for the TID provided from user space and attach to
870 * it after doing proper sanity checks.
871 */
872static int attach_to_pi_owner(u32 uval, union futex_key *key,
873 struct futex_pi_state **ps)
874{
875 pid_t pid = uval & FUTEX_TID_MASK;
876 struct futex_pi_state *pi_state;
877 struct task_struct *p;
878
879 /*
883 * We are the first waiter - try to look up the real owner and attach 880 * We are the first waiter - try to look up the real owner and attach
884 * the new pi_state to it, but bail out when TID = 0 [1] 881 * the new pi_state to it, but bail out when TID = 0 [1]
885 */ 882 */
@@ -920,7 +917,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
920 pi_state = alloc_pi_state(); 917 pi_state = alloc_pi_state();
921 918
922 /* 919 /*
923 * Initialize the pi_mutex in locked state and make 'p' 920 * Initialize the pi_mutex in locked state and make @p
924 * the owner of it: 921 * the owner of it:
925 */ 922 */
926 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p); 923 rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
@@ -940,6 +937,36 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
940 return 0; 937 return 0;
941} 938}
942 939
940static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
941 union futex_key *key, struct futex_pi_state **ps)
942{
943 struct futex_q *match = futex_top_waiter(hb, key);
944
945 /*
946 * If there is a waiter on that futex, validate it and
947 * attach to the pi_state when the validation succeeds.
948 */
949 if (match)
950 return attach_to_pi_state(uval, match->pi_state, ps);
951
952 /*
953 * We are the first waiter - try to look up the owner based on
954 * @uval and attach to it.
955 */
956 return attach_to_pi_owner(uval, key, ps);
957}
958
959static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
960{
961 u32 uninitialized_var(curval);
962
963 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
964 return -EFAULT;
965
966 /*If user space value changed, let the caller retry */
967 return curval != uval ? -EAGAIN : 0;
968}
969
943/** 970/**
944 * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex 971 * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
945 * @uaddr: the pi futex user address 972 * @uaddr: the pi futex user address
@@ -963,113 +990,69 @@ static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
963 struct futex_pi_state **ps, 990 struct futex_pi_state **ps,
964 struct task_struct *task, int set_waiters) 991 struct task_struct *task, int set_waiters)
965{ 992{
966 int lock_taken, ret, force_take = 0; 993 u32 uval, newval, vpid = task_pid_vnr(task);
967 u32 uval, newval, curval, vpid = task_pid_vnr(task); 994 struct futex_q *match;
968 995 int ret;
969retry:
970 ret = lock_taken = 0;
971 996
972 /* 997 /*
973 * To avoid races, we attempt to take the lock here again 998 * Read the user space value first so we can validate a few
974 * (by doing a 0 -> TID atomic cmpxchg), while holding all 999 * things before proceeding further.
975 * the locks. It will most likely not succeed.
976 */ 1000 */
977 newval = vpid; 1001 if (get_futex_value_locked(&uval, uaddr))
978 if (set_waiters)
979 newval |= FUTEX_WAITERS;
980
981 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, 0, newval)))
982 return -EFAULT; 1002 return -EFAULT;
983 1003
984 /* 1004 /*
985 * Detect deadlocks. 1005 * Detect deadlocks.
986 */ 1006 */
987 if ((unlikely((curval & FUTEX_TID_MASK) == vpid))) 1007 if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))
988 return -EDEADLK; 1008 return -EDEADLK;
989 1009
990 /* 1010 /*
991 * Surprise - we got the lock, but we do not trust user space at all. 1011 * Lookup existing state first. If it exists, try to attach to
992 */ 1012 * its pi_state.
993 if (unlikely(!curval)) {
994 /*
995 * We verify whether there is kernel state for this
996 * futex. If not, we can safely assume, that the 0 ->
997 * TID transition is correct. If state exists, we do
998 * not bother to fixup the user space state as it was
999 * corrupted already.
1000 */
1001 return futex_top_waiter(hb, key) ? -EINVAL : 1;
1002 }
1003
1004 uval = curval;
1005
1006 /*
1007 * Set the FUTEX_WAITERS flag, so the owner will know it has someone
1008 * to wake at the next unlock.
1009 */ 1013 */
1010 newval = curval | FUTEX_WAITERS; 1014 match = futex_top_waiter(hb, key);
1015 if (match)
1016 return attach_to_pi_state(uval, match->pi_state, ps);
1011 1017
1012 /* 1018 /*
1013 * Should we force take the futex? See below. 1019 * No waiter and user TID is 0. We are here because the
1020 * waiters or the owner died bit is set or called from
1021 * requeue_cmp_pi or for whatever reason something took the
1022 * syscall.
1014 */ 1023 */
1015 if (unlikely(force_take)) { 1024 if (!(uval & FUTEX_TID_MASK)) {
1016 /* 1025 /*
1017 * Keep the OWNER_DIED and the WAITERS bit and set the 1026 * We take over the futex. No other waiters and the user space
1018 * new TID value. 1027 * TID is 0. We preserve the owner died bit.
1019 */ 1028 */
1020 newval = (curval & ~FUTEX_TID_MASK) | vpid; 1029 newval = uval & FUTEX_OWNER_DIED;
1021 force_take = 0; 1030 newval |= vpid;
1022 lock_taken = 1;
1023 }
1024 1031
1025 if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval))) 1032 /* The futex requeue_pi code can enforce the waiters bit */
1026 return -EFAULT; 1033 if (set_waiters)
1027 if (unlikely(curval != uval)) 1034 newval |= FUTEX_WAITERS;
1028 goto retry; 1035
1036 ret = lock_pi_update_atomic(uaddr, uval, newval);
1037 /* If the take over worked, return 1 */
1038 return ret < 0 ? ret : 1;
1039 }
1029 1040
1030 /* 1041 /*
1031 * We took the lock due to forced take over. 1042 * First waiter. Set the waiters bit before attaching ourself to
1043 * the owner. If owner tries to unlock, it will be forced into
1044 * the kernel and blocked on hb->lock.
1032 */ 1045 */
1033 if (unlikely(lock_taken)) 1046 newval = uval | FUTEX_WAITERS;
1034 return 1; 1047 ret = lock_pi_update_atomic(uaddr, uval, newval);
1035 1048 if (ret)
1049 return ret;
1036 /* 1050 /*
1037 * We dont have the lock. Look up the PI state (or create it if 1051 * If the update of the user space value succeeded, we try to
1038 * we are the first waiter): 1052 * attach to the owner. If that fails, no harm done, we only
1053 * set the FUTEX_WAITERS bit in the user space variable.
1039 */ 1054 */
1040 ret = lookup_pi_state(uval, hb, key, ps); 1055 return attach_to_pi_owner(uval, key, ps);
1041
1042 if (unlikely(ret)) {
1043 switch (ret) {
1044 case -ESRCH:
1045 /*
1046 * We failed to find an owner for this
1047 * futex. So we have no pi_state to block
1048 * on. This can happen in two cases:
1049 *
1050 * 1) The owner died
1051 * 2) A stale FUTEX_WAITERS bit
1052 *
1053 * Re-read the futex value.
1054 */
1055 if (get_futex_value_locked(&curval, uaddr))
1056 return -EFAULT;
1057
1058 /*
1059 * If the owner died or we have a stale
1060 * WAITERS bit the owner TID in the user space
1061 * futex is 0.
1062 */
1063 if (!(curval & FUTEX_TID_MASK)) {
1064 force_take = 1;
1065 goto retry;
1066 }
1067 default:
1068 break;
1069 }
1070 }
1071
1072 return ret;
1073} 1056}
1074 1057
1075/** 1058/**
@@ -1186,22 +1169,6 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
1186 return 0; 1169 return 0;
1187} 1170}
1188 1171
1189static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
1190{
1191 u32 uninitialized_var(oldval);
1192
1193 /*
1194 * There is no waiter, so we unlock the futex. The owner died
1195 * bit has not to be preserved here. We are the owner:
1196 */
1197 if (cmpxchg_futex_value_locked(&oldval, uaddr, uval, 0))
1198 return -EFAULT;
1199 if (oldval != uval)
1200 return -EAGAIN;
1201
1202 return 0;
1203}
1204
1205/* 1172/*
1206 * Express the locking dependencies for lockdep: 1173 * Express the locking dependencies for lockdep:
1207 */ 1174 */
@@ -1659,7 +1626,12 @@ retry_private:
1659 goto retry; 1626 goto retry;
1660 goto out; 1627 goto out;
1661 case -EAGAIN: 1628 case -EAGAIN:
1662 /* The owner was exiting, try again. */ 1629 /*
1630 * Two reasons for this:
1631 * - Owner is exiting and we just wait for the
1632 * exit to complete.
1633 * - The user space value changed.
1634 */
1663 double_unlock_hb(hb1, hb2); 1635 double_unlock_hb(hb1, hb2);
1664 hb_waiters_dec(hb2); 1636 hb_waiters_dec(hb2);
1665 put_futex_key(&key2); 1637 put_futex_key(&key2);
@@ -1718,7 +1690,7 @@ retry_private:
1718 this->pi_state = pi_state; 1690 this->pi_state = pi_state;
1719 ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex, 1691 ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
1720 this->rt_waiter, 1692 this->rt_waiter,
1721 this->task, 1); 1693 this->task);
1722 if (ret == 1) { 1694 if (ret == 1) {
1723 /* We got the lock. */ 1695 /* We got the lock. */
1724 requeue_pi_wake_futex(this, &key2, hb2); 1696 requeue_pi_wake_futex(this, &key2, hb2);
@@ -2316,8 +2288,10 @@ retry_private:
2316 goto uaddr_faulted; 2288 goto uaddr_faulted;
2317 case -EAGAIN: 2289 case -EAGAIN:
2318 /* 2290 /*
2319 * Task is exiting and we just wait for the 2291 * Two reasons for this:
2320 * exit to complete. 2292 * - Task is exiting and we just wait for the
2293 * exit to complete.
2294 * - The user space value changed.
2321 */ 2295 */
2322 queue_unlock(hb); 2296 queue_unlock(hb);
2323 put_futex_key(&q.key); 2297 put_futex_key(&q.key);
@@ -2337,9 +2311,9 @@ retry_private:
2337 /* 2311 /*
2338 * Block on the PI mutex: 2312 * Block on the PI mutex:
2339 */ 2313 */
2340 if (!trylock) 2314 if (!trylock) {
2341 ret = rt_mutex_timed_lock(&q.pi_state->pi_mutex, to, 1); 2315 ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
2342 else { 2316 } else {
2343 ret = rt_mutex_trylock(&q.pi_state->pi_mutex); 2317 ret = rt_mutex_trylock(&q.pi_state->pi_mutex);
2344 /* Fixup the trylock return value: */ 2318 /* Fixup the trylock return value: */
2345 ret = ret ? 0 : -EWOULDBLOCK; 2319 ret = ret ? 0 : -EWOULDBLOCK;
@@ -2401,10 +2375,10 @@ uaddr_faulted:
2401 */ 2375 */
2402static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) 2376static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
2403{ 2377{
2404 struct futex_hash_bucket *hb; 2378 u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
2405 struct futex_q *this, *next;
2406 union futex_key key = FUTEX_KEY_INIT; 2379 union futex_key key = FUTEX_KEY_INIT;
2407 u32 uval, vpid = task_pid_vnr(current); 2380 struct futex_hash_bucket *hb;
2381 struct futex_q *match;
2408 int ret; 2382 int ret;
2409 2383
2410retry: 2384retry:
@@ -2417,57 +2391,47 @@ retry:
2417 return -EPERM; 2391 return -EPERM;
2418 2392
2419 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE); 2393 ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
2420 if (unlikely(ret != 0)) 2394 if (ret)
2421 goto out; 2395 return ret;
2422 2396
2423 hb = hash_futex(&key); 2397 hb = hash_futex(&key);
2424 spin_lock(&hb->lock); 2398 spin_lock(&hb->lock);
2425 2399
2426 /* 2400 /*
2427 * To avoid races, try to do the TID -> 0 atomic transition 2401 * Check waiters first. We do not trust user space values at
2428 * again. If it succeeds then we can return without waking 2402 * all and we at least want to know if user space fiddled
2429 * anyone else up. We only try this if neither the waiters nor 2403 * with the futex value instead of blindly unlocking.
2430 * the owner died bit are set.
2431 */
2432 if (!(uval & ~FUTEX_TID_MASK) &&
2433 cmpxchg_futex_value_locked(&uval, uaddr, vpid, 0))
2434 goto pi_faulted;
2435 /*
2436 * Rare case: we managed to release the lock atomically,
2437 * no need to wake anyone else up:
2438 */
2439 if (unlikely(uval == vpid))
2440 goto out_unlock;
2441
2442 /*
2443 * Ok, other tasks may need to be woken up - check waiters
2444 * and do the wakeup if necessary:
2445 */ 2404 */
2446 plist_for_each_entry_safe(this, next, &hb->chain, list) { 2405 match = futex_top_waiter(hb, &key);
2447 if (!match_futex (&this->key, &key)) 2406 if (match) {
2448 continue; 2407 ret = wake_futex_pi(uaddr, uval, match);
2449 ret = wake_futex_pi(uaddr, uval, this);
2450 /* 2408 /*
2451 * The atomic access to the futex value 2409 * The atomic access to the futex value generated a
2452 * generated a pagefault, so retry the 2410 * pagefault, so retry the user-access and the wakeup:
2453 * user-access and the wakeup:
2454 */ 2411 */
2455 if (ret == -EFAULT) 2412 if (ret == -EFAULT)
2456 goto pi_faulted; 2413 goto pi_faulted;
2457 goto out_unlock; 2414 goto out_unlock;
2458 } 2415 }
2416
2459 /* 2417 /*
2460 * No waiters - kernel unlocks the futex: 2418 * We have no kernel internal state, i.e. no waiters in the
2419 * kernel. Waiters which are about to queue themselves are stuck
2420 * on hb->lock. So we can safely ignore them. We do neither
2421 * preserve the WAITERS bit not the OWNER_DIED one. We are the
2422 * owner.
2461 */ 2423 */
2462 ret = unlock_futex_pi(uaddr, uval); 2424 if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))
2463 if (ret == -EFAULT)
2464 goto pi_faulted; 2425 goto pi_faulted;
2465 2426
2427 /*
2428 * If uval has changed, let user space handle it.
2429 */
2430 ret = (curval == uval) ? 0 : -EAGAIN;
2431
2466out_unlock: 2432out_unlock:
2467 spin_unlock(&hb->lock); 2433 spin_unlock(&hb->lock);
2468 put_futex_key(&key); 2434 put_futex_key(&key);
2469
2470out:
2471 return ret; 2435 return ret;
2472 2436
2473pi_faulted: 2437pi_faulted:
@@ -2669,7 +2633,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
2669 */ 2633 */
2670 WARN_ON(!q.pi_state); 2634 WARN_ON(!q.pi_state);
2671 pi_mutex = &q.pi_state->pi_mutex; 2635 pi_mutex = &q.pi_state->pi_mutex;
2672 ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter, 1); 2636 ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter);
2673 debug_rt_mutex_free_waiter(&rt_waiter); 2637 debug_rt_mutex_free_waiter(&rt_waiter);
2674 2638
2675 spin_lock(q.lock_ptr); 2639 spin_lock(q.lock_ptr);
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index d24e4339b46d..88d0d4420ad2 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -384,7 +384,9 @@ static void print_lockdep_off(const char *bug_msg)
384{ 384{
385 printk(KERN_DEBUG "%s\n", bug_msg); 385 printk(KERN_DEBUG "%s\n", bug_msg);
386 printk(KERN_DEBUG "turning off the locking correctness validator.\n"); 386 printk(KERN_DEBUG "turning off the locking correctness validator.\n");
387#ifdef CONFIG_LOCK_STAT
387 printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n"); 388 printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");
389#endif
388} 390}
389 391
390static int save_trace(struct stack_trace *trace) 392static int save_trace(struct stack_trace *trace)
diff --git a/kernel/locking/mcs_spinlock.c b/kernel/locking/mcs_spinlock.c
index be9ee1559fca..9887a905a762 100644
--- a/kernel/locking/mcs_spinlock.c
+++ b/kernel/locking/mcs_spinlock.c
@@ -1,6 +1,4 @@
1
2#include <linux/percpu.h> 1#include <linux/percpu.h>
3#include <linux/mutex.h>
4#include <linux/sched.h> 2#include <linux/sched.h>
5#include "mcs_spinlock.h" 3#include "mcs_spinlock.h"
6 4
@@ -79,7 +77,7 @@ osq_wait_next(struct optimistic_spin_queue *lock,
79 break; 77 break;
80 } 78 }
81 79
82 arch_mutex_cpu_relax(); 80 cpu_relax_lowlatency();
83 } 81 }
84 82
85 return next; 83 return next;
@@ -120,7 +118,7 @@ bool osq_lock(struct optimistic_spin_queue *lock)
120 if (need_resched()) 118 if (need_resched())
121 goto unqueue; 119 goto unqueue;
122 120
123 arch_mutex_cpu_relax(); 121 cpu_relax_lowlatency();
124 } 122 }
125 return true; 123 return true;
126 124
@@ -146,7 +144,7 @@ unqueue:
146 if (smp_load_acquire(&node->locked)) 144 if (smp_load_acquire(&node->locked))
147 return true; 145 return true;
148 146
149 arch_mutex_cpu_relax(); 147 cpu_relax_lowlatency();
150 148
151 /* 149 /*
152 * Or we race against a concurrent unqueue()'s step-B, in which 150 * Or we race against a concurrent unqueue()'s step-B, in which
diff --git a/kernel/locking/mcs_spinlock.h b/kernel/locking/mcs_spinlock.h
index 74356dc0ce29..23e89c5930e9 100644
--- a/kernel/locking/mcs_spinlock.h
+++ b/kernel/locking/mcs_spinlock.h
@@ -27,7 +27,7 @@ struct mcs_spinlock {
27#define arch_mcs_spin_lock_contended(l) \ 27#define arch_mcs_spin_lock_contended(l) \
28do { \ 28do { \
29 while (!(smp_load_acquire(l))) \ 29 while (!(smp_load_acquire(l))) \
30 arch_mutex_cpu_relax(); \ 30 cpu_relax_lowlatency(); \
31} while (0) 31} while (0)
32#endif 32#endif
33 33
@@ -104,7 +104,7 @@ void mcs_spin_unlock(struct mcs_spinlock **lock, struct mcs_spinlock *node)
104 return; 104 return;
105 /* Wait until the next pointer is set */ 105 /* Wait until the next pointer is set */
106 while (!(next = ACCESS_ONCE(node->next))) 106 while (!(next = ACCESS_ONCE(node->next)))
107 arch_mutex_cpu_relax(); 107 cpu_relax_lowlatency();
108 } 108 }
109 109
110 /* Pass lock to next waiter. */ 110 /* Pass lock to next waiter. */
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index acca2c1a3c5e..ae712b25e492 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -46,12 +46,6 @@
46# include <asm/mutex.h> 46# include <asm/mutex.h>
47#endif 47#endif
48 48
49/*
50 * A negative mutex count indicates that waiters are sleeping waiting for the
51 * mutex.
52 */
53#define MUTEX_SHOW_NO_WAITER(mutex) (atomic_read(&(mutex)->count) >= 0)
54
55void 49void
56__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) 50__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
57{ 51{
@@ -152,7 +146,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
152 if (need_resched()) 146 if (need_resched())
153 break; 147 break;
154 148
155 arch_mutex_cpu_relax(); 149 cpu_relax_lowlatency();
156 } 150 }
157 rcu_read_unlock(); 151 rcu_read_unlock();
158 152
@@ -388,12 +382,10 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
388 /* 382 /*
389 * Optimistic spinning. 383 * Optimistic spinning.
390 * 384 *
391 * We try to spin for acquisition when we find that there are no 385 * We try to spin for acquisition when we find that the lock owner
392 * pending waiters and the lock owner is currently running on a 386 * is currently running on a (different) CPU and while we don't
393 * (different) CPU. 387 * need to reschedule. The rationale is that if the lock owner is
394 * 388 * running, it is likely to release the lock soon.
395 * The rationale is that if the lock owner is running, it is likely to
396 * release the lock soon.
397 * 389 *
398 * Since this needs the lock owner, and this mutex implementation 390 * Since this needs the lock owner, and this mutex implementation
399 * doesn't track the owner atomically in the lock field, we need to 391 * doesn't track the owner atomically in the lock field, we need to
@@ -440,7 +432,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
440 if (owner && !mutex_spin_on_owner(lock, owner)) 432 if (owner && !mutex_spin_on_owner(lock, owner))
441 break; 433 break;
442 434
443 if ((atomic_read(&lock->count) == 1) && 435 /* Try to acquire the mutex if it is unlocked. */
436 if (!mutex_is_locked(lock) &&
444 (atomic_cmpxchg(&lock->count, 1, 0) == 1)) { 437 (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
445 lock_acquired(&lock->dep_map, ip); 438 lock_acquired(&lock->dep_map, ip);
446 if (use_ww_ctx) { 439 if (use_ww_ctx) {
@@ -471,7 +464,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
471 * memory barriers as we'll eventually observe the right 464 * memory barriers as we'll eventually observe the right
472 * values at the cost of a few extra spins. 465 * values at the cost of a few extra spins.
473 */ 466 */
474 arch_mutex_cpu_relax(); 467 cpu_relax_lowlatency();
475 } 468 }
476 osq_unlock(&lock->osq); 469 osq_unlock(&lock->osq);
477slowpath: 470slowpath:
@@ -485,8 +478,11 @@ slowpath:
485#endif 478#endif
486 spin_lock_mutex(&lock->wait_lock, flags); 479 spin_lock_mutex(&lock->wait_lock, flags);
487 480
488 /* once more, can we acquire the lock? */ 481 /*
489 if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1)) 482 * Once more, try to acquire the lock. Only try-lock the mutex if
483 * it is unlocked to reduce unnecessary xchg() operations.
484 */
485 if (!mutex_is_locked(lock) && (atomic_xchg(&lock->count, 0) == 1))
490 goto skip_wait; 486 goto skip_wait;
491 487
492 debug_mutex_lock_common(lock, &waiter); 488 debug_mutex_lock_common(lock, &waiter);
@@ -506,9 +502,10 @@ slowpath:
506 * it's unlocked. Later on, if we sleep, this is the 502 * it's unlocked. Later on, if we sleep, this is the
507 * operation that gives us the lock. We xchg it to -1, so 503 * operation that gives us the lock. We xchg it to -1, so
508 * that when we release the lock, we properly wake up the 504 * that when we release the lock, we properly wake up the
509 * other waiters: 505 * other waiters. We only attempt the xchg if the count is
506 * non-negative in order to avoid unnecessary xchg operations:
510 */ 507 */
511 if (MUTEX_SHOW_NO_WAITER(lock) && 508 if (atomic_read(&lock->count) >= 0 &&
512 (atomic_xchg(&lock->count, -1) == 1)) 509 (atomic_xchg(&lock->count, -1) == 1))
513 break; 510 break;
514 511
@@ -823,6 +820,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
823 unsigned long flags; 820 unsigned long flags;
824 int prev; 821 int prev;
825 822
823 /* No need to trylock if the mutex is locked. */
824 if (mutex_is_locked(lock))
825 return 0;
826
826 spin_lock_mutex(&lock->wait_lock, flags); 827 spin_lock_mutex(&lock->wait_lock, flags);
827 828
828 prev = atomic_xchg(&lock->count, -1); 829 prev = atomic_xchg(&lock->count, -1);
diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c
index fb5b8ac411a5..f956ede7f90d 100644
--- a/kernel/locking/qrwlock.c
+++ b/kernel/locking/qrwlock.c
@@ -20,7 +20,6 @@
20#include <linux/cpumask.h> 20#include <linux/cpumask.h>
21#include <linux/percpu.h> 21#include <linux/percpu.h>
22#include <linux/hardirq.h> 22#include <linux/hardirq.h>
23#include <linux/mutex.h>
24#include <asm/qrwlock.h> 23#include <asm/qrwlock.h>
25 24
26/** 25/**
@@ -35,7 +34,7 @@ static __always_inline void
35rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts) 34rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
36{ 35{
37 while ((cnts & _QW_WMASK) == _QW_LOCKED) { 36 while ((cnts & _QW_WMASK) == _QW_LOCKED) {
38 arch_mutex_cpu_relax(); 37 cpu_relax_lowlatency();
39 cnts = smp_load_acquire((u32 *)&lock->cnts); 38 cnts = smp_load_acquire((u32 *)&lock->cnts);
40 } 39 }
41} 40}
@@ -75,7 +74,7 @@ void queue_read_lock_slowpath(struct qrwlock *lock)
75 * to make sure that the write lock isn't taken. 74 * to make sure that the write lock isn't taken.
76 */ 75 */
77 while (atomic_read(&lock->cnts) & _QW_WMASK) 76 while (atomic_read(&lock->cnts) & _QW_WMASK)
78 arch_mutex_cpu_relax(); 77 cpu_relax_lowlatency();
79 78
80 cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS; 79 cnts = atomic_add_return(_QR_BIAS, &lock->cnts) - _QR_BIAS;
81 rspin_until_writer_unlock(lock, cnts); 80 rspin_until_writer_unlock(lock, cnts);
@@ -114,7 +113,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock)
114 cnts | _QW_WAITING) == cnts)) 113 cnts | _QW_WAITING) == cnts))
115 break; 114 break;
116 115
117 arch_mutex_cpu_relax(); 116 cpu_relax_lowlatency();
118 } 117 }
119 118
120 /* When no more readers, set the locked flag */ 119 /* When no more readers, set the locked flag */
@@ -125,7 +124,7 @@ void queue_write_lock_slowpath(struct qrwlock *lock)
125 _QW_LOCKED) == _QW_WAITING)) 124 _QW_LOCKED) == _QW_WAITING))
126 break; 125 break;
127 126
128 arch_mutex_cpu_relax(); 127 cpu_relax_lowlatency();
129 } 128 }
130unlock: 129unlock:
131 arch_spin_unlock(&lock->lock); 130 arch_spin_unlock(&lock->lock);
diff --git a/kernel/locking/rtmutex-debug.c b/kernel/locking/rtmutex-debug.c
index 49b2ed3dced8..62b6cee8ea7f 100644
--- a/kernel/locking/rtmutex-debug.c
+++ b/kernel/locking/rtmutex-debug.c
@@ -66,12 +66,13 @@ void rt_mutex_debug_task_free(struct task_struct *task)
66 * the deadlock. We print when we return. act_waiter can be NULL in 66 * the deadlock. We print when we return. act_waiter can be NULL in
67 * case of a remove waiter operation. 67 * case of a remove waiter operation.
68 */ 68 */
69void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *act_waiter, 69void debug_rt_mutex_deadlock(enum rtmutex_chainwalk chwalk,
70 struct rt_mutex_waiter *act_waiter,
70 struct rt_mutex *lock) 71 struct rt_mutex *lock)
71{ 72{
72 struct task_struct *task; 73 struct task_struct *task;
73 74
74 if (!debug_locks || detect || !act_waiter) 75 if (!debug_locks || chwalk == RT_MUTEX_FULL_CHAINWALK || !act_waiter)
75 return; 76 return;
76 77
77 task = rt_mutex_owner(act_waiter->lock); 78 task = rt_mutex_owner(act_waiter->lock);
diff --git a/kernel/locking/rtmutex-debug.h b/kernel/locking/rtmutex-debug.h
index ab29b6a22669..d0519c3432b6 100644
--- a/kernel/locking/rtmutex-debug.h
+++ b/kernel/locking/rtmutex-debug.h
@@ -20,14 +20,15 @@ extern void debug_rt_mutex_unlock(struct rt_mutex *lock);
20extern void debug_rt_mutex_proxy_lock(struct rt_mutex *lock, 20extern void debug_rt_mutex_proxy_lock(struct rt_mutex *lock,
21 struct task_struct *powner); 21 struct task_struct *powner);
22extern void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock); 22extern void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock);
23extern void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *waiter, 23extern void debug_rt_mutex_deadlock(enum rtmutex_chainwalk chwalk,
24 struct rt_mutex_waiter *waiter,
24 struct rt_mutex *lock); 25 struct rt_mutex *lock);
25extern void debug_rt_mutex_print_deadlock(struct rt_mutex_waiter *waiter); 26extern void debug_rt_mutex_print_deadlock(struct rt_mutex_waiter *waiter);
26# define debug_rt_mutex_reset_waiter(w) \ 27# define debug_rt_mutex_reset_waiter(w) \
27 do { (w)->deadlock_lock = NULL; } while (0) 28 do { (w)->deadlock_lock = NULL; } while (0)
28 29
29static inline int debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *waiter, 30static inline bool debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *waiter,
30 int detect) 31 enum rtmutex_chainwalk walk)
31{ 32{
32 return (waiter != NULL); 33 return (waiter != NULL);
33} 34}
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index fc605941b9b8..a0ea2a141b3b 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -308,6 +308,32 @@ static void rt_mutex_adjust_prio(struct task_struct *task)
308} 308}
309 309
310/* 310/*
311 * Deadlock detection is conditional:
312 *
313 * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
314 * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
315 *
316 * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
317 * conducted independent of the detect argument.
318 *
319 * If the waiter argument is NULL this indicates the deboost path and
320 * deadlock detection is disabled independent of the detect argument
321 * and the config settings.
322 */
323static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
324 enum rtmutex_chainwalk chwalk)
325{
326 /*
327 * This is just a wrapper function for the following call,
328 * because debug_rt_mutex_detect_deadlock() smells like a magic
329 * debug feature and I wanted to keep the cond function in the
330 * main source file along with the comments instead of having
331 * two of the same in the headers.
332 */
333 return debug_rt_mutex_detect_deadlock(waiter, chwalk);
334}
335
336/*
311 * Max number of times we'll walk the boosting chain: 337 * Max number of times we'll walk the boosting chain:
312 */ 338 */
313int max_lock_depth = 1024; 339int max_lock_depth = 1024;
@@ -337,21 +363,65 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
337 * @top_task: the current top waiter 363 * @top_task: the current top waiter
338 * 364 *
339 * Returns 0 or -EDEADLK. 365 * Returns 0 or -EDEADLK.
366 *
367 * Chain walk basics and protection scope
368 *
369 * [R] refcount on task
370 * [P] task->pi_lock held
371 * [L] rtmutex->wait_lock held
372 *
373 * Step Description Protected by
374 * function arguments:
375 * @task [R]
376 * @orig_lock if != NULL @top_task is blocked on it
377 * @next_lock Unprotected. Cannot be
378 * dereferenced. Only used for
379 * comparison.
380 * @orig_waiter if != NULL @top_task is blocked on it
381 * @top_task current, or in case of proxy
382 * locking protected by calling
383 * code
384 * again:
385 * loop_sanity_check();
386 * retry:
387 * [1] lock(task->pi_lock); [R] acquire [P]
388 * [2] waiter = task->pi_blocked_on; [P]
389 * [3] check_exit_conditions_1(); [P]
390 * [4] lock = waiter->lock; [P]
391 * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
392 * unlock(task->pi_lock); release [P]
393 * goto retry;
394 * }
395 * [6] check_exit_conditions_2(); [P] + [L]
396 * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
397 * [8] unlock(task->pi_lock); release [P]
398 * put_task_struct(task); release [R]
399 * [9] check_exit_conditions_3(); [L]
400 * [10] task = owner(lock); [L]
401 * get_task_struct(task); [L] acquire [R]
402 * lock(task->pi_lock); [L] acquire [P]
403 * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
404 * [12] check_exit_conditions_4(); [P] + [L]
405 * [13] unlock(task->pi_lock); release [P]
406 * unlock(lock->wait_lock); release [L]
407 * goto again;
340 */ 408 */
341static int rt_mutex_adjust_prio_chain(struct task_struct *task, 409static int rt_mutex_adjust_prio_chain(struct task_struct *task,
342 int deadlock_detect, 410 enum rtmutex_chainwalk chwalk,
343 struct rt_mutex *orig_lock, 411 struct rt_mutex *orig_lock,
344 struct rt_mutex *next_lock, 412 struct rt_mutex *next_lock,
345 struct rt_mutex_waiter *orig_waiter, 413 struct rt_mutex_waiter *orig_waiter,
346 struct task_struct *top_task) 414 struct task_struct *top_task)
347{ 415{
348 struct rt_mutex *lock;
349 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter; 416 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
350 int detect_deadlock, ret = 0, depth = 0; 417 struct rt_mutex_waiter *prerequeue_top_waiter;
418 int ret = 0, depth = 0;
419 struct rt_mutex *lock;
420 bool detect_deadlock;
351 unsigned long flags; 421 unsigned long flags;
422 bool requeue = true;
352 423
353 detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter, 424 detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
354 deadlock_detect);
355 425
356 /* 426 /*
357 * The (de)boosting is a step by step approach with a lot of 427 * The (de)boosting is a step by step approach with a lot of
@@ -360,6 +430,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
360 * carefully whether things change under us. 430 * carefully whether things change under us.
361 */ 431 */
362 again: 432 again:
433 /*
434 * We limit the lock chain length for each invocation.
435 */
363 if (++depth > max_lock_depth) { 436 if (++depth > max_lock_depth) {
364 static int prev_max; 437 static int prev_max;
365 438
@@ -377,13 +450,28 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
377 450
378 return -EDEADLK; 451 return -EDEADLK;
379 } 452 }
453
454 /*
455 * We are fully preemptible here and only hold the refcount on
456 * @task. So everything can have changed under us since the
457 * caller or our own code below (goto retry/again) dropped all
458 * locks.
459 */
380 retry: 460 retry:
381 /* 461 /*
382 * Task can not go away as we did a get_task() before ! 462 * [1] Task cannot go away as we did a get_task() before !
383 */ 463 */
384 raw_spin_lock_irqsave(&task->pi_lock, flags); 464 raw_spin_lock_irqsave(&task->pi_lock, flags);
385 465
466 /*
467 * [2] Get the waiter on which @task is blocked on.
468 */
386 waiter = task->pi_blocked_on; 469 waiter = task->pi_blocked_on;
470
471 /*
472 * [3] check_exit_conditions_1() protected by task->pi_lock.
473 */
474
387 /* 475 /*
388 * Check whether the end of the boosting chain has been 476 * Check whether the end of the boosting chain has been
389 * reached or the state of the chain has changed while we 477 * reached or the state of the chain has changed while we
@@ -421,20 +509,41 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
421 goto out_unlock_pi; 509 goto out_unlock_pi;
422 /* 510 /*
423 * If deadlock detection is off, we stop here if we 511 * If deadlock detection is off, we stop here if we
424 * are not the top pi waiter of the task. 512 * are not the top pi waiter of the task. If deadlock
513 * detection is enabled we continue, but stop the
514 * requeueing in the chain walk.
425 */ 515 */
426 if (!detect_deadlock && top_waiter != task_top_pi_waiter(task)) 516 if (top_waiter != task_top_pi_waiter(task)) {
427 goto out_unlock_pi; 517 if (!detect_deadlock)
518 goto out_unlock_pi;
519 else
520 requeue = false;
521 }
428 } 522 }
429 523
430 /* 524 /*
431 * When deadlock detection is off then we check, if further 525 * If the waiter priority is the same as the task priority
432 * priority adjustment is necessary. 526 * then there is no further priority adjustment necessary. If
527 * deadlock detection is off, we stop the chain walk. If its
528 * enabled we continue, but stop the requeueing in the chain
529 * walk.
433 */ 530 */
434 if (!detect_deadlock && waiter->prio == task->prio) 531 if (waiter->prio == task->prio) {
435 goto out_unlock_pi; 532 if (!detect_deadlock)
533 goto out_unlock_pi;
534 else
535 requeue = false;
536 }
436 537
538 /*
539 * [4] Get the next lock
540 */
437 lock = waiter->lock; 541 lock = waiter->lock;
542 /*
543 * [5] We need to trylock here as we are holding task->pi_lock,
544 * which is the reverse lock order versus the other rtmutex
545 * operations.
546 */
438 if (!raw_spin_trylock(&lock->wait_lock)) { 547 if (!raw_spin_trylock(&lock->wait_lock)) {
439 raw_spin_unlock_irqrestore(&task->pi_lock, flags); 548 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
440 cpu_relax(); 549 cpu_relax();
@@ -442,79 +551,180 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
442 } 551 }
443 552
444 /* 553 /*
554 * [6] check_exit_conditions_2() protected by task->pi_lock and
555 * lock->wait_lock.
556 *
445 * Deadlock detection. If the lock is the same as the original 557 * Deadlock detection. If the lock is the same as the original
446 * lock which caused us to walk the lock chain or if the 558 * lock which caused us to walk the lock chain or if the
447 * current lock is owned by the task which initiated the chain 559 * current lock is owned by the task which initiated the chain
448 * walk, we detected a deadlock. 560 * walk, we detected a deadlock.
449 */ 561 */
450 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { 562 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
451 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); 563 debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);
452 raw_spin_unlock(&lock->wait_lock); 564 raw_spin_unlock(&lock->wait_lock);
453 ret = -EDEADLK; 565 ret = -EDEADLK;
454 goto out_unlock_pi; 566 goto out_unlock_pi;
455 } 567 }
456 568
457 top_waiter = rt_mutex_top_waiter(lock); 569 /*
570 * If we just follow the lock chain for deadlock detection, no
571 * need to do all the requeue operations. To avoid a truckload
572 * of conditionals around the various places below, just do the
573 * minimum chain walk checks.
574 */
575 if (!requeue) {
576 /*
577 * No requeue[7] here. Just release @task [8]
578 */
579 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
580 put_task_struct(task);
581
582 /*
583 * [9] check_exit_conditions_3 protected by lock->wait_lock.
584 * If there is no owner of the lock, end of chain.
585 */
586 if (!rt_mutex_owner(lock)) {
587 raw_spin_unlock(&lock->wait_lock);
588 return 0;
589 }
590
591 /* [10] Grab the next task, i.e. owner of @lock */
592 task = rt_mutex_owner(lock);
593 get_task_struct(task);
594 raw_spin_lock_irqsave(&task->pi_lock, flags);
595
596 /*
597 * No requeue [11] here. We just do deadlock detection.
598 *
599 * [12] Store whether owner is blocked
600 * itself. Decision is made after dropping the locks
601 */
602 next_lock = task_blocked_on_lock(task);
603 /*
604 * Get the top waiter for the next iteration
605 */
606 top_waiter = rt_mutex_top_waiter(lock);
607
608 /* [13] Drop locks */
609 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
610 raw_spin_unlock(&lock->wait_lock);
611
612 /* If owner is not blocked, end of chain. */
613 if (!next_lock)
614 goto out_put_task;
615 goto again;
616 }
458 617
459 /* Requeue the waiter */ 618 /*
619 * Store the current top waiter before doing the requeue
620 * operation on @lock. We need it for the boost/deboost
621 * decision below.
622 */
623 prerequeue_top_waiter = rt_mutex_top_waiter(lock);
624
625 /* [7] Requeue the waiter in the lock waiter list. */
460 rt_mutex_dequeue(lock, waiter); 626 rt_mutex_dequeue(lock, waiter);
461 waiter->prio = task->prio; 627 waiter->prio = task->prio;
462 rt_mutex_enqueue(lock, waiter); 628 rt_mutex_enqueue(lock, waiter);
463 629
464 /* Release the task */ 630 /* [8] Release the task */
465 raw_spin_unlock_irqrestore(&task->pi_lock, flags); 631 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
632 put_task_struct(task);
633
634 /*
635 * [9] check_exit_conditions_3 protected by lock->wait_lock.
636 *
637 * We must abort the chain walk if there is no lock owner even
638 * in the dead lock detection case, as we have nothing to
639 * follow here. This is the end of the chain we are walking.
640 */
466 if (!rt_mutex_owner(lock)) { 641 if (!rt_mutex_owner(lock)) {
467 /* 642 /*
468 * If the requeue above changed the top waiter, then we need 643 * If the requeue [7] above changed the top waiter,
469 * to wake the new top waiter up to try to get the lock. 644 * then we need to wake the new top waiter up to try
645 * to get the lock.
470 */ 646 */
471 647 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
472 if (top_waiter != rt_mutex_top_waiter(lock))
473 wake_up_process(rt_mutex_top_waiter(lock)->task); 648 wake_up_process(rt_mutex_top_waiter(lock)->task);
474 raw_spin_unlock(&lock->wait_lock); 649 raw_spin_unlock(&lock->wait_lock);
475 goto out_put_task; 650 return 0;
476 } 651 }
477 put_task_struct(task);
478 652
479 /* Grab the next task */ 653 /* [10] Grab the next task, i.e. the owner of @lock */
480 task = rt_mutex_owner(lock); 654 task = rt_mutex_owner(lock);
481 get_task_struct(task); 655 get_task_struct(task);
482 raw_spin_lock_irqsave(&task->pi_lock, flags); 656 raw_spin_lock_irqsave(&task->pi_lock, flags);
483 657
658 /* [11] requeue the pi waiters if necessary */
484 if (waiter == rt_mutex_top_waiter(lock)) { 659 if (waiter == rt_mutex_top_waiter(lock)) {
485 /* Boost the owner */ 660 /*
486 rt_mutex_dequeue_pi(task, top_waiter); 661 * The waiter became the new top (highest priority)
662 * waiter on the lock. Replace the previous top waiter
663 * in the owner tasks pi waiters list with this waiter
664 * and adjust the priority of the owner.
665 */
666 rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
487 rt_mutex_enqueue_pi(task, waiter); 667 rt_mutex_enqueue_pi(task, waiter);
488 __rt_mutex_adjust_prio(task); 668 __rt_mutex_adjust_prio(task);
489 669
490 } else if (top_waiter == waiter) { 670 } else if (prerequeue_top_waiter == waiter) {
491 /* Deboost the owner */ 671 /*
672 * The waiter was the top waiter on the lock, but is
673 * no longer the top prority waiter. Replace waiter in
674 * the owner tasks pi waiters list with the new top
675 * (highest priority) waiter and adjust the priority
676 * of the owner.
677 * The new top waiter is stored in @waiter so that
678 * @waiter == @top_waiter evaluates to true below and
679 * we continue to deboost the rest of the chain.
680 */
492 rt_mutex_dequeue_pi(task, waiter); 681 rt_mutex_dequeue_pi(task, waiter);
493 waiter = rt_mutex_top_waiter(lock); 682 waiter = rt_mutex_top_waiter(lock);
494 rt_mutex_enqueue_pi(task, waiter); 683 rt_mutex_enqueue_pi(task, waiter);
495 __rt_mutex_adjust_prio(task); 684 __rt_mutex_adjust_prio(task);
685 } else {
686 /*
687 * Nothing changed. No need to do any priority
688 * adjustment.
689 */
496 } 690 }
497 691
498 /* 692 /*
693 * [12] check_exit_conditions_4() protected by task->pi_lock
694 * and lock->wait_lock. The actual decisions are made after we
695 * dropped the locks.
696 *
499 * Check whether the task which owns the current lock is pi 697 * Check whether the task which owns the current lock is pi
500 * blocked itself. If yes we store a pointer to the lock for 698 * blocked itself. If yes we store a pointer to the lock for
501 * the lock chain change detection above. After we dropped 699 * the lock chain change detection above. After we dropped
502 * task->pi_lock next_lock cannot be dereferenced anymore. 700 * task->pi_lock next_lock cannot be dereferenced anymore.
503 */ 701 */
504 next_lock = task_blocked_on_lock(task); 702 next_lock = task_blocked_on_lock(task);
703 /*
704 * Store the top waiter of @lock for the end of chain walk
705 * decision below.
706 */
707 top_waiter = rt_mutex_top_waiter(lock);
505 708
709 /* [13] Drop the locks */
506 raw_spin_unlock_irqrestore(&task->pi_lock, flags); 710 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
507
508 top_waiter = rt_mutex_top_waiter(lock);
509 raw_spin_unlock(&lock->wait_lock); 711 raw_spin_unlock(&lock->wait_lock);
510 712
511 /* 713 /*
714 * Make the actual exit decisions [12], based on the stored
715 * values.
716 *
512 * We reached the end of the lock chain. Stop right here. No 717 * We reached the end of the lock chain. Stop right here. No
513 * point to go back just to figure that out. 718 * point to go back just to figure that out.
514 */ 719 */
515 if (!next_lock) 720 if (!next_lock)
516 goto out_put_task; 721 goto out_put_task;
517 722
723 /*
724 * If the current waiter is not the top waiter on the lock,
725 * then we can stop the chain walk here if we are not in full
726 * deadlock detection mode.
727 */
518 if (!detect_deadlock && waiter != top_waiter) 728 if (!detect_deadlock && waiter != top_waiter)
519 goto out_put_task; 729 goto out_put_task;
520 730
@@ -533,76 +743,119 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
533 * 743 *
534 * Must be called with lock->wait_lock held. 744 * Must be called with lock->wait_lock held.
535 * 745 *
536 * @lock: the lock to be acquired. 746 * @lock: The lock to be acquired.
537 * @task: the task which wants to acquire the lock 747 * @task: The task which wants to acquire the lock
538 * @waiter: the waiter that is queued to the lock's wait list. (could be NULL) 748 * @waiter: The waiter that is queued to the lock's wait list if the
749 * callsite called task_blocked_on_lock(), otherwise NULL
539 */ 750 */
540static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, 751static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
541 struct rt_mutex_waiter *waiter) 752 struct rt_mutex_waiter *waiter)
542{ 753{
754 unsigned long flags;
755
543 /* 756 /*
544 * We have to be careful here if the atomic speedups are 757 * Before testing whether we can acquire @lock, we set the
545 * enabled, such that, when 758 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
546 * - no other waiter is on the lock 759 * other tasks which try to modify @lock into the slow path
547 * - the lock has been released since we did the cmpxchg 760 * and they serialize on @lock->wait_lock.
548 * the lock can be released or taken while we are doing the 761 *
549 * checks and marking the lock with RT_MUTEX_HAS_WAITERS. 762 * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
763 * as explained at the top of this file if and only if:
550 * 764 *
551 * The atomic acquire/release aware variant of 765 * - There is a lock owner. The caller must fixup the
552 * mark_rt_mutex_waiters uses a cmpxchg loop. After setting 766 * transient state if it does a trylock or leaves the lock
553 * the WAITERS bit, the atomic release / acquire can not 767 * function due to a signal or timeout.
554 * happen anymore and lock->wait_lock protects us from the
555 * non-atomic case.
556 * 768 *
557 * Note, that this might set lock->owner = 769 * - @task acquires the lock and there are no other
558 * RT_MUTEX_HAS_WAITERS in the case the lock is not contended 770 * waiters. This is undone in rt_mutex_set_owner(@task) at
559 * any more. This is fixed up when we take the ownership. 771 * the end of this function.
560 * This is the transitional state explained at the top of this file.
561 */ 772 */
562 mark_rt_mutex_waiters(lock); 773 mark_rt_mutex_waiters(lock);
563 774
775 /*
776 * If @lock has an owner, give up.
777 */
564 if (rt_mutex_owner(lock)) 778 if (rt_mutex_owner(lock))
565 return 0; 779 return 0;
566 780
567 /* 781 /*
568 * It will get the lock because of one of these conditions: 782 * If @waiter != NULL, @task has already enqueued the waiter
569 * 1) there is no waiter 783 * into @lock waiter list. If @waiter == NULL then this is a
570 * 2) higher priority than waiters 784 * trylock attempt.
571 * 3) it is top waiter
572 */ 785 */
573 if (rt_mutex_has_waiters(lock)) { 786 if (waiter) {
574 if (task->prio >= rt_mutex_top_waiter(lock)->prio) { 787 /*
575 if (!waiter || waiter != rt_mutex_top_waiter(lock)) 788 * If waiter is not the highest priority waiter of
576 return 0; 789 * @lock, give up.
577 } 790 */
578 } 791 if (waiter != rt_mutex_top_waiter(lock))
579 792 return 0;
580 if (waiter || rt_mutex_has_waiters(lock)) {
581 unsigned long flags;
582 struct rt_mutex_waiter *top;
583
584 raw_spin_lock_irqsave(&task->pi_lock, flags);
585 793
586 /* remove the queued waiter. */ 794 /*
587 if (waiter) { 795 * We can acquire the lock. Remove the waiter from the
588 rt_mutex_dequeue(lock, waiter); 796 * lock waiters list.
589 task->pi_blocked_on = NULL; 797 */
590 } 798 rt_mutex_dequeue(lock, waiter);
591 799
800 } else {
592 /* 801 /*
593 * We have to enqueue the top waiter(if it exists) into 802 * If the lock has waiters already we check whether @task is
594 * task->pi_waiters list. 803 * eligible to take over the lock.
804 *
805 * If there are no other waiters, @task can acquire
806 * the lock. @task->pi_blocked_on is NULL, so it does
807 * not need to be dequeued.
595 */ 808 */
596 if (rt_mutex_has_waiters(lock)) { 809 if (rt_mutex_has_waiters(lock)) {
597 top = rt_mutex_top_waiter(lock); 810 /*
598 rt_mutex_enqueue_pi(task, top); 811 * If @task->prio is greater than or equal to
812 * the top waiter priority (kernel view),
813 * @task lost.
814 */
815 if (task->prio >= rt_mutex_top_waiter(lock)->prio)
816 return 0;
817
818 /*
819 * The current top waiter stays enqueued. We
820 * don't have to change anything in the lock
821 * waiters order.
822 */
823 } else {
824 /*
825 * No waiters. Take the lock without the
826 * pi_lock dance.@task->pi_blocked_on is NULL
827 * and we have no waiters to enqueue in @task
828 * pi waiters list.
829 */
830 goto takeit;
599 } 831 }
600 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
601 } 832 }
602 833
834 /*
835 * Clear @task->pi_blocked_on. Requires protection by
836 * @task->pi_lock. Redundant operation for the @waiter == NULL
837 * case, but conditionals are more expensive than a redundant
838 * store.
839 */
840 raw_spin_lock_irqsave(&task->pi_lock, flags);
841 task->pi_blocked_on = NULL;
842 /*
843 * Finish the lock acquisition. @task is the new owner. If
844 * other waiters exist we have to insert the highest priority
845 * waiter into @task->pi_waiters list.
846 */
847 if (rt_mutex_has_waiters(lock))
848 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
849 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
850
851takeit:
603 /* We got the lock. */ 852 /* We got the lock. */
604 debug_rt_mutex_lock(lock); 853 debug_rt_mutex_lock(lock);
605 854
855 /*
856 * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
857 * are still waiters or clears it.
858 */
606 rt_mutex_set_owner(lock, task); 859 rt_mutex_set_owner(lock, task);
607 860
608 rt_mutex_deadlock_account_lock(lock, task); 861 rt_mutex_deadlock_account_lock(lock, task);
@@ -620,7 +873,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
620static int task_blocks_on_rt_mutex(struct rt_mutex *lock, 873static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
621 struct rt_mutex_waiter *waiter, 874 struct rt_mutex_waiter *waiter,
622 struct task_struct *task, 875 struct task_struct *task,
623 int detect_deadlock) 876 enum rtmutex_chainwalk chwalk)
624{ 877{
625 struct task_struct *owner = rt_mutex_owner(lock); 878 struct task_struct *owner = rt_mutex_owner(lock);
626 struct rt_mutex_waiter *top_waiter = waiter; 879 struct rt_mutex_waiter *top_waiter = waiter;
@@ -666,7 +919,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
666 __rt_mutex_adjust_prio(owner); 919 __rt_mutex_adjust_prio(owner);
667 if (owner->pi_blocked_on) 920 if (owner->pi_blocked_on)
668 chain_walk = 1; 921 chain_walk = 1;
669 } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) { 922 } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
670 chain_walk = 1; 923 chain_walk = 1;
671 } 924 }
672 925
@@ -691,7 +944,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
691 944
692 raw_spin_unlock(&lock->wait_lock); 945 raw_spin_unlock(&lock->wait_lock);
693 946
694 res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, 947 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
695 next_lock, waiter, task); 948 next_lock, waiter, task);
696 949
697 raw_spin_lock(&lock->wait_lock); 950 raw_spin_lock(&lock->wait_lock);
@@ -753,9 +1006,9 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
753static void remove_waiter(struct rt_mutex *lock, 1006static void remove_waiter(struct rt_mutex *lock,
754 struct rt_mutex_waiter *waiter) 1007 struct rt_mutex_waiter *waiter)
755{ 1008{
756 int first = (waiter == rt_mutex_top_waiter(lock)); 1009 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
757 struct task_struct *owner = rt_mutex_owner(lock); 1010 struct task_struct *owner = rt_mutex_owner(lock);
758 struct rt_mutex *next_lock = NULL; 1011 struct rt_mutex *next_lock;
759 unsigned long flags; 1012 unsigned long flags;
760 1013
761 raw_spin_lock_irqsave(&current->pi_lock, flags); 1014 raw_spin_lock_irqsave(&current->pi_lock, flags);
@@ -763,29 +1016,31 @@ static void remove_waiter(struct rt_mutex *lock,
763 current->pi_blocked_on = NULL; 1016 current->pi_blocked_on = NULL;
764 raw_spin_unlock_irqrestore(&current->pi_lock, flags); 1017 raw_spin_unlock_irqrestore(&current->pi_lock, flags);
765 1018
766 if (!owner) 1019 /*
1020 * Only update priority if the waiter was the highest priority
1021 * waiter of the lock and there is an owner to update.
1022 */
1023 if (!owner || !is_top_waiter)
767 return; 1024 return;
768 1025
769 if (first) { 1026 raw_spin_lock_irqsave(&owner->pi_lock, flags);
770
771 raw_spin_lock_irqsave(&owner->pi_lock, flags);
772 1027
773 rt_mutex_dequeue_pi(owner, waiter); 1028 rt_mutex_dequeue_pi(owner, waiter);
774 1029
775 if (rt_mutex_has_waiters(lock)) { 1030 if (rt_mutex_has_waiters(lock))
776 struct rt_mutex_waiter *next; 1031 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
777 1032
778 next = rt_mutex_top_waiter(lock); 1033 __rt_mutex_adjust_prio(owner);
779 rt_mutex_enqueue_pi(owner, next);
780 }
781 __rt_mutex_adjust_prio(owner);
782 1034
783 /* Store the lock on which owner is blocked or NULL */ 1035 /* Store the lock on which owner is blocked or NULL */
784 next_lock = task_blocked_on_lock(owner); 1036 next_lock = task_blocked_on_lock(owner);
785 1037
786 raw_spin_unlock_irqrestore(&owner->pi_lock, flags); 1038 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
787 }
788 1039
1040 /*
1041 * Don't walk the chain, if the owner task is not blocked
1042 * itself.
1043 */
789 if (!next_lock) 1044 if (!next_lock)
790 return; 1045 return;
791 1046
@@ -794,7 +1049,8 @@ static void remove_waiter(struct rt_mutex *lock,
794 1049
795 raw_spin_unlock(&lock->wait_lock); 1050 raw_spin_unlock(&lock->wait_lock);
796 1051
797 rt_mutex_adjust_prio_chain(owner, 0, lock, next_lock, NULL, current); 1052 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
1053 next_lock, NULL, current);
798 1054
799 raw_spin_lock(&lock->wait_lock); 1055 raw_spin_lock(&lock->wait_lock);
800} 1056}
@@ -824,7 +1080,8 @@ void rt_mutex_adjust_pi(struct task_struct *task)
824 /* gets dropped in rt_mutex_adjust_prio_chain()! */ 1080 /* gets dropped in rt_mutex_adjust_prio_chain()! */
825 get_task_struct(task); 1081 get_task_struct(task);
826 1082
827 rt_mutex_adjust_prio_chain(task, 0, NULL, next_lock, NULL, task); 1083 rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
1084 next_lock, NULL, task);
828} 1085}
829 1086
830/** 1087/**
@@ -902,7 +1159,7 @@ static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
902static int __sched 1159static int __sched
903rt_mutex_slowlock(struct rt_mutex *lock, int state, 1160rt_mutex_slowlock(struct rt_mutex *lock, int state,
904 struct hrtimer_sleeper *timeout, 1161 struct hrtimer_sleeper *timeout,
905 int detect_deadlock) 1162 enum rtmutex_chainwalk chwalk)
906{ 1163{
907 struct rt_mutex_waiter waiter; 1164 struct rt_mutex_waiter waiter;
908 int ret = 0; 1165 int ret = 0;
@@ -928,7 +1185,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
928 timeout->task = NULL; 1185 timeout->task = NULL;
929 } 1186 }
930 1187
931 ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock); 1188 ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
932 1189
933 if (likely(!ret)) 1190 if (likely(!ret))
934 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); 1191 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
@@ -937,7 +1194,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
937 1194
938 if (unlikely(ret)) { 1195 if (unlikely(ret)) {
939 remove_waiter(lock, &waiter); 1196 remove_waiter(lock, &waiter);
940 rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter); 1197 rt_mutex_handle_deadlock(ret, chwalk, &waiter);
941 } 1198 }
942 1199
943 /* 1200 /*
@@ -960,22 +1217,31 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
960/* 1217/*
961 * Slow path try-lock function: 1218 * Slow path try-lock function:
962 */ 1219 */
963static inline int 1220static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
964rt_mutex_slowtrylock(struct rt_mutex *lock)
965{ 1221{
966 int ret = 0; 1222 int ret;
1223
1224 /*
1225 * If the lock already has an owner we fail to get the lock.
1226 * This can be done without taking the @lock->wait_lock as
1227 * it is only being read, and this is a trylock anyway.
1228 */
1229 if (rt_mutex_owner(lock))
1230 return 0;
967 1231
1232 /*
1233 * The mutex has currently no owner. Lock the wait lock and
1234 * try to acquire the lock.
1235 */
968 raw_spin_lock(&lock->wait_lock); 1236 raw_spin_lock(&lock->wait_lock);
969 1237
970 if (likely(rt_mutex_owner(lock) != current)) { 1238 ret = try_to_take_rt_mutex(lock, current, NULL);
971 1239
972 ret = try_to_take_rt_mutex(lock, current, NULL); 1240 /*
973 /* 1241 * try_to_take_rt_mutex() sets the lock waiters bit
974 * try_to_take_rt_mutex() sets the lock waiters 1242 * unconditionally. Clean this up.
975 * bit unconditionally. Clean this up. 1243 */
976 */ 1244 fixup_rt_mutex_waiters(lock);
977 fixup_rt_mutex_waiters(lock);
978 }
979 1245
980 raw_spin_unlock(&lock->wait_lock); 1246 raw_spin_unlock(&lock->wait_lock);
981 1247
@@ -1053,30 +1319,31 @@ rt_mutex_slowunlock(struct rt_mutex *lock)
1053 */ 1319 */
1054static inline int 1320static inline int
1055rt_mutex_fastlock(struct rt_mutex *lock, int state, 1321rt_mutex_fastlock(struct rt_mutex *lock, int state,
1056 int detect_deadlock,
1057 int (*slowfn)(struct rt_mutex *lock, int state, 1322 int (*slowfn)(struct rt_mutex *lock, int state,
1058 struct hrtimer_sleeper *timeout, 1323 struct hrtimer_sleeper *timeout,
1059 int detect_deadlock)) 1324 enum rtmutex_chainwalk chwalk))
1060{ 1325{
1061 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { 1326 if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
1062 rt_mutex_deadlock_account_lock(lock, current); 1327 rt_mutex_deadlock_account_lock(lock, current);
1063 return 0; 1328 return 0;
1064 } else 1329 } else
1065 return slowfn(lock, state, NULL, detect_deadlock); 1330 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
1066} 1331}
1067 1332
1068static inline int 1333static inline int
1069rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, 1334rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
1070 struct hrtimer_sleeper *timeout, int detect_deadlock, 1335 struct hrtimer_sleeper *timeout,
1336 enum rtmutex_chainwalk chwalk,
1071 int (*slowfn)(struct rt_mutex *lock, int state, 1337 int (*slowfn)(struct rt_mutex *lock, int state,
1072 struct hrtimer_sleeper *timeout, 1338 struct hrtimer_sleeper *timeout,
1073 int detect_deadlock)) 1339 enum rtmutex_chainwalk chwalk))
1074{ 1340{
1075 if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) { 1341 if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
1342 likely(rt_mutex_cmpxchg(lock, NULL, current))) {
1076 rt_mutex_deadlock_account_lock(lock, current); 1343 rt_mutex_deadlock_account_lock(lock, current);
1077 return 0; 1344 return 0;
1078 } else 1345 } else
1079 return slowfn(lock, state, timeout, detect_deadlock); 1346 return slowfn(lock, state, timeout, chwalk);
1080} 1347}
1081 1348
1082static inline int 1349static inline int
@@ -1109,54 +1376,61 @@ void __sched rt_mutex_lock(struct rt_mutex *lock)
1109{ 1376{
1110 might_sleep(); 1377 might_sleep();
1111 1378
1112 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock); 1379 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
1113} 1380}
1114EXPORT_SYMBOL_GPL(rt_mutex_lock); 1381EXPORT_SYMBOL_GPL(rt_mutex_lock);
1115 1382
1116/** 1383/**
1117 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible 1384 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
1118 * 1385 *
1119 * @lock: the rt_mutex to be locked 1386 * @lock: the rt_mutex to be locked
1120 * @detect_deadlock: deadlock detection on/off
1121 * 1387 *
1122 * Returns: 1388 * Returns:
1123 * 0 on success 1389 * 0 on success
1124 * -EINTR when interrupted by a signal 1390 * -EINTR when interrupted by a signal
1125 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
1126 */ 1391 */
1127int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock, 1392int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
1128 int detect_deadlock)
1129{ 1393{
1130 might_sleep(); 1394 might_sleep();
1131 1395
1132 return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, 1396 return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
1133 detect_deadlock, rt_mutex_slowlock);
1134} 1397}
1135EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); 1398EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
1136 1399
1400/*
1401 * Futex variant with full deadlock detection.
1402 */
1403int rt_mutex_timed_futex_lock(struct rt_mutex *lock,
1404 struct hrtimer_sleeper *timeout)
1405{
1406 might_sleep();
1407
1408 return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
1409 RT_MUTEX_FULL_CHAINWALK,
1410 rt_mutex_slowlock);
1411}
1412
1137/** 1413/**
1138 * rt_mutex_timed_lock - lock a rt_mutex interruptible 1414 * rt_mutex_timed_lock - lock a rt_mutex interruptible
1139 * the timeout structure is provided 1415 * the timeout structure is provided
1140 * by the caller 1416 * by the caller
1141 * 1417 *
1142 * @lock: the rt_mutex to be locked 1418 * @lock: the rt_mutex to be locked
1143 * @timeout: timeout structure or NULL (no timeout) 1419 * @timeout: timeout structure or NULL (no timeout)
1144 * @detect_deadlock: deadlock detection on/off
1145 * 1420 *
1146 * Returns: 1421 * Returns:
1147 * 0 on success 1422 * 0 on success
1148 * -EINTR when interrupted by a signal 1423 * -EINTR when interrupted by a signal
1149 * -ETIMEDOUT when the timeout expired 1424 * -ETIMEDOUT when the timeout expired
1150 * -EDEADLK when the lock would deadlock (when deadlock detection is on)
1151 */ 1425 */
1152int 1426int
1153rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout, 1427rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
1154 int detect_deadlock)
1155{ 1428{
1156 might_sleep(); 1429 might_sleep();
1157 1430
1158 return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 1431 return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
1159 detect_deadlock, rt_mutex_slowlock); 1432 RT_MUTEX_MIN_CHAINWALK,
1433 rt_mutex_slowlock);
1160} 1434}
1161EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); 1435EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
1162 1436
@@ -1262,7 +1536,6 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock,
1262 * @lock: the rt_mutex to take 1536 * @lock: the rt_mutex to take
1263 * @waiter: the pre-initialized rt_mutex_waiter 1537 * @waiter: the pre-initialized rt_mutex_waiter
1264 * @task: the task to prepare 1538 * @task: the task to prepare
1265 * @detect_deadlock: perform deadlock detection (1) or not (0)
1266 * 1539 *
1267 * Returns: 1540 * Returns:
1268 * 0 - task blocked on lock 1541 * 0 - task blocked on lock
@@ -1273,7 +1546,7 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock,
1273 */ 1546 */
1274int rt_mutex_start_proxy_lock(struct rt_mutex *lock, 1547int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1275 struct rt_mutex_waiter *waiter, 1548 struct rt_mutex_waiter *waiter,
1276 struct task_struct *task, int detect_deadlock) 1549 struct task_struct *task)
1277{ 1550{
1278 int ret; 1551 int ret;
1279 1552
@@ -1285,7 +1558,8 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1285 } 1558 }
1286 1559
1287 /* We enforce deadlock detection for futexes */ 1560 /* We enforce deadlock detection for futexes */
1288 ret = task_blocks_on_rt_mutex(lock, waiter, task, 1); 1561 ret = task_blocks_on_rt_mutex(lock, waiter, task,
1562 RT_MUTEX_FULL_CHAINWALK);
1289 1563
1290 if (ret && !rt_mutex_owner(lock)) { 1564 if (ret && !rt_mutex_owner(lock)) {
1291 /* 1565 /*
@@ -1331,22 +1605,20 @@ struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
1331 * rt_mutex_finish_proxy_lock() - Complete lock acquisition 1605 * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1332 * @lock: the rt_mutex we were woken on 1606 * @lock: the rt_mutex we were woken on
1333 * @to: the timeout, null if none. hrtimer should already have 1607 * @to: the timeout, null if none. hrtimer should already have
1334 * been started. 1608 * been started.
1335 * @waiter: the pre-initialized rt_mutex_waiter 1609 * @waiter: the pre-initialized rt_mutex_waiter
1336 * @detect_deadlock: perform deadlock detection (1) or not (0)
1337 * 1610 *
1338 * Complete the lock acquisition started our behalf by another thread. 1611 * Complete the lock acquisition started our behalf by another thread.
1339 * 1612 *
1340 * Returns: 1613 * Returns:
1341 * 0 - success 1614 * 0 - success
1342 * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK 1615 * <0 - error, one of -EINTR, -ETIMEDOUT
1343 * 1616 *
1344 * Special API call for PI-futex requeue support 1617 * Special API call for PI-futex requeue support
1345 */ 1618 */
1346int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, 1619int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1347 struct hrtimer_sleeper *to, 1620 struct hrtimer_sleeper *to,
1348 struct rt_mutex_waiter *waiter, 1621 struct rt_mutex_waiter *waiter)
1349 int detect_deadlock)
1350{ 1622{
1351 int ret; 1623 int ret;
1352 1624
diff --git a/kernel/locking/rtmutex.h b/kernel/locking/rtmutex.h
index f6a1f3c133b1..c4060584c407 100644
--- a/kernel/locking/rtmutex.h
+++ b/kernel/locking/rtmutex.h
@@ -22,10 +22,15 @@
22#define debug_rt_mutex_init(m, n) do { } while (0) 22#define debug_rt_mutex_init(m, n) do { } while (0)
23#define debug_rt_mutex_deadlock(d, a ,l) do { } while (0) 23#define debug_rt_mutex_deadlock(d, a ,l) do { } while (0)
24#define debug_rt_mutex_print_deadlock(w) do { } while (0) 24#define debug_rt_mutex_print_deadlock(w) do { } while (0)
25#define debug_rt_mutex_detect_deadlock(w,d) (d)
26#define debug_rt_mutex_reset_waiter(w) do { } while (0) 25#define debug_rt_mutex_reset_waiter(w) do { } while (0)
27 26
28static inline void rt_mutex_print_deadlock(struct rt_mutex_waiter *w) 27static inline void rt_mutex_print_deadlock(struct rt_mutex_waiter *w)
29{ 28{
30 WARN(1, "rtmutex deadlock detected\n"); 29 WARN(1, "rtmutex deadlock detected\n");
31} 30}
31
32static inline bool debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *w,
33 enum rtmutex_chainwalk walk)
34{
35 return walk == RT_MUTEX_FULL_CHAINWALK;
36}
diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index 7431a9c86f35..855212501407 100644
--- a/kernel/locking/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
@@ -102,6 +102,21 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex *lock)
102} 102}
103 103
104/* 104/*
105 * Constants for rt mutex functions which have a selectable deadlock
106 * detection.
107 *
108 * RT_MUTEX_MIN_CHAINWALK: Stops the lock chain walk when there are
109 * no further PI adjustments to be made.
110 *
111 * RT_MUTEX_FULL_CHAINWALK: Invoke deadlock detection with a full
112 * walk of the lock chain.
113 */
114enum rtmutex_chainwalk {
115 RT_MUTEX_MIN_CHAINWALK,
116 RT_MUTEX_FULL_CHAINWALK,
117};
118
119/*
105 * PI-futex support (proxy locking functions, etc.): 120 * PI-futex support (proxy locking functions, etc.):
106 */ 121 */
107extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); 122extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock);
@@ -111,12 +126,11 @@ extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
111 struct task_struct *proxy_owner); 126 struct task_struct *proxy_owner);
112extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock, 127extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
113 struct rt_mutex_waiter *waiter, 128 struct rt_mutex_waiter *waiter,
114 struct task_struct *task, 129 struct task_struct *task);
115 int detect_deadlock);
116extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, 130extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
117 struct hrtimer_sleeper *to, 131 struct hrtimer_sleeper *to,
118 struct rt_mutex_waiter *waiter, 132 struct rt_mutex_waiter *waiter);
119 int detect_deadlock); 133extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
120 134
121#ifdef CONFIG_DEBUG_RT_MUTEXES 135#ifdef CONFIG_DEBUG_RT_MUTEXES
122# include "rtmutex-debug.h" 136# include "rtmutex-debug.h"
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index a2391ac135c8..d6203faf2eb1 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -329,7 +329,7 @@ bool rwsem_spin_on_owner(struct rw_semaphore *sem, struct task_struct *owner)
329 if (need_resched()) 329 if (need_resched())
330 break; 330 break;
331 331
332 arch_mutex_cpu_relax(); 332 cpu_relax_lowlatency();
333 } 333 }
334 rcu_read_unlock(); 334 rcu_read_unlock();
335 335
@@ -381,7 +381,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
381 * memory barriers as we'll eventually observe the right 381 * memory barriers as we'll eventually observe the right
382 * values at the cost of a few extra spins. 382 * values at the cost of a few extra spins.
383 */ 383 */
384 arch_mutex_cpu_relax(); 384 cpu_relax_lowlatency();
385 } 385 }
386 osq_unlock(&sem->osq); 386 osq_unlock(&sem->osq);
387done: 387done:
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 12132e433e30..901096d31c66 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -835,7 +835,7 @@ config DEBUG_RT_MUTEXES
835 835
836config RT_MUTEX_TESTER 836config RT_MUTEX_TESTER
837 bool "Built-in scriptable tester for rt-mutexes" 837 bool "Built-in scriptable tester for rt-mutexes"
838 depends on DEBUG_KERNEL && RT_MUTEXES 838 depends on DEBUG_KERNEL && RT_MUTEXES && BROKEN
839 help 839 help
840 This option enables a rt-mutex tester. 840 This option enables a rt-mutex tester.
841 841
diff --git a/lib/lockref.c b/lib/lockref.c
index f07a40d33871..d2233de9a86e 100644
--- a/lib/lockref.c
+++ b/lib/lockref.c
@@ -1,6 +1,5 @@
1#include <linux/export.h> 1#include <linux/export.h>
2#include <linux/lockref.h> 2#include <linux/lockref.h>
3#include <linux/mutex.h>
4 3
5#if USE_CMPXCHG_LOCKREF 4#if USE_CMPXCHG_LOCKREF
6 5
@@ -29,7 +28,7 @@
29 if (likely(old.lock_count == prev.lock_count)) { \ 28 if (likely(old.lock_count == prev.lock_count)) { \
30 SUCCESS; \ 29 SUCCESS; \
31 } \ 30 } \
32 arch_mutex_cpu_relax(); \ 31 cpu_relax_lowlatency(); \
33 } \ 32 } \
34} while (0) 33} while (0)
35 34
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-25 23:10:58 -0500