aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm/spinlock.h
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@goop.org>2013-08-09 10:21:49 -0400
committerH. Peter Anvin <hpa@linux.intel.com>2013-08-09 10:53:05 -0400
commit545ac13892ab391049a92108cf59a0d05de7e28c (patch)
treee993b90bcbedd44b77c895cf7fcee89ee5fe9d51 /arch/x86/include/asm/spinlock.h
parentc095ba7224d8edc71dcef0d655911399a8bd4a3f (diff)
x86, spinlock: Replace pv spinlocks with pv ticketlocks
Rather than outright replacing the entire spinlock implementation in order to paravirtualize it, keep the ticket lock implementation but add a couple of pvops hooks on the slow patch (long spin on lock, unlocking a contended lock). Ticket locks have a number of nice properties, but they also have some surprising behaviours in virtual environments. They enforce a strict FIFO ordering on cpus trying to take a lock; however, if the hypervisor scheduler does not schedule the cpus in the correct order, the system can waste a huge amount of time spinning until the next cpu can take the lock. (See Thomas Friebel's talk "Prevent Guests from Spinning Around" http://www.xen.org/files/xensummitboston08/LHP.pdf for more details.) To address this, we add two hooks: - __ticket_spin_lock which is called after the cpu has been spinning on the lock for a significant number of iterations but has failed to take the lock (presumably because the cpu holding the lock has been descheduled). The lock_spinning pvop is expected to block the cpu until it has been kicked by the current lock holder. - __ticket_spin_unlock, which on releasing a contended lock (there are more cpus with tail tickets), it looks to see if the next cpu is blocked and wakes it if so. When compiled with CONFIG_PARAVIRT_SPINLOCKS disabled, a set of stub functions causes all the extra code to go away. Results: ======= setup: 32 core machine with 32 vcpu KVM guest (HT off) with 8GB RAM base = 3.11-rc patched = base + pvspinlock V12 +-----------------+----------------+--------+ dbench (Throughput in MB/sec. Higher is better) +-----------------+----------------+--------+ | base (stdev %)|patched(stdev%) | %gain | +-----------------+----------------+--------+ | 15035.3 (0.3) |15150.0 (0.6) | 0.8 | | 1470.0 (2.2) | 1713.7 (1.9) | 16.6 | | 848.6 (4.3) | 967.8 (4.3) | 14.0 | | 652.9 (3.5) | 685.3 (3.7) | 5.0 | +-----------------+----------------+--------+ pvspinlock shows benefits for overcommit ratio > 1 for PLE enabled cases, and undercommits results are flat Signed-off-by: Jeremy Fitzhardinge <jeremy@goop.org> Link: http://lkml.kernel.org/r/1376058122-8248-2-git-send-email-raghavendra.kt@linux.vnet.ibm.com Reviewed-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> Tested-by: Attilio Rao <attilio.rao@citrix.com> [ Raghavendra: Changed SPIN_THRESHOLD, fixed redefinition of arch_spinlock_t] Signed-off-by: Raghavendra K T <raghavendra.kt@linux.vnet.ibm.com> Acked-by: Ingo Molnar <mingo@kernel.org> Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Diffstat (limited to 'arch/x86/include/asm/spinlock.h')
-rw-r--r--arch/x86/include/asm/spinlock.h53
1 files changed, 43 insertions, 10 deletions
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 33692eaabab5..4d542444bea3 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -34,6 +34,35 @@
34# define UNLOCK_LOCK_PREFIX 34# define UNLOCK_LOCK_PREFIX
35#endif 35#endif
36 36
37/* How long a lock should spin before we consider blocking */
38#define SPIN_THRESHOLD (1 << 15)
39
40#ifndef CONFIG_PARAVIRT_SPINLOCKS
41
42static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
43 __ticket_t ticket)
44{
45}
46
47static __always_inline void ____ticket_unlock_kick(struct arch_spinlock *lock,
48 __ticket_t ticket)
49{
50}
51
52#endif /* CONFIG_PARAVIRT_SPINLOCKS */
53
54
55/*
56 * If a spinlock has someone waiting on it, then kick the appropriate
57 * waiting cpu.
58 */
59static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
60 __ticket_t next)
61{
62 if (unlikely(lock->tickets.tail != next))
63 ____ticket_unlock_kick(lock, next);
64}
65
37/* 66/*
38 * Ticket locks are conceptually two parts, one indicating the current head of 67 * Ticket locks are conceptually two parts, one indicating the current head of
39 * the queue, and the other indicating the current tail. The lock is acquired 68 * the queue, and the other indicating the current tail. The lock is acquired
@@ -47,19 +76,24 @@
47 * in the high part, because a wide xadd increment of the low part would carry 76 * in the high part, because a wide xadd increment of the low part would carry
48 * up and contaminate the high part. 77 * up and contaminate the high part.
49 */ 78 */
50static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock) 79static __always_inline void __ticket_spin_lock(struct arch_spinlock *lock)
51{ 80{
52 register struct __raw_tickets inc = { .tail = 1 }; 81 register struct __raw_tickets inc = { .tail = 1 };
53 82
54 inc = xadd(&lock->tickets, inc); 83 inc = xadd(&lock->tickets, inc);
55 84
56 for (;;) { 85 for (;;) {
57 if (inc.head == inc.tail) 86 unsigned count = SPIN_THRESHOLD;
58 break; 87
59 cpu_relax(); 88 do {
60 inc.head = ACCESS_ONCE(lock->tickets.head); 89 if (inc.head == inc.tail)
90 goto out;
91 cpu_relax();
92 inc.head = ACCESS_ONCE(lock->tickets.head);
93 } while (--count);
94 __ticket_lock_spinning(lock, inc.tail);
61 } 95 }
62 barrier(); /* make sure nothing creeps before the lock is taken */ 96out: barrier(); /* make sure nothing creeps before the lock is taken */
63} 97}
64 98
65static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock) 99static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
@@ -78,7 +112,10 @@ static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
78 112
79static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) 113static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
80{ 114{
115 __ticket_t next = lock->tickets.head + 1;
116
81 __add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX); 117 __add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
118 __ticket_unlock_kick(lock, next);
82} 119}
83 120
84static inline int __ticket_spin_is_locked(arch_spinlock_t *lock) 121static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)
@@ -95,8 +132,6 @@ static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)
95 return (__ticket_t)(tmp.tail - tmp.head) > 1; 132 return (__ticket_t)(tmp.tail - tmp.head) > 1;
96} 133}
97 134
98#ifndef CONFIG_PARAVIRT_SPINLOCKS
99
100static inline int arch_spin_is_locked(arch_spinlock_t *lock) 135static inline int arch_spin_is_locked(arch_spinlock_t *lock)
101{ 136{
102 return __ticket_spin_is_locked(lock); 137 return __ticket_spin_is_locked(lock);
@@ -129,8 +164,6 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock,
129 arch_spin_lock(lock); 164 arch_spin_lock(lock);
130} 165}
131 166
132#endif /* CONFIG_PARAVIRT_SPINLOCKS */
133
134static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) 167static inline void arch_spin_unlock_wait(arch_spinlock_t *lock)
135{ 168{
136 while (arch_spin_is_locked(lock)) 169 while (arch_spin_is_locked(lock))