diff options
Diffstat (limited to 'arch/x86/xen')
-rw-r--r-- | arch/x86/xen/spinlock.c | 30 |
1 files changed, 27 insertions, 3 deletions
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c index 0d8f3b2d9bec..e6061f2e64f2 100644 --- a/arch/x86/xen/spinlock.c +++ b/arch/x86/xen/spinlock.c | |||
@@ -23,6 +23,7 @@ static struct xen_spinlock_stats | |||
23 | u32 taken_slow_nested; | 23 | u32 taken_slow_nested; |
24 | u32 taken_slow_pickup; | 24 | u32 taken_slow_pickup; |
25 | u32 taken_slow_spurious; | 25 | u32 taken_slow_spurious; |
26 | u32 taken_slow_irqenable; | ||
26 | 27 | ||
27 | u64 released; | 28 | u64 released; |
28 | u32 released_slow; | 29 | u32 released_slow; |
@@ -167,12 +168,13 @@ static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock | |||
167 | __get_cpu_var(lock_spinners) = prev; | 168 | __get_cpu_var(lock_spinners) = prev; |
168 | } | 169 | } |
169 | 170 | ||
170 | static noinline int xen_spin_lock_slow(struct raw_spinlock *lock) | 171 | static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enable) |
171 | { | 172 | { |
172 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | 173 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; |
173 | struct xen_spinlock *prev; | 174 | struct xen_spinlock *prev; |
174 | int irq = __get_cpu_var(lock_kicker_irq); | 175 | int irq = __get_cpu_var(lock_kicker_irq); |
175 | int ret; | 176 | int ret; |
177 | unsigned long flags; | ||
176 | 178 | ||
177 | /* If kicker interrupts not initialized yet, just spin */ | 179 | /* If kicker interrupts not initialized yet, just spin */ |
178 | if (irq == -1) | 180 | if (irq == -1) |
@@ -181,6 +183,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock) | |||
181 | /* announce we're spinning */ | 183 | /* announce we're spinning */ |
182 | prev = spinning_lock(xl); | 184 | prev = spinning_lock(xl); |
183 | 185 | ||
186 | flags = __raw_local_save_flags(); | ||
187 | if (irq_enable) { | ||
188 | ADD_STATS(taken_slow_irqenable, 1); | ||
189 | raw_local_irq_enable(); | ||
190 | } | ||
191 | |||
184 | ADD_STATS(taken_slow, 1); | 192 | ADD_STATS(taken_slow, 1); |
185 | ADD_STATS(taken_slow_nested, prev != NULL); | 193 | ADD_STATS(taken_slow_nested, prev != NULL); |
186 | 194 | ||
@@ -220,11 +228,12 @@ static noinline int xen_spin_lock_slow(struct raw_spinlock *lock) | |||
220 | kstat_this_cpu.irqs[irq]++; | 228 | kstat_this_cpu.irqs[irq]++; |
221 | 229 | ||
222 | out: | 230 | out: |
231 | raw_local_irq_restore(flags); | ||
223 | unspinning_lock(xl, prev); | 232 | unspinning_lock(xl, prev); |
224 | return ret; | 233 | return ret; |
225 | } | 234 | } |
226 | 235 | ||
227 | static void xen_spin_lock(struct raw_spinlock *lock) | 236 | static inline void __xen_spin_lock(struct raw_spinlock *lock, bool irq_enable) |
228 | { | 237 | { |
229 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; | 238 | struct xen_spinlock *xl = (struct xen_spinlock *)lock; |
230 | unsigned timeout; | 239 | unsigned timeout; |
@@ -254,11 +263,23 @@ static void xen_spin_lock(struct raw_spinlock *lock) | |||
254 | : "memory"); | 263 | : "memory"); |
255 | 264 | ||
256 | spin_time_accum_fast(start_spin_fast); | 265 | spin_time_accum_fast(start_spin_fast); |
257 | } while (unlikely(oldval != 0 && (TIMEOUT == ~0 || !xen_spin_lock_slow(lock)))); | 266 | |
267 | } while (unlikely(oldval != 0 && | ||
268 | (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable)))); | ||
258 | 269 | ||
259 | spin_time_accum(start_spin); | 270 | spin_time_accum(start_spin); |
260 | } | 271 | } |
261 | 272 | ||
273 | static void xen_spin_lock(struct raw_spinlock *lock) | ||
274 | { | ||
275 | __xen_spin_lock(lock, false); | ||
276 | } | ||
277 | |||
278 | static void xen_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags) | ||
279 | { | ||
280 | __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags)); | ||
281 | } | ||
282 | |||
262 | static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) | 283 | static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl) |
263 | { | 284 | { |
264 | int cpu; | 285 | int cpu; |
@@ -323,6 +344,7 @@ void __init xen_init_spinlocks(void) | |||
323 | pv_lock_ops.spin_is_locked = xen_spin_is_locked; | 344 | pv_lock_ops.spin_is_locked = xen_spin_is_locked; |
324 | pv_lock_ops.spin_is_contended = xen_spin_is_contended; | 345 | pv_lock_ops.spin_is_contended = xen_spin_is_contended; |
325 | pv_lock_ops.spin_lock = xen_spin_lock; | 346 | pv_lock_ops.spin_lock = xen_spin_lock; |
347 | pv_lock_ops.spin_lock_flags = xen_spin_lock_flags; | ||
326 | pv_lock_ops.spin_trylock = xen_spin_trylock; | 348 | pv_lock_ops.spin_trylock = xen_spin_trylock; |
327 | pv_lock_ops.spin_unlock = xen_spin_unlock; | 349 | pv_lock_ops.spin_unlock = xen_spin_unlock; |
328 | } | 350 | } |
@@ -353,6 +375,8 @@ static int __init xen_spinlock_debugfs(void) | |||
353 | &spinlock_stats.taken_slow_pickup); | 375 | &spinlock_stats.taken_slow_pickup); |
354 | debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, | 376 | debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug, |
355 | &spinlock_stats.taken_slow_spurious); | 377 | &spinlock_stats.taken_slow_spurious); |
378 | debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug, | ||
379 | &spinlock_stats.taken_slow_irqenable); | ||
356 | 380 | ||
357 | debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); | 381 | debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released); |
358 | debugfs_create_u32("released_slow", 0444, d_spin_debug, | 382 | debugfs_create_u32("released_slow", 0444, d_spin_debug, |