aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorMikulas Patocka <mpatocka@redhat.com>2012-10-22 19:37:47 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-10-28 13:59:36 -0400
commit5c1eabe68501d1e1b1586c7f4c46cc531828c4ab (patch)
treed97aab24d8d0abbd8378e7c851f7a55d266bb851 /include
parente657e078d3dfa9f96976db7a2b5fd7d7c9f1f1a6 (diff)
percpu-rw-semaphores: use light/heavy barriers
This patch introduces new barrier pair light_mb() and heavy_mb() for percpu rw semaphores. This patch fixes a bug in percpu-rw-semaphores where a barrier was missing in percpu_up_write. This patch improves performance on the read path of percpu-rw-semaphores: on non-x86 cpus, there was a smp_mb() in percpu_up_read. This patch changes it to a compiler barrier and removes the "#if defined(X86) ..." condition. From: Lai Jiangshan <laijs@cn.fujitsu.com> Signed-off-by: Mikulas Patocka <mpatocka@redhat.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include')
-rw-r--r--include/linux/percpu-rwsem.h20
1 files changed, 7 insertions, 13 deletions
diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index cf80f7e5277f..18f35b54286c 100644
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -12,6 +12,9 @@ struct percpu_rw_semaphore {
12 struct mutex mtx; 12 struct mutex mtx;
13}; 13};
14 14
15#define light_mb() barrier()
16#define heavy_mb() synchronize_sched()
17
15static inline void percpu_down_read(struct percpu_rw_semaphore *p) 18static inline void percpu_down_read(struct percpu_rw_semaphore *p)
16{ 19{
17 rcu_read_lock(); 20 rcu_read_lock();
@@ -24,22 +27,12 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *p)
24 } 27 }
25 this_cpu_inc(*p->counters); 28 this_cpu_inc(*p->counters);
26 rcu_read_unlock(); 29 rcu_read_unlock();
30 light_mb(); /* A, between read of p->locked and read of data, paired with D */
27} 31}
28 32
29static inline void percpu_up_read(struct percpu_rw_semaphore *p) 33static inline void percpu_up_read(struct percpu_rw_semaphore *p)
30{ 34{
31 /* 35 light_mb(); /* B, between read of the data and write to p->counter, paired with C */
32 * On X86, write operation in this_cpu_dec serves as a memory unlock
33 * barrier (i.e. memory accesses may be moved before the write, but
34 * no memory accesses are moved past the write).
35 * On other architectures this may not be the case, so we need smp_mb()
36 * there.
37 */
38#if defined(CONFIG_X86) && (!defined(CONFIG_X86_PPRO_FENCE) && !defined(CONFIG_X86_OOSTORE))
39 barrier();
40#else
41 smp_mb();
42#endif
43 this_cpu_dec(*p->counters); 36 this_cpu_dec(*p->counters);
44} 37}
45 38
@@ -61,11 +54,12 @@ static inline void percpu_down_write(struct percpu_rw_semaphore *p)
61 synchronize_rcu(); 54 synchronize_rcu();
62 while (__percpu_count(p->counters)) 55 while (__percpu_count(p->counters))
63 msleep(1); 56 msleep(1);
64 smp_rmb(); /* paired with smp_mb() in percpu_sem_up_read() */ 57 heavy_mb(); /* C, between read of p->counter and write to data, paired with B */
65} 58}
66 59
67static inline void percpu_up_write(struct percpu_rw_semaphore *p) 60static inline void percpu_up_write(struct percpu_rw_semaphore *p)
68{ 61{
62 heavy_mb(); /* D, between write to data and write to p->locked, paired with A */
69 p->locked = false; 63 p->locked = false;
70 mutex_unlock(&p->mtx); 64 mutex_unlock(&p->mtx);
71} 65}