aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTejun Heo <tj@kernel.org>2014-09-24 13:31:48 -0400
committerTejun Heo <tj@kernel.org>2014-09-24 13:31:48 -0400
commiteecc16ba9a49b05dd847a317af166a6728eb56ca (patch)
treee61dbe61074cfe6e09593dc3f60d3fb7bdd454e7
parent6251f9976af7656b6970a8820153f356430f5de2 (diff)
percpu_ref: replace pcpu_ prefix with percpu_
percpu_ref uses pcpu_ prefix for internal stuff and percpu_ for externally visible ones. This is the same convention used in the percpu allocator implementation. It works fine there but percpu_ref doesn't have too much internal-only stuff and scattered usages of pcpu_ prefix are confusing than helpful. This patch replaces all pcpu_ prefixes with percpu_. This is pure rename and there's no functional change. Note that PCPU_REF_DEAD is renamed to __PERCPU_REF_DEAD to signify that the flag is internal. Signed-off-by: Tejun Heo <tj@kernel.org> Reviewed-by: Kent Overstreet <kmo@daterainc.com>
-rw-r--r--include/linux/percpu-refcount.h46
-rw-r--r--lib/percpu-refcount.c56
2 files changed, 52 insertions, 50 deletions
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index d44b027f74fd..3d463a39e0f7 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -13,7 +13,7 @@
13 * 13 *
14 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less 14 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
15 * than an atomic_t - this is because of the way shutdown works, see 15 * than an atomic_t - this is because of the way shutdown works, see
16 * percpu_ref_kill()/PCPU_COUNT_BIAS. 16 * percpu_ref_kill()/PERCPU_COUNT_BIAS.
17 * 17 *
18 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the 18 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
19 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() 19 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
@@ -60,7 +60,7 @@ struct percpu_ref {
60 * The low bit of the pointer indicates whether the ref is in percpu 60 * The low bit of the pointer indicates whether the ref is in percpu
61 * mode; if set, then get/put will manipulate the atomic_t. 61 * mode; if set, then get/put will manipulate the atomic_t.
62 */ 62 */
63 unsigned long pcpu_count_ptr; 63 unsigned long percpu_count_ptr;
64 percpu_ref_func_t *release; 64 percpu_ref_func_t *release;
65 percpu_ref_func_t *confirm_kill; 65 percpu_ref_func_t *confirm_kill;
66 struct rcu_head rcu; 66 struct rcu_head rcu;
@@ -88,26 +88,26 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
88 return percpu_ref_kill_and_confirm(ref, NULL); 88 return percpu_ref_kill_and_confirm(ref, NULL);
89} 89}
90 90
91#define PCPU_REF_DEAD 1 91#define __PERCPU_REF_DEAD 1
92 92
93/* 93/*
94 * Internal helper. Don't use outside percpu-refcount proper. The 94 * Internal helper. Don't use outside percpu-refcount proper. The
95 * function doesn't return the pointer and let the caller test it for NULL 95 * function doesn't return the pointer and let the caller test it for NULL
96 * because doing so forces the compiler to generate two conditional 96 * because doing so forces the compiler to generate two conditional
97 * branches as it can't assume that @ref->pcpu_count is not NULL. 97 * branches as it can't assume that @ref->percpu_count is not NULL.
98 */ 98 */
99static inline bool __pcpu_ref_alive(struct percpu_ref *ref, 99static inline bool __percpu_ref_alive(struct percpu_ref *ref,
100 unsigned long __percpu **pcpu_countp) 100 unsigned long __percpu **percpu_countp)
101{ 101{
102 unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); 102 unsigned long percpu_ptr = ACCESS_ONCE(ref->percpu_count_ptr);
103 103
104 /* paired with smp_store_release() in percpu_ref_reinit() */ 104 /* paired with smp_store_release() in percpu_ref_reinit() */
105 smp_read_barrier_depends(); 105 smp_read_barrier_depends();
106 106
107 if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) 107 if (unlikely(percpu_ptr & __PERCPU_REF_DEAD))
108 return false; 108 return false;
109 109
110 *pcpu_countp = (unsigned long __percpu *)pcpu_ptr; 110 *percpu_countp = (unsigned long __percpu *)percpu_ptr;
111 return true; 111 return true;
112} 112}
113 113
@@ -121,12 +121,12 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
121 */ 121 */
122static inline void percpu_ref_get(struct percpu_ref *ref) 122static inline void percpu_ref_get(struct percpu_ref *ref)
123{ 123{
124 unsigned long __percpu *pcpu_count; 124 unsigned long __percpu *percpu_count;
125 125
126 rcu_read_lock_sched(); 126 rcu_read_lock_sched();
127 127
128 if (__pcpu_ref_alive(ref, &pcpu_count)) 128 if (__percpu_ref_alive(ref, &percpu_count))
129 this_cpu_inc(*pcpu_count); 129 this_cpu_inc(*percpu_count);
130 else 130 else
131 atomic_long_inc(&ref->count); 131 atomic_long_inc(&ref->count);
132 132
@@ -144,13 +144,13 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
144 */ 144 */
145static inline bool percpu_ref_tryget(struct percpu_ref *ref) 145static inline bool percpu_ref_tryget(struct percpu_ref *ref)
146{ 146{
147 unsigned long __percpu *pcpu_count; 147 unsigned long __percpu *percpu_count;
148 int ret; 148 int ret;
149 149
150 rcu_read_lock_sched(); 150 rcu_read_lock_sched();
151 151
152 if (__pcpu_ref_alive(ref, &pcpu_count)) { 152 if (__percpu_ref_alive(ref, &percpu_count)) {
153 this_cpu_inc(*pcpu_count); 153 this_cpu_inc(*percpu_count);
154 ret = true; 154 ret = true;
155 } else { 155 } else {
156 ret = atomic_long_inc_not_zero(&ref->count); 156 ret = atomic_long_inc_not_zero(&ref->count);
@@ -178,13 +178,13 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
178 */ 178 */
179static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) 179static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
180{ 180{
181 unsigned long __percpu *pcpu_count; 181 unsigned long __percpu *percpu_count;
182 int ret = false; 182 int ret = false;
183 183
184 rcu_read_lock_sched(); 184 rcu_read_lock_sched();
185 185
186 if (__pcpu_ref_alive(ref, &pcpu_count)) { 186 if (__percpu_ref_alive(ref, &percpu_count)) {
187 this_cpu_inc(*pcpu_count); 187 this_cpu_inc(*percpu_count);
188 ret = true; 188 ret = true;
189 } 189 }
190 190
@@ -204,12 +204,12 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
204 */ 204 */
205static inline void percpu_ref_put(struct percpu_ref *ref) 205static inline void percpu_ref_put(struct percpu_ref *ref)
206{ 206{
207 unsigned long __percpu *pcpu_count; 207 unsigned long __percpu *percpu_count;
208 208
209 rcu_read_lock_sched(); 209 rcu_read_lock_sched();
210 210
211 if (__pcpu_ref_alive(ref, &pcpu_count)) 211 if (__percpu_ref_alive(ref, &percpu_count))
212 this_cpu_dec(*pcpu_count); 212 this_cpu_dec(*percpu_count);
213 else if (unlikely(atomic_long_dec_and_test(&ref->count))) 213 else if (unlikely(atomic_long_dec_and_test(&ref->count)))
214 ref->release(ref); 214 ref->release(ref);
215 215
@@ -226,9 +226,9 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
226 */ 226 */
227static inline bool percpu_ref_is_zero(struct percpu_ref *ref) 227static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
228{ 228{
229 unsigned long __percpu *pcpu_count; 229 unsigned long __percpu *percpu_count;
230 230
231 if (__pcpu_ref_alive(ref, &pcpu_count)) 231 if (__percpu_ref_alive(ref, &percpu_count))
232 return false; 232 return false;
233 return !atomic_long_read(&ref->count); 233 return !atomic_long_read(&ref->count);
234} 234}
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 8ef3f5c20df6..5aea6b7356c7 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -11,8 +11,8 @@
11 * percpu counters will all sum to the correct value 11 * percpu counters will all sum to the correct value
12 * 12 *
13 * (More precisely: because moduler arithmatic is commutative the sum of all the 13 * (More precisely: because moduler arithmatic is commutative the sum of all the
14 * pcpu_count vars will be equal to what it would have been if all the gets and 14 * percpu_count vars will be equal to what it would have been if all the gets
15 * puts were done to a single integer, even if some of the percpu integers 15 * and puts were done to a single integer, even if some of the percpu integers
16 * overflow or underflow). 16 * overflow or underflow).
17 * 17 *
18 * The real trick to implementing percpu refcounts is shutdown. We can't detect 18 * The real trick to implementing percpu refcounts is shutdown. We can't detect
@@ -29,11 +29,12 @@
29 * atomic_long_t can't hit 0 before we've added up all the percpu refs. 29 * atomic_long_t can't hit 0 before we've added up all the percpu refs.
30 */ 30 */
31 31
32#define PCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1)) 32#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
33 33
34static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref) 34static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref)
35{ 35{
36 return (unsigned long __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD); 36 return (unsigned long __percpu *)
37 (ref->percpu_count_ptr & ~__PERCPU_REF_DEAD);
37} 38}
38 39
39/** 40/**
@@ -51,10 +52,11 @@ static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref)
51int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, 52int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
52 gfp_t gfp) 53 gfp_t gfp)
53{ 54{
54 atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS); 55 atomic_long_set(&ref->count, 1 + PERCPU_COUNT_BIAS);
55 56
56 ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned long, gfp); 57 ref->percpu_count_ptr =
57 if (!ref->pcpu_count_ptr) 58 (unsigned long)alloc_percpu_gfp(unsigned long, gfp);
59 if (!ref->percpu_count_ptr)
58 return -ENOMEM; 60 return -ENOMEM;
59 61
60 ref->release = release; 62 ref->release = release;
@@ -74,11 +76,11 @@ EXPORT_SYMBOL_GPL(percpu_ref_init);
74 */ 76 */
75void percpu_ref_exit(struct percpu_ref *ref) 77void percpu_ref_exit(struct percpu_ref *ref)
76{ 78{
77 unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); 79 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
78 80
79 if (pcpu_count) { 81 if (percpu_count) {
80 free_percpu(pcpu_count); 82 free_percpu(percpu_count);
81 ref->pcpu_count_ptr = PCPU_REF_DEAD; 83 ref->percpu_count_ptr = __PERCPU_REF_DEAD;
82 } 84 }
83} 85}
84EXPORT_SYMBOL_GPL(percpu_ref_exit); 86EXPORT_SYMBOL_GPL(percpu_ref_exit);
@@ -86,14 +88,14 @@ EXPORT_SYMBOL_GPL(percpu_ref_exit);
86static void percpu_ref_kill_rcu(struct rcu_head *rcu) 88static void percpu_ref_kill_rcu(struct rcu_head *rcu)
87{ 89{
88 struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); 90 struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
89 unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); 91 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
90 unsigned long count = 0; 92 unsigned long count = 0;
91 int cpu; 93 int cpu;
92 94
93 for_each_possible_cpu(cpu) 95 for_each_possible_cpu(cpu)
94 count += *per_cpu_ptr(pcpu_count, cpu); 96 count += *per_cpu_ptr(percpu_count, cpu);
95 97
96 pr_debug("global %ld pcpu %ld", 98 pr_debug("global %ld percpu %ld",
97 atomic_long_read(&ref->count), (long)count); 99 atomic_long_read(&ref->count), (long)count);
98 100
99 /* 101 /*
@@ -108,7 +110,7 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
108 * reaching 0 before we add the percpu counts. But doing it at the same 110 * reaching 0 before we add the percpu counts. But doing it at the same
109 * time is equivalent and saves us atomic operations: 111 * time is equivalent and saves us atomic operations:
110 */ 112 */
111 atomic_long_add((long)count - PCPU_COUNT_BIAS, &ref->count); 113 atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count);
112 114
113 WARN_ONCE(atomic_long_read(&ref->count) <= 0, 115 WARN_ONCE(atomic_long_read(&ref->count) <= 0,
114 "percpu ref (%pf) <= 0 (%ld) after killed", 116 "percpu ref (%pf) <= 0 (%ld) after killed",
@@ -143,10 +145,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
143void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 145void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
144 percpu_ref_func_t *confirm_kill) 146 percpu_ref_func_t *confirm_kill)
145{ 147{
146 WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD, 148 WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD,
147 "%s called more than once on %pf!", __func__, ref->release); 149 "%s called more than once on %pf!", __func__, ref->release);
148 150
149 ref->pcpu_count_ptr |= PCPU_REF_DEAD; 151 ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
150 ref->confirm_kill = confirm_kill; 152 ref->confirm_kill = confirm_kill;
151 153
152 call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu); 154 call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);
@@ -166,24 +168,24 @@ EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
166 */ 168 */
167void percpu_ref_reinit(struct percpu_ref *ref) 169void percpu_ref_reinit(struct percpu_ref *ref)
168{ 170{
169 unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref); 171 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
170 int cpu; 172 int cpu;
171 173
172 BUG_ON(!pcpu_count); 174 BUG_ON(!percpu_count);
173 WARN_ON_ONCE(!percpu_ref_is_zero(ref)); 175 WARN_ON_ONCE(!percpu_ref_is_zero(ref));
174 176
175 atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS); 177 atomic_long_set(&ref->count, 1 + PERCPU_COUNT_BIAS);
176 178
177 /* 179 /*
178 * Restore per-cpu operation. smp_store_release() is paired with 180 * Restore per-cpu operation. smp_store_release() is paired with
179 * smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees 181 * smp_read_barrier_depends() in __percpu_ref_alive() and
180 * that the zeroing is visible to all percpu accesses which can see 182 * guarantees that the zeroing is visible to all percpu accesses
181 * the following PCPU_REF_DEAD clearing. 183 * which can see the following __PERCPU_REF_DEAD clearing.
182 */ 184 */
183 for_each_possible_cpu(cpu) 185 for_each_possible_cpu(cpu)
184 *per_cpu_ptr(pcpu_count, cpu) = 0; 186 *per_cpu_ptr(percpu_count, cpu) = 0;
185 187
186 smp_store_release(&ref->pcpu_count_ptr, 188 smp_store_release(&ref->percpu_count_ptr,
187 ref->pcpu_count_ptr & ~PCPU_REF_DEAD); 189 ref->percpu_count_ptr & ~__PERCPU_REF_DEAD);
188} 190}
189EXPORT_SYMBOL_GPL(percpu_ref_reinit); 191EXPORT_SYMBOL_GPL(percpu_ref_reinit);