aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux/percpu-refcount.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/percpu-refcount.h')
-rw-r--r--include/linux/percpu-refcount.h46
1 files changed, 23 insertions, 23 deletions
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index d44b027f74fd..3d463a39e0f7 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -13,7 +13,7 @@
13 * 13 *
14 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less 14 * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less
15 * than an atomic_t - this is because of the way shutdown works, see 15 * than an atomic_t - this is because of the way shutdown works, see
16 * percpu_ref_kill()/PCPU_COUNT_BIAS. 16 * percpu_ref_kill()/PERCPU_COUNT_BIAS.
17 * 17 *
18 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the 18 * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the
19 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() 19 * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill()
@@ -60,7 +60,7 @@ struct percpu_ref {
60 * The low bit of the pointer indicates whether the ref is in percpu 60 * The low bit of the pointer indicates whether the ref is in percpu
61 * mode; if set, then get/put will manipulate the atomic_t. 61 * mode; if set, then get/put will manipulate the atomic_t.
62 */ 62 */
63 unsigned long pcpu_count_ptr; 63 unsigned long percpu_count_ptr;
64 percpu_ref_func_t *release; 64 percpu_ref_func_t *release;
65 percpu_ref_func_t *confirm_kill; 65 percpu_ref_func_t *confirm_kill;
66 struct rcu_head rcu; 66 struct rcu_head rcu;
@@ -88,26 +88,26 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
88 return percpu_ref_kill_and_confirm(ref, NULL); 88 return percpu_ref_kill_and_confirm(ref, NULL);
89} 89}
90 90
91#define PCPU_REF_DEAD 1 91#define __PERCPU_REF_DEAD 1
92 92
93/* 93/*
94 * Internal helper. Don't use outside percpu-refcount proper. The 94 * Internal helper. Don't use outside percpu-refcount proper. The
95 * function doesn't return the pointer and let the caller test it for NULL 95 * function doesn't return the pointer and let the caller test it for NULL
96 * because doing so forces the compiler to generate two conditional 96 * because doing so forces the compiler to generate two conditional
97 * branches as it can't assume that @ref->pcpu_count is not NULL. 97 * branches as it can't assume that @ref->percpu_count is not NULL.
98 */ 98 */
99static inline bool __pcpu_ref_alive(struct percpu_ref *ref, 99static inline bool __percpu_ref_alive(struct percpu_ref *ref,
100 unsigned long __percpu **pcpu_countp) 100 unsigned long __percpu **percpu_countp)
101{ 101{
102 unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); 102 unsigned long percpu_ptr = ACCESS_ONCE(ref->percpu_count_ptr);
103 103
104 /* paired with smp_store_release() in percpu_ref_reinit() */ 104 /* paired with smp_store_release() in percpu_ref_reinit() */
105 smp_read_barrier_depends(); 105 smp_read_barrier_depends();
106 106
107 if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) 107 if (unlikely(percpu_ptr & __PERCPU_REF_DEAD))
108 return false; 108 return false;
109 109
110 *pcpu_countp = (unsigned long __percpu *)pcpu_ptr; 110 *percpu_countp = (unsigned long __percpu *)percpu_ptr;
111 return true; 111 return true;
112} 112}
113 113
@@ -121,12 +121,12 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
121 */ 121 */
122static inline void percpu_ref_get(struct percpu_ref *ref) 122static inline void percpu_ref_get(struct percpu_ref *ref)
123{ 123{
124 unsigned long __percpu *pcpu_count; 124 unsigned long __percpu *percpu_count;
125 125
126 rcu_read_lock_sched(); 126 rcu_read_lock_sched();
127 127
128 if (__pcpu_ref_alive(ref, &pcpu_count)) 128 if (__percpu_ref_alive(ref, &percpu_count))
129 this_cpu_inc(*pcpu_count); 129 this_cpu_inc(*percpu_count);
130 else 130 else
131 atomic_long_inc(&ref->count); 131 atomic_long_inc(&ref->count);
132 132
@@ -144,13 +144,13 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
144 */ 144 */
145static inline bool percpu_ref_tryget(struct percpu_ref *ref) 145static inline bool percpu_ref_tryget(struct percpu_ref *ref)
146{ 146{
147 unsigned long __percpu *pcpu_count; 147 unsigned long __percpu *percpu_count;
148 int ret; 148 int ret;
149 149
150 rcu_read_lock_sched(); 150 rcu_read_lock_sched();
151 151
152 if (__pcpu_ref_alive(ref, &pcpu_count)) { 152 if (__percpu_ref_alive(ref, &percpu_count)) {
153 this_cpu_inc(*pcpu_count); 153 this_cpu_inc(*percpu_count);
154 ret = true; 154 ret = true;
155 } else { 155 } else {
156 ret = atomic_long_inc_not_zero(&ref->count); 156 ret = atomic_long_inc_not_zero(&ref->count);
@@ -178,13 +178,13 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
178 */ 178 */
179static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) 179static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
180{ 180{
181 unsigned long __percpu *pcpu_count; 181 unsigned long __percpu *percpu_count;
182 int ret = false; 182 int ret = false;
183 183
184 rcu_read_lock_sched(); 184 rcu_read_lock_sched();
185 185
186 if (__pcpu_ref_alive(ref, &pcpu_count)) { 186 if (__percpu_ref_alive(ref, &percpu_count)) {
187 this_cpu_inc(*pcpu_count); 187 this_cpu_inc(*percpu_count);
188 ret = true; 188 ret = true;
189 } 189 }
190 190
@@ -204,12 +204,12 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
204 */ 204 */
205static inline void percpu_ref_put(struct percpu_ref *ref) 205static inline void percpu_ref_put(struct percpu_ref *ref)
206{ 206{
207 unsigned long __percpu *pcpu_count; 207 unsigned long __percpu *percpu_count;
208 208
209 rcu_read_lock_sched(); 209 rcu_read_lock_sched();
210 210
211 if (__pcpu_ref_alive(ref, &pcpu_count)) 211 if (__percpu_ref_alive(ref, &percpu_count))
212 this_cpu_dec(*pcpu_count); 212 this_cpu_dec(*percpu_count);
213 else if (unlikely(atomic_long_dec_and_test(&ref->count))) 213 else if (unlikely(atomic_long_dec_and_test(&ref->count)))
214 ref->release(ref); 214 ref->release(ref);
215 215
@@ -226,9 +226,9 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
226 */ 226 */
227static inline bool percpu_ref_is_zero(struct percpu_ref *ref) 227static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
228{ 228{
229 unsigned long __percpu *pcpu_count; 229 unsigned long __percpu *percpu_count;
230 230
231 if (__pcpu_ref_alive(ref, &pcpu_count)) 231 if (__percpu_ref_alive(ref, &percpu_count))
232 return false; 232 return false;
233 return !atomic_long_read(&ref->count); 233 return !atomic_long_read(&ref->count);
234} 234}