diff options
Diffstat (limited to 'include/linux/percpu-refcount.h')
| -rw-r--r-- | include/linux/percpu-refcount.h | 46 |
1 files changed, 23 insertions, 23 deletions
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d44b027f74fd..3d463a39e0f7 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h | |||
| @@ -13,7 +13,7 @@ | |||
| 13 | * | 13 | * |
| 14 | * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less | 14 | * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less |
| 15 | * than an atomic_t - this is because of the way shutdown works, see | 15 | * than an atomic_t - this is because of the way shutdown works, see |
| 16 | * percpu_ref_kill()/PCPU_COUNT_BIAS. | 16 | * percpu_ref_kill()/PERCPU_COUNT_BIAS. |
| 17 | * | 17 | * |
| 18 | * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the | 18 | * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the |
| 19 | * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() | 19 | * refcount hitting 0 - it can't, if it was in percpu mode. percpu_ref_kill() |
| @@ -60,7 +60,7 @@ struct percpu_ref { | |||
| 60 | * The low bit of the pointer indicates whether the ref is in percpu | 60 | * The low bit of the pointer indicates whether the ref is in percpu |
| 61 | * mode; if set, then get/put will manipulate the atomic_t. | 61 | * mode; if set, then get/put will manipulate the atomic_t. |
| 62 | */ | 62 | */ |
| 63 | unsigned long pcpu_count_ptr; | 63 | unsigned long percpu_count_ptr; |
| 64 | percpu_ref_func_t *release; | 64 | percpu_ref_func_t *release; |
| 65 | percpu_ref_func_t *confirm_kill; | 65 | percpu_ref_func_t *confirm_kill; |
| 66 | struct rcu_head rcu; | 66 | struct rcu_head rcu; |
| @@ -88,26 +88,26 @@ static inline void percpu_ref_kill(struct percpu_ref *ref) | |||
| 88 | return percpu_ref_kill_and_confirm(ref, NULL); | 88 | return percpu_ref_kill_and_confirm(ref, NULL); |
| 89 | } | 89 | } |
| 90 | 90 | ||
| 91 | #define PCPU_REF_DEAD 1 | 91 | #define __PERCPU_REF_DEAD 1 |
| 92 | 92 | ||
| 93 | /* | 93 | /* |
| 94 | * Internal helper. Don't use outside percpu-refcount proper. The | 94 | * Internal helper. Don't use outside percpu-refcount proper. The |
| 95 | * function doesn't return the pointer and let the caller test it for NULL | 95 | * function doesn't return the pointer and let the caller test it for NULL |
| 96 | * because doing so forces the compiler to generate two conditional | 96 | * because doing so forces the compiler to generate two conditional |
| 97 | * branches as it can't assume that @ref->pcpu_count is not NULL. | 97 | * branches as it can't assume that @ref->percpu_count is not NULL. |
| 98 | */ | 98 | */ |
| 99 | static inline bool __pcpu_ref_alive(struct percpu_ref *ref, | 99 | static inline bool __percpu_ref_alive(struct percpu_ref *ref, |
| 100 | unsigned long __percpu **pcpu_countp) | 100 | unsigned long __percpu **percpu_countp) |
| 101 | { | 101 | { |
| 102 | unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); | 102 | unsigned long percpu_ptr = ACCESS_ONCE(ref->percpu_count_ptr); |
| 103 | 103 | ||
| 104 | /* paired with smp_store_release() in percpu_ref_reinit() */ | 104 | /* paired with smp_store_release() in percpu_ref_reinit() */ |
| 105 | smp_read_barrier_depends(); | 105 | smp_read_barrier_depends(); |
| 106 | 106 | ||
| 107 | if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) | 107 | if (unlikely(percpu_ptr & __PERCPU_REF_DEAD)) |
| 108 | return false; | 108 | return false; |
| 109 | 109 | ||
| 110 | *pcpu_countp = (unsigned long __percpu *)pcpu_ptr; | 110 | *percpu_countp = (unsigned long __percpu *)percpu_ptr; |
| 111 | return true; | 111 | return true; |
| 112 | } | 112 | } |
| 113 | 113 | ||
| @@ -121,12 +121,12 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref, | |||
| 121 | */ | 121 | */ |
| 122 | static inline void percpu_ref_get(struct percpu_ref *ref) | 122 | static inline void percpu_ref_get(struct percpu_ref *ref) |
| 123 | { | 123 | { |
| 124 | unsigned long __percpu *pcpu_count; | 124 | unsigned long __percpu *percpu_count; |
| 125 | 125 | ||
| 126 | rcu_read_lock_sched(); | 126 | rcu_read_lock_sched(); |
| 127 | 127 | ||
| 128 | if (__pcpu_ref_alive(ref, &pcpu_count)) | 128 | if (__percpu_ref_alive(ref, &percpu_count)) |
| 129 | this_cpu_inc(*pcpu_count); | 129 | this_cpu_inc(*percpu_count); |
| 130 | else | 130 | else |
| 131 | atomic_long_inc(&ref->count); | 131 | atomic_long_inc(&ref->count); |
| 132 | 132 | ||
| @@ -144,13 +144,13 @@ static inline void percpu_ref_get(struct percpu_ref *ref) | |||
| 144 | */ | 144 | */ |
| 145 | static inline bool percpu_ref_tryget(struct percpu_ref *ref) | 145 | static inline bool percpu_ref_tryget(struct percpu_ref *ref) |
| 146 | { | 146 | { |
| 147 | unsigned long __percpu *pcpu_count; | 147 | unsigned long __percpu *percpu_count; |
| 148 | int ret; | 148 | int ret; |
| 149 | 149 | ||
| 150 | rcu_read_lock_sched(); | 150 | rcu_read_lock_sched(); |
| 151 | 151 | ||
| 152 | if (__pcpu_ref_alive(ref, &pcpu_count)) { | 152 | if (__percpu_ref_alive(ref, &percpu_count)) { |
| 153 | this_cpu_inc(*pcpu_count); | 153 | this_cpu_inc(*percpu_count); |
| 154 | ret = true; | 154 | ret = true; |
| 155 | } else { | 155 | } else { |
| 156 | ret = atomic_long_inc_not_zero(&ref->count); | 156 | ret = atomic_long_inc_not_zero(&ref->count); |
| @@ -178,13 +178,13 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) | |||
| 178 | */ | 178 | */ |
| 179 | static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) | 179 | static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) |
| 180 | { | 180 | { |
| 181 | unsigned long __percpu *pcpu_count; | 181 | unsigned long __percpu *percpu_count; |
| 182 | int ret = false; | 182 | int ret = false; |
| 183 | 183 | ||
| 184 | rcu_read_lock_sched(); | 184 | rcu_read_lock_sched(); |
| 185 | 185 | ||
| 186 | if (__pcpu_ref_alive(ref, &pcpu_count)) { | 186 | if (__percpu_ref_alive(ref, &percpu_count)) { |
| 187 | this_cpu_inc(*pcpu_count); | 187 | this_cpu_inc(*percpu_count); |
| 188 | ret = true; | 188 | ret = true; |
| 189 | } | 189 | } |
| 190 | 190 | ||
| @@ -204,12 +204,12 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) | |||
| 204 | */ | 204 | */ |
| 205 | static inline void percpu_ref_put(struct percpu_ref *ref) | 205 | static inline void percpu_ref_put(struct percpu_ref *ref) |
| 206 | { | 206 | { |
| 207 | unsigned long __percpu *pcpu_count; | 207 | unsigned long __percpu *percpu_count; |
| 208 | 208 | ||
| 209 | rcu_read_lock_sched(); | 209 | rcu_read_lock_sched(); |
| 210 | 210 | ||
| 211 | if (__pcpu_ref_alive(ref, &pcpu_count)) | 211 | if (__percpu_ref_alive(ref, &percpu_count)) |
| 212 | this_cpu_dec(*pcpu_count); | 212 | this_cpu_dec(*percpu_count); |
| 213 | else if (unlikely(atomic_long_dec_and_test(&ref->count))) | 213 | else if (unlikely(atomic_long_dec_and_test(&ref->count))) |
| 214 | ref->release(ref); | 214 | ref->release(ref); |
| 215 | 215 | ||
| @@ -226,9 +226,9 @@ static inline void percpu_ref_put(struct percpu_ref *ref) | |||
| 226 | */ | 226 | */ |
| 227 | static inline bool percpu_ref_is_zero(struct percpu_ref *ref) | 227 | static inline bool percpu_ref_is_zero(struct percpu_ref *ref) |
| 228 | { | 228 | { |
| 229 | unsigned long __percpu *pcpu_count; | 229 | unsigned long __percpu *percpu_count; |
| 230 | 230 | ||
| 231 | if (__pcpu_ref_alive(ref, &pcpu_count)) | 231 | if (__percpu_ref_alive(ref, &percpu_count)) |
| 232 | return false; | 232 | return false; |
| 233 | return !atomic_long_read(&ref->count); | 233 | return !atomic_long_read(&ref->count); |
| 234 | } | 234 | } |
