aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2014-07-01 12:19:04 -0400
committerJens Axboe <axboe@fb.com>2014-07-01 12:19:04 -0400
commit17737d3b5997ac9f810967f0c6014d124ec39490 (patch)
tree5ee591a925382668d669fe5db3c3d1067f7340e8 /include/linux
parent4c834452aad01531db949414f94f817a86348d59 (diff)
parent2d7227828e1475c7b272e55bd70c4cec8eea219a (diff)
Merge branch 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu into for-3.17/core
Merge the percpu_ref changes from Tejun, he says they are stable now.
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/percpu-defs.h380
-rw-r--r--include/linux/percpu-refcount.h64
-rw-r--r--include/linux/percpu.h673
3 files changed, 410 insertions, 707 deletions
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index a5fc7d01aad6..c93fff16776c 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -1,6 +1,40 @@
1/*
2 * linux/percpu-defs.h - basic definitions for percpu areas
3 *
4 * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
5 *
6 * This file is separate from linux/percpu.h to avoid cyclic inclusion
7 * dependency from arch header files. Only to be included from
8 * asm/percpu.h.
9 *
10 * This file includes macros necessary to declare percpu sections and
11 * variables, and definitions of percpu accessors and operations. It
12 * should provide enough percpu features to arch header files even when
13 * they can only include asm/percpu.h to avoid cyclic inclusion dependency.
14 */
15
1#ifndef _LINUX_PERCPU_DEFS_H 16#ifndef _LINUX_PERCPU_DEFS_H
2#define _LINUX_PERCPU_DEFS_H 17#define _LINUX_PERCPU_DEFS_H
3 18
19#ifdef CONFIG_SMP
20
21#ifdef MODULE
22#define PER_CPU_SHARED_ALIGNED_SECTION ""
23#define PER_CPU_ALIGNED_SECTION ""
24#else
25#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
26#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
27#endif
28#define PER_CPU_FIRST_SECTION "..first"
29
30#else
31
32#define PER_CPU_SHARED_ALIGNED_SECTION ""
33#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
34#define PER_CPU_FIRST_SECTION ""
35
36#endif
37
4/* 38/*
5 * Base implementations of per-CPU variable declarations and definitions, where 39 * Base implementations of per-CPU variable declarations and definitions, where
6 * the section in which the variable is to be placed is provided by the 40 * the section in which the variable is to be placed is provided by the
@@ -19,19 +53,6 @@
19 __attribute__((section(".discard"), unused)) 53 __attribute__((section(".discard"), unused))
20 54
21/* 55/*
22 * Macro which verifies @ptr is a percpu pointer without evaluating
23 * @ptr. This is to be used in percpu accessors to verify that the
24 * input parameter is a percpu pointer.
25 *
26 * + 0 is required in order to convert the pointer type from a
27 * potential array type to a pointer to a single item of the array.
28 */
29#define __verify_pcpu_ptr(ptr) do { \
30 const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
31 (void)__vpp_verify; \
32} while (0)
33
34/*
35 * s390 and alpha modules require percpu variables to be defined as 56 * s390 and alpha modules require percpu variables to be defined as
36 * weak to force the compiler to generate GOT based external 57 * weak to force the compiler to generate GOT based external
37 * references for them. This is necessary because percpu sections 58 * references for them. This is necessary because percpu sections
@@ -164,4 +185,337 @@
164#define EXPORT_PER_CPU_SYMBOL_GPL(var) 185#define EXPORT_PER_CPU_SYMBOL_GPL(var)
165#endif 186#endif
166 187
188/*
189 * Accessors and operations.
190 */
191#ifndef __ASSEMBLY__
192
193/*
194 * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
195 * @ptr and is invoked once before a percpu area is accessed by all
196 * accessors and operations. This is performed in the generic part of
197 * percpu and arch overrides don't need to worry about it; however, if an
198 * arch wants to implement an arch-specific percpu accessor or operation,
199 * it may use __verify_pcpu_ptr() to verify the parameters.
200 *
201 * + 0 is required in order to convert the pointer type from a
202 * potential array type to a pointer to a single item of the array.
203 */
204#define __verify_pcpu_ptr(ptr) \
205do { \
206 const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
207 (void)__vpp_verify; \
208} while (0)
209
210#ifdef CONFIG_SMP
211
212/*
213 * Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
214 * to prevent the compiler from making incorrect assumptions about the
215 * pointer value. The weird cast keeps both GCC and sparse happy.
216 */
217#define SHIFT_PERCPU_PTR(__p, __offset) \
218 RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
219
220#define per_cpu_ptr(ptr, cpu) \
221({ \
222 __verify_pcpu_ptr(ptr); \
223 SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
224})
225
226#define raw_cpu_ptr(ptr) \
227({ \
228 __verify_pcpu_ptr(ptr); \
229 arch_raw_cpu_ptr(ptr); \
230})
231
232#ifdef CONFIG_DEBUG_PREEMPT
233#define this_cpu_ptr(ptr) \
234({ \
235 __verify_pcpu_ptr(ptr); \
236 SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
237})
238#else
239#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
240#endif
241
242#else /* CONFIG_SMP */
243
244#define VERIFY_PERCPU_PTR(__p) \
245({ \
246 __verify_pcpu_ptr(__p); \
247 (typeof(*(__p)) __kernel __force *)(__p); \
248})
249
250#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
251#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
252#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
253
254#endif /* CONFIG_SMP */
255
256#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
257#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
258#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
259
260/* keep until we have removed all uses of __this_cpu_ptr */
261#define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
262
263/*
264 * Must be an lvalue. Since @var must be a simple identifier,
265 * we force a syntax error here if it isn't.
266 */
267#define get_cpu_var(var) \
268(*({ \
269 preempt_disable(); \
270 this_cpu_ptr(&var); \
271}))
272
273/*
274 * The weird & is necessary because sparse considers (void)(var) to be
275 * a direct dereference of percpu variable (var).
276 */
277#define put_cpu_var(var) \
278do { \
279 (void)&(var); \
280 preempt_enable(); \
281} while (0)
282
283#define get_cpu_ptr(var) \
284({ \
285 preempt_disable(); \
286 this_cpu_ptr(var); \
287})
288
289#define put_cpu_ptr(var) \
290do { \
291 (void)(var); \
292 preempt_enable(); \
293} while (0)
294
295/*
296 * Branching function to split up a function into a set of functions that
297 * are called for different scalar sizes of the objects handled.
298 */
299
300extern void __bad_size_call_parameter(void);
301
302#ifdef CONFIG_DEBUG_PREEMPT
303extern void __this_cpu_preempt_check(const char *op);
304#else
305static inline void __this_cpu_preempt_check(const char *op) { }
306#endif
307
308#define __pcpu_size_call_return(stem, variable) \
309({ \
310 typeof(variable) pscr_ret__; \
311 __verify_pcpu_ptr(&(variable)); \
312 switch(sizeof(variable)) { \
313 case 1: pscr_ret__ = stem##1(variable); break; \
314 case 2: pscr_ret__ = stem##2(variable); break; \
315 case 4: pscr_ret__ = stem##4(variable); break; \
316 case 8: pscr_ret__ = stem##8(variable); break; \
317 default: \
318 __bad_size_call_parameter(); break; \
319 } \
320 pscr_ret__; \
321})
322
323#define __pcpu_size_call_return2(stem, variable, ...) \
324({ \
325 typeof(variable) pscr2_ret__; \
326 __verify_pcpu_ptr(&(variable)); \
327 switch(sizeof(variable)) { \
328 case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
329 case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
330 case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
331 case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
332 default: \
333 __bad_size_call_parameter(); break; \
334 } \
335 pscr2_ret__; \
336})
337
338/*
339 * Special handling for cmpxchg_double. cmpxchg_double is passed two
340 * percpu variables. The first has to be aligned to a double word
341 * boundary and the second has to follow directly thereafter.
342 * We enforce this on all architectures even if they don't support
343 * a double cmpxchg instruction, since it's a cheap requirement, and it
344 * avoids breaking the requirement for architectures with the instruction.
345 */
346#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
347({ \
348 bool pdcrb_ret__; \
349 __verify_pcpu_ptr(&(pcp1)); \
350 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
351 VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
352 VM_BUG_ON((unsigned long)(&(pcp2)) != \
353 (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
354 switch(sizeof(pcp1)) { \
355 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
356 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
357 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
358 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
359 default: \
360 __bad_size_call_parameter(); break; \
361 } \
362 pdcrb_ret__; \
363})
364
365#define __pcpu_size_call(stem, variable, ...) \
366do { \
367 __verify_pcpu_ptr(&(variable)); \
368 switch(sizeof(variable)) { \
369 case 1: stem##1(variable, __VA_ARGS__);break; \
370 case 2: stem##2(variable, __VA_ARGS__);break; \
371 case 4: stem##4(variable, __VA_ARGS__);break; \
372 case 8: stem##8(variable, __VA_ARGS__);break; \
373 default: \
374 __bad_size_call_parameter();break; \
375 } \
376} while (0)
377
378/*
379 * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
380 *
381 * Optimized manipulation for memory allocated through the per cpu
382 * allocator or for addresses of per cpu variables.
383 *
384 * These operation guarantee exclusivity of access for other operations
385 * on the *same* processor. The assumption is that per cpu data is only
386 * accessed by a single processor instance (the current one).
387 *
388 * The arch code can provide optimized implementation by defining macros
389 * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
390 * cpu atomic operations for 2 byte sized RMW actions. If arch code does
391 * not provide operations for a scalar size then the fallback in the
392 * generic code will be used.
393 *
394 * cmpxchg_double replaces two adjacent scalars at once. The first two
395 * parameters are per cpu variables which have to be of the same size. A
396 * truth value is returned to indicate success or failure (since a double
397 * register result is difficult to handle). There is very limited hardware
398 * support for these operations, so only certain sizes may work.
399 */
400
401/*
402 * Operations for contexts where we do not want to do any checks for
403 * preemptions. Unless strictly necessary, always use [__]this_cpu_*()
404 * instead.
405 *
406 * If there is no other protection through preempt disable and/or disabling
407 * interupts then one of these RMW operations can show unexpected behavior
408 * because the execution thread was rescheduled on another processor or an
409 * interrupt occurred and the same percpu variable was modified from the
410 * interrupt context.
411 */
412#define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
413#define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
414#define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
415#define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
416#define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
417#define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
418#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
419#define raw_cpu_cmpxchg(pcp, oval, nval) \
420 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
421#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
422 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
423
424#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
425#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
426#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
427#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
428#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
429#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
430
431/*
432 * Operations for contexts that are safe from preemption/interrupts. These
433 * operations verify that preemption is disabled.
434 */
435#define __this_cpu_read(pcp) \
436({ \
437 __this_cpu_preempt_check("read"); \
438 raw_cpu_read(pcp); \
439})
440
441#define __this_cpu_write(pcp, val) \
442({ \
443 __this_cpu_preempt_check("write"); \
444 raw_cpu_write(pcp, val); \
445})
446
447#define __this_cpu_add(pcp, val) \
448({ \
449 __this_cpu_preempt_check("add"); \
450 raw_cpu_add(pcp, val); \
451})
452
453#define __this_cpu_and(pcp, val) \
454({ \
455 __this_cpu_preempt_check("and"); \
456 raw_cpu_and(pcp, val); \
457})
458
459#define __this_cpu_or(pcp, val) \
460({ \
461 __this_cpu_preempt_check("or"); \
462 raw_cpu_or(pcp, val); \
463})
464
465#define __this_cpu_add_return(pcp, val) \
466({ \
467 __this_cpu_preempt_check("add_return"); \
468 raw_cpu_add_return(pcp, val); \
469})
470
471#define __this_cpu_xchg(pcp, nval) \
472({ \
473 __this_cpu_preempt_check("xchg"); \
474 raw_cpu_xchg(pcp, nval); \
475})
476
477#define __this_cpu_cmpxchg(pcp, oval, nval) \
478({ \
479 __this_cpu_preempt_check("cmpxchg"); \
480 raw_cpu_cmpxchg(pcp, oval, nval); \
481})
482
483#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
484({ __this_cpu_preempt_check("cmpxchg_double"); \
485 raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
486})
487
488#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
489#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
490#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
491#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
492#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
493#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
494
495/*
496 * Operations with implied preemption protection. These operations can be
497 * used without worrying about preemption. Note that interrupts may still
498 * occur while an operation is in progress and if the interrupt modifies
499 * the variable too then RMW actions may not be reliable.
500 */
501#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
502#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
503#define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
504#define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
505#define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
506#define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
507#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
508#define this_cpu_cmpxchg(pcp, oval, nval) \
509 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
510#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
511 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
512
513#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
514#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
515#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
516#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
517#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
518#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
519
520#endif /* __ASSEMBLY__ */
167#endif /* _LINUX_PERCPU_DEFS_H */ 521#endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 5d8920e23073..3dfbf237cd8f 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -57,11 +57,9 @@ struct percpu_ref {
57 atomic_t count; 57 atomic_t count;
58 /* 58 /*
59 * The low bit of the pointer indicates whether the ref is in percpu 59 * The low bit of the pointer indicates whether the ref is in percpu
60 * mode; if set, then get/put will manipulate the atomic_t (this is a 60 * mode; if set, then get/put will manipulate the atomic_t.
61 * hack because we need to keep the pointer around for
62 * percpu_ref_kill_rcu())
63 */ 61 */
64 unsigned __percpu *pcpu_count; 62 unsigned long pcpu_count_ptr;
65 percpu_ref_func_t *release; 63 percpu_ref_func_t *release;
66 percpu_ref_func_t *confirm_kill; 64 percpu_ref_func_t *confirm_kill;
67 struct rcu_head rcu; 65 struct rcu_head rcu;
@@ -69,7 +67,8 @@ struct percpu_ref {
69 67
70int __must_check percpu_ref_init(struct percpu_ref *ref, 68int __must_check percpu_ref_init(struct percpu_ref *ref,
71 percpu_ref_func_t *release); 69 percpu_ref_func_t *release);
72void percpu_ref_cancel_init(struct percpu_ref *ref); 70void percpu_ref_reinit(struct percpu_ref *ref);
71void percpu_ref_exit(struct percpu_ref *ref);
73void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 72void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
74 percpu_ref_func_t *confirm_kill); 73 percpu_ref_func_t *confirm_kill);
75 74
@@ -88,12 +87,28 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
88 return percpu_ref_kill_and_confirm(ref, NULL); 87 return percpu_ref_kill_and_confirm(ref, NULL);
89} 88}
90 89
91#define PCPU_STATUS_BITS 2
92#define PCPU_STATUS_MASK ((1 << PCPU_STATUS_BITS) - 1)
93#define PCPU_REF_PTR 0
94#define PCPU_REF_DEAD 1 90#define PCPU_REF_DEAD 1
95 91
96#define REF_STATUS(count) (((unsigned long) count) & PCPU_STATUS_MASK) 92/*
93 * Internal helper. Don't use outside percpu-refcount proper. The
94 * function doesn't return the pointer and let the caller test it for NULL
95 * because doing so forces the compiler to generate two conditional
96 * branches as it can't assume that @ref->pcpu_count is not NULL.
97 */
98static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
99 unsigned __percpu **pcpu_countp)
100{
101 unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
102
103 /* paired with smp_store_release() in percpu_ref_reinit() */
104 smp_read_barrier_depends();
105
106 if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
107 return false;
108
109 *pcpu_countp = (unsigned __percpu *)pcpu_ptr;
110 return true;
111}
97 112
98/** 113/**
99 * percpu_ref_get - increment a percpu refcount 114 * percpu_ref_get - increment a percpu refcount
@@ -107,9 +122,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
107 122
108 rcu_read_lock_sched(); 123 rcu_read_lock_sched();
109 124
110 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 125 if (__pcpu_ref_alive(ref, &pcpu_count))
111
112 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
113 this_cpu_inc(*pcpu_count); 126 this_cpu_inc(*pcpu_count);
114 else 127 else
115 atomic_inc(&ref->count); 128 atomic_inc(&ref->count);
@@ -133,9 +146,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
133 146
134 rcu_read_lock_sched(); 147 rcu_read_lock_sched();
135 148
136 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 149 if (__pcpu_ref_alive(ref, &pcpu_count)) {
137
138 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
139 this_cpu_inc(*pcpu_count); 150 this_cpu_inc(*pcpu_count);
140 ret = true; 151 ret = true;
141 } else { 152 } else {
@@ -168,9 +179,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
168 179
169 rcu_read_lock_sched(); 180 rcu_read_lock_sched();
170 181
171 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 182 if (__pcpu_ref_alive(ref, &pcpu_count)) {
172
173 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
174 this_cpu_inc(*pcpu_count); 183 this_cpu_inc(*pcpu_count);
175 ret = true; 184 ret = true;
176 } 185 }
@@ -193,9 +202,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
193 202
194 rcu_read_lock_sched(); 203 rcu_read_lock_sched();
195 204
196 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 205 if (__pcpu_ref_alive(ref, &pcpu_count))
197
198 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
199 this_cpu_dec(*pcpu_count); 206 this_cpu_dec(*pcpu_count);
200 else if (unlikely(atomic_dec_and_test(&ref->count))) 207 else if (unlikely(atomic_dec_and_test(&ref->count)))
201 ref->release(ref); 208 ref->release(ref);
@@ -203,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
203 rcu_read_unlock_sched(); 210 rcu_read_unlock_sched();
204} 211}
205 212
213/**
214 * percpu_ref_is_zero - test whether a percpu refcount reached zero
215 * @ref: percpu_ref to test
216 *
217 * Returns %true if @ref reached zero.
218 */
219static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
220{
221 unsigned __percpu *pcpu_count;
222
223 if (__pcpu_ref_alive(ref, &pcpu_count))
224 return false;
225 return !atomic_read(&ref->count);
226}
227
206#endif 228#endif
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 8419053d0f2e..6f61b61b7996 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -23,32 +23,6 @@
23 PERCPU_MODULE_RESERVE) 23 PERCPU_MODULE_RESERVE)
24#endif 24#endif
25 25
26/*
27 * Must be an lvalue. Since @var must be a simple identifier,
28 * we force a syntax error here if it isn't.
29 */
30#define get_cpu_var(var) (*({ \
31 preempt_disable(); \
32 this_cpu_ptr(&var); }))
33
34/*
35 * The weird & is necessary because sparse considers (void)(var) to be
36 * a direct dereference of percpu variable (var).
37 */
38#define put_cpu_var(var) do { \
39 (void)&(var); \
40 preempt_enable(); \
41} while (0)
42
43#define get_cpu_ptr(var) ({ \
44 preempt_disable(); \
45 this_cpu_ptr(var); })
46
47#define put_cpu_ptr(var) do { \
48 (void)(var); \
49 preempt_enable(); \
50} while (0)
51
52/* minimum unit size, also is the maximum supported allocation size */ 26/* minimum unit size, also is the maximum supported allocation size */
53#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 27#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
54 28
@@ -140,17 +114,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
140 pcpu_fc_populate_pte_fn_t populate_pte_fn); 114 pcpu_fc_populate_pte_fn_t populate_pte_fn);
141#endif 115#endif
142 116
143/*
144 * Use this to get to a cpu's version of the per-cpu object
145 * dynamically allocated. Non-atomic access to the current CPU's
146 * version should probably be combined with get_cpu()/put_cpu().
147 */
148#ifdef CONFIG_SMP
149#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
150#else
151#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
152#endif
153
154extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); 117extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
155extern bool is_kernel_percpu_address(unsigned long addr); 118extern bool is_kernel_percpu_address(unsigned long addr);
156 119
@@ -166,640 +129,4 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
166#define alloc_percpu(type) \ 129#define alloc_percpu(type) \
167 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type)) 130 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
168 131
169/*
170 * Branching function to split up a function into a set of functions that
171 * are called for different scalar sizes of the objects handled.
172 */
173
174extern void __bad_size_call_parameter(void);
175
176#ifdef CONFIG_DEBUG_PREEMPT
177extern void __this_cpu_preempt_check(const char *op);
178#else
179static inline void __this_cpu_preempt_check(const char *op) { }
180#endif
181
182#define __pcpu_size_call_return(stem, variable) \
183({ typeof(variable) pscr_ret__; \
184 __verify_pcpu_ptr(&(variable)); \
185 switch(sizeof(variable)) { \
186 case 1: pscr_ret__ = stem##1(variable);break; \
187 case 2: pscr_ret__ = stem##2(variable);break; \
188 case 4: pscr_ret__ = stem##4(variable);break; \
189 case 8: pscr_ret__ = stem##8(variable);break; \
190 default: \
191 __bad_size_call_parameter();break; \
192 } \
193 pscr_ret__; \
194})
195
196#define __pcpu_size_call_return2(stem, variable, ...) \
197({ \
198 typeof(variable) pscr2_ret__; \
199 __verify_pcpu_ptr(&(variable)); \
200 switch(sizeof(variable)) { \
201 case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
202 case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
203 case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
204 case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
205 default: \
206 __bad_size_call_parameter(); break; \
207 } \
208 pscr2_ret__; \
209})
210
211/*
212 * Special handling for cmpxchg_double. cmpxchg_double is passed two
213 * percpu variables. The first has to be aligned to a double word
214 * boundary and the second has to follow directly thereafter.
215 * We enforce this on all architectures even if they don't support
216 * a double cmpxchg instruction, since it's a cheap requirement, and it
217 * avoids breaking the requirement for architectures with the instruction.
218 */
219#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
220({ \
221 bool pdcrb_ret__; \
222 __verify_pcpu_ptr(&pcp1); \
223 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
224 VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
225 VM_BUG_ON((unsigned long)(&pcp2) != \
226 (unsigned long)(&pcp1) + sizeof(pcp1)); \
227 switch(sizeof(pcp1)) { \
228 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
229 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
230 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
231 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
232 default: \
233 __bad_size_call_parameter(); break; \
234 } \
235 pdcrb_ret__; \
236})
237
238#define __pcpu_size_call(stem, variable, ...) \
239do { \
240 __verify_pcpu_ptr(&(variable)); \
241 switch(sizeof(variable)) { \
242 case 1: stem##1(variable, __VA_ARGS__);break; \
243 case 2: stem##2(variable, __VA_ARGS__);break; \
244 case 4: stem##4(variable, __VA_ARGS__);break; \
245 case 8: stem##8(variable, __VA_ARGS__);break; \
246 default: \
247 __bad_size_call_parameter();break; \
248 } \
249} while (0)
250
251/*
252 * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
253 *
254 * Optimized manipulation for memory allocated through the per cpu
255 * allocator or for addresses of per cpu variables.
256 *
257 * These operation guarantee exclusivity of access for other operations
258 * on the *same* processor. The assumption is that per cpu data is only
259 * accessed by a single processor instance (the current one).
260 *
261 * The first group is used for accesses that must be done in a
262 * preemption safe way since we know that the context is not preempt
263 * safe. Interrupts may occur. If the interrupt modifies the variable
264 * too then RMW actions will not be reliable.
265 *
266 * The arch code can provide optimized functions in two ways:
267 *
268 * 1. Override the function completely. F.e. define this_cpu_add().
269 * The arch must then ensure that the various scalar format passed
270 * are handled correctly.
271 *
272 * 2. Provide functions for certain scalar sizes. F.e. provide
273 * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
274 * sized RMW actions. If arch code does not provide operations for
275 * a scalar size then the fallback in the generic code will be
276 * used.
277 */
278
279#define _this_cpu_generic_read(pcp) \
280({ typeof(pcp) ret__; \
281 preempt_disable(); \
282 ret__ = *this_cpu_ptr(&(pcp)); \
283 preempt_enable(); \
284 ret__; \
285})
286
287#ifndef this_cpu_read
288# ifndef this_cpu_read_1
289# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
290# endif
291# ifndef this_cpu_read_2
292# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
293# endif
294# ifndef this_cpu_read_4
295# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
296# endif
297# ifndef this_cpu_read_8
298# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
299# endif
300# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
301#endif
302
303#define _this_cpu_generic_to_op(pcp, val, op) \
304do { \
305 unsigned long flags; \
306 raw_local_irq_save(flags); \
307 *raw_cpu_ptr(&(pcp)) op val; \
308 raw_local_irq_restore(flags); \
309} while (0)
310
311#ifndef this_cpu_write
312# ifndef this_cpu_write_1
313# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
314# endif
315# ifndef this_cpu_write_2
316# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
317# endif
318# ifndef this_cpu_write_4
319# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
320# endif
321# ifndef this_cpu_write_8
322# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
323# endif
324# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
325#endif
326
327#ifndef this_cpu_add
328# ifndef this_cpu_add_1
329# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
330# endif
331# ifndef this_cpu_add_2
332# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
333# endif
334# ifndef this_cpu_add_4
335# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
336# endif
337# ifndef this_cpu_add_8
338# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
339# endif
340# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
341#endif
342
343#ifndef this_cpu_sub
344# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val))
345#endif
346
347#ifndef this_cpu_inc
348# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
349#endif
350
351#ifndef this_cpu_dec
352# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
353#endif
354
355#ifndef this_cpu_and
356# ifndef this_cpu_and_1
357# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
358# endif
359# ifndef this_cpu_and_2
360# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
361# endif
362# ifndef this_cpu_and_4
363# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
364# endif
365# ifndef this_cpu_and_8
366# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
367# endif
368# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
369#endif
370
371#ifndef this_cpu_or
372# ifndef this_cpu_or_1
373# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
374# endif
375# ifndef this_cpu_or_2
376# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
377# endif
378# ifndef this_cpu_or_4
379# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
380# endif
381# ifndef this_cpu_or_8
382# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
383# endif
384# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
385#endif
386
387#define _this_cpu_generic_add_return(pcp, val) \
388({ \
389 typeof(pcp) ret__; \
390 unsigned long flags; \
391 raw_local_irq_save(flags); \
392 raw_cpu_add(pcp, val); \
393 ret__ = raw_cpu_read(pcp); \
394 raw_local_irq_restore(flags); \
395 ret__; \
396})
397
398#ifndef this_cpu_add_return
399# ifndef this_cpu_add_return_1
400# define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
401# endif
402# ifndef this_cpu_add_return_2
403# define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
404# endif
405# ifndef this_cpu_add_return_4
406# define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
407# endif
408# ifndef this_cpu_add_return_8
409# define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
410# endif
411# define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
412#endif
413
414#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
415#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
416#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
417
418#define _this_cpu_generic_xchg(pcp, nval) \
419({ typeof(pcp) ret__; \
420 unsigned long flags; \
421 raw_local_irq_save(flags); \
422 ret__ = raw_cpu_read(pcp); \
423 raw_cpu_write(pcp, nval); \
424 raw_local_irq_restore(flags); \
425 ret__; \
426})
427
428#ifndef this_cpu_xchg
429# ifndef this_cpu_xchg_1
430# define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
431# endif
432# ifndef this_cpu_xchg_2
433# define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
434# endif
435# ifndef this_cpu_xchg_4
436# define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
437# endif
438# ifndef this_cpu_xchg_8
439# define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
440# endif
441# define this_cpu_xchg(pcp, nval) \
442 __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
443#endif
444
445#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
446({ \
447 typeof(pcp) ret__; \
448 unsigned long flags; \
449 raw_local_irq_save(flags); \
450 ret__ = raw_cpu_read(pcp); \
451 if (ret__ == (oval)) \
452 raw_cpu_write(pcp, nval); \
453 raw_local_irq_restore(flags); \
454 ret__; \
455})
456
457#ifndef this_cpu_cmpxchg
458# ifndef this_cpu_cmpxchg_1
459# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
460# endif
461# ifndef this_cpu_cmpxchg_2
462# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
463# endif
464# ifndef this_cpu_cmpxchg_4
465# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
466# endif
467# ifndef this_cpu_cmpxchg_8
468# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
469# endif
470# define this_cpu_cmpxchg(pcp, oval, nval) \
471 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
472#endif
473
474/*
475 * cmpxchg_double replaces two adjacent scalars at once. The first
476 * two parameters are per cpu variables which have to be of the same
477 * size. A truth value is returned to indicate success or failure
478 * (since a double register result is difficult to handle). There is
479 * very limited hardware support for these operations, so only certain
480 * sizes may work.
481 */
482#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
483({ \
484 int ret__; \
485 unsigned long flags; \
486 raw_local_irq_save(flags); \
487 ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
488 oval1, oval2, nval1, nval2); \
489 raw_local_irq_restore(flags); \
490 ret__; \
491})
492
493#ifndef this_cpu_cmpxchg_double
494# ifndef this_cpu_cmpxchg_double_1
495# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
496 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
497# endif
498# ifndef this_cpu_cmpxchg_double_2
499# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
500 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
501# endif
502# ifndef this_cpu_cmpxchg_double_4
503# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
504 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
505# endif
506# ifndef this_cpu_cmpxchg_double_8
507# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
508 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
509# endif
510# define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
511 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
512#endif
513
514/*
515 * Generic percpu operations for contexts where we do not want to do
516 * any checks for preemptiosn.
517 *
518 * If there is no other protection through preempt disable and/or
519 * disabling interupts then one of these RMW operations can show unexpected
520 * behavior because the execution thread was rescheduled on another processor
521 * or an interrupt occurred and the same percpu variable was modified from
522 * the interrupt context.
523 */
524#ifndef raw_cpu_read
525# ifndef raw_cpu_read_1
526# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
527# endif
528# ifndef raw_cpu_read_2
529# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
530# endif
531# ifndef raw_cpu_read_4
532# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
533# endif
534# ifndef raw_cpu_read_8
535# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
536# endif
537# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
538#endif
539
540#define raw_cpu_generic_to_op(pcp, val, op) \
541do { \
542 *raw_cpu_ptr(&(pcp)) op val; \
543} while (0)
544
545
546#ifndef raw_cpu_write
547# ifndef raw_cpu_write_1
548# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
549# endif
550# ifndef raw_cpu_write_2
551# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
552# endif
553# ifndef raw_cpu_write_4
554# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
555# endif
556# ifndef raw_cpu_write_8
557# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
558# endif
559# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
560#endif
561
562#ifndef raw_cpu_add
563# ifndef raw_cpu_add_1
564# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
565# endif
566# ifndef raw_cpu_add_2
567# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
568# endif
569# ifndef raw_cpu_add_4
570# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
571# endif
572# ifndef raw_cpu_add_8
573# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
574# endif
575# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
576#endif
577
578#ifndef raw_cpu_sub
579# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
580#endif
581
582#ifndef raw_cpu_inc
583# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
584#endif
585
586#ifndef raw_cpu_dec
587# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
588#endif
589
590#ifndef raw_cpu_and
591# ifndef raw_cpu_and_1
592# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
593# endif
594# ifndef raw_cpu_and_2
595# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
596# endif
597# ifndef raw_cpu_and_4
598# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
599# endif
600# ifndef raw_cpu_and_8
601# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
602# endif
603# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
604#endif
605
606#ifndef raw_cpu_or
607# ifndef raw_cpu_or_1
608# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
609# endif
610# ifndef raw_cpu_or_2
611# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
612# endif
613# ifndef raw_cpu_or_4
614# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
615# endif
616# ifndef raw_cpu_or_8
617# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
618# endif
619# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
620#endif
621
622#define raw_cpu_generic_add_return(pcp, val) \
623({ \
624 raw_cpu_add(pcp, val); \
625 raw_cpu_read(pcp); \
626})
627
628#ifndef raw_cpu_add_return
629# ifndef raw_cpu_add_return_1
630# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
631# endif
632# ifndef raw_cpu_add_return_2
633# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
634# endif
635# ifndef raw_cpu_add_return_4
636# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
637# endif
638# ifndef raw_cpu_add_return_8
639# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
640# endif
641# define raw_cpu_add_return(pcp, val) \
642 __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
643#endif
644
645#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
646#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
647#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
648
649#define raw_cpu_generic_xchg(pcp, nval) \
650({ typeof(pcp) ret__; \
651 ret__ = raw_cpu_read(pcp); \
652 raw_cpu_write(pcp, nval); \
653 ret__; \
654})
655
656#ifndef raw_cpu_xchg
657# ifndef raw_cpu_xchg_1
658# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
659# endif
660# ifndef raw_cpu_xchg_2
661# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
662# endif
663# ifndef raw_cpu_xchg_4
664# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
665# endif
666# ifndef raw_cpu_xchg_8
667# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
668# endif
669# define raw_cpu_xchg(pcp, nval) \
670 __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
671#endif
672
673#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
674({ \
675 typeof(pcp) ret__; \
676 ret__ = raw_cpu_read(pcp); \
677 if (ret__ == (oval)) \
678 raw_cpu_write(pcp, nval); \
679 ret__; \
680})
681
682#ifndef raw_cpu_cmpxchg
683# ifndef raw_cpu_cmpxchg_1
684# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
685# endif
686# ifndef raw_cpu_cmpxchg_2
687# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
688# endif
689# ifndef raw_cpu_cmpxchg_4
690# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
691# endif
692# ifndef raw_cpu_cmpxchg_8
693# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
694# endif
695# define raw_cpu_cmpxchg(pcp, oval, nval) \
696 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
697#endif
698
699#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
700({ \
701 int __ret = 0; \
702 if (raw_cpu_read(pcp1) == (oval1) && \
703 raw_cpu_read(pcp2) == (oval2)) { \
704 raw_cpu_write(pcp1, (nval1)); \
705 raw_cpu_write(pcp2, (nval2)); \
706 __ret = 1; \
707 } \
708 (__ret); \
709})
710
711#ifndef raw_cpu_cmpxchg_double
712# ifndef raw_cpu_cmpxchg_double_1
713# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
714 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
715# endif
716# ifndef raw_cpu_cmpxchg_double_2
717# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
718 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
719# endif
720# ifndef raw_cpu_cmpxchg_double_4
721# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
722 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
723# endif
724# ifndef raw_cpu_cmpxchg_double_8
725# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
726 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
727# endif
728# define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
729 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
730#endif
731
732/*
733 * Generic percpu operations for context that are safe from preemption/interrupts.
734 */
735#ifndef __this_cpu_read
736# define __this_cpu_read(pcp) \
737 (__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))
738#endif
739
740#ifndef __this_cpu_write
741# define __this_cpu_write(pcp, val) \
742do { __this_cpu_preempt_check("write"); \
743 __pcpu_size_call(raw_cpu_write_, (pcp), (val)); \
744} while (0)
745#endif
746
747#ifndef __this_cpu_add
748# define __this_cpu_add(pcp, val) \
749do { __this_cpu_preempt_check("add"); \
750 __pcpu_size_call(raw_cpu_add_, (pcp), (val)); \
751} while (0)
752#endif
753
754#ifndef __this_cpu_sub
755# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
756#endif
757
758#ifndef __this_cpu_inc
759# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
760#endif
761
762#ifndef __this_cpu_dec
763# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
764#endif
765
766#ifndef __this_cpu_and
767# define __this_cpu_and(pcp, val) \
768do { __this_cpu_preempt_check("and"); \
769 __pcpu_size_call(raw_cpu_and_, (pcp), (val)); \
770} while (0)
771
772#endif
773
774#ifndef __this_cpu_or
775# define __this_cpu_or(pcp, val) \
776do { __this_cpu_preempt_check("or"); \
777 __pcpu_size_call(raw_cpu_or_, (pcp), (val)); \
778} while (0)
779#endif
780
781#ifndef __this_cpu_add_return
782# define __this_cpu_add_return(pcp, val) \
783 (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))
784#endif
785
786#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
787#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
788#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
789
790#ifndef __this_cpu_xchg
791# define __this_cpu_xchg(pcp, nval) \
792 (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))
793#endif
794
795#ifndef __this_cpu_cmpxchg
796# define __this_cpu_cmpxchg(pcp, oval, nval) \
797 (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))
798#endif
799
800#ifndef __this_cpu_cmpxchg_double
801# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
802 (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))
803#endif
804
805#endif /* __LINUX_PERCPU_H */ 132#endif /* __LINUX_PERCPU_H */
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-15 11:03:28 -0400