aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-08-04 13:09:27 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-08-04 13:09:27 -0400
commitf2a84170ede80e4b80f636e3700ef4d4d5dc7d33 (patch)
tree68a51fd83da747173200e06b046cbeddd80251a4
parentc4c3f5fba01e189fb3618f09545abdb4cf8ec8ee (diff)
parent2d7227828e1475c7b272e55bd70c4cec8eea219a (diff)
Merge branch 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
Pull percpu updates from Tejun Heo: - Major reorganization of percpu header files which I think makes things a lot more readable and logical than before. - percpu-refcount is updated so that it requires explicit destruction and can be reinitialized if necessary. This was pulled into the block tree to replace the custom percpu refcnting implemented in blk-mq. - In the process, percpu and percpu-refcount got cleaned up a bit * 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: (21 commits) percpu-refcount: implement percpu_ref_reinit() and percpu_ref_is_zero() percpu-refcount: require percpu_ref to be exited explicitly percpu-refcount: use unsigned long for pcpu_count pointer percpu-refcount: add helpers for ->percpu_count accesses percpu-refcount: one bit is enough for REF_STATUS percpu-refcount, aio: use percpu_ref_cancel_init() in ioctx_alloc() workqueue: stronger test in process_one_work() workqueue: clear POOL_DISASSOCIATED in rebind_workers() percpu: Use ALIGN macro instead of hand coding alignment calculation percpu: invoke __verify_pcpu_ptr() from the generic part of accessors and operations percpu: preffity percpu header files percpu: use raw_cpu_*() to define __this_cpu_*() percpu: reorder macros in percpu header files percpu: move {raw|this}_cpu_*() definitions to include/linux/percpu-defs.h percpu: move generic {raw|this}_cpu_*_N() definitions to include/asm-generic/percpu.h percpu: only allow sized arch overrides for {raw|this}_cpu_*() ops percpu: reorganize include/linux/percpu-defs.h percpu: move accessors from include/linux/percpu.h to percpu-defs.h percpu: include/asm-generic/percpu.h should contain only arch-overridable parts percpu: introduce arch_raw_cpu_ptr() ...
Diffstat
-rw-r--r--arch/x86/include/asm/percpu.h3
-rw-r--r--drivers/target/target_core_tpg.c4
-rw-r--r--fs/aio.c6
-rw-r--r--include/asm-generic/percpu.h410
-rw-r--r--include/linux/percpu-defs.h380
-rw-r--r--include/linux/percpu-refcount.h64
-rw-r--r--include/linux/percpu.h673
-rw-r--r--kernel/cgroup.c8
-rw-r--r--kernel/workqueue.c6
-rw-r--r--lib/percpu-refcount.c86
-rw-r--r--mm/percpu.c3
11 files changed, 832 insertions, 811 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 851bcdc5db04..fd472181a1d0 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -52,10 +52,9 @@
52 * Compared to the generic __my_cpu_offset version, the following 52 * Compared to the generic __my_cpu_offset version, the following
53 * saves one instruction and avoids clobbering a temp register. 53 * saves one instruction and avoids clobbering a temp register.
54 */ 54 */
55#define raw_cpu_ptr(ptr) \ 55#define arch_raw_cpu_ptr(ptr) \
56({ \ 56({ \
57 unsigned long tcp_ptr__; \ 57 unsigned long tcp_ptr__; \
58 __verify_pcpu_ptr(ptr); \
59 asm volatile("add " __percpu_arg(1) ", %0" \ 58 asm volatile("add " __percpu_arg(1) ", %0" \
60 : "=r" (tcp_ptr__) \ 59 : "=r" (tcp_ptr__) \
61 : "m" (this_cpu_off), "0" (ptr)); \ 60 : "m" (this_cpu_off), "0" (ptr)); \
diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c
index c036595b17cf..fddfae61222f 100644
--- a/drivers/target/target_core_tpg.c
+++ b/drivers/target/target_core_tpg.c
@@ -825,7 +825,7 @@ int core_tpg_add_lun(
825 825
826 ret = core_dev_export(dev, tpg, lun); 826 ret = core_dev_export(dev, tpg, lun);
827 if (ret < 0) { 827 if (ret < 0) {
828 percpu_ref_cancel_init(&lun->lun_ref); 828 percpu_ref_exit(&lun->lun_ref);
829 return ret; 829 return ret;
830 } 830 }
831 831
@@ -880,5 +880,7 @@ int core_tpg_post_dellun(
880 lun->lun_status = TRANSPORT_LUN_STATUS_FREE; 880 lun->lun_status = TRANSPORT_LUN_STATUS_FREE;
881 spin_unlock(&tpg->tpg_lun_lock); 881 spin_unlock(&tpg->tpg_lun_lock);
882 882
883 percpu_ref_exit(&lun->lun_ref);
884
883 return 0; 885 return 0;
884} 886}
diff --git a/fs/aio.c b/fs/aio.c
index 1c9c5f0a9e2b..bd7ec2cc2674 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -506,6 +506,8 @@ static void free_ioctx(struct work_struct *work)
506 506
507 aio_free_ring(ctx); 507 aio_free_ring(ctx);
508 free_percpu(ctx->cpu); 508 free_percpu(ctx->cpu);
509 percpu_ref_exit(&ctx->reqs);
510 percpu_ref_exit(&ctx->users);
509 kmem_cache_free(kioctx_cachep, ctx); 511 kmem_cache_free(kioctx_cachep, ctx);
510} 512}
511 513
@@ -715,8 +717,8 @@ err_ctx:
715err: 717err:
716 mutex_unlock(&ctx->ring_lock); 718 mutex_unlock(&ctx->ring_lock);
717 free_percpu(ctx->cpu); 719 free_percpu(ctx->cpu);
718 free_percpu(ctx->reqs.pcpu_count); 720 percpu_ref_exit(&ctx->reqs);
719 free_percpu(ctx->users.pcpu_count); 721 percpu_ref_exit(&ctx->users);
720 kmem_cache_free(kioctx_cachep, ctx); 722 kmem_cache_free(kioctx_cachep, ctx);
721 pr_debug("error allocating ioctx %d\n", err); 723 pr_debug("error allocating ioctx %d\n", err);
722 return ERR_PTR(err); 724 return ERR_PTR(err);
diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 0703aa75b5e8..4d9f233c4ba8 100644
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -36,93 +36,385 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
36#endif 36#endif
37 37
38/* 38/*
39 * Add a offset to a pointer but keep the pointer as is. 39 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
40 * 40 * translations for raw_cpu_ptr().
41 * Only S390 provides its own means of moving the pointer.
42 */ 41 */
43#ifndef SHIFT_PERCPU_PTR 42#ifndef arch_raw_cpu_ptr
44/* Weird cast keeps both GCC and sparse happy. */ 43#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
45#define SHIFT_PERCPU_PTR(__p, __offset) ({ \
46 __verify_pcpu_ptr((__p)); \
47 RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
48})
49#endif 44#endif
50 45
51/* 46#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
52 * A percpu variable may point to a discarded regions. The following are 47extern void setup_per_cpu_areas(void);
53 * established ways to produce a usable pointer from the percpu variable
54 * offset.
55 */
56#define per_cpu(var, cpu) \
57 (*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
58
59#ifndef raw_cpu_ptr
60#define raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
61#endif 48#endif
62#ifdef CONFIG_DEBUG_PREEMPT 49
63#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset) 50#endif /* SMP */
51
52#ifndef PER_CPU_BASE_SECTION
53#ifdef CONFIG_SMP
54#define PER_CPU_BASE_SECTION ".data..percpu"
64#else 55#else
65#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) 56#define PER_CPU_BASE_SECTION ".data"
57#endif
66#endif 58#endif
67 59
68#define __get_cpu_var(var) (*this_cpu_ptr(&(var))) 60#ifndef PER_CPU_ATTRIBUTES
69#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var))) 61#define PER_CPU_ATTRIBUTES
62#endif
70 63
71#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 64#ifndef PER_CPU_DEF_ATTRIBUTES
72extern void setup_per_cpu_areas(void); 65#define PER_CPU_DEF_ATTRIBUTES
73#endif 66#endif
74 67
75#else /* ! SMP */ 68#define raw_cpu_generic_to_op(pcp, val, op) \
69do { \
70 *raw_cpu_ptr(&(pcp)) op val; \
71} while (0)
76 72
77#define VERIFY_PERCPU_PTR(__p) ({ \ 73#define raw_cpu_generic_add_return(pcp, val) \
78 __verify_pcpu_ptr((__p)); \ 74({ \
79 (typeof(*(__p)) __kernel __force *)(__p); \ 75 raw_cpu_add(pcp, val); \
76 raw_cpu_read(pcp); \
80}) 77})
81 78
82#define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var)))) 79#define raw_cpu_generic_xchg(pcp, nval) \
83#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) 80({ \
84#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var))) 81 typeof(pcp) __ret; \
85#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0) 82 __ret = raw_cpu_read(pcp); \
86#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr) 83 raw_cpu_write(pcp, nval); \
84 __ret; \
85})
87 86
88#endif /* SMP */ 87#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
88({ \
89 typeof(pcp) __ret; \
90 __ret = raw_cpu_read(pcp); \
91 if (__ret == (oval)) \
92 raw_cpu_write(pcp, nval); \
93 __ret; \
94})
89 95
90#ifndef PER_CPU_BASE_SECTION 96#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
91#ifdef CONFIG_SMP 97({ \
92#define PER_CPU_BASE_SECTION ".data..percpu" 98 int __ret = 0; \
93#else 99 if (raw_cpu_read(pcp1) == (oval1) && \
94#define PER_CPU_BASE_SECTION ".data" 100 raw_cpu_read(pcp2) == (oval2)) { \
101 raw_cpu_write(pcp1, nval1); \
102 raw_cpu_write(pcp2, nval2); \
103 __ret = 1; \
104 } \
105 (__ret); \
106})
107
108#define this_cpu_generic_read(pcp) \
109({ \
110 typeof(pcp) __ret; \
111 preempt_disable(); \
112 __ret = *this_cpu_ptr(&(pcp)); \
113 preempt_enable(); \
114 __ret; \
115})
116
117#define this_cpu_generic_to_op(pcp, val, op) \
118do { \
119 unsigned long __flags; \
120 raw_local_irq_save(__flags); \
121 *raw_cpu_ptr(&(pcp)) op val; \
122 raw_local_irq_restore(__flags); \
123} while (0)
124
125#define this_cpu_generic_add_return(pcp, val) \
126({ \
127 typeof(pcp) __ret; \
128 unsigned long __flags; \
129 raw_local_irq_save(__flags); \
130 raw_cpu_add(pcp, val); \
131 __ret = raw_cpu_read(pcp); \
132 raw_local_irq_restore(__flags); \
133 __ret; \
134})
135
136#define this_cpu_generic_xchg(pcp, nval) \
137({ \
138 typeof(pcp) __ret; \
139 unsigned long __flags; \
140 raw_local_irq_save(__flags); \
141 __ret = raw_cpu_read(pcp); \
142 raw_cpu_write(pcp, nval); \
143 raw_local_irq_restore(__flags); \
144 __ret; \
145})
146
147#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
148({ \
149 typeof(pcp) __ret; \
150 unsigned long __flags; \
151 raw_local_irq_save(__flags); \
152 __ret = raw_cpu_read(pcp); \
153 if (__ret == (oval)) \
154 raw_cpu_write(pcp, nval); \
155 raw_local_irq_restore(__flags); \
156 __ret; \
157})
158
159#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
160({ \
161 int __ret; \
162 unsigned long __flags; \
163 raw_local_irq_save(__flags); \
164 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
165 oval1, oval2, nval1, nval2); \
166 raw_local_irq_restore(__flags); \
167 __ret; \
168})
169
170#ifndef raw_cpu_read_1
171#define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
95#endif 172#endif
173#ifndef raw_cpu_read_2
174#define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
175#endif
176#ifndef raw_cpu_read_4
177#define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
178#endif
179#ifndef raw_cpu_read_8
180#define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
96#endif 181#endif
97 182
98#ifdef CONFIG_SMP 183#ifndef raw_cpu_write_1
184#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
185#endif
186#ifndef raw_cpu_write_2
187#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
188#endif
189#ifndef raw_cpu_write_4
190#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
191#endif
192#ifndef raw_cpu_write_8
193#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
194#endif
99 195
100#ifdef MODULE 196#ifndef raw_cpu_add_1
101#define PER_CPU_SHARED_ALIGNED_SECTION "" 197#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
102#define PER_CPU_ALIGNED_SECTION "" 198#endif
103#else 199#ifndef raw_cpu_add_2
104#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned" 200#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
105#define PER_CPU_ALIGNED_SECTION "..shared_aligned" 201#endif
202#ifndef raw_cpu_add_4
203#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
204#endif
205#ifndef raw_cpu_add_8
206#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
106#endif 207#endif
107#define PER_CPU_FIRST_SECTION "..first"
108 208
109#else 209#ifndef raw_cpu_and_1
210#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
211#endif
212#ifndef raw_cpu_and_2
213#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
214#endif
215#ifndef raw_cpu_and_4
216#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
217#endif
218#ifndef raw_cpu_and_8
219#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
220#endif
221
222#ifndef raw_cpu_or_1
223#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
224#endif
225#ifndef raw_cpu_or_2
226#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
227#endif
228#ifndef raw_cpu_or_4
229#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
230#endif
231#ifndef raw_cpu_or_8
232#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
233#endif
110 234
111#define PER_CPU_SHARED_ALIGNED_SECTION "" 235#ifndef raw_cpu_add_return_1
112#define PER_CPU_ALIGNED_SECTION "..shared_aligned" 236#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
113#define PER_CPU_FIRST_SECTION "" 237#endif
238#ifndef raw_cpu_add_return_2
239#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
240#endif
241#ifndef raw_cpu_add_return_4
242#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
243#endif
244#ifndef raw_cpu_add_return_8
245#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
246#endif
114 247
248#ifndef raw_cpu_xchg_1
249#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
250#endif
251#ifndef raw_cpu_xchg_2
252#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
253#endif
254#ifndef raw_cpu_xchg_4
255#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
256#endif
257#ifndef raw_cpu_xchg_8
258#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
115#endif 259#endif
116 260
117#ifndef PER_CPU_ATTRIBUTES 261#ifndef raw_cpu_cmpxchg_1
118#define PER_CPU_ATTRIBUTES 262#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
263 raw_cpu_generic_cmpxchg(pcp, oval, nval)
264#endif
265#ifndef raw_cpu_cmpxchg_2
266#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
267 raw_cpu_generic_cmpxchg(pcp, oval, nval)
268#endif
269#ifndef raw_cpu_cmpxchg_4
270#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
271 raw_cpu_generic_cmpxchg(pcp, oval, nval)
272#endif
273#ifndef raw_cpu_cmpxchg_8
274#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
275 raw_cpu_generic_cmpxchg(pcp, oval, nval)
119#endif 276#endif
120 277
121#ifndef PER_CPU_DEF_ATTRIBUTES 278#ifndef raw_cpu_cmpxchg_double_1
122#define PER_CPU_DEF_ATTRIBUTES 279#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
280 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
281#endif
282#ifndef raw_cpu_cmpxchg_double_2
283#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
284 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
285#endif
286#ifndef raw_cpu_cmpxchg_double_4
287#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
288 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
289#endif
290#ifndef raw_cpu_cmpxchg_double_8
291#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
292 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
293#endif
294
295#ifndef this_cpu_read_1
296#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
297#endif
298#ifndef this_cpu_read_2
299#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
300#endif
301#ifndef this_cpu_read_4
302#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
303#endif
304#ifndef this_cpu_read_8
305#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
123#endif 306#endif
124 307
125/* Keep until we have removed all uses of __this_cpu_ptr */ 308#ifndef this_cpu_write_1
126#define __this_cpu_ptr raw_cpu_ptr 309#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
310#endif
311#ifndef this_cpu_write_2
312#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
313#endif
314#ifndef this_cpu_write_4
315#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
316#endif
317#ifndef this_cpu_write_8
318#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
319#endif
320
321#ifndef this_cpu_add_1
322#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
323#endif
324#ifndef this_cpu_add_2
325#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
326#endif
327#ifndef this_cpu_add_4
328#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
329#endif
330#ifndef this_cpu_add_8
331#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
332#endif
333
334#ifndef this_cpu_and_1
335#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
336#endif
337#ifndef this_cpu_and_2
338#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
339#endif
340#ifndef this_cpu_and_4
341#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
342#endif
343#ifndef this_cpu_and_8
344#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
345#endif
346
347#ifndef this_cpu_or_1
348#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
349#endif
350#ifndef this_cpu_or_2
351#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
352#endif
353#ifndef this_cpu_or_4
354#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
355#endif
356#ifndef this_cpu_or_8
357#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
358#endif
359
360#ifndef this_cpu_add_return_1
361#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
362#endif
363#ifndef this_cpu_add_return_2
364#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
365#endif
366#ifndef this_cpu_add_return_4
367#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
368#endif
369#ifndef this_cpu_add_return_8
370#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
371#endif
372
373#ifndef this_cpu_xchg_1
374#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
375#endif
376#ifndef this_cpu_xchg_2
377#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
378#endif
379#ifndef this_cpu_xchg_4
380#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
381#endif
382#ifndef this_cpu_xchg_8
383#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
384#endif
385
386#ifndef this_cpu_cmpxchg_1
387#define this_cpu_cmpxchg_1(pcp, oval, nval) \
388 this_cpu_generic_cmpxchg(pcp, oval, nval)
389#endif
390#ifndef this_cpu_cmpxchg_2
391#define this_cpu_cmpxchg_2(pcp, oval, nval) \
392 this_cpu_generic_cmpxchg(pcp, oval, nval)
393#endif
394#ifndef this_cpu_cmpxchg_4
395#define this_cpu_cmpxchg_4(pcp, oval, nval) \
396 this_cpu_generic_cmpxchg(pcp, oval, nval)
397#endif
398#ifndef this_cpu_cmpxchg_8
399#define this_cpu_cmpxchg_8(pcp, oval, nval) \
400 this_cpu_generic_cmpxchg(pcp, oval, nval)
401#endif
402
403#ifndef this_cpu_cmpxchg_double_1
404#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
405 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
406#endif
407#ifndef this_cpu_cmpxchg_double_2
408#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
409 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
410#endif
411#ifndef this_cpu_cmpxchg_double_4
412#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
413 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
414#endif
415#ifndef this_cpu_cmpxchg_double_8
416#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
417 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
418#endif
127 419
128#endif /* _ASM_GENERIC_PERCPU_H_ */ 420#endif /* _ASM_GENERIC_PERCPU_H_ */
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index dec01d6c3f80..cfd56046ecec 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -1,6 +1,40 @@
1/*
2 * linux/percpu-defs.h - basic definitions for percpu areas
3 *
4 * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
5 *
6 * This file is separate from linux/percpu.h to avoid cyclic inclusion
7 * dependency from arch header files. Only to be included from
8 * asm/percpu.h.
9 *
10 * This file includes macros necessary to declare percpu sections and
11 * variables, and definitions of percpu accessors and operations. It
12 * should provide enough percpu features to arch header files even when
13 * they can only include asm/percpu.h to avoid cyclic inclusion dependency.
14 */
15
1#ifndef _LINUX_PERCPU_DEFS_H 16#ifndef _LINUX_PERCPU_DEFS_H
2#define _LINUX_PERCPU_DEFS_H 17#define _LINUX_PERCPU_DEFS_H
3 18
19#ifdef CONFIG_SMP
20
21#ifdef MODULE
22#define PER_CPU_SHARED_ALIGNED_SECTION ""
23#define PER_CPU_ALIGNED_SECTION ""
24#else
25#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
26#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
27#endif
28#define PER_CPU_FIRST_SECTION "..first"
29
30#else
31
32#define PER_CPU_SHARED_ALIGNED_SECTION ""
33#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
34#define PER_CPU_FIRST_SECTION ""
35
36#endif
37
4/* 38/*
5 * Base implementations of per-CPU variable declarations and definitions, where 39 * Base implementations of per-CPU variable declarations and definitions, where
6 * the section in which the variable is to be placed is provided by the 40 * the section in which the variable is to be placed is provided by the
@@ -19,19 +53,6 @@
19 __attribute__((section(".discard"), unused)) 53 __attribute__((section(".discard"), unused))
20 54
21/* 55/*
22 * Macro which verifies @ptr is a percpu pointer without evaluating
23 * @ptr. This is to be used in percpu accessors to verify that the
24 * input parameter is a percpu pointer.
25 *
26 * + 0 is required in order to convert the pointer type from a
27 * potential array type to a pointer to a single item of the array.
28 */
29#define __verify_pcpu_ptr(ptr) do { \
30 const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
31 (void)__vpp_verify; \
32} while (0)
33
34/*
35 * s390 and alpha modules require percpu variables to be defined as 56 * s390 and alpha modules require percpu variables to be defined as
36 * weak to force the compiler to generate GOT based external 57 * weak to force the compiler to generate GOT based external
37 * references for them. This is necessary because percpu sections 58 * references for them. This is necessary because percpu sections
@@ -164,4 +185,337 @@
164#define EXPORT_PER_CPU_SYMBOL_GPL(var) 185#define EXPORT_PER_CPU_SYMBOL_GPL(var)
165#endif 186#endif
166 187
188/*
189 * Accessors and operations.
190 */
191#ifndef __ASSEMBLY__
192
193/*
194 * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
195 * @ptr and is invoked once before a percpu area is accessed by all
196 * accessors and operations. This is performed in the generic part of
197 * percpu and arch overrides don't need to worry about it; however, if an
198 * arch wants to implement an arch-specific percpu accessor or operation,
199 * it may use __verify_pcpu_ptr() to verify the parameters.
200 *
201 * + 0 is required in order to convert the pointer type from a
202 * potential array type to a pointer to a single item of the array.
203 */
204#define __verify_pcpu_ptr(ptr) \
205do { \
206 const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
207 (void)__vpp_verify; \
208} while (0)
209
210#ifdef CONFIG_SMP
211
212/*
213 * Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
214 * to prevent the compiler from making incorrect assumptions about the
215 * pointer value. The weird cast keeps both GCC and sparse happy.
216 */
217#define SHIFT_PERCPU_PTR(__p, __offset) \
218 RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
219
220#define per_cpu_ptr(ptr, cpu) \
221({ \
222 __verify_pcpu_ptr(ptr); \
223 SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
224})
225
226#define raw_cpu_ptr(ptr) \
227({ \
228 __verify_pcpu_ptr(ptr); \
229 arch_raw_cpu_ptr(ptr); \
230})
231
232#ifdef CONFIG_DEBUG_PREEMPT
233#define this_cpu_ptr(ptr) \
234({ \
235 __verify_pcpu_ptr(ptr); \
236 SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
237})
238#else
239#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
240#endif
241
242#else /* CONFIG_SMP */
243
244#define VERIFY_PERCPU_PTR(__p) \
245({ \
246 __verify_pcpu_ptr(__p); \
247 (typeof(*(__p)) __kernel __force *)(__p); \
248})
249
250#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
251#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
252#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
253
254#endif /* CONFIG_SMP */
255
256#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
257#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
258#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
259
260/* keep until we have removed all uses of __this_cpu_ptr */
261#define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
262
263/*
264 * Must be an lvalue. Since @var must be a simple identifier,
265 * we force a syntax error here if it isn't.
266 */
267#define get_cpu_var(var) \
268(*({ \
269 preempt_disable(); \
270 this_cpu_ptr(&var); \
271}))
272
273/*
274 * The weird & is necessary because sparse considers (void)(var) to be
275 * a direct dereference of percpu variable (var).
276 */
277#define put_cpu_var(var) \
278do { \
279 (void)&(var); \
280 preempt_enable(); \
281} while (0)
282
283#define get_cpu_ptr(var) \
284({ \
285 preempt_disable(); \
286 this_cpu_ptr(var); \
287})
288
289#define put_cpu_ptr(var) \
290do { \
291 (void)(var); \
292 preempt_enable(); \
293} while (0)
294
295/*
296 * Branching function to split up a function into a set of functions that
297 * are called for different scalar sizes of the objects handled.
298 */
299
300extern void __bad_size_call_parameter(void);
301
302#ifdef CONFIG_DEBUG_PREEMPT
303extern void __this_cpu_preempt_check(const char *op);
304#else
305static inline void __this_cpu_preempt_check(const char *op) { }
306#endif
307
308#define __pcpu_size_call_return(stem, variable) \
309({ \
310 typeof(variable) pscr_ret__; \
311 __verify_pcpu_ptr(&(variable)); \
312 switch(sizeof(variable)) { \
313 case 1: pscr_ret__ = stem##1(variable); break; \
314 case 2: pscr_ret__ = stem##2(variable); break; \
315 case 4: pscr_ret__ = stem##4(variable); break; \
316 case 8: pscr_ret__ = stem##8(variable); break; \
317 default: \
318 __bad_size_call_parameter(); break; \
319 } \
320 pscr_ret__; \
321})
322
323#define __pcpu_size_call_return2(stem, variable, ...) \
324({ \
325 typeof(variable) pscr2_ret__; \
326 __verify_pcpu_ptr(&(variable)); \
327 switch(sizeof(variable)) { \
328 case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
329 case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
330 case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
331 case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
332 default: \
333 __bad_size_call_parameter(); break; \
334 } \
335 pscr2_ret__; \
336})
337
338/*
339 * Special handling for cmpxchg_double. cmpxchg_double is passed two
340 * percpu variables. The first has to be aligned to a double word
341 * boundary and the second has to follow directly thereafter.
342 * We enforce this on all architectures even if they don't support
343 * a double cmpxchg instruction, since it's a cheap requirement, and it
344 * avoids breaking the requirement for architectures with the instruction.
345 */
346#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
347({ \
348 bool pdcrb_ret__; \
349 __verify_pcpu_ptr(&(pcp1)); \
350 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
351 VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
352 VM_BUG_ON((unsigned long)(&(pcp2)) != \
353 (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
354 switch(sizeof(pcp1)) { \
355 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
356 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
357 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
358 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
359 default: \
360 __bad_size_call_parameter(); break; \
361 } \
362 pdcrb_ret__; \
363})
364
365#define __pcpu_size_call(stem, variable, ...) \
366do { \
367 __verify_pcpu_ptr(&(variable)); \
368 switch(sizeof(variable)) { \
369 case 1: stem##1(variable, __VA_ARGS__);break; \
370 case 2: stem##2(variable, __VA_ARGS__);break; \
371 case 4: stem##4(variable, __VA_ARGS__);break; \
372 case 8: stem##8(variable, __VA_ARGS__);break; \
373 default: \
374 __bad_size_call_parameter();break; \
375 } \
376} while (0)
377
378/*
379 * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
380 *
381 * Optimized manipulation for memory allocated through the per cpu
382 * allocator or for addresses of per cpu variables.
383 *
384 * These operation guarantee exclusivity of access for other operations
385 * on the *same* processor. The assumption is that per cpu data is only
386 * accessed by a single processor instance (the current one).
387 *
388 * The arch code can provide optimized implementation by defining macros
389 * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
390 * cpu atomic operations for 2 byte sized RMW actions. If arch code does
391 * not provide operations for a scalar size then the fallback in the
392 * generic code will be used.
393 *
394 * cmpxchg_double replaces two adjacent scalars at once. The first two
395 * parameters are per cpu variables which have to be of the same size. A
396 * truth value is returned to indicate success or failure (since a double
397 * register result is difficult to handle). There is very limited hardware
398 * support for these operations, so only certain sizes may work.
399 */
400
401/*
402 * Operations for contexts where we do not want to do any checks for
403 * preemptions. Unless strictly necessary, always use [__]this_cpu_*()
404 * instead.
405 *
406 * If there is no other protection through preempt disable and/or disabling
407 * interupts then one of these RMW operations can show unexpected behavior
408 * because the execution thread was rescheduled on another processor or an
409 * interrupt occurred and the same percpu variable was modified from the
410 * interrupt context.
411 */
412#define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
413#define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
414#define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
415#define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
416#define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
417#define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
418#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
419#define raw_cpu_cmpxchg(pcp, oval, nval) \
420 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
421#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
422 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
423
424#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
425#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
426#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
427#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
428#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
429#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
430
431/*
432 * Operations for contexts that are safe from preemption/interrupts. These
433 * operations verify that preemption is disabled.
434 */
435#define __this_cpu_read(pcp) \
436({ \
437 __this_cpu_preempt_check("read"); \
438 raw_cpu_read(pcp); \
439})
440
441#define __this_cpu_write(pcp, val) \
442({ \
443 __this_cpu_preempt_check("write"); \
444 raw_cpu_write(pcp, val); \
445})
446
447#define __this_cpu_add(pcp, val) \
448({ \
449 __this_cpu_preempt_check("add"); \
450 raw_cpu_add(pcp, val); \
451})
452
453#define __this_cpu_and(pcp, val) \
454({ \
455 __this_cpu_preempt_check("and"); \
456 raw_cpu_and(pcp, val); \
457})
458
459#define __this_cpu_or(pcp, val) \
460({ \
461 __this_cpu_preempt_check("or"); \
462 raw_cpu_or(pcp, val); \
463})
464
465#define __this_cpu_add_return(pcp, val) \
466({ \
467 __this_cpu_preempt_check("add_return"); \
468 raw_cpu_add_return(pcp, val); \
469})
470
471#define __this_cpu_xchg(pcp, nval) \
472({ \
473 __this_cpu_preempt_check("xchg"); \
474 raw_cpu_xchg(pcp, nval); \
475})
476
477#define __this_cpu_cmpxchg(pcp, oval, nval) \
478({ \
479 __this_cpu_preempt_check("cmpxchg"); \
480 raw_cpu_cmpxchg(pcp, oval, nval); \
481})
482
483#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
484({ __this_cpu_preempt_check("cmpxchg_double"); \
485 raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
486})
487
488#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
489#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
490#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
491#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
492#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
493#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
494
495/*
496 * Operations with implied preemption protection. These operations can be
497 * used without worrying about preemption. Note that interrupts may still
498 * occur while an operation is in progress and if the interrupt modifies
499 * the variable too then RMW actions may not be reliable.
500 */
501#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
502#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
503#define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
504#define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
505#define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
506#define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
507#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
508#define this_cpu_cmpxchg(pcp, oval, nval) \
509 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
510#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
511 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
512
513#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
514#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
515#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
516#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
517#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
518#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
519
520#endif /* __ASSEMBLY__ */
167#endif /* _LINUX_PERCPU_DEFS_H */ 521#endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 5d8920e23073..3dfbf237cd8f 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -57,11 +57,9 @@ struct percpu_ref {
57 atomic_t count; 57 atomic_t count;
58 /* 58 /*
59 * The low bit of the pointer indicates whether the ref is in percpu 59 * The low bit of the pointer indicates whether the ref is in percpu
60 * mode; if set, then get/put will manipulate the atomic_t (this is a 60 * mode; if set, then get/put will manipulate the atomic_t.
61 * hack because we need to keep the pointer around for
62 * percpu_ref_kill_rcu())
63 */ 61 */
64 unsigned __percpu *pcpu_count; 62 unsigned long pcpu_count_ptr;
65 percpu_ref_func_t *release; 63 percpu_ref_func_t *release;
66 percpu_ref_func_t *confirm_kill; 64 percpu_ref_func_t *confirm_kill;
67 struct rcu_head rcu; 65 struct rcu_head rcu;
@@ -69,7 +67,8 @@ struct percpu_ref {
69 67
70int __must_check percpu_ref_init(struct percpu_ref *ref, 68int __must_check percpu_ref_init(struct percpu_ref *ref,
71 percpu_ref_func_t *release); 69 percpu_ref_func_t *release);
72void percpu_ref_cancel_init(struct percpu_ref *ref); 70void percpu_ref_reinit(struct percpu_ref *ref);
71void percpu_ref_exit(struct percpu_ref *ref);
73void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 72void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
74 percpu_ref_func_t *confirm_kill); 73 percpu_ref_func_t *confirm_kill);
75 74
@@ -88,12 +87,28 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
88 return percpu_ref_kill_and_confirm(ref, NULL); 87 return percpu_ref_kill_and_confirm(ref, NULL);
89} 88}
90 89
91#define PCPU_STATUS_BITS 2
92#define PCPU_STATUS_MASK ((1 << PCPU_STATUS_BITS) - 1)
93#define PCPU_REF_PTR 0
94#define PCPU_REF_DEAD 1 90#define PCPU_REF_DEAD 1
95 91
96#define REF_STATUS(count) (((unsigned long) count) & PCPU_STATUS_MASK) 92/*
93 * Internal helper. Don't use outside percpu-refcount proper. The
94 * function doesn't return the pointer and let the caller test it for NULL
95 * because doing so forces the compiler to generate two conditional
96 * branches as it can't assume that @ref->pcpu_count is not NULL.
97 */
98static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
99 unsigned __percpu **pcpu_countp)
100{
101 unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
102
103 /* paired with smp_store_release() in percpu_ref_reinit() */
104 smp_read_barrier_depends();
105
106 if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
107 return false;
108
109 *pcpu_countp = (unsigned __percpu *)pcpu_ptr;
110 return true;
111}
97 112
98/** 113/**
99 * percpu_ref_get - increment a percpu refcount 114 * percpu_ref_get - increment a percpu refcount
@@ -107,9 +122,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
107 122
108 rcu_read_lock_sched(); 123 rcu_read_lock_sched();
109 124
110 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 125 if (__pcpu_ref_alive(ref, &pcpu_count))
111
112 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
113 this_cpu_inc(*pcpu_count); 126 this_cpu_inc(*pcpu_count);
114 else 127 else
115 atomic_inc(&ref->count); 128 atomic_inc(&ref->count);
@@ -133,9 +146,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
133 146
134 rcu_read_lock_sched(); 147 rcu_read_lock_sched();
135 148
136 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 149 if (__pcpu_ref_alive(ref, &pcpu_count)) {
137
138 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
139 this_cpu_inc(*pcpu_count); 150 this_cpu_inc(*pcpu_count);
140 ret = true; 151 ret = true;
141 } else { 152 } else {
@@ -168,9 +179,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
168 179
169 rcu_read_lock_sched(); 180 rcu_read_lock_sched();
170 181
171 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 182 if (__pcpu_ref_alive(ref, &pcpu_count)) {
172
173 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
174 this_cpu_inc(*pcpu_count); 183 this_cpu_inc(*pcpu_count);
175 ret = true; 184 ret = true;
176 } 185 }
@@ -193,9 +202,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
193 202
194 rcu_read_lock_sched(); 203 rcu_read_lock_sched();
195 204
196 pcpu_count = ACCESS_ONCE(ref->pcpu_count); 205 if (__pcpu_ref_alive(ref, &pcpu_count))
197
198 if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
199 this_cpu_dec(*pcpu_count); 206 this_cpu_dec(*pcpu_count);
200 else if (unlikely(atomic_dec_and_test(&ref->count))) 207 else if (unlikely(atomic_dec_and_test(&ref->count)))
201 ref->release(ref); 208 ref->release(ref);
@@ -203,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
203 rcu_read_unlock_sched(); 210 rcu_read_unlock_sched();
204} 211}
205 212
213/**
214 * percpu_ref_is_zero - test whether a percpu refcount reached zero
215 * @ref: percpu_ref to test
216 *
217 * Returns %true if @ref reached zero.
218 */
219static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
220{
221 unsigned __percpu *pcpu_count;
222
223 if (__pcpu_ref_alive(ref, &pcpu_count))
224 return false;
225 return !atomic_read(&ref->count);
226}
227
206#endif 228#endif
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 8419053d0f2e..6f61b61b7996 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -23,32 +23,6 @@
23 PERCPU_MODULE_RESERVE) 23 PERCPU_MODULE_RESERVE)
24#endif 24#endif
25 25
26/*
27 * Must be an lvalue. Since @var must be a simple identifier,
28 * we force a syntax error here if it isn't.
29 */
30#define get_cpu_var(var) (*({ \
31 preempt_disable(); \
32 this_cpu_ptr(&var); }))
33
34/*
35 * The weird & is necessary because sparse considers (void)(var) to be
36 * a direct dereference of percpu variable (var).
37 */
38#define put_cpu_var(var) do { \
39 (void)&(var); \
40 preempt_enable(); \
41} while (0)
42
43#define get_cpu_ptr(var) ({ \
44 preempt_disable(); \
45 this_cpu_ptr(var); })
46
47#define put_cpu_ptr(var) do { \
48 (void)(var); \
49 preempt_enable(); \
50} while (0)
51
52/* minimum unit size, also is the maximum supported allocation size */ 26/* minimum unit size, also is the maximum supported allocation size */
53#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 27#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
54 28
@@ -140,17 +114,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
140 pcpu_fc_populate_pte_fn_t populate_pte_fn); 114 pcpu_fc_populate_pte_fn_t populate_pte_fn);
141#endif 115#endif
142 116
143/*
144 * Use this to get to a cpu's version of the per-cpu object
145 * dynamically allocated. Non-atomic access to the current CPU's
146 * version should probably be combined with get_cpu()/put_cpu().
147 */
148#ifdef CONFIG_SMP
149#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
150#else
151#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
152#endif
153
154extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); 117extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
155extern bool is_kernel_percpu_address(unsigned long addr); 118extern bool is_kernel_percpu_address(unsigned long addr);
156 119
@@ -166,640 +129,4 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
166#define alloc_percpu(type) \ 129#define alloc_percpu(type) \
167 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type)) 130 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
168 131
169/*
170 * Branching function to split up a function into a set of functions that
171 * are called for different scalar sizes of the objects handled.
172 */
173
174extern void __bad_size_call_parameter(void);
175
176#ifdef CONFIG_DEBUG_PREEMPT
177extern void __this_cpu_preempt_check(const char *op);
178#else
179static inline void __this_cpu_preempt_check(const char *op) { }
180#endif
181
182#define __pcpu_size_call_return(stem, variable) \
183({ typeof(variable) pscr_ret__; \
184 __verify_pcpu_ptr(&(variable)); \
185 switch(sizeof(variable)) { \
186 case 1: pscr_ret__ = stem##1(variable);break; \
187 case 2: pscr_ret__ = stem##2(variable);break; \
188 case 4: pscr_ret__ = stem##4(variable);break; \
189 case 8: pscr_ret__ = stem##8(variable);break; \
190 default: \
191 __bad_size_call_parameter();break; \
192 } \
193 pscr_ret__; \
194})
195
196#define __pcpu_size_call_return2(stem, variable, ...) \
197({ \
198 typeof(variable) pscr2_ret__; \
199 __verify_pcpu_ptr(&(variable)); \
200 switch(sizeof(variable)) { \
201 case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
202 case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
203 case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
204 case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
205 default: \
206 __bad_size_call_parameter(); break; \
207 } \
208 pscr2_ret__; \
209})
210
211/*
212 * Special handling for cmpxchg_double. cmpxchg_double is passed two
213 * percpu variables. The first has to be aligned to a double word
214 * boundary and the second has to follow directly thereafter.
215 * We enforce this on all architectures even if they don't support
216 * a double cmpxchg instruction, since it's a cheap requirement, and it
217 * avoids breaking the requirement for architectures with the instruction.
218 */
219#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
220({ \
221 bool pdcrb_ret__; \
222 __verify_pcpu_ptr(&pcp1); \
223 BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
224 VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
225 VM_BUG_ON((unsigned long)(&pcp2) != \
226 (unsigned long)(&pcp1) + sizeof(pcp1)); \
227 switch(sizeof(pcp1)) { \
228 case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
229 case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
230 case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
231 case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
232 default: \
233 __bad_size_call_parameter(); break; \
234 } \
235 pdcrb_ret__; \
236})
237
238#define __pcpu_size_call(stem, variable, ...) \
239do { \
240 __verify_pcpu_ptr(&(variable)); \
241 switch(sizeof(variable)) { \
242 case 1: stem##1(variable, __VA_ARGS__);break; \
243 case 2: stem##2(variable, __VA_ARGS__);break; \
244 case 4: stem##4(variable, __VA_ARGS__);break; \
245 case 8: stem##8(variable, __VA_ARGS__);break; \
246 default: \
247 __bad_size_call_parameter();break; \
248 } \
249} while (0)
250
251/*
252 * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
253 *
254 * Optimized manipulation for memory allocated through the per cpu
255 * allocator or for addresses of per cpu variables.
256 *
257 * These operation guarantee exclusivity of access for other operations
258 * on the *same* processor. The assumption is that per cpu data is only
259 * accessed by a single processor instance (the current one).
260 *
261 * The first group is used for accesses that must be done in a
262 * preemption safe way since we know that the context is not preempt
263 * safe. Interrupts may occur. If the interrupt modifies the variable
264 * too then RMW actions will not be reliable.
265 *
266 * The arch code can provide optimized functions in two ways:
267 *
268 * 1. Override the function completely. F.e. define this_cpu_add().
269 * The arch must then ensure that the various scalar format passed
270 * are handled correctly.
271 *
272 * 2. Provide functions for certain scalar sizes. F.e. provide
273 * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
274 * sized RMW actions. If arch code does not provide operations for
275 * a scalar size then the fallback in the generic code will be
276 * used.
277 */
278
279#define _this_cpu_generic_read(pcp) \
280({ typeof(pcp) ret__; \
281 preempt_disable(); \
282 ret__ = *this_cpu_ptr(&(pcp)); \
283 preempt_enable(); \
284 ret__; \
285})
286
287#ifndef this_cpu_read
288# ifndef this_cpu_read_1
289# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
290# endif
291# ifndef this_cpu_read_2
292# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
293# endif
294# ifndef this_cpu_read_4
295# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
296# endif
297# ifndef this_cpu_read_8
298# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
299# endif
300# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
301#endif
302
303#define _this_cpu_generic_to_op(pcp, val, op) \
304do { \
305 unsigned long flags; \
306 raw_local_irq_save(flags); \
307 *raw_cpu_ptr(&(pcp)) op val; \
308 raw_local_irq_restore(flags); \
309} while (0)
310
311#ifndef this_cpu_write
312# ifndef this_cpu_write_1
313# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
314# endif
315# ifndef this_cpu_write_2
316# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
317# endif
318# ifndef this_cpu_write_4
319# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
320# endif
321# ifndef this_cpu_write_8
322# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
323# endif
324# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
325#endif
326
327#ifndef this_cpu_add
328# ifndef this_cpu_add_1
329# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
330# endif
331# ifndef this_cpu_add_2
332# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
333# endif
334# ifndef this_cpu_add_4
335# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
336# endif
337# ifndef this_cpu_add_8
338# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
339# endif
340# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
341#endif
342
343#ifndef this_cpu_sub
344# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val))
345#endif
346
347#ifndef this_cpu_inc
348# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
349#endif
350
351#ifndef this_cpu_dec
352# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
353#endif
354
355#ifndef this_cpu_and
356# ifndef this_cpu_and_1
357# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
358# endif
359# ifndef this_cpu_and_2
360# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
361# endif
362# ifndef this_cpu_and_4
363# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
364# endif
365# ifndef this_cpu_and_8
366# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
367# endif
368# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
369#endif
370
371#ifndef this_cpu_or
372# ifndef this_cpu_or_1
373# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
374# endif
375# ifndef this_cpu_or_2
376# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
377# endif
378# ifndef this_cpu_or_4
379# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
380# endif
381# ifndef this_cpu_or_8
382# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
383# endif
384# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
385#endif
386
387#define _this_cpu_generic_add_return(pcp, val) \
388({ \
389 typeof(pcp) ret__; \
390 unsigned long flags; \
391 raw_local_irq_save(flags); \
392 raw_cpu_add(pcp, val); \
393 ret__ = raw_cpu_read(pcp); \
394 raw_local_irq_restore(flags); \
395 ret__; \
396})
397
398#ifndef this_cpu_add_return
399# ifndef this_cpu_add_return_1
400# define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
401# endif
402# ifndef this_cpu_add_return_2
403# define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
404# endif
405# ifndef this_cpu_add_return_4
406# define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
407# endif
408# ifndef this_cpu_add_return_8
409# define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
410# endif
411# define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
412#endif
413
414#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
415#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
416#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
417
418#define _this_cpu_generic_xchg(pcp, nval) \
419({ typeof(pcp) ret__; \
420 unsigned long flags; \
421 raw_local_irq_save(flags); \
422 ret__ = raw_cpu_read(pcp); \
423 raw_cpu_write(pcp, nval); \
424 raw_local_irq_restore(flags); \
425 ret__; \
426})
427
428#ifndef this_cpu_xchg
429# ifndef this_cpu_xchg_1
430# define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
431# endif
432# ifndef this_cpu_xchg_2
433# define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
434# endif
435# ifndef this_cpu_xchg_4
436# define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
437# endif
438# ifndef this_cpu_xchg_8
439# define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
440# endif
441# define this_cpu_xchg(pcp, nval) \
442 __pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
443#endif
444
445#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
446({ \
447 typeof(pcp) ret__; \
448 unsigned long flags; \
449 raw_local_irq_save(flags); \
450 ret__ = raw_cpu_read(pcp); \
451 if (ret__ == (oval)) \
452 raw_cpu_write(pcp, nval); \
453 raw_local_irq_restore(flags); \
454 ret__; \
455})
456
457#ifndef this_cpu_cmpxchg
458# ifndef this_cpu_cmpxchg_1
459# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
460# endif
461# ifndef this_cpu_cmpxchg_2
462# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
463# endif
464# ifndef this_cpu_cmpxchg_4
465# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
466# endif
467# ifndef this_cpu_cmpxchg_8
468# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
469# endif
470# define this_cpu_cmpxchg(pcp, oval, nval) \
471 __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
472#endif
473
474/*
475 * cmpxchg_double replaces two adjacent scalars at once. The first
476 * two parameters are per cpu variables which have to be of the same
477 * size. A truth value is returned to indicate success or failure
478 * (since a double register result is difficult to handle). There is
479 * very limited hardware support for these operations, so only certain
480 * sizes may work.
481 */
482#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
483({ \
484 int ret__; \
485 unsigned long flags; \
486 raw_local_irq_save(flags); \
487 ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
488 oval1, oval2, nval1, nval2); \
489 raw_local_irq_restore(flags); \
490 ret__; \
491})
492
493#ifndef this_cpu_cmpxchg_double
494# ifndef this_cpu_cmpxchg_double_1
495# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
496 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
497# endif
498# ifndef this_cpu_cmpxchg_double_2
499# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
500 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
501# endif
502# ifndef this_cpu_cmpxchg_double_4
503# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
504 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
505# endif
506# ifndef this_cpu_cmpxchg_double_8
507# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
508 _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
509# endif
510# define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
511 __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
512#endif
513
514/*
515 * Generic percpu operations for contexts where we do not want to do
516 * any checks for preemptiosn.
517 *
518 * If there is no other protection through preempt disable and/or
519 * disabling interupts then one of these RMW operations can show unexpected
520 * behavior because the execution thread was rescheduled on another processor
521 * or an interrupt occurred and the same percpu variable was modified from
522 * the interrupt context.
523 */
524#ifndef raw_cpu_read
525# ifndef raw_cpu_read_1
526# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
527# endif
528# ifndef raw_cpu_read_2
529# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
530# endif
531# ifndef raw_cpu_read_4
532# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
533# endif
534# ifndef raw_cpu_read_8
535# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
536# endif
537# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
538#endif
539
540#define raw_cpu_generic_to_op(pcp, val, op) \
541do { \
542 *raw_cpu_ptr(&(pcp)) op val; \
543} while (0)
544
545
546#ifndef raw_cpu_write
547# ifndef raw_cpu_write_1
548# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
549# endif
550# ifndef raw_cpu_write_2
551# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
552# endif
553# ifndef raw_cpu_write_4
554# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
555# endif
556# ifndef raw_cpu_write_8
557# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
558# endif
559# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
560#endif
561
562#ifndef raw_cpu_add
563# ifndef raw_cpu_add_1
564# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
565# endif
566# ifndef raw_cpu_add_2
567# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
568# endif
569# ifndef raw_cpu_add_4
570# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
571# endif
572# ifndef raw_cpu_add_8
573# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
574# endif
575# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
576#endif
577
578#ifndef raw_cpu_sub
579# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
580#endif
581
582#ifndef raw_cpu_inc
583# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
584#endif
585
586#ifndef raw_cpu_dec
587# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
588#endif
589
590#ifndef raw_cpu_and
591# ifndef raw_cpu_and_1
592# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
593# endif
594# ifndef raw_cpu_and_2
595# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
596# endif
597# ifndef raw_cpu_and_4
598# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
599# endif
600# ifndef raw_cpu_and_8
601# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
602# endif
603# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
604#endif
605
606#ifndef raw_cpu_or
607# ifndef raw_cpu_or_1
608# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
609# endif
610# ifndef raw_cpu_or_2
611# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
612# endif
613# ifndef raw_cpu_or_4
614# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
615# endif
616# ifndef raw_cpu_or_8
617# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
618# endif
619# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
620#endif
621
622#define raw_cpu_generic_add_return(pcp, val) \
623({ \
624 raw_cpu_add(pcp, val); \
625 raw_cpu_read(pcp); \
626})
627
628#ifndef raw_cpu_add_return
629# ifndef raw_cpu_add_return_1
630# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
631# endif
632# ifndef raw_cpu_add_return_2
633# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
634# endif
635# ifndef raw_cpu_add_return_4
636# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
637# endif
638# ifndef raw_cpu_add_return_8
639# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
640# endif
641# define raw_cpu_add_return(pcp, val) \
642 __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
643#endif
644
645#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
646#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
647#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
648
649#define raw_cpu_generic_xchg(pcp, nval) \
650({ typeof(pcp) ret__; \
651 ret__ = raw_cpu_read(pcp); \
652 raw_cpu_write(pcp, nval); \
653 ret__; \
654})
655
656#ifndef raw_cpu_xchg
657# ifndef raw_cpu_xchg_1
658# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
659# endif
660# ifndef raw_cpu_xchg_2
661# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
662# endif
663# ifndef raw_cpu_xchg_4
664# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
665# endif
666# ifndef raw_cpu_xchg_8
667# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
668# endif
669# define raw_cpu_xchg(pcp, nval) \
670 __pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
671#endif
672
673#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
674({ \
675 typeof(pcp) ret__; \
676 ret__ = raw_cpu_read(pcp); \
677 if (ret__ == (oval)) \
678 raw_cpu_write(pcp, nval); \
679 ret__; \
680})
681
682#ifndef raw_cpu_cmpxchg
683# ifndef raw_cpu_cmpxchg_1
684# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
685# endif
686# ifndef raw_cpu_cmpxchg_2
687# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
688# endif
689# ifndef raw_cpu_cmpxchg_4
690# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
691# endif
692# ifndef raw_cpu_cmpxchg_8
693# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
694# endif
695# define raw_cpu_cmpxchg(pcp, oval, nval) \
696 __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
697#endif
698
699#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
700({ \
701 int __ret = 0; \
702 if (raw_cpu_read(pcp1) == (oval1) && \
703 raw_cpu_read(pcp2) == (oval2)) { \
704 raw_cpu_write(pcp1, (nval1)); \
705 raw_cpu_write(pcp2, (nval2)); \
706 __ret = 1; \
707 } \
708 (__ret); \
709})
710
711#ifndef raw_cpu_cmpxchg_double
712# ifndef raw_cpu_cmpxchg_double_1
713# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
714 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
715# endif
716# ifndef raw_cpu_cmpxchg_double_2
717# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
718 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
719# endif
720# ifndef raw_cpu_cmpxchg_double_4
721# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
722 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
723# endif
724# ifndef raw_cpu_cmpxchg_double_8
725# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
726 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
727# endif
728# define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
729 __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
730#endif
731
732/*
733 * Generic percpu operations for context that are safe from preemption/interrupts.
734 */
735#ifndef __this_cpu_read
736# define __this_cpu_read(pcp) \
737 (__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))
738#endif
739
740#ifndef __this_cpu_write
741# define __this_cpu_write(pcp, val) \
742do { __this_cpu_preempt_check("write"); \
743 __pcpu_size_call(raw_cpu_write_, (pcp), (val)); \
744} while (0)
745#endif
746
747#ifndef __this_cpu_add
748# define __this_cpu_add(pcp, val) \
749do { __this_cpu_preempt_check("add"); \
750 __pcpu_size_call(raw_cpu_add_, (pcp), (val)); \
751} while (0)
752#endif
753
754#ifndef __this_cpu_sub
755# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
756#endif
757
758#ifndef __this_cpu_inc
759# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
760#endif
761
762#ifndef __this_cpu_dec
763# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
764#endif
765
766#ifndef __this_cpu_and
767# define __this_cpu_and(pcp, val) \
768do { __this_cpu_preempt_check("and"); \
769 __pcpu_size_call(raw_cpu_and_, (pcp), (val)); \
770} while (0)
771
772#endif
773
774#ifndef __this_cpu_or
775# define __this_cpu_or(pcp, val) \
776do { __this_cpu_preempt_check("or"); \
777 __pcpu_size_call(raw_cpu_or_, (pcp), (val)); \
778} while (0)
779#endif
780
781#ifndef __this_cpu_add_return
782# define __this_cpu_add_return(pcp, val) \
783 (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))
784#endif
785
786#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
787#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
788#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
789
790#ifndef __this_cpu_xchg
791# define __this_cpu_xchg(pcp, nval) \
792 (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))
793#endif
794
795#ifndef __this_cpu_cmpxchg
796# define __this_cpu_cmpxchg(pcp, oval, nval) \
797 (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))
798#endif
799
800#ifndef __this_cpu_cmpxchg_double
801# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
802 (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))
803#endif
804
805#endif /* __LINUX_PERCPU_H */ 132#endif /* __LINUX_PERCPU_H */
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 70776aec2562..aad41f06901b 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -1638,7 +1638,7 @@ destroy_root:
1638exit_root_id: 1638exit_root_id:
1639 cgroup_exit_root_id(root); 1639 cgroup_exit_root_id(root);
1640cancel_ref: 1640cancel_ref:
1641 percpu_ref_cancel_init(&root_cgrp->self.refcnt); 1641 percpu_ref_exit(&root_cgrp->self.refcnt);
1642out: 1642out:
1643 free_cgrp_cset_links(&tmp_links); 1643 free_cgrp_cset_links(&tmp_links);
1644 return ret; 1644 return ret;
@@ -4175,6 +4175,8 @@ static void css_free_work_fn(struct work_struct *work)
4175 container_of(work, struct cgroup_subsys_state, destroy_work); 4175 container_of(work, struct cgroup_subsys_state, destroy_work);
4176 struct cgroup *cgrp = css->cgroup; 4176 struct cgroup *cgrp = css->cgroup;
4177 4177
4178 percpu_ref_exit(&css->refcnt);
4179
4178 if (css->ss) { 4180 if (css->ss) {
4179 /* css free path */ 4181 /* css free path */
4180 if (css->parent) 4182 if (css->parent)
@@ -4372,7 +4374,7 @@ err_list_del:
4372err_free_id: 4374err_free_id:
4373 cgroup_idr_remove(&ss->css_idr, css->id); 4375 cgroup_idr_remove(&ss->css_idr, css->id);
4374err_free_percpu_ref: 4376err_free_percpu_ref:
4375 percpu_ref_cancel_init(&css->refcnt); 4377 percpu_ref_exit(&css->refcnt);
4376err_free_css: 4378err_free_css:
4377 call_rcu(&css->rcu_head, css_free_rcu_fn); 4379 call_rcu(&css->rcu_head, css_free_rcu_fn);
4378 return err; 4380 return err;
@@ -4483,7 +4485,7 @@ static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
4483out_free_id: 4485out_free_id:
4484 cgroup_idr_remove(&root->cgroup_idr, cgrp->id); 4486 cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
4485out_cancel_ref: 4487out_cancel_ref:
4486 percpu_ref_cancel_init(&cgrp->self.refcnt); 4488 percpu_ref_exit(&cgrp->self.refcnt);
4487out_free_cgrp: 4489out_free_cgrp:
4488 kfree(cgrp); 4490 kfree(cgrp);
4489out_unlock: 4491out_unlock:
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 7a2e449a96b1..5dbe22aa3efd 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1962,6 +1962,7 @@ __acquires(&pool->lock)
1962 1962
1963 lockdep_copy_map(&lockdep_map, &work->lockdep_map); 1963 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
1964#endif 1964#endif
1965 /* ensure we're on the correct CPU */
1965 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) && 1966 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1966 raw_smp_processor_id() != pool->cpu); 1967 raw_smp_processor_id() != pool->cpu);
1967 1968
@@ -4574,11 +4575,10 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4574 for_each_pool(pool, pi) { 4575 for_each_pool(pool, pi) {
4575 mutex_lock(&pool->attach_mutex); 4576 mutex_lock(&pool->attach_mutex);
4576 4577
4577 if (pool->cpu == cpu) { 4578 if (pool->cpu == cpu)
4578 rebind_workers(pool); 4579 rebind_workers(pool);
4579 } else if (pool->cpu < 0) { 4580 else if (pool->cpu < 0)
4580 restore_unbound_workers_cpumask(pool, cpu); 4581 restore_unbound_workers_cpumask(pool, cpu);
4581 }
4582 4582
4583 mutex_unlock(&pool->attach_mutex); 4583 mutex_unlock(&pool->attach_mutex);
4584 } 4584 }
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 963b7034a51b..fe5a3342e960 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -31,6 +31,11 @@
31 31
32#define PCPU_COUNT_BIAS (1U << 31) 32#define PCPU_COUNT_BIAS (1U << 31)
33 33
34static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
35{
36 return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
37}
38
34/** 39/**
35 * percpu_ref_init - initialize a percpu refcount 40 * percpu_ref_init - initialize a percpu refcount
36 * @ref: percpu_ref to initialize 41 * @ref: percpu_ref to initialize
@@ -46,8 +51,8 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
46{ 51{
47 atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); 52 atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
48 53
49 ref->pcpu_count = alloc_percpu(unsigned); 54 ref->pcpu_count_ptr = (unsigned long)alloc_percpu(unsigned);
50 if (!ref->pcpu_count) 55 if (!ref->pcpu_count_ptr)
51 return -ENOMEM; 56 return -ENOMEM;
52 57
53 ref->release = release; 58 ref->release = release;
@@ -56,53 +61,71 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
56EXPORT_SYMBOL_GPL(percpu_ref_init); 61EXPORT_SYMBOL_GPL(percpu_ref_init);
57 62
58/** 63/**
59 * percpu_ref_cancel_init - cancel percpu_ref_init() 64 * percpu_ref_reinit - re-initialize a percpu refcount
60 * @ref: percpu_ref to cancel init for 65 * @ref: perpcu_ref to re-initialize
61 * 66 *
62 * Once a percpu_ref is initialized, its destruction is initiated by 67 * Re-initialize @ref so that it's in the same state as when it finished
63 * percpu_ref_kill() and completes asynchronously, which can be painful to 68 * percpu_ref_init(). @ref must have been initialized successfully, killed
64 * do when destroying a half-constructed object in init failure path. 69 * and reached 0 but not exited.
65 * 70 *
66 * This function destroys @ref without invoking @ref->release and the 71 * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
67 * memory area containing it can be freed immediately on return. To 72 * this function is in progress.
68 * prevent accidental misuse, it's required that @ref has finished
69 * percpu_ref_init(), whether successful or not, but never used.
70 *
71 * The weird name and usage restriction are to prevent people from using
72 * this function by mistake for normal shutdown instead of
73 * percpu_ref_kill().
74 */ 73 */
75void percpu_ref_cancel_init(struct percpu_ref *ref) 74void percpu_ref_reinit(struct percpu_ref *ref)
76{ 75{
77 unsigned __percpu *pcpu_count = ref->pcpu_count; 76 unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
78 int cpu; 77 int cpu;
79 78
80 WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS); 79 BUG_ON(!pcpu_count);
80 WARN_ON(!percpu_ref_is_zero(ref));
81
82 atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
83
84 /*
85 * Restore per-cpu operation. smp_store_release() is paired with
86 * smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees
87 * that the zeroing is visible to all percpu accesses which can see
88 * the following PCPU_REF_DEAD clearing.
89 */
90 for_each_possible_cpu(cpu)
91 *per_cpu_ptr(pcpu_count, cpu) = 0;
92
93 smp_store_release(&ref->pcpu_count_ptr,
94 ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
95}
96EXPORT_SYMBOL_GPL(percpu_ref_reinit);
97
98/**
99 * percpu_ref_exit - undo percpu_ref_init()
100 * @ref: percpu_ref to exit
101 *
102 * This function exits @ref. The caller is responsible for ensuring that
103 * @ref is no longer in active use. The usual places to invoke this
104 * function from are the @ref->release() callback or in init failure path
105 * where percpu_ref_init() succeeded but other parts of the initialization
106 * of the embedding object failed.
107 */
108void percpu_ref_exit(struct percpu_ref *ref)
109{
110 unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
81 111
82 if (pcpu_count) { 112 if (pcpu_count) {
83 for_each_possible_cpu(cpu) 113 free_percpu(pcpu_count);
84 WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu)); 114 ref->pcpu_count_ptr = PCPU_REF_DEAD;
85 free_percpu(ref->pcpu_count);
86 } 115 }
87} 116}
88EXPORT_SYMBOL_GPL(percpu_ref_cancel_init); 117EXPORT_SYMBOL_GPL(percpu_ref_exit);
89 118
90static void percpu_ref_kill_rcu(struct rcu_head *rcu) 119static void percpu_ref_kill_rcu(struct rcu_head *rcu)
91{ 120{
92 struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); 121 struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
93 unsigned __percpu *pcpu_count = ref->pcpu_count; 122 unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
94 unsigned count = 0; 123 unsigned count = 0;
95 int cpu; 124 int cpu;
96 125
97 /* Mask out PCPU_REF_DEAD */
98 pcpu_count = (unsigned __percpu *)
99 (((unsigned long) pcpu_count) & ~PCPU_STATUS_MASK);
100
101 for_each_possible_cpu(cpu) 126 for_each_possible_cpu(cpu)
102 count += *per_cpu_ptr(pcpu_count, cpu); 127 count += *per_cpu_ptr(pcpu_count, cpu);
103 128
104 free_percpu(pcpu_count);
105
106 pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count); 129 pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);
107 130
108 /* 131 /*
@@ -152,11 +175,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
152void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 175void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
153 percpu_ref_func_t *confirm_kill) 176 percpu_ref_func_t *confirm_kill)
154{ 177{
155 WARN_ONCE(REF_STATUS(ref->pcpu_count) == PCPU_REF_DEAD, 178 WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD,
156 "percpu_ref_kill() called more than once!\n"); 179 "percpu_ref_kill() called more than once!\n");
157 180
158 ref->pcpu_count = (unsigned __percpu *) 181 ref->pcpu_count_ptr |= PCPU_REF_DEAD;
159 (((unsigned long) ref->pcpu_count)|PCPU_REF_DEAD);
160 ref->confirm_kill = confirm_kill; 182 ref->confirm_kill = confirm_kill;
161 183
162 call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu); 184 call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);
diff --git a/mm/percpu.c b/mm/percpu.c
index 2ddf9a990dbd..2139e30a4b44 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -720,8 +720,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
720 if (unlikely(align < 2)) 720 if (unlikely(align < 2))
721 align = 2; 721 align = 2;
722 722
723 if (unlikely(size & 1)) 723 size = ALIGN(size, 2);
724 size++;
725 724
726 if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { 725 if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
727 WARN(true, "illegal size (%zu) or align (%zu) for " 726 WARN(true, "illegal size (%zu) or align (%zu) for "
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-21 21:10:19 -0400