aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/include/asm/percpu.h26
-rw-r--r--include/linux/percpu-defs.h1
-rw-r--r--include/linux/percpu.h48
-rw-r--r--mm/percpu.c1
4 files changed, 39 insertions, 37 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 8b5ec19bdef4..0c44196b78ac 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -74,31 +74,31 @@ extern void __bad_percpu_size(void);
74 74
75#define percpu_to_op(op, var, val) \ 75#define percpu_to_op(op, var, val) \
76do { \ 76do { \
77 typedef typeof(var) T__; \ 77 typedef typeof(var) pto_T__; \
78 if (0) { \ 78 if (0) { \
79 T__ tmp__; \ 79 pto_T__ pto_tmp__; \
80 tmp__ = (val); \ 80 pto_tmp__ = (val); \
81 } \ 81 } \
82 switch (sizeof(var)) { \ 82 switch (sizeof(var)) { \
83 case 1: \ 83 case 1: \
84 asm(op "b %1,"__percpu_arg(0) \ 84 asm(op "b %1,"__percpu_arg(0) \
85 : "+m" (var) \ 85 : "+m" (var) \
86 : "qi" ((T__)(val))); \ 86 : "qi" ((pto_T__)(val))); \
87 break; \ 87 break; \
88 case 2: \ 88 case 2: \
89 asm(op "w %1,"__percpu_arg(0) \ 89 asm(op "w %1,"__percpu_arg(0) \
90 : "+m" (var) \ 90 : "+m" (var) \
91 : "ri" ((T__)(val))); \ 91 : "ri" ((pto_T__)(val))); \
92 break; \ 92 break; \
93 case 4: \ 93 case 4: \
94 asm(op "l %1,"__percpu_arg(0) \ 94 asm(op "l %1,"__percpu_arg(0) \
95 : "+m" (var) \ 95 : "+m" (var) \
96 : "ri" ((T__)(val))); \ 96 : "ri" ((pto_T__)(val))); \
97 break; \ 97 break; \
98 case 8: \ 98 case 8: \
99 asm(op "q %1,"__percpu_arg(0) \ 99 asm(op "q %1,"__percpu_arg(0) \
100 : "+m" (var) \ 100 : "+m" (var) \
101 : "re" ((T__)(val))); \ 101 : "re" ((pto_T__)(val))); \
102 break; \ 102 break; \
103 default: __bad_percpu_size(); \ 103 default: __bad_percpu_size(); \
104 } \ 104 } \
@@ -106,31 +106,31 @@ do { \
106 106
107#define percpu_from_op(op, var, constraint) \ 107#define percpu_from_op(op, var, constraint) \
108({ \ 108({ \
109 typeof(var) ret__; \ 109 typeof(var) pfo_ret__; \
110 switch (sizeof(var)) { \ 110 switch (sizeof(var)) { \
111 case 1: \ 111 case 1: \
112 asm(op "b "__percpu_arg(1)",%0" \ 112 asm(op "b "__percpu_arg(1)",%0" \
113 : "=q" (ret__) \ 113 : "=q" (pfo_ret__) \
114 : constraint); \ 114 : constraint); \
115 break; \ 115 break; \
116 case 2: \ 116 case 2: \
117 asm(op "w "__percpu_arg(1)",%0" \ 117 asm(op "w "__percpu_arg(1)",%0" \
118 : "=r" (ret__) \ 118 : "=r" (pfo_ret__) \
119 : constraint); \ 119 : constraint); \
120 break; \ 120 break; \
121 case 4: \ 121 case 4: \
122 asm(op "l "__percpu_arg(1)",%0" \ 122 asm(op "l "__percpu_arg(1)",%0" \
123 : "=r" (ret__) \ 123 : "=r" (pfo_ret__) \
124 : constraint); \ 124 : constraint); \
125 break; \ 125 break; \
126 case 8: \ 126 case 8: \
127 asm(op "q "__percpu_arg(1)",%0" \ 127 asm(op "q "__percpu_arg(1)",%0" \
128 : "=r" (ret__) \ 128 : "=r" (pfo_ret__) \
129 : constraint); \ 129 : constraint); \
130 break; \ 130 break; \
131 default: __bad_percpu_size(); \ 131 default: __bad_percpu_size(); \
132 } \ 132 } \
133 ret__; \ 133 pfo_ret__; \
134}) 134})
135 135
136/* 136/*
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 9bd03193ecd4..5a5d6ce4bd55 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -60,6 +60,7 @@
60 60
61#define DEFINE_PER_CPU_SECTION(type, name, sec) \ 61#define DEFINE_PER_CPU_SECTION(type, name, sec) \
62 __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \ 62 __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
63 extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
63 __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \ 64 __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
64 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \ 65 __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak \
65 __typeof__(type) per_cpu__##name 66 __typeof__(type) per_cpu__##name
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 519d6876590f..522f421ec213 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -226,20 +226,20 @@ do { \
226 226
227extern void __bad_size_call_parameter(void); 227extern void __bad_size_call_parameter(void);
228 228
229#define __size_call_return(stem, variable) \ 229#define __pcpu_size_call_return(stem, variable) \
230({ typeof(variable) ret__; \ 230({ typeof(variable) pscr_ret__; \
231 switch(sizeof(variable)) { \ 231 switch(sizeof(variable)) { \
232 case 1: ret__ = stem##1(variable);break; \ 232 case 1: pscr_ret__ = stem##1(variable);break; \
233 case 2: ret__ = stem##2(variable);break; \ 233 case 2: pscr_ret__ = stem##2(variable);break; \
234 case 4: ret__ = stem##4(variable);break; \ 234 case 4: pscr_ret__ = stem##4(variable);break; \
235 case 8: ret__ = stem##8(variable);break; \ 235 case 8: pscr_ret__ = stem##8(variable);break; \
236 default: \ 236 default: \
237 __bad_size_call_parameter();break; \ 237 __bad_size_call_parameter();break; \
238 } \ 238 } \
239 ret__; \ 239 pscr_ret__; \
240}) 240})
241 241
242#define __size_call(stem, variable, ...) \ 242#define __pcpu_size_call(stem, variable, ...) \
243do { \ 243do { \
244 switch(sizeof(variable)) { \ 244 switch(sizeof(variable)) { \
245 case 1: stem##1(variable, __VA_ARGS__);break; \ 245 case 1: stem##1(variable, __VA_ARGS__);break; \
@@ -299,7 +299,7 @@ do { \
299# ifndef this_cpu_read_8 299# ifndef this_cpu_read_8
300# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp) 300# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
301# endif 301# endif
302# define this_cpu_read(pcp) __size_call_return(this_cpu_read_, (pcp)) 302# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
303#endif 303#endif
304 304
305#define _this_cpu_generic_to_op(pcp, val, op) \ 305#define _this_cpu_generic_to_op(pcp, val, op) \
@@ -322,7 +322,7 @@ do { \
322# ifndef this_cpu_write_8 322# ifndef this_cpu_write_8
323# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =) 323# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
324# endif 324# endif
325# define this_cpu_write(pcp, val) __size_call(this_cpu_write_, (pcp), (val)) 325# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
326#endif 326#endif
327 327
328#ifndef this_cpu_add 328#ifndef this_cpu_add
@@ -338,7 +338,7 @@ do { \
338# ifndef this_cpu_add_8 338# ifndef this_cpu_add_8
339# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=) 339# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
340# endif 340# endif
341# define this_cpu_add(pcp, val) __size_call(this_cpu_add_, (pcp), (val)) 341# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
342#endif 342#endif
343 343
344#ifndef this_cpu_sub 344#ifndef this_cpu_sub
@@ -366,7 +366,7 @@ do { \
366# ifndef this_cpu_and_8 366# ifndef this_cpu_and_8
367# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=) 367# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
368# endif 368# endif
369# define this_cpu_and(pcp, val) __size_call(this_cpu_and_, (pcp), (val)) 369# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
370#endif 370#endif
371 371
372#ifndef this_cpu_or 372#ifndef this_cpu_or
@@ -382,7 +382,7 @@ do { \
382# ifndef this_cpu_or_8 382# ifndef this_cpu_or_8
383# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=) 383# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
384# endif 384# endif
385# define this_cpu_or(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) 385# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
386#endif 386#endif
387 387
388#ifndef this_cpu_xor 388#ifndef this_cpu_xor
@@ -398,7 +398,7 @@ do { \
398# ifndef this_cpu_xor_8 398# ifndef this_cpu_xor_8
399# define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=) 399# define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
400# endif 400# endif
401# define this_cpu_xor(pcp, val) __size_call(this_cpu_or_, (pcp), (val)) 401# define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
402#endif 402#endif
403 403
404/* 404/*
@@ -428,7 +428,7 @@ do { \
428# ifndef __this_cpu_read_8 428# ifndef __this_cpu_read_8
429# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp))) 429# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
430# endif 430# endif
431# define __this_cpu_read(pcp) __size_call_return(__this_cpu_read_, (pcp)) 431# define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
432#endif 432#endif
433 433
434#define __this_cpu_generic_to_op(pcp, val, op) \ 434#define __this_cpu_generic_to_op(pcp, val, op) \
@@ -449,7 +449,7 @@ do { \
449# ifndef __this_cpu_write_8 449# ifndef __this_cpu_write_8
450# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =) 450# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
451# endif 451# endif
452# define __this_cpu_write(pcp, val) __size_call(__this_cpu_write_, (pcp), (val)) 452# define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
453#endif 453#endif
454 454
455#ifndef __this_cpu_add 455#ifndef __this_cpu_add
@@ -465,7 +465,7 @@ do { \
465# ifndef __this_cpu_add_8 465# ifndef __this_cpu_add_8
466# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=) 466# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
467# endif 467# endif
468# define __this_cpu_add(pcp, val) __size_call(__this_cpu_add_, (pcp), (val)) 468# define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
469#endif 469#endif
470 470
471#ifndef __this_cpu_sub 471#ifndef __this_cpu_sub
@@ -493,7 +493,7 @@ do { \
493# ifndef __this_cpu_and_8 493# ifndef __this_cpu_and_8
494# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=) 494# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
495# endif 495# endif
496# define __this_cpu_and(pcp, val) __size_call(__this_cpu_and_, (pcp), (val)) 496# define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
497#endif 497#endif
498 498
499#ifndef __this_cpu_or 499#ifndef __this_cpu_or
@@ -509,7 +509,7 @@ do { \
509# ifndef __this_cpu_or_8 509# ifndef __this_cpu_or_8
510# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=) 510# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
511# endif 511# endif
512# define __this_cpu_or(pcp, val) __size_call(__this_cpu_or_, (pcp), (val)) 512# define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
513#endif 513#endif
514 514
515#ifndef __this_cpu_xor 515#ifndef __this_cpu_xor
@@ -525,7 +525,7 @@ do { \
525# ifndef __this_cpu_xor_8 525# ifndef __this_cpu_xor_8
526# define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=) 526# define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
527# endif 527# endif
528# define __this_cpu_xor(pcp, val) __size_call(__this_cpu_xor_, (pcp), (val)) 528# define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
529#endif 529#endif
530 530
531/* 531/*
@@ -556,7 +556,7 @@ do { \
556# ifndef irqsafe_cpu_add_8 556# ifndef irqsafe_cpu_add_8
557# define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=) 557# define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
558# endif 558# endif
559# define irqsafe_cpu_add(pcp, val) __size_call(irqsafe_cpu_add_, (pcp), (val)) 559# define irqsafe_cpu_add(pcp, val) __pcpu_size_call(irqsafe_cpu_add_, (pcp), (val))
560#endif 560#endif
561 561
562#ifndef irqsafe_cpu_sub 562#ifndef irqsafe_cpu_sub
@@ -584,7 +584,7 @@ do { \
584# ifndef irqsafe_cpu_and_8 584# ifndef irqsafe_cpu_and_8
585# define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=) 585# define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
586# endif 586# endif
587# define irqsafe_cpu_and(pcp, val) __size_call(irqsafe_cpu_and_, (val)) 587# define irqsafe_cpu_and(pcp, val) __pcpu_size_call(irqsafe_cpu_and_, (val))
588#endif 588#endif
589 589
590#ifndef irqsafe_cpu_or 590#ifndef irqsafe_cpu_or
@@ -600,7 +600,7 @@ do { \
600# ifndef irqsafe_cpu_or_8 600# ifndef irqsafe_cpu_or_8
601# define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=) 601# define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
602# endif 602# endif
603# define irqsafe_cpu_or(pcp, val) __size_call(irqsafe_cpu_or_, (val)) 603# define irqsafe_cpu_or(pcp, val) __pcpu_size_call(irqsafe_cpu_or_, (val))
604#endif 604#endif
605 605
606#ifndef irqsafe_cpu_xor 606#ifndef irqsafe_cpu_xor
@@ -616,7 +616,7 @@ do { \
616# ifndef irqsafe_cpu_xor_8 616# ifndef irqsafe_cpu_xor_8
617# define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=) 617# define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
618# endif 618# endif
619# define irqsafe_cpu_xor(pcp, val) __size_call(irqsafe_cpu_xor_, (val)) 619# define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
620#endif 620#endif
621 621
622#endif /* __LINUX_PERCPU_H */ 622#endif /* __LINUX_PERCPU_H */
diff --git a/mm/percpu.c b/mm/percpu.c
index ec158bb5f86d..e2e80fc78601 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -365,6 +365,7 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
365 * 0 if noop, 1 if successfully extended, -errno on failure. 365 * 0 if noop, 1 if successfully extended, -errno on failure.
366 */ 366 */
367static int pcpu_extend_area_map(struct pcpu_chunk *chunk) 367static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
368 __releases(lock) __acquires(lock)
368{ 369{
369 int new_alloc; 370 int new_alloc;
370 int *new; 371 int *new;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-03 19:55:41 -0500