1 files changed, 78 insertions, 91 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ece72053ba63..aee103b26d01 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -2,53 +2,12 @@
 #define _ASM_X86_PERCPU_H
 #ifdef CONFIG_X86_64
-#include <linux/compiler.h>
+#define __percpu_seg            gs
+#define __percpu_mov_op         movq
-/* Same as asm-generic/percpu.h, except that we store the per cpu offset
+#else
-   in the PDA. Longer term the PDA and every per cpu variable
+#define __percpu_seg            fs
-   should be just put into a single section and referenced directly
+#define __percpu_mov_op         movl
-   from %gs */
-#ifdef CONFIG_SMP
-#include <asm/pda.h>
-#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
-#define __my_cpu_offset read_pda(data_offset)
-#define per_cpu_offset(x) (__per_cpu_offset(x))
 #endif
-#include <asm-generic/percpu.h>
-DECLARE_PER_CPU(struct x8664_pda, pda);
-/*
- * These are supposed to be implemented as a single instruction which
- * operates on the per-cpu data base segment.  x86-64 doesn't have
- * that yet, so this is a fairly inefficient workaround for the
- * meantime.  The single instruction is atomic with respect to
- * preemption and interrupts, so we need to explicitly disable
- * interrupts here to achieve the same effect.  However, because it
- * can be used from within interrupt-disable/enable, we can't actually
- * disable interrupts; disabling preemption is enough.
- */
-#define x86_read_percpu(var)                                            \
-        ({                                                              \
-                typeof(per_cpu_var(var)) __tmp;                         \
-                preempt_disable();                                      \
-                __tmp = __get_cpu_var(var);                             \
-                preempt_enable();                                       \
-                __tmp;                                                  \
-        })
-#define x86_write_percpu(var, val)                                      \
-        do {                                                            \
-                preempt_disable();                                      \
-                __get_cpu_var(var) = (val);                             \
-                preempt_enable();                                       \
-        } while(0)
-#else /* CONFIG_X86_64 */
 #ifdef __ASSEMBLY__
@@ -65,47 +24,48 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
 #ifdef CONFIG_SMP
-#define PER_CPU(var, reg)                               \
+#define PER_CPU(var, reg)                                               \
-        movl %fs:per_cpu__##this_cpu_off, reg;          \
+        __percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg;       \
        lea per_cpu__##var(reg), reg
-#define PER_CPU_VAR(var)        %fs:per_cpu__##var
+#define PER_CPU_VAR(var)        %__percpu_seg:per_cpu__##var
 #else /* ! SMP */
-#define PER_CPU(var, reg)                       \
+#define PER_CPU(var, reg)                                               \
-        movl $per_cpu__##var, reg
+        __percpu_mov_op $per_cpu__##var, reg
 #define PER_CPU_VAR(var)        per_cpu__##var
 #endif  /* SMP */
+#ifdef CONFIG_X86_64_SMP
+#define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
+#else
+#define INIT_PER_CPU_VAR(var)  per_cpu__##var
+#endif
 #else /* ...!ASSEMBLY */
+#include <linux/stringify.h>
+#ifdef CONFIG_SMP
+#define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
+#define __my_cpu_offset         percpu_read(this_cpu_off)
+#else
+#define __percpu_arg(x)         "%" #x
+#endif
 /*
- * PER_CPU finds an address of a per-cpu variable.
+ * Initialized pointers to per-cpu variables needed for the boot
+ * processor need to use these macros to get the proper address
+ * offset from __per_cpu_load on SMP.
 *
- * Args:
+ * There also must be an entry in vmlinux_64.lds.S
- *    var - variable name
- *    cpu - 32bit register containing the current CPU number
- *
- * The resulting address is stored in the "cpu" argument.
- *
- * Example:
- *    PER_CPU(cpu_gdt_descr, %ebx)
 */
-#ifdef CONFIG_SMP
+#define DECLARE_INIT_PER_CPU(var) \
+       extern typeof(per_cpu_var(var)) init_per_cpu_var(var)
-#define __my_cpu_offset x86_read_percpu(this_cpu_off)
-/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
-#define __percpu_seg "%%fs:"
-#else  /* !SMP */
-#define __percpu_seg ""
-#endif  /* SMP */
-#include <asm-generic/percpu.h>
-/* We can use this directly for local CPU (faster). */
+#ifdef CONFIG_X86_64_SMP
-DECLARE_PER_CPU(unsigned long, this_cpu_off);
+#define init_per_cpu_var(var)  init_per_cpu__##var
+#else
+#define init_per_cpu_var(var)  per_cpu_var(var)
+#endif
 /* For arch-specific code, we can use direct single-insn ops (they
 * don't give an lvalue though). */
@@ -120,20 +80,25 @@ do {							\
        }                                               \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b %1,"__percpu_seg"%0"          \
+                asm(op "b %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 2:                                         \
-                asm(op "w %1,"__percpu_seg"%0"          \
+                asm(op "w %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 4:                                         \
-                asm(op "l %1,"__percpu_seg"%0"          \
+                asm(op "l %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
+        case 8:                                         \
+                asm(op "q %1,"__percpu_arg(0)           \
+                    : "+m" (var)                        \
+                    : "re" ((T__)val));                 \
+                break;                                  \
        default: __bad_percpu_size();                   \
        }                                               \
 } while (0)
@@ -143,17 +108,22 @@ do {							\
        typeof(var) ret__;                              \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b "__percpu_seg"%1,%0"          \
+                asm(op "b "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 2:                                         \
-                asm(op "w "__percpu_seg"%1,%0"          \
+                asm(op "w "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 4:                                         \
-                asm(op "l "__percpu_seg"%1,%0"          \
+                asm(op "l "__percpu_arg(1)",%0"         \
+                    : "=r" (ret__)                      \
+                    : "m" (var));                       \
+                break;                                  \
+        case 8:                                         \
+                asm(op "q "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
@@ -162,13 +132,30 @@ do {							\
        ret__;                                          \
 })
-#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
+#define percpu_read(var)        percpu_from_op("mov", per_cpu__##var)
-#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)
+#define percpu_write(var, val)  percpu_to_op("mov", per_cpu__##var, val)
-#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
+#define percpu_add(var, val)    percpu_to_op("add", per_cpu__##var, val)
-#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
+#define percpu_sub(var, val)    percpu_to_op("sub", per_cpu__##var, val)
-#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
+#define percpu_and(var, val)    percpu_to_op("and", per_cpu__##var, val)
+#define percpu_or(var, val)     percpu_to_op("or", per_cpu__##var, val)
+#define percpu_xor(var, val)    percpu_to_op("xor", per_cpu__##var, val)
+/* This is not atomic against other CPUs -- CPU preemption needs to be off */
+#define x86_test_and_clear_bit_percpu(bit, var)                         \
+({                                                                      \
+        int old__;                                                      \
+        asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
+                     : "=r" (old__), "+m" (per_cpu__##var)              \
+                     : "dIr" (bit));                                    \
+        old__;                                                          \
+})
+#include <asm-generic/percpu.h>
+/* We can use this directly for local CPU (faster). */
+DECLARE_PER_CPU(unsigned long, this_cpu_off);
 #endif /* !__ASSEMBLY__ */
-#endif /* !CONFIG_X86_64 */
 #ifdef CONFIG_SMP
@@ -195,9 +182,9 @@ do {							\
 #define early_per_cpu_ptr(_name) (_name##_early_ptr)
 #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
 #define early_per_cpu(_name, _cpu)                              \
-        (early_per_cpu_ptr(_name) ?                             \
+        *(early_per_cpu_ptr(_name) ?                            \
-                early_per_cpu_ptr(_name)[_cpu] :                \
+                &early_per_cpu_ptr(_name)[_cpu] :               \
-                per_cpu(_name, _cpu))
+                &per_cpu(_name, _cpu))
 #else   /* !CONFIG_SMP */
 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)          \

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index ece72053ba63..aee103b26d01 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h
@@ -2,53 +2,12 @@
2	#define _ASM_X86_PERCPU_H	2	#define _ASM_X86_PERCPU_H
3		3
4	#ifdef CONFIG_X86_64	4	#ifdef CONFIG_X86_64
5	#include <linux/compiler.h>	5	#define __percpu_seg gs
6		6	#define __percpu_mov_op movq
7	/* Same as asm-generic/percpu.h, except that we store the per cpu offset	7	#else
8	in the PDA. Longer term the PDA and every per cpu variable	8	#define __percpu_seg fs
9	should be just put into a single section and referenced directly	9	#define __percpu_mov_op movl
10	from %gs */
11
12	#ifdef CONFIG_SMP
13	#include <asm/pda.h>
14
15	#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
16	#define __my_cpu_offset read_pda(data_offset)
17
18	#define per_cpu_offset(x) (__per_cpu_offset(x))
19
20	#endif	10	#endif
21	#include <asm-generic/percpu.h>
22
23	DECLARE_PER_CPU(struct x8664_pda, pda);
24
25	/*
26	* These are supposed to be implemented as a single instruction which
27	* operates on the per-cpu data base segment. x86-64 doesn't have
28	* that yet, so this is a fairly inefficient workaround for the
29	* meantime. The single instruction is atomic with respect to
30	* preemption and interrupts, so we need to explicitly disable
31	* interrupts here to achieve the same effect. However, because it
32	* can be used from within interrupt-disable/enable, we can't actually
33	* disable interrupts; disabling preemption is enough.
34	*/
35	#define x86_read_percpu(var) \
36	({ \
37	typeof(per_cpu_var(var)) __tmp; \
38	preempt_disable(); \
39	__tmp = __get_cpu_var(var); \
40	preempt_enable(); \
41	__tmp; \
42	})
43
44	#define x86_write_percpu(var, val) \
45	do { \
46	preempt_disable(); \
47	__get_cpu_var(var) = (val); \
48	preempt_enable(); \
49	} while(0)
50
51	#else /* CONFIG_X86_64 */
52		11
53	#ifdef __ASSEMBLY__	12	#ifdef __ASSEMBLY__
54		13
@@ -65,47 +24,48 @@ DECLARE_PER_CPU(struct x8664_pda, pda);
65	* PER_CPU(cpu_gdt_descr, %ebx)	24	* PER_CPU(cpu_gdt_descr, %ebx)
66	*/	25	*/
67	#ifdef CONFIG_SMP	26	#ifdef CONFIG_SMP
68	#define PER_CPU(var, reg) \	27	#define PER_CPU(var, reg) \
69	movl %fs:per_cpu__##this_cpu_off, reg; \	28	__percpu_mov_op %__percpu_seg:per_cpu__this_cpu_off, reg; \
70	lea per_cpu__##var(reg), reg	29	lea per_cpu__##var(reg), reg
71	#define PER_CPU_VAR(var) %fs:per_cpu__##var	30	#define PER_CPU_VAR(var) %__percpu_seg:per_cpu__##var
72	#else /* ! SMP */	31	#else /* ! SMP */
73	#define PER_CPU(var, reg) \	32	#define PER_CPU(var, reg) \
74	movl $per_cpu__##var, reg	33	__percpu_mov_op $per_cpu__##var, reg
75	#define PER_CPU_VAR(var) per_cpu__##var	34	#define PER_CPU_VAR(var) per_cpu__##var
76	#endif /* SMP */	35	#endif /* SMP */
77		36
		37	#ifdef CONFIG_X86_64_SMP
		38	#define INIT_PER_CPU_VAR(var) init_per_cpu__##var
		39	#else
		40	#define INIT_PER_CPU_VAR(var) per_cpu__##var
		41	#endif
		42
78	#else /* ...!ASSEMBLY */	43	#else /* ...!ASSEMBLY */
79		44
		45	#include <linux/stringify.h>
		46
		47	#ifdef CONFIG_SMP
		48	#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
		49	#define __my_cpu_offset percpu_read(this_cpu_off)
		50	#else
		51	#define __percpu_arg(x) "%" #x
		52	#endif
		53
80	/*	54	/*
81	* PER_CPU finds an address of a per-cpu variable.	55	* Initialized pointers to per-cpu variables needed for the boot
		56	* processor need to use these macros to get the proper address
		57	* offset from __per_cpu_load on SMP.
82	*	58	*
83	* Args:	59	* There also must be an entry in vmlinux_64.lds.S
84	* var - variable name
85	* cpu - 32bit register containing the current CPU number
86	*
87	* The resulting address is stored in the "cpu" argument.
88	*
89	* Example:
90	* PER_CPU(cpu_gdt_descr, %ebx)
91	*/	60	*/
92	#ifdef CONFIG_SMP	61	#define DECLARE_INIT_PER_CPU(var) \
93		62	extern typeof(per_cpu_var(var)) init_per_cpu_var(var)
94	#define __my_cpu_offset x86_read_percpu(this_cpu_off)
95
96	/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
97	#define __percpu_seg "%%fs:"
98
99	#else /* !SMP */
100
101	#define __percpu_seg ""
102
103	#endif /* SMP */
104
105	#include <asm-generic/percpu.h>
106		63
107	/* We can use this directly for local CPU (faster). */	64	#ifdef CONFIG_X86_64_SMP
108	DECLARE_PER_CPU(unsigned long, this_cpu_off);	65	#define init_per_cpu_var(var) init_per_cpu__##var
		66	#else
		67	#define init_per_cpu_var(var) per_cpu_var(var)
		68	#endif
109		69
110	/* For arch-specific code, we can use direct single-insn ops (they	70	/* For arch-specific code, we can use direct single-insn ops (they
111	* don't give an lvalue though). */	71	* don't give an lvalue though). */
@@ -120,20 +80,25 @@ do { \
120	} \	80	} \
121	switch (sizeof(var)) { \	81	switch (sizeof(var)) { \
122	case 1: \	82	case 1: \
123	asm(op "b %1,"__percpu_seg"%0" \	83	asm(op "b %1,"__percpu_arg(0) \
124	: "+m" (var) \	84	: "+m" (var) \
125	: "ri" ((T__)val)); \	85	: "ri" ((T__)val)); \
126	break; \	86	break; \
127	case 2: \	87	case 2: \
128	asm(op "w %1,"__percpu_seg"%0" \	88	asm(op "w %1,"__percpu_arg(0) \
129	: "+m" (var) \	89	: "+m" (var) \
130	: "ri" ((T__)val)); \	90	: "ri" ((T__)val)); \
131	break; \	91	break; \
132	case 4: \	92	case 4: \
133	asm(op "l %1,"__percpu_seg"%0" \	93	asm(op "l %1,"__percpu_arg(0) \
134	: "+m" (var) \	94	: "+m" (var) \
135	: "ri" ((T__)val)); \	95	: "ri" ((T__)val)); \
136	break; \	96	break; \
		97	case 8: \
		98	asm(op "q %1,"__percpu_arg(0) \
		99	: "+m" (var) \
		100	: "re" ((T__)val)); \
		101	break; \
137	default: __bad_percpu_size(); \	102	default: __bad_percpu_size(); \
138	} \	103	} \
139	} while (0)	104	} while (0)
@@ -143,17 +108,22 @@ do { \
143	typeof(var) ret__; \	108	typeof(var) ret__; \
144	switch (sizeof(var)) { \	109	switch (sizeof(var)) { \
145	case 1: \	110	case 1: \
146	asm(op "b "__percpu_seg"%1,%0" \	111	asm(op "b "__percpu_arg(1)",%0" \
147	: "=r" (ret__) \	112	: "=r" (ret__) \
148	: "m" (var)); \	113	: "m" (var)); \
149	break; \	114	break; \
150	case 2: \	115	case 2: \
151	asm(op "w "__percpu_seg"%1,%0" \	116	asm(op "w "__percpu_arg(1)",%0" \
152	: "=r" (ret__) \	117	: "=r" (ret__) \
153	: "m" (var)); \	118	: "m" (var)); \
154	break; \	119	break; \
155	case 4: \	120	case 4: \
156	asm(op "l "__percpu_seg"%1,%0" \	121	asm(op "l "__percpu_arg(1)",%0" \
		122	: "=r" (ret__) \
		123	: "m" (var)); \
		124	break; \
		125	case 8: \
		126	asm(op "q "__percpu_arg(1)",%0" \
157	: "=r" (ret__) \	127	: "=r" (ret__) \
158	: "m" (var)); \	128	: "m" (var)); \
159	break; \	129	break; \
@@ -162,13 +132,30 @@ do { \
162	ret__; \	132	ret__; \
163	})	133	})
164		134
165	#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)	135	#define percpu_read(var) percpu_from_op("mov", per_cpu__##var)
166	#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)	136	#define percpu_write(var, val) percpu_to_op("mov", per_cpu__##var, val)
167	#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)	137	#define percpu_add(var, val) percpu_to_op("add", per_cpu__##var, val)
168	#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)	138	#define percpu_sub(var, val) percpu_to_op("sub", per_cpu__##var, val)
169	#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)	139	#define percpu_and(var, val) percpu_to_op("and", per_cpu__##var, val)
		140	#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
		141	#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
		142
		143	/* This is not atomic against other CPUs -- CPU preemption needs to be off */
		144	#define x86_test_and_clear_bit_percpu(bit, var) \
		145	({ \
		146	int old__; \
		147	asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
		148	: "=r" (old__), "+m" (per_cpu__##var) \
		149	: "dIr" (bit)); \
		150	old__; \
		151	})
		152
		153	#include <asm-generic/percpu.h>
		154
		155	/* We can use this directly for local CPU (faster). */
		156	DECLARE_PER_CPU(unsigned long, this_cpu_off);
		157
170	#endif /* !__ASSEMBLY__ */	158	#endif /* !__ASSEMBLY__ */
171	#endif /* !CONFIG_X86_64 */
172		159
173	#ifdef CONFIG_SMP	160	#ifdef CONFIG_SMP
174		161
@@ -195,9 +182,9 @@ do { \
195	#define early_per_cpu_ptr(_name) (_name##_early_ptr)	182	#define early_per_cpu_ptr(_name) (_name##_early_ptr)
196	#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])	183	#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
197	#define early_per_cpu(_name, _cpu) \	184	#define early_per_cpu(_name, _cpu) \
198	(early_per_cpu_ptr(_name) ? \	185	*(early_per_cpu_ptr(_name) ? \
199	early_per_cpu_ptr(_name)[_cpu] : \	186	&early_per_cpu_ptr(_name)[_cpu] : \
200	per_cpu(_name, _cpu))	187	&per_cpu(_name, _cpu))
201		188
202	#else /* !CONFIG_SMP */	189	#else /* !CONFIG_SMP */
203	#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \	190	#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \