1 files changed, 127 insertions, 9 deletions
diff --git a/include/asm-i386/percpu.h b/include/asm-i386/percpu.h
index 510ae1d3486c..f54830b5d5ac 100644
--- a/include/asm-i386/percpu.h
+++ b/include/asm-i386/percpu.h
@@ -1,9 +1,32 @@
 #ifndef __ARCH_I386_PERCPU__
 #define __ARCH_I386_PERCPU__
-#ifndef __ASSEMBLY__
+#ifdef __ASSEMBLY__
-#include <asm-generic/percpu.h>
-#else
+/*
+ * PER_CPU finds an address of a per-cpu variable.
+ *
+ * Args:
+ *    var - variable name
+ *    reg - 32bit register
+ *
+ * The resulting address is stored in the "reg" argument.
+ *
+ * Example:
+ *    PER_CPU(cpu_gdt_descr, %ebx)
+ */
+#ifdef CONFIG_SMP
+#define PER_CPU(var, reg)                               \
+        movl %fs:per_cpu__##this_cpu_off, reg;          \
+        lea per_cpu__##var(reg), reg
+#define PER_CPU_VAR(var)        %fs:per_cpu__##var
+#else /* ! SMP */
+#define PER_CPU(var, reg)                       \
+        movl $per_cpu__##var, reg
+#define PER_CPU_VAR(var)        per_cpu__##var
+#endif  /* SMP */
+#else /* ...!ASSEMBLY */
 /*
 * PER_CPU finds an address of a per-cpu variable.
@@ -18,14 +41,109 @@
 *    PER_CPU(cpu_gdt_descr, %ebx)
 */
 #ifdef CONFIG_SMP
-#define PER_CPU(var, cpu) \
+/* Same as generic implementation except for optimized local access. */
-        movl __per_cpu_offset(,cpu,4), cpu;     \
+#define __GENERIC_PER_CPU
-        addl $per_cpu__/**/var, cpu;
-#else /* ! SMP */
+/* This is used for other cpus to find our section. */
-#define PER_CPU(var, cpu) \
+extern unsigned long __per_cpu_offset[];
-        movl $per_cpu__/**/var, cpu;
+#define per_cpu_offset(x) (__per_cpu_offset[x])
+/* Separate out the type, so (int[3], foo) works. */
+#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
+#define DEFINE_PER_CPU(type, name) \
+    __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
+/* We can use this directly for local CPU (faster). */
+DECLARE_PER_CPU(unsigned long, this_cpu_off);
+/* var is in discarded region: offset to particular copy we want */
+#define per_cpu(var, cpu) (*({                          \
+        extern int simple_indentifier_##var(void);      \
+        RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))
+#define __raw_get_cpu_var(var) (*({                                     \
+        extern int simple_indentifier_##var(void);                      \
+        RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off));     \
+}))
+#define __get_cpu_var(var) __raw_get_cpu_var(var)
+/* A macro to avoid #include hell... */
+#define percpu_modcopy(pcpudst, src, size)                      \
+do {                                                            \
+        unsigned int __i;                                       \
+        for_each_possible_cpu(__i)                              \
+                memcpy((pcpudst)+__per_cpu_offset[__i],         \
+                       (src), (size));                          \
+} while (0)
+#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
+#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
+/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
+#define __percpu_seg "%%fs:"
+#else  /* !SMP */
+#include <asm-generic/percpu.h>
+#define __percpu_seg ""
 #endif  /* SMP */
+/* For arch-specific code, we can use direct single-insn ops (they
+ * don't give an lvalue though). */
+extern void __bad_percpu_size(void);
+#define percpu_to_op(op,var,val)                                \
+        do {                                                    \
+                typedef typeof(var) T__;                        \
+                if (0) { T__ tmp__; tmp__ = (val); }            \
+                switch (sizeof(var)) {                          \
+                case 1:                                         \
+                        asm(op "b %1,"__percpu_seg"%0"          \
+                            : "+m" (var)                        \
+                            :"ri" ((T__)val));                  \
+                        break;                                  \
+                case 2:                                         \
+                        asm(op "w %1,"__percpu_seg"%0"          \
+                            : "+m" (var)                        \
+                            :"ri" ((T__)val));                  \
+                        break;                                  \
+                case 4:                                         \
+                        asm(op "l %1,"__percpu_seg"%0"          \
+                            : "+m" (var)                        \
+                            :"ri" ((T__)val));                  \
+                        break;                                  \
+                default: __bad_percpu_size();                   \
+                }                                               \
+        } while (0)
+#define percpu_from_op(op,var)                                  \
+        ({                                                      \
+                typeof(var) ret__;                              \
+                switch (sizeof(var)) {                          \
+                case 1:                                         \
+                        asm(op "b "__percpu_seg"%1,%0"          \
+                            : "=r" (ret__)                      \
+                            : "m" (var));                       \
+                        break;                                  \
+                case 2:                                         \
+                        asm(op "w "__percpu_seg"%1,%0"          \
+                            : "=r" (ret__)                      \
+                            : "m" (var));                       \
+                        break;                                  \
+                case 4:                                         \
+                        asm(op "l "__percpu_seg"%1,%0"          \
+                            : "=r" (ret__)                      \
+                            : "m" (var));                       \
+                        break;                                  \
+                default: __bad_percpu_size();                   \
+                }                                               \
+                ret__; })
+#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
+#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
+#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
+#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
+#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
 #endif /* !__ASSEMBLY__ */
 #endif /* __ARCH_I386_PERCPU__ */

diff --git a/include/asm-i386/percpu.h b/include/asm-i386/percpu.h index 510ae1d3486c..f54830b5d5ac 100644 --- a/include/asm-i386/percpu.h +++ b/include/asm-i386/percpu.h
@@ -1,9 +1,32 @@
1	#ifndef __ARCH_I386_PERCPU__	1	#ifndef __ARCH_I386_PERCPU__
2	#define __ARCH_I386_PERCPU__	2	#define __ARCH_I386_PERCPU__
3		3
4	#ifndef __ASSEMBLY__	4	#ifdef __ASSEMBLY__
5	#include <asm-generic/percpu.h>	5
6	#else	6	/*
		7	* PER_CPU finds an address of a per-cpu variable.
		8	*
		9	* Args:
		10	* var - variable name
		11	* reg - 32bit register
		12	*
		13	* The resulting address is stored in the "reg" argument.
		14	*
		15	* Example:
		16	* PER_CPU(cpu_gdt_descr, %ebx)
		17	*/
		18	#ifdef CONFIG_SMP
		19	#define PER_CPU(var, reg) \
		20	movl %fs:per_cpu__##this_cpu_off, reg; \
		21	lea per_cpu__##var(reg), reg
		22	#define PER_CPU_VAR(var) %fs:per_cpu__##var
		23	#else /* ! SMP */
		24	#define PER_CPU(var, reg) \
		25	movl $per_cpu__##var, reg
		26	#define PER_CPU_VAR(var) per_cpu__##var
		27	#endif /* SMP */
		28
		29	#else /* ...!ASSEMBLY */
7		30
8	/*	31	/*
9	* PER_CPU finds an address of a per-cpu variable.	32	* PER_CPU finds an address of a per-cpu variable.
@@ -18,14 +41,109 @@
18	* PER_CPU(cpu_gdt_descr, %ebx)	41	* PER_CPU(cpu_gdt_descr, %ebx)
19	*/	42	*/
20	#ifdef CONFIG_SMP	43	#ifdef CONFIG_SMP
21	#define PER_CPU(var, cpu) \	44	/* Same as generic implementation except for optimized local access. */
22	movl __per_cpu_offset(,cpu,4), cpu; \	45	#define __GENERIC_PER_CPU
23	addl $per_cpu__/**/var, cpu;	46
24	#else /* ! SMP */	47	/* This is used for other cpus to find our section. */
25	#define PER_CPU(var, cpu) \	48	extern unsigned long __per_cpu_offset[];
26	movl $per_cpu__/**/var, cpu;	49
		50	#define per_cpu_offset(x) (__per_cpu_offset[x])
		51
		52	/* Separate out the type, so (int[3], foo) works. */
		53	#define DECLARE_PER_CPU(type, name) extern __typeof__(type) per_cpu__##name
		54	#define DEFINE_PER_CPU(type, name) \
		55	__attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name
		56
		57	/* We can use this directly for local CPU (faster). */
		58	DECLARE_PER_CPU(unsigned long, this_cpu_off);
		59
		60	/* var is in discarded region: offset to particular copy we want */
		61	#define per_cpu(var, cpu) (*({ \
		62	extern int simple_indentifier_##var(void); \
		63	RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); }))
		64
		65	#define __raw_get_cpu_var(var) (*({ \
		66	extern int simple_indentifier_##var(void); \
		67	RELOC_HIDE(&per_cpu__##var, x86_read_percpu(this_cpu_off)); \
		68	}))
		69
		70	#define __get_cpu_var(var) __raw_get_cpu_var(var)
		71
		72	/* A macro to avoid #include hell... */
		73	#define percpu_modcopy(pcpudst, src, size) \
		74	do { \
		75	unsigned int __i; \
		76	for_each_possible_cpu(__i) \
		77	memcpy((pcpudst)+__per_cpu_offset[__i], \
		78	(src), (size)); \
		79	} while (0)
		80
		81	#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
		82	#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
		83
		84	/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
		85	#define __percpu_seg "%%fs:"
		86	#else /* !SMP */
		87	#include <asm-generic/percpu.h>
		88	#define __percpu_seg ""
27	#endif /* SMP */	89	#endif /* SMP */
28		90
		91	/* For arch-specific code, we can use direct single-insn ops (they
		92	* don't give an lvalue though). */
		93	extern void __bad_percpu_size(void);
		94
		95	#define percpu_to_op(op,var,val) \
		96	do { \
		97	typedef typeof(var) T__; \
		98	if (0) { T__ tmp__; tmp__ = (val); } \
		99	switch (sizeof(var)) { \
		100	case 1: \
		101	asm(op "b %1,"__percpu_seg"%0" \
		102	: "+m" (var) \
		103	:"ri" ((T__)val)); \
		104	break; \
		105	case 2: \
		106	asm(op "w %1,"__percpu_seg"%0" \
		107	: "+m" (var) \
		108	:"ri" ((T__)val)); \
		109	break; \
		110	case 4: \
		111	asm(op "l %1,"__percpu_seg"%0" \
		112	: "+m" (var) \
		113	:"ri" ((T__)val)); \
		114	break; \
		115	default: __bad_percpu_size(); \
		116	} \
		117	} while (0)
		118
		119	#define percpu_from_op(op,var) \
		120	({ \
		121	typeof(var) ret__; \
		122	switch (sizeof(var)) { \
		123	case 1: \
		124	asm(op "b "__percpu_seg"%1,%0" \
		125	: "=r" (ret__) \
		126	: "m" (var)); \
		127	break; \
		128	case 2: \
		129	asm(op "w "__percpu_seg"%1,%0" \
		130	: "=r" (ret__) \
		131	: "m" (var)); \
		132	break; \
		133	case 4: \
		134	asm(op "l "__percpu_seg"%1,%0" \
		135	: "=r" (ret__) \
		136	: "m" (var)); \
		137	break; \
		138	default: __bad_percpu_size(); \
		139	} \
		140	ret__; })
		141
		142	#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
		143	#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
		144	#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
		145	#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
		146	#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
29	#endif /* !__ASSEMBLY__ */	147	#endif /* !__ASSEMBLY__ */
30		148
31	#endif /* __ARCH_I386_PERCPU__ */	149	#endif /* __ARCH_I386_PERCPU__ */