x86-64: Use absolute displacements for per-cpu accesses.

Accessing memory through %gs should not use rip-relative addressing. Adding a P prefix for the argument tells gcc to not add (%rip) to the memory references. Signed-off-by: Brian Gerst <brgerst@gmail.com> Signed-off-by: Tejun Heo <tj@kernel.org>
author: Brian Gerst <brgerst@gmail.com> 2009-01-18 10:38:59 -0500
committer: Tejun Heo <tj@kernel.org> 2009-01-18 10:38:59 -0500
commit: 87b264065880fa696c121dad8498a60524e0f6de (patch)
tree: 5a40a9ca966e5feda1040f9ef2c7798ac4d1e234
parent: c2558e0eba66b49993e619da66c95a50a97830a3 (diff)
2 files changed, 14 insertions, 14 deletions
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 03aa4b00a1c3..165d5272ece1 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -39,10 +39,10 @@
 #include <linux/stringify.h>
 #ifdef CONFIG_SMP
-#define __percpu_seg_str        "%%"__stringify(__percpu_seg)":"
+#define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
 #define __my_cpu_offset         percpu_read(this_cpu_off)
 #else
-#define __percpu_seg_str
+#define __percpu_arg(x)         "%" #x
 #endif
 /* For arch-specific code, we can use direct single-insn ops (they
@@ -58,22 +58,22 @@ do {							\
        }                                               \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b %1,"__percpu_seg_str"%0"      \
+                asm(op "b %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 2:                                         \
-                asm(op "w %1,"__percpu_seg_str"%0"      \
+                asm(op "w %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 4:                                         \
-                asm(op "l %1,"__percpu_seg_str"%0"      \
+                asm(op "l %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "ri" ((T__)val));                 \
                break;                                  \
        case 8:                                         \
-                asm(op "q %1,"__percpu_seg_str"%0"      \
+                asm(op "q %1,"__percpu_arg(0)           \
                    : "+m" (var)                        \
                    : "r" ((T__)val));                  \
                break;                                  \
@@ -86,22 +86,22 @@ do {							\
        typeof(var) ret__;                              \
        switch (sizeof(var)) {                          \
        case 1:                                         \
-                asm(op "b "__percpu_seg_str"%1,%0"      \
+                asm(op "b "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 2:                                         \
-                asm(op "w "__percpu_seg_str"%1,%0"      \
+                asm(op "w "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 4:                                         \
-                asm(op "l "__percpu_seg_str"%1,%0"      \
+                asm(op "l "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
        case 8:                                         \
-                asm(op "q "__percpu_seg_str"%1,%0"      \
+                asm(op "q "__percpu_arg(1)",%0"         \
                    : "=r" (ret__)                      \
                    : "m" (var));                       \
                break;                                  \
@@ -122,9 +122,9 @@ do {							\
 #define x86_test_and_clear_bit_percpu(bit, var)                         \
 ({                                                                      \
        int old__;                                                      \
-        asm volatile("btr %1,"__percpu_seg_str"%c2\n\tsbbl %0,%0"       \
+        asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
-                     : "=r" (old__)                                     \
+                     : "=r" (old__), "+m" (per_cpu__##var)              \
-                     : "dIr" (bit), "i" (&per_cpu__##var) : "memory");  \
+                     : "dIr" (bit));                                    \
        old__;                                                          \
 })
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 4399aac680e9..d1dc27dba36d 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -94,7 +94,7 @@ do {									\
             "call __switch_to\n\t"                                       \
             ".globl thread_return\n"                                     \
             "thread_return:\n\t"                                         \
-             "movq "__percpu_seg_str"%P[current_task],%%rsi\n\t"          \
+             "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
             "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
             LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"        \
             "movq %%rax,%%rdi\n\t"                                       \
author	Brian Gerst <brgerst@gmail.com>	2009-01-18 10:38:59 -0500
committer	Tejun Heo <tj@kernel.org>	2009-01-18 10:38:59 -0500
commit	87b264065880fa696c121dad8498a60524e0f6de (patch)
tree	5a40a9ca966e5feda1040f9ef2c7798ac4d1e234
parent	c2558e0eba66b49993e619da66c95a50a97830a3 (diff)

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index 03aa4b00a1c3..165d5272ece1 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h
@@ -39,10 +39,10 @@
39	#include <linux/stringify.h>	39	#include <linux/stringify.h>
40		40
41	#ifdef CONFIG_SMP	41	#ifdef CONFIG_SMP
42	#define __percpu_seg_str "%%"__stringify(__percpu_seg)":"	42	#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
43	#define __my_cpu_offset percpu_read(this_cpu_off)	43	#define __my_cpu_offset percpu_read(this_cpu_off)
44	#else	44	#else
45	#define __percpu_seg_str	45	#define __percpu_arg(x) "%" #x
46	#endif	46	#endif
47		47
48	/* For arch-specific code, we can use direct single-insn ops (they	48	/* For arch-specific code, we can use direct single-insn ops (they
@@ -58,22 +58,22 @@ do { \
58	} \	58	} \
59	switch (sizeof(var)) { \	59	switch (sizeof(var)) { \
60	case 1: \	60	case 1: \
61	asm(op "b %1,"__percpu_seg_str"%0" \	61	asm(op "b %1,"__percpu_arg(0) \
62	: "+m" (var) \	62	: "+m" (var) \
63	: "ri" ((T__)val)); \	63	: "ri" ((T__)val)); \
64	break; \	64	break; \
65	case 2: \	65	case 2: \
66	asm(op "w %1,"__percpu_seg_str"%0" \	66	asm(op "w %1,"__percpu_arg(0) \
67	: "+m" (var) \	67	: "+m" (var) \
68	: "ri" ((T__)val)); \	68	: "ri" ((T__)val)); \
69	break; \	69	break; \
70	case 4: \	70	case 4: \
71	asm(op "l %1,"__percpu_seg_str"%0" \	71	asm(op "l %1,"__percpu_arg(0) \
72	: "+m" (var) \	72	: "+m" (var) \
73	: "ri" ((T__)val)); \	73	: "ri" ((T__)val)); \
74	break; \	74	break; \
75	case 8: \	75	case 8: \
76	asm(op "q %1,"__percpu_seg_str"%0" \	76	asm(op "q %1,"__percpu_arg(0) \
77	: "+m" (var) \	77	: "+m" (var) \
78	: "r" ((T__)val)); \	78	: "r" ((T__)val)); \
79	break; \	79	break; \
@@ -86,22 +86,22 @@ do { \
86	typeof(var) ret__; \	86	typeof(var) ret__; \
87	switch (sizeof(var)) { \	87	switch (sizeof(var)) { \
88	case 1: \	88	case 1: \
89	asm(op "b "__percpu_seg_str"%1,%0" \	89	asm(op "b "__percpu_arg(1)",%0" \
90	: "=r" (ret__) \	90	: "=r" (ret__) \
91	: "m" (var)); \	91	: "m" (var)); \
92	break; \	92	break; \
93	case 2: \	93	case 2: \
94	asm(op "w "__percpu_seg_str"%1,%0" \	94	asm(op "w "__percpu_arg(1)",%0" \
95	: "=r" (ret__) \	95	: "=r" (ret__) \
96	: "m" (var)); \	96	: "m" (var)); \
97	break; \	97	break; \
98	case 4: \	98	case 4: \
99	asm(op "l "__percpu_seg_str"%1,%0" \	99	asm(op "l "__percpu_arg(1)",%0" \
100	: "=r" (ret__) \	100	: "=r" (ret__) \
101	: "m" (var)); \	101	: "m" (var)); \
102	break; \	102	break; \
103	case 8: \	103	case 8: \
104	asm(op "q "__percpu_seg_str"%1,%0" \	104	asm(op "q "__percpu_arg(1)",%0" \
105	: "=r" (ret__) \	105	: "=r" (ret__) \
106	: "m" (var)); \	106	: "m" (var)); \
107	break; \	107	break; \
@@ -122,9 +122,9 @@ do { \
122	#define x86_test_and_clear_bit_percpu(bit, var) \	122	#define x86_test_and_clear_bit_percpu(bit, var) \
123	({ \	123	({ \
124	int old__; \	124	int old__; \
125	asm volatile("btr %1,"__percpu_seg_str"%c2\n\tsbbl %0,%0" \	125	asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
126	: "=r" (old__) \	126	: "=r" (old__), "+m" (per_cpu__##var) \
127	: "dIr" (bit), "i" (&per_cpu__##var) : "memory"); \	127	: "dIr" (bit)); \
128	old__; \	128	old__; \
129	})	129	})
130		130


diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h index 4399aac680e9..d1dc27dba36d 100644 --- a/arch/x86/include/asm/system.h +++ b/arch/x86/include/asm/system.h
@@ -94,7 +94,7 @@ do { \
94	"call __switch_to\n\t" \	94	"call __switch_to\n\t" \
95	".globl thread_return\n" \	95	".globl thread_return\n" \
96	"thread_return:\n\t" \	96	"thread_return:\n\t" \
97	"movq "__percpu_seg_str"%P[current_task],%%rsi\n\t" \	97	"movq "__percpu_arg([current_task])",%%rsi\n\t" \
98	"movq %P[thread_info](%%rsi),%%r8\n\t" \	98	"movq %P[thread_info](%%rsi),%%r8\n\t" \
99	LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \	99	LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \
100	"movq %%rax,%%rdi\n\t" \	100	"movq %%rax,%%rdi\n\t" \