aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/lib
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-07-22 20:02:24 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-07-22 20:02:24 -0400
commiteb47418dc56baaca33d270a868d8ddaa81150952 (patch)
tree16c0b09bacf3f6c25e5786afcfa741fa2a550ae5 /arch/x86/lib
parent2c9e88a1085b3183e5f92170a74980e5654f817b (diff)
parenta750036f35cda160ef77408ec92c3dc41f8feebb (diff)
Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: x86: Fix write lock scalability 64-bit issue x86: Unify rwsem assembly implementation x86: Unify rwlock assembly implementation x86, asm: Fix binutils 2.16 issue with __USER32_CS x86, asm: Cleanup thunk_64.S x86, asm: Flip RESTORE_ARGS arguments logic x86, asm: Flip SAVE_ARGS arguments logic x86, asm: Thin down SAVE/RESTORE_* asm macros
Diffstat (limited to 'arch/x86/lib')
-rw-r--r--arch/x86/lib/Makefile7
-rw-r--r--arch/x86/lib/rwlock.S44
-rw-r--r--arch/x86/lib/rwlock_64.S38
-rw-r--r--arch/x86/lib/rwsem.S (renamed from arch/x86/lib/rwsem_64.S)75
-rw-r--r--arch/x86/lib/semaphore_32.S124
-rw-r--r--arch/x86/lib/thunk_64.S45
6 files changed, 125 insertions, 208 deletions
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index 6ba477342b8e..b00f6785da74 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -20,6 +20,8 @@ lib-y := delay.o
20lib-y += thunk_$(BITS).o 20lib-y += thunk_$(BITS).o
21lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o 21lib-y += usercopy_$(BITS).o usercopy.o getuser.o putuser.o
22lib-y += memcpy_$(BITS).o 22lib-y += memcpy_$(BITS).o
23lib-$(CONFIG_SMP) += rwlock.o
24lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
23lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o 25lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o
24 26
25obj-y += msr.o msr-reg.o msr-reg-export.o 27obj-y += msr.o msr-reg.o msr-reg-export.o
@@ -29,7 +31,7 @@ ifeq ($(CONFIG_X86_32),y)
29 lib-y += atomic64_cx8_32.o 31 lib-y += atomic64_cx8_32.o
30 lib-y += checksum_32.o 32 lib-y += checksum_32.o
31 lib-y += strstr_32.o 33 lib-y += strstr_32.o
32 lib-y += semaphore_32.o string_32.o 34 lib-y += string_32.o
33 lib-y += cmpxchg.o 35 lib-y += cmpxchg.o
34ifneq ($(CONFIG_X86_CMPXCHG64),y) 36ifneq ($(CONFIG_X86_CMPXCHG64),y)
35 lib-y += cmpxchg8b_emu.o atomic64_386_32.o 37 lib-y += cmpxchg8b_emu.o atomic64_386_32.o
@@ -40,7 +42,6 @@ else
40 lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o 42 lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o
41 lib-y += thunk_64.o clear_page_64.o copy_page_64.o 43 lib-y += thunk_64.o clear_page_64.o copy_page_64.o
42 lib-y += memmove_64.o memset_64.o 44 lib-y += memmove_64.o memset_64.o
43 lib-y += copy_user_64.o rwlock_64.o copy_user_nocache_64.o 45 lib-y += copy_user_64.o copy_user_nocache_64.o
44 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem_64.o
45 lib-y += cmpxchg16b_emu.o 46 lib-y += cmpxchg16b_emu.o
46endif 47endif
diff --git a/arch/x86/lib/rwlock.S b/arch/x86/lib/rwlock.S
new file mode 100644
index 000000000000..1cad22139c88
--- /dev/null
+++ b/arch/x86/lib/rwlock.S
@@ -0,0 +1,44 @@
1/* Slow paths of read/write spinlocks. */
2
3#include <linux/linkage.h>
4#include <asm/alternative-asm.h>
5#include <asm/frame.h>
6#include <asm/rwlock.h>
7
8#ifdef CONFIG_X86_32
9# define __lock_ptr eax
10#else
11# define __lock_ptr rdi
12#endif
13
14ENTRY(__write_lock_failed)
15 CFI_STARTPROC
16 FRAME
170: LOCK_PREFIX
18 WRITE_LOCK_ADD($RW_LOCK_BIAS) (%__lock_ptr)
191: rep; nop
20 cmpl $WRITE_LOCK_CMP, (%__lock_ptr)
21 jne 1b
22 LOCK_PREFIX
23 WRITE_LOCK_SUB($RW_LOCK_BIAS) (%__lock_ptr)
24 jnz 0b
25 ENDFRAME
26 ret
27 CFI_ENDPROC
28END(__write_lock_failed)
29
30ENTRY(__read_lock_failed)
31 CFI_STARTPROC
32 FRAME
330: LOCK_PREFIX
34 READ_LOCK_SIZE(inc) (%__lock_ptr)
351: rep; nop
36 READ_LOCK_SIZE(cmp) $1, (%__lock_ptr)
37 js 1b
38 LOCK_PREFIX
39 READ_LOCK_SIZE(dec) (%__lock_ptr)
40 js 0b
41 ENDFRAME
42 ret
43 CFI_ENDPROC
44END(__read_lock_failed)
diff --git a/arch/x86/lib/rwlock_64.S b/arch/x86/lib/rwlock_64.S
deleted file mode 100644
index 05ea55f71405..000000000000
--- a/arch/x86/lib/rwlock_64.S
+++ /dev/null
@@ -1,38 +0,0 @@
1/* Slow paths of read/write spinlocks. */
2
3#include <linux/linkage.h>
4#include <asm/rwlock.h>
5#include <asm/alternative-asm.h>
6#include <asm/dwarf2.h>
7
8/* rdi: pointer to rwlock_t */
9ENTRY(__write_lock_failed)
10 CFI_STARTPROC
11 LOCK_PREFIX
12 addl $RW_LOCK_BIAS,(%rdi)
131: rep
14 nop
15 cmpl $RW_LOCK_BIAS,(%rdi)
16 jne 1b
17 LOCK_PREFIX
18 subl $RW_LOCK_BIAS,(%rdi)
19 jnz __write_lock_failed
20 ret
21 CFI_ENDPROC
22END(__write_lock_failed)
23
24/* rdi: pointer to rwlock_t */
25ENTRY(__read_lock_failed)
26 CFI_STARTPROC
27 LOCK_PREFIX
28 incl (%rdi)
291: rep
30 nop
31 cmpl $1,(%rdi)
32 js 1b
33 LOCK_PREFIX
34 decl (%rdi)
35 js __read_lock_failed
36 ret
37 CFI_ENDPROC
38END(__read_lock_failed)
diff --git a/arch/x86/lib/rwsem_64.S b/arch/x86/lib/rwsem.S
index 67743977398b..5dff5f042468 100644
--- a/arch/x86/lib/rwsem_64.S
+++ b/arch/x86/lib/rwsem.S
@@ -1,4 +1,51 @@
1/* 1/*
2 * x86 semaphore implementation.
3 *
4 * (C) Copyright 1999 Linus Torvalds
5 *
6 * Portions Copyright 1999 Red Hat, Inc.
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 *
13 * rw semaphores implemented November 1999 by Benjamin LaHaise <bcrl@kvack.org>
14 */
15
16#include <linux/linkage.h>
17#include <asm/alternative-asm.h>
18#include <asm/dwarf2.h>
19
20#define __ASM_HALF_REG(reg) __ASM_SEL(reg, e##reg)
21#define __ASM_HALF_SIZE(inst) __ASM_SEL(inst##w, inst##l)
22
23#ifdef CONFIG_X86_32
24
25/*
26 * The semaphore operations have a special calling sequence that
27 * allow us to do a simpler in-line version of them. These routines
28 * need to convert that sequence back into the C sequence when
29 * there is contention on the semaphore.
30 *
31 * %eax contains the semaphore pointer on entry. Save the C-clobbered
32 * registers (%eax, %edx and %ecx) except %eax whish is either a return
33 * value or just clobbered..
34 */
35
36#define save_common_regs \
37 pushl_cfi %ecx; CFI_REL_OFFSET ecx, 0
38
39#define restore_common_regs \
40 popl_cfi %ecx; CFI_RESTORE ecx
41
42 /* Avoid uglifying the argument copying x86-64 needs to do. */
43 .macro movq src, dst
44 .endm
45
46#else
47
48/*
2 * x86-64 rwsem wrappers 49 * x86-64 rwsem wrappers
3 * 50 *
4 * This interfaces the inline asm code to the slow-path 51 * This interfaces the inline asm code to the slow-path
@@ -16,12 +63,6 @@
16 * but %rdi, %rsi, %rcx, %r8-r11 always need saving. 63 * but %rdi, %rsi, %rcx, %r8-r11 always need saving.
17 */ 64 */
18 65
19#include <linux/linkage.h>
20#include <asm/rwlock.h>
21#include <asm/alternative-asm.h>
22#include <asm/frame.h>
23#include <asm/dwarf2.h>
24
25#define save_common_regs \ 66#define save_common_regs \
26 pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \ 67 pushq_cfi %rdi; CFI_REL_OFFSET rdi, 0; \
27 pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \ 68 pushq_cfi %rsi; CFI_REL_OFFSET rsi, 0; \
@@ -40,16 +81,18 @@
40 popq_cfi %rsi; CFI_RESTORE rsi; \ 81 popq_cfi %rsi; CFI_RESTORE rsi; \
41 popq_cfi %rdi; CFI_RESTORE rdi 82 popq_cfi %rdi; CFI_RESTORE rdi
42 83
84#endif
85
43/* Fix up special calling conventions */ 86/* Fix up special calling conventions */
44ENTRY(call_rwsem_down_read_failed) 87ENTRY(call_rwsem_down_read_failed)
45 CFI_STARTPROC 88 CFI_STARTPROC
46 save_common_regs 89 save_common_regs
47 pushq_cfi %rdx 90 __ASM_SIZE(push,_cfi) %__ASM_REG(dx)
48 CFI_REL_OFFSET rdx, 0 91 CFI_REL_OFFSET __ASM_REG(dx), 0
49 movq %rax,%rdi 92 movq %rax,%rdi
50 call rwsem_down_read_failed 93 call rwsem_down_read_failed
51 popq_cfi %rdx 94 __ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
52 CFI_RESTORE rdx 95 CFI_RESTORE __ASM_REG(dx)
53 restore_common_regs 96 restore_common_regs
54 ret 97 ret
55 CFI_ENDPROC 98 CFI_ENDPROC
@@ -67,7 +110,8 @@ ENDPROC(call_rwsem_down_write_failed)
67 110
68ENTRY(call_rwsem_wake) 111ENTRY(call_rwsem_wake)
69 CFI_STARTPROC 112 CFI_STARTPROC
70 decl %edx /* do nothing if still outstanding active readers */ 113 /* do nothing if still outstanding active readers */
114 __ASM_HALF_SIZE(dec) %__ASM_HALF_REG(dx)
71 jnz 1f 115 jnz 1f
72 save_common_regs 116 save_common_regs
73 movq %rax,%rdi 117 movq %rax,%rdi
@@ -77,16 +121,15 @@ ENTRY(call_rwsem_wake)
77 CFI_ENDPROC 121 CFI_ENDPROC
78ENDPROC(call_rwsem_wake) 122ENDPROC(call_rwsem_wake)
79 123
80/* Fix up special calling conventions */
81ENTRY(call_rwsem_downgrade_wake) 124ENTRY(call_rwsem_downgrade_wake)
82 CFI_STARTPROC 125 CFI_STARTPROC
83 save_common_regs 126 save_common_regs
84 pushq_cfi %rdx 127 __ASM_SIZE(push,_cfi) %__ASM_REG(dx)
85 CFI_REL_OFFSET rdx, 0 128 CFI_REL_OFFSET __ASM_REG(dx), 0
86 movq %rax,%rdi 129 movq %rax,%rdi
87 call rwsem_downgrade_wake 130 call rwsem_downgrade_wake
88 popq_cfi %rdx 131 __ASM_SIZE(pop,_cfi) %__ASM_REG(dx)
89 CFI_RESTORE rdx 132 CFI_RESTORE __ASM_REG(dx)
90 restore_common_regs 133 restore_common_regs
91 ret 134 ret
92 CFI_ENDPROC 135 CFI_ENDPROC
diff --git a/arch/x86/lib/semaphore_32.S b/arch/x86/lib/semaphore_32.S
deleted file mode 100644
index 06691daa4108..000000000000
--- a/arch/x86/lib/semaphore_32.S
+++ /dev/null
@@ -1,124 +0,0 @@
1/*
2 * i386 semaphore implementation.
3 *
4 * (C) Copyright 1999 Linus Torvalds
5 *
6 * Portions Copyright 1999 Red Hat, Inc.
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 *
13 * rw semaphores implemented November 1999 by Benjamin LaHaise <bcrl@kvack.org>
14 */
15
16#include <linux/linkage.h>
17#include <asm/rwlock.h>
18#include <asm/alternative-asm.h>
19#include <asm/frame.h>
20#include <asm/dwarf2.h>
21
22/*
23 * The semaphore operations have a special calling sequence that
24 * allow us to do a simpler in-line version of them. These routines
25 * need to convert that sequence back into the C sequence when
26 * there is contention on the semaphore.
27 *
28 * %eax contains the semaphore pointer on entry. Save the C-clobbered
29 * registers (%eax, %edx and %ecx) except %eax whish is either a return
30 * value or just clobbered..
31 */
32 .section .sched.text, "ax"
33
34/*
35 * rw spinlock fallbacks
36 */
37#ifdef CONFIG_SMP
38ENTRY(__write_lock_failed)
39 CFI_STARTPROC
40 FRAME
412: LOCK_PREFIX
42 addl $ RW_LOCK_BIAS,(%eax)
431: rep; nop
44 cmpl $ RW_LOCK_BIAS,(%eax)
45 jne 1b
46 LOCK_PREFIX
47 subl $ RW_LOCK_BIAS,(%eax)
48 jnz 2b
49 ENDFRAME
50 ret
51 CFI_ENDPROC
52 ENDPROC(__write_lock_failed)
53
54ENTRY(__read_lock_failed)
55 CFI_STARTPROC
56 FRAME
572: LOCK_PREFIX
58 incl (%eax)
591: rep; nop
60 cmpl $1,(%eax)
61 js 1b
62 LOCK_PREFIX
63 decl (%eax)
64 js 2b
65 ENDFRAME
66 ret
67 CFI_ENDPROC
68 ENDPROC(__read_lock_failed)
69
70#endif
71
72#ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM
73
74/* Fix up special calling conventions */
75ENTRY(call_rwsem_down_read_failed)
76 CFI_STARTPROC
77 pushl_cfi %ecx
78 CFI_REL_OFFSET ecx,0
79 pushl_cfi %edx
80 CFI_REL_OFFSET edx,0
81 call rwsem_down_read_failed
82 popl_cfi %edx
83 popl_cfi %ecx
84 ret
85 CFI_ENDPROC
86 ENDPROC(call_rwsem_down_read_failed)
87
88ENTRY(call_rwsem_down_write_failed)
89 CFI_STARTPROC
90 pushl_cfi %ecx
91 CFI_REL_OFFSET ecx,0
92 calll rwsem_down_write_failed
93 popl_cfi %ecx
94 ret
95 CFI_ENDPROC
96 ENDPROC(call_rwsem_down_write_failed)
97
98ENTRY(call_rwsem_wake)
99 CFI_STARTPROC
100 decw %dx /* do nothing if still outstanding active readers */
101 jnz 1f
102 pushl_cfi %ecx
103 CFI_REL_OFFSET ecx,0
104 call rwsem_wake
105 popl_cfi %ecx
1061: ret
107 CFI_ENDPROC
108 ENDPROC(call_rwsem_wake)
109
110/* Fix up special calling conventions */
111ENTRY(call_rwsem_downgrade_wake)
112 CFI_STARTPROC
113 pushl_cfi %ecx
114 CFI_REL_OFFSET ecx,0
115 pushl_cfi %edx
116 CFI_REL_OFFSET edx,0
117 call rwsem_downgrade_wake
118 popl_cfi %edx
119 popl_cfi %ecx
120 ret
121 CFI_ENDPROC
122 ENDPROC(call_rwsem_downgrade_wake)
123
124#endif
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
index 782b082c9ff7..a63efd6bb6a5 100644
--- a/arch/x86/lib/thunk_64.S
+++ b/arch/x86/lib/thunk_64.S
@@ -5,50 +5,41 @@
5 * Added trace_hardirqs callers - Copyright 2007 Steven Rostedt, Red Hat, Inc. 5 * Added trace_hardirqs callers - Copyright 2007 Steven Rostedt, Red Hat, Inc.
6 * Subject to the GNU public license, v.2. No warranty of any kind. 6 * Subject to the GNU public license, v.2. No warranty of any kind.
7 */ 7 */
8#include <linux/linkage.h>
9#include <asm/dwarf2.h>
10#include <asm/calling.h>
8 11
9 #include <linux/linkage.h> 12 /* rdi: arg1 ... normal C conventions. rax is saved/restored. */
10 #include <asm/dwarf2.h> 13 .macro THUNK name, func, put_ret_addr_in_rdi=0
11 #include <asm/calling.h>
12 #include <asm/rwlock.h>
13
14 /* rdi: arg1 ... normal C conventions. rax is saved/restored. */
15 .macro thunk name,func
16 .globl \name
17\name:
18 CFI_STARTPROC
19 SAVE_ARGS
20 call \func
21 jmp restore
22 CFI_ENDPROC
23 .endm
24
25#ifdef CONFIG_TRACE_IRQFLAGS
26 /* put return address in rdi (arg1) */
27 .macro thunk_ra name,func
28 .globl \name 14 .globl \name
29\name: 15\name:
30 CFI_STARTPROC 16 CFI_STARTPROC
17
18 /* this one pushes 9 elems, the next one would be %rIP */
31 SAVE_ARGS 19 SAVE_ARGS
32 /* SAVE_ARGS pushs 9 elements */ 20
33 /* the next element would be the rip */ 21 .if \put_ret_addr_in_rdi
34 movq 9*8(%rsp), %rdi 22 movq_cfi_restore 9*8, rdi
23 .endif
24
35 call \func 25 call \func
36 jmp restore 26 jmp restore
37 CFI_ENDPROC 27 CFI_ENDPROC
38 .endm 28 .endm
39 29
40 thunk_ra trace_hardirqs_on_thunk,trace_hardirqs_on_caller 30#ifdef CONFIG_TRACE_IRQFLAGS
41 thunk_ra trace_hardirqs_off_thunk,trace_hardirqs_off_caller 31 THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1
32 THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1
42#endif 33#endif
43 34
44#ifdef CONFIG_DEBUG_LOCK_ALLOC 35#ifdef CONFIG_DEBUG_LOCK_ALLOC
45 thunk lockdep_sys_exit_thunk,lockdep_sys_exit 36 THUNK lockdep_sys_exit_thunk,lockdep_sys_exit
46#endif 37#endif
47 38
48 /* SAVE_ARGS below is used only for the .cfi directives it contains. */ 39 /* SAVE_ARGS below is used only for the .cfi directives it contains. */
49 CFI_STARTPROC 40 CFI_STARTPROC
50 SAVE_ARGS 41 SAVE_ARGS
51restore: 42restore:
52 RESTORE_ARGS 43 RESTORE_ARGS
53 ret 44 ret
54 CFI_ENDPROC 45 CFI_ENDPROC
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-27 16:54:36 -0500