1 files changed, 283 insertions, 160 deletions
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 65efd42454be..e99206831459 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -30,12 +30,13 @@
 *      1C(%esp) - %ds
 *      20(%esp) - %es
 *      24(%esp) - %fs
- *      28(%esp) - orig_eax
+ *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
- *      2C(%esp) - %eip
+ *      2C(%esp) - orig_eax
- *      30(%esp) - %cs
+ *      30(%esp) - %eip
- *      34(%esp) - %eflags
+ *      34(%esp) - %cs
- *      38(%esp) - %oldesp
+ *      38(%esp) - %eflags
- *      3C(%esp) - %oldss
+ *      3C(%esp) - %oldesp
+ *      40(%esp) - %oldss
 *
 * "current" is in register %ebx during any slow entries.
 */
@@ -101,121 +102,221 @@
 #define resume_userspace_sig    resume_userspace
 #endif
-#define SAVE_ALL \
+/*
-        cld; \
+ * User gs save/restore
-        pushl %fs; \
+ *
-        CFI_ADJUST_CFA_OFFSET 4;\
+ * %gs is used for userland TLS and kernel only uses it for stack
-        /*CFI_REL_OFFSET fs, 0;*/\
+ * canary which is required to be at %gs:20 by gcc.  Read the comment
-        pushl %es; \
+ * at the top of stackprotector.h for more info.
-        CFI_ADJUST_CFA_OFFSET 4;\
+ *
-        /*CFI_REL_OFFSET es, 0;*/\
+ * Local labels 98 and 99 are used.
-        pushl %ds; \
+ */
-        CFI_ADJUST_CFA_OFFSET 4;\
+#ifdef CONFIG_X86_32_LAZY_GS
-        /*CFI_REL_OFFSET ds, 0;*/\
-        pushl %eax; \
+ /* unfortunately push/pop can't be no-op */
-        CFI_ADJUST_CFA_OFFSET 4;\
+.macro PUSH_GS
-        CFI_REL_OFFSET eax, 0;\
+        pushl $0
-        pushl %ebp; \
+        CFI_ADJUST_CFA_OFFSET 4
-        CFI_ADJUST_CFA_OFFSET 4;\
+.endm
-        CFI_REL_OFFSET ebp, 0;\
+.macro POP_GS pop=0
-        pushl %edi; \
+        addl $(4 + \pop), %esp
-        CFI_ADJUST_CFA_OFFSET 4;\
+        CFI_ADJUST_CFA_OFFSET -(4 + \pop)
-        CFI_REL_OFFSET edi, 0;\
+.endm
-        pushl %esi; \
+.macro POP_GS_EX
-        CFI_ADJUST_CFA_OFFSET 4;\
+.endm
-        CFI_REL_OFFSET esi, 0;\
-        pushl %edx; \
+ /* all the rest are no-op */
-        CFI_ADJUST_CFA_OFFSET 4;\
+.macro PTGS_TO_GS
-        CFI_REL_OFFSET edx, 0;\
+.endm
-        pushl %ecx; \
+.macro PTGS_TO_GS_EX
-        CFI_ADJUST_CFA_OFFSET 4;\
+.endm
-        CFI_REL_OFFSET ecx, 0;\
+.macro GS_TO_REG reg
-        pushl %ebx; \
+.endm
-        CFI_ADJUST_CFA_OFFSET 4;\
+.macro REG_TO_PTGS reg
-        CFI_REL_OFFSET ebx, 0;\
+.endm
-        movl $(__USER_DS), %edx; \
+.macro SET_KERNEL_GS reg
-        movl %edx, %ds; \
+.endm
-        movl %edx, %es; \
-        movl $(__KERNEL_PERCPU), %edx; \
+#else   /* CONFIG_X86_32_LAZY_GS */
+.macro PUSH_GS
+        pushl %gs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET gs, 0*/
+.endm
+.macro POP_GS pop=0
+98:     popl %gs
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_RESTORE gs*/
+  .if \pop <> 0
+        add $\pop, %esp
+        CFI_ADJUST_CFA_OFFSET -\pop
+  .endif
+.endm
+.macro POP_GS_EX
+.pushsection .fixup, "ax"
+99:     movl $0, (%esp)
+        jmp 98b
+.section __ex_table, "a"
+        .align 4
+        .long 98b, 99b
+.popsection
+.endm
+.macro PTGS_TO_GS
+98:     mov PT_GS(%esp), %gs
+.endm
+.macro PTGS_TO_GS_EX
+.pushsection .fixup, "ax"
+99:     movl $0, PT_GS(%esp)
+        jmp 98b
+.section __ex_table, "a"
+        .align 4
+        .long 98b, 99b
+.popsection
+.endm
+.macro GS_TO_REG reg
+        movl %gs, \reg
+        /*CFI_REGISTER gs, \reg*/
+.endm
+.macro REG_TO_PTGS reg
+        movl \reg, PT_GS(%esp)
+        /*CFI_REL_OFFSET gs, PT_GS*/
+.endm
+.macro SET_KERNEL_GS reg
+        movl $(__KERNEL_STACK_CANARY), \reg
+        movl \reg, %gs
+.endm
+#endif  /* CONFIG_X86_32_LAZY_GS */
+.macro SAVE_ALL
+        cld
+        PUSH_GS
+        pushl %fs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET fs, 0;*/
+        pushl %es
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET es, 0;*/
+        pushl %ds
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET ds, 0;*/
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET eax, 0
+        pushl %ebp
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebp, 0
+        pushl %edi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edi, 0
+        pushl %esi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET esi, 0
+        pushl %edx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edx, 0
+        pushl %ecx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ecx, 0
+        pushl %ebx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebx, 0
+        movl $(__USER_DS), %edx
+        movl %edx, %ds
+        movl %edx, %es
+        movl $(__KERNEL_PERCPU), %edx
        movl %edx, %fs
+        SET_KERNEL_GS %edx
+.endm
-#define RESTORE_INT_REGS \
+.macro RESTORE_INT_REGS
-        popl %ebx;      \
+        popl %ebx
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE ebx;\
+        CFI_RESTORE ebx
-        popl %ecx;      \
+        popl %ecx
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE ecx;\
+        CFI_RESTORE ecx
-        popl %edx;      \
+        popl %edx
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE edx;\
+        CFI_RESTORE edx
-        popl %esi;      \
+        popl %esi
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE esi;\
+        CFI_RESTORE esi
-        popl %edi;      \
+        popl %edi
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE edi;\
+        CFI_RESTORE edi
-        popl %ebp;      \
+        popl %ebp
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        CFI_RESTORE ebp;\
+        CFI_RESTORE ebp
-        popl %eax;      \
+        popl %eax
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
        CFI_RESTORE eax
+.endm
-#define RESTORE_REGS    \
+.macro RESTORE_REGS pop=0
-        RESTORE_INT_REGS; \
+        RESTORE_INT_REGS
-1:      popl %ds;       \
+1:      popl %ds
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        /*CFI_RESTORE ds;*/\
+        /*CFI_RESTORE ds;*/
-2:      popl %es;       \
+2:      popl %es
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        /*CFI_RESTORE es;*/\
+        /*CFI_RESTORE es;*/
-3:      popl %fs;       \
+3:      popl %fs
-        CFI_ADJUST_CFA_OFFSET -4;\
+        CFI_ADJUST_CFA_OFFSET -4
-        /*CFI_RESTORE fs;*/\
+        /*CFI_RESTORE fs;*/
-.pushsection .fixup,"ax";       \
+        POP_GS \pop
-4:      movl $0,(%esp); \
+.pushsection .fixup, "ax"
-        jmp 1b;         \
+4:      movl $0, (%esp)
-5:      movl $0,(%esp); \
+        jmp 1b
-        jmp 2b;         \
+5:      movl $0, (%esp)
-6:      movl $0,(%esp); \
+        jmp 2b
-        jmp 3b;         \
+6:      movl $0, (%esp)
-.section __ex_table,"a";\
+        jmp 3b
-        .align 4;       \
+.section __ex_table, "a"
-        .long 1b,4b;    \
+        .align 4
-        .long 2b,5b;    \
+        .long 1b, 4b
-        .long 3b,6b;    \
+        .long 2b, 5b
+        .long 3b, 6b
 .popsection
+        POP_GS_EX
+.endm
-#define RING0_INT_FRAME \
+.macro RING0_INT_FRAME
-        CFI_STARTPROC simple;\
+        CFI_STARTPROC simple
-        CFI_SIGNAL_FRAME;\
+        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA esp, 3*4;\
+        CFI_DEF_CFA esp, 3*4
-        /*CFI_OFFSET cs, -2*4;*/\
+        /*CFI_OFFSET cs, -2*4;*/
        CFI_OFFSET eip, -3*4
+.endm
-#define RING0_EC_FRAME \
+.macro RING0_EC_FRAME
-        CFI_STARTPROC simple;\
+        CFI_STARTPROC simple
-        CFI_SIGNAL_FRAME;\
+        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA esp, 4*4;\
+        CFI_DEF_CFA esp, 4*4
-        /*CFI_OFFSET cs, -2*4;*/\
+        /*CFI_OFFSET cs, -2*4;*/
        CFI_OFFSET eip, -3*4
+.endm
-#define RING0_PTREGS_FRAME \
+.macro RING0_PTREGS_FRAME
-        CFI_STARTPROC simple;\
+        CFI_STARTPROC simple
-        CFI_SIGNAL_FRAME;\
+        CFI_SIGNAL_FRAME
-        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
+        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
-        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
+        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
-        CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
+        CFI_OFFSET eip, PT_EIP-PT_OLDESP
-        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
+        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
-        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
+        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
-        CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
+        CFI_OFFSET eax, PT_EAX-PT_OLDESP
-        CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
+        CFI_OFFSET ebp, PT_EBP-PT_OLDESP
-        CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
+        CFI_OFFSET edi, PT_EDI-PT_OLDESP
-        CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
+        CFI_OFFSET esi, PT_ESI-PT_OLDESP
-        CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
+        CFI_OFFSET edx, PT_EDX-PT_OLDESP
-        CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
+        CFI_OFFSET ecx, PT_ECX-PT_OLDESP
        CFI_OFFSET ebx, PT_EBX-PT_OLDESP
+.endm
 ENTRY(ret_from_fork)
        CFI_STARTPROC
@@ -362,6 +463,7 @@ sysenter_exit:
        xorl %ebp,%ebp
        TRACE_IRQS_ON
 1:      mov  PT_FS(%esp), %fs
+        PTGS_TO_GS
        ENABLE_INTERRUPTS_SYSEXIT
 #ifdef CONFIG_AUDITSYSCALL
@@ -410,6 +512,7 @@ sysexit_audit:
        .align 4
        .long 1b,2b
 .popsection
+        PTGS_TO_GS_EX
 ENDPROC(ia32_sysenter_target)
        # system call handler stub
@@ -452,8 +555,7 @@ restore_all:
 restore_nocheck:
        TRACE_IRQS_IRET
 restore_nocheck_notrace:
-        RESTORE_REGS
+        RESTORE_REGS 4                  # skip orig_eax/error_code
-        addl $4, %esp                   # skip orig_eax/error_code
        CFI_ADJUST_CFA_OFFSET -4
 irq_return:
        INTERRUPT_RETURN
@@ -595,28 +697,50 @@ syscall_badsys:
 END(syscall_badsys)
        CFI_ENDPROC
-#define FIXUP_ESPFIX_STACK \
+/*
-        /* since we are on a wrong stack, we cant make it a C code :( */ \
+ * System calls that need a pt_regs pointer.
-        PER_CPU(gdt_page, %ebx); \
+ */
-        GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
+#define PTREGSCALL(name) \
-        addl %esp, %eax; \
+        ALIGN; \
-        pushl $__KERNEL_DS; \
+ptregs_##name: \
-        CFI_ADJUST_CFA_OFFSET 4; \
+        leal 4(%esp),%eax; \
-        pushl %eax; \
+        jmp sys_##name;
-        CFI_ADJUST_CFA_OFFSET 4; \
-        lss (%esp), %esp; \
+PTREGSCALL(iopl)
-        CFI_ADJUST_CFA_OFFSET -8;
+PTREGSCALL(fork)
-#define UNWIND_ESPFIX_STACK \
+PTREGSCALL(clone)
-        movl %ss, %eax; \
+PTREGSCALL(vfork)
-        /* see if on espfix stack */ \
+PTREGSCALL(execve)
-        cmpw $__ESPFIX_SS, %ax; \
+PTREGSCALL(sigaltstack)
-        jne 27f; \
+PTREGSCALL(sigreturn)
-        movl $__KERNEL_DS, %eax; \
+PTREGSCALL(rt_sigreturn)
-        movl %eax, %ds; \
+PTREGSCALL(vm86)
-        movl %eax, %es; \
+PTREGSCALL(vm86old)
-        /* switch to normal stack */ \
-        FIXUP_ESPFIX_STACK; \
+.macro FIXUP_ESPFIX_STACK
-27:;
+        /* since we are on a wrong stack, we cant make it a C code :( */
+        PER_CPU(gdt_page, %ebx)
+        GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
+        addl %esp, %eax
+        pushl $__KERNEL_DS
+        CFI_ADJUST_CFA_OFFSET 4
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        lss (%esp), %esp
+        CFI_ADJUST_CFA_OFFSET -8
+.endm
+.macro UNWIND_ESPFIX_STACK
+        movl %ss, %eax
+        /* see if on espfix stack */
+        cmpw $__ESPFIX_SS, %ax
+        jne 27f
+        movl $__KERNEL_DS, %eax
+        movl %eax, %ds
+        movl %eax, %es
+        /* switch to normal stack */
+        FIXUP_ESPFIX_STACK
+27:
+.endm
 /*
 * Build the entry stubs and pointer table with some assembler magic.
@@ -1070,7 +1194,10 @@ ENTRY(page_fault)
        CFI_ADJUST_CFA_OFFSET 4
        ALIGN
 error_code:
-        /* the function address is in %fs's slot on the stack */
+        /* the function address is in %gs's slot on the stack */
+        pushl %fs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET fs, 0*/
        pushl %es
        CFI_ADJUST_CFA_OFFSET 4
        /*CFI_REL_OFFSET es, 0*/
@@ -1099,20 +1226,15 @@ error_code:
        CFI_ADJUST_CFA_OFFSET 4
        CFI_REL_OFFSET ebx, 0
        cld
-        pushl %fs
-        CFI_ADJUST_CFA_OFFSET 4
-        /*CFI_REL_OFFSET fs, 0*/
        movl $(__KERNEL_PERCPU), %ecx
        movl %ecx, %fs
        UNWIND_ESPFIX_STACK
-        popl %ecx
+        GS_TO_REG %ecx
-        CFI_ADJUST_CFA_OFFSET -4
+        movl PT_GS(%esp), %edi          # get the function address
-        /*CFI_REGISTER es, ecx*/
-        movl PT_FS(%esp), %edi          # get the function address
        movl PT_ORIG_EAX(%esp), %edx    # get the error code
        movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
-        mov  %ecx, PT_FS(%esp)
+        REG_TO_PTGS %ecx
-        /*CFI_REL_OFFSET fs, ES*/
+        SET_KERNEL_GS %ecx
        movl $(__USER_DS), %ecx
        movl %ecx, %ds
        movl %ecx, %es
@@ -1136,26 +1258,27 @@ END(page_fault)
 * by hand onto the new stack - while updating the return eip past
 * the instruction that would have done it for sysenter.
 */
-#define FIX_STACK(offset, ok, label)            \
+.macro FIX_STACK offset ok label
-        cmpw $__KERNEL_CS,4(%esp);              \
+        cmpw $__KERNEL_CS, 4(%esp)
-        jne ok;                                 \
+        jne \ok
-label:                                          \
+\label:
-        movl TSS_sysenter_sp0+offset(%esp),%esp;        \
+        movl TSS_sysenter_sp0 + \offset(%esp), %esp
-        CFI_DEF_CFA esp, 0;                     \
+        CFI_DEF_CFA esp, 0
-        CFI_UNDEFINED eip;                      \
+        CFI_UNDEFINED eip
-        pushfl;                                 \
+        pushfl
-        CFI_ADJUST_CFA_OFFSET 4;                \
+        CFI_ADJUST_CFA_OFFSET 4
-        pushl $__KERNEL_CS;                     \
+        pushl $__KERNEL_CS
-        CFI_ADJUST_CFA_OFFSET 4;                \
+        CFI_ADJUST_CFA_OFFSET 4
-        pushl $sysenter_past_esp;               \
+        pushl $sysenter_past_esp
-        CFI_ADJUST_CFA_OFFSET 4;                \
+        CFI_ADJUST_CFA_OFFSET 4
        CFI_REL_OFFSET eip, 0
+.endm
 ENTRY(debug)
        RING0_INT_FRAME
        cmpl $ia32_sysenter_target,(%esp)
        jne debug_stack_correct
-        FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
+        FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
 debug_stack_correct:
        pushl $-1                       # mark this as an int
        CFI_ADJUST_CFA_OFFSET 4
@@ -1213,7 +1336,7 @@ nmi_stack_correct:
 nmi_stack_fixup:
        RING0_INT_FRAME
-        FIX_STACK(12,nmi_stack_correct, 1)
+        FIX_STACK 12, nmi_stack_correct, 1
        jmp nmi_stack_correct
 nmi_debug_stack_check:
@@ -1224,7 +1347,7 @@ nmi_debug_stack_check:
        jb nmi_stack_correct
        cmpl $debug_esp_fix_insn,(%esp)
        ja nmi_stack_correct
-        FIX_STACK(24,nmi_stack_correct, 1)
+        FIX_STACK 24, nmi_stack_correct, 1
        jmp nmi_stack_correct
 nmi_espfix_stack:

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 65efd42454be..e99206831459 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S
@@ -30,12 +30,13 @@
30	* 1C(%esp) - %ds	30	* 1C(%esp) - %ds
31	* 20(%esp) - %es	31	* 20(%esp) - %es
32	* 24(%esp) - %fs	32	* 24(%esp) - %fs
33	* 28(%esp) - orig_eax	33	* 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
34	* 2C(%esp) - %eip	34	* 2C(%esp) - orig_eax
35	* 30(%esp) - %cs	35	* 30(%esp) - %eip
36	* 34(%esp) - %eflags	36	* 34(%esp) - %cs
37	* 38(%esp) - %oldesp	37	* 38(%esp) - %eflags
38	* 3C(%esp) - %oldss	38	* 3C(%esp) - %oldesp
		39	* 40(%esp) - %oldss
39	*	40	*
40	* "current" is in register %ebx during any slow entries.	41	* "current" is in register %ebx during any slow entries.
41	*/	42	*/
@@ -101,121 +102,221 @@
101	#define resume_userspace_sig resume_userspace	102	#define resume_userspace_sig resume_userspace
102	#endif	103	#endif
103		104
104	#define SAVE_ALL \	105	/*
105	cld; \	106	* User gs save/restore
106	pushl %fs; \	107	*
107	CFI_ADJUST_CFA_OFFSET 4;\	108	* %gs is used for userland TLS and kernel only uses it for stack
108	/CFI_REL_OFFSET fs, 0;/\	109	* canary which is required to be at %gs:20 by gcc. Read the comment
109	pushl %es; \	110	* at the top of stackprotector.h for more info.
110	CFI_ADJUST_CFA_OFFSET 4;\	111	*
111	/CFI_REL_OFFSET es, 0;/\	112	* Local labels 98 and 99 are used.
112	pushl %ds; \	113	*/
113	CFI_ADJUST_CFA_OFFSET 4;\	114	#ifdef CONFIG_X86_32_LAZY_GS
114	/CFI_REL_OFFSET ds, 0;/\	115
115	pushl %eax; \	116	/* unfortunately push/pop can't be no-op */
116	CFI_ADJUST_CFA_OFFSET 4;\	117	.macro PUSH_GS
117	CFI_REL_OFFSET eax, 0;\	118	pushl $0
118	pushl %ebp; \	119	CFI_ADJUST_CFA_OFFSET 4
119	CFI_ADJUST_CFA_OFFSET 4;\	120	.endm
120	CFI_REL_OFFSET ebp, 0;\	121	.macro POP_GS pop=0
121	pushl %edi; \	122	addl $(4 + \pop), %esp
122	CFI_ADJUST_CFA_OFFSET 4;\	123	CFI_ADJUST_CFA_OFFSET -(4 + \pop)
123	CFI_REL_OFFSET edi, 0;\	124	.endm
124	pushl %esi; \	125	.macro POP_GS_EX
125	CFI_ADJUST_CFA_OFFSET 4;\	126	.endm
126	CFI_REL_OFFSET esi, 0;\	127
127	pushl %edx; \	128	/* all the rest are no-op */
128	CFI_ADJUST_CFA_OFFSET 4;\	129	.macro PTGS_TO_GS
129	CFI_REL_OFFSET edx, 0;\	130	.endm
130	pushl %ecx; \	131	.macro PTGS_TO_GS_EX
131	CFI_ADJUST_CFA_OFFSET 4;\	132	.endm
132	CFI_REL_OFFSET ecx, 0;\	133	.macro GS_TO_REG reg
133	pushl %ebx; \	134	.endm
134	CFI_ADJUST_CFA_OFFSET 4;\	135	.macro REG_TO_PTGS reg
135	CFI_REL_OFFSET ebx, 0;\	136	.endm
136	movl $(__USER_DS), %edx; \	137	.macro SET_KERNEL_GS reg
137	movl %edx, %ds; \	138	.endm
138	movl %edx, %es; \	139
139	movl $(__KERNEL_PERCPU), %edx; \	140	#else /* CONFIG_X86_32_LAZY_GS */
		141
		142	.macro PUSH_GS
		143	pushl %gs
		144	CFI_ADJUST_CFA_OFFSET 4
		145	/CFI_REL_OFFSET gs, 0/
		146	.endm
		147
		148	.macro POP_GS pop=0
		149	98: popl %gs
		150	CFI_ADJUST_CFA_OFFSET -4
		151	/CFI_RESTORE gs/
		152	.if \pop <> 0
		153	add $\pop, %esp
		154	CFI_ADJUST_CFA_OFFSET -\pop
		155	.endif
		156	.endm
		157	.macro POP_GS_EX
		158	.pushsection .fixup, "ax"
		159	99: movl $0, (%esp)
		160	jmp 98b
		161	.section __ex_table, "a"
		162	.align 4
		163	.long 98b, 99b
		164	.popsection
		165	.endm
		166
		167	.macro PTGS_TO_GS
		168	98: mov PT_GS(%esp), %gs
		169	.endm
		170	.macro PTGS_TO_GS_EX
		171	.pushsection .fixup, "ax"
		172	99: movl $0, PT_GS(%esp)
		173	jmp 98b
		174	.section __ex_table, "a"
		175	.align 4
		176	.long 98b, 99b
		177	.popsection
		178	.endm
		179
		180	.macro GS_TO_REG reg
		181	movl %gs, \reg
		182	/CFI_REGISTER gs, \reg/
		183	.endm
		184	.macro REG_TO_PTGS reg
		185	movl \reg, PT_GS(%esp)
		186	/CFI_REL_OFFSET gs, PT_GS/
		187	.endm
		188	.macro SET_KERNEL_GS reg
		189	movl $(__KERNEL_STACK_CANARY), \reg
		190	movl \reg, %gs
		191	.endm
		192
		193	#endif /* CONFIG_X86_32_LAZY_GS */
		194
		195	.macro SAVE_ALL
		196	cld
		197	PUSH_GS
		198	pushl %fs
		199	CFI_ADJUST_CFA_OFFSET 4
		200	/CFI_REL_OFFSET fs, 0;/
		201	pushl %es
		202	CFI_ADJUST_CFA_OFFSET 4
		203	/CFI_REL_OFFSET es, 0;/
		204	pushl %ds
		205	CFI_ADJUST_CFA_OFFSET 4
		206	/CFI_REL_OFFSET ds, 0;/
		207	pushl %eax
		208	CFI_ADJUST_CFA_OFFSET 4
		209	CFI_REL_OFFSET eax, 0
		210	pushl %ebp
		211	CFI_ADJUST_CFA_OFFSET 4
		212	CFI_REL_OFFSET ebp, 0
		213	pushl %edi
		214	CFI_ADJUST_CFA_OFFSET 4
		215	CFI_REL_OFFSET edi, 0
		216	pushl %esi
		217	CFI_ADJUST_CFA_OFFSET 4
		218	CFI_REL_OFFSET esi, 0
		219	pushl %edx
		220	CFI_ADJUST_CFA_OFFSET 4
		221	CFI_REL_OFFSET edx, 0
		222	pushl %ecx
		223	CFI_ADJUST_CFA_OFFSET 4
		224	CFI_REL_OFFSET ecx, 0
		225	pushl %ebx
		226	CFI_ADJUST_CFA_OFFSET 4
		227	CFI_REL_OFFSET ebx, 0
		228	movl $(__USER_DS), %edx
		229	movl %edx, %ds
		230	movl %edx, %es
		231	movl $(__KERNEL_PERCPU), %edx
140	movl %edx, %fs	232	movl %edx, %fs
		233	SET_KERNEL_GS %edx
		234	.endm
141		235
142	#define RESTORE_INT_REGS \	236	.macro RESTORE_INT_REGS
143	popl %ebx; \	237	popl %ebx
144	CFI_ADJUST_CFA_OFFSET -4;\	238	CFI_ADJUST_CFA_OFFSET -4
145	CFI_RESTORE ebx;\	239	CFI_RESTORE ebx
146	popl %ecx; \	240	popl %ecx
147	CFI_ADJUST_CFA_OFFSET -4;\	241	CFI_ADJUST_CFA_OFFSET -4
148	CFI_RESTORE ecx;\	242	CFI_RESTORE ecx
149	popl %edx; \	243	popl %edx
150	CFI_ADJUST_CFA_OFFSET -4;\	244	CFI_ADJUST_CFA_OFFSET -4
151	CFI_RESTORE edx;\	245	CFI_RESTORE edx
152	popl %esi; \	246	popl %esi
153	CFI_ADJUST_CFA_OFFSET -4;\	247	CFI_ADJUST_CFA_OFFSET -4
154	CFI_RESTORE esi;\	248	CFI_RESTORE esi
155	popl %edi; \	249	popl %edi
156	CFI_ADJUST_CFA_OFFSET -4;\	250	CFI_ADJUST_CFA_OFFSET -4
157	CFI_RESTORE edi;\	251	CFI_RESTORE edi
158	popl %ebp; \	252	popl %ebp
159	CFI_ADJUST_CFA_OFFSET -4;\	253	CFI_ADJUST_CFA_OFFSET -4
160	CFI_RESTORE ebp;\	254	CFI_RESTORE ebp
161	popl %eax; \	255	popl %eax
162	CFI_ADJUST_CFA_OFFSET -4;\	256	CFI_ADJUST_CFA_OFFSET -4
163	CFI_RESTORE eax	257	CFI_RESTORE eax
		258	.endm
164		259
165	#define RESTORE_REGS \	260	.macro RESTORE_REGS pop=0
166	RESTORE_INT_REGS; \	261	RESTORE_INT_REGS
167	1: popl %ds; \	262	1: popl %ds
168	CFI_ADJUST_CFA_OFFSET -4;\	263	CFI_ADJUST_CFA_OFFSET -4
169	/CFI_RESTORE ds;/\	264	/CFI_RESTORE ds;/
170	2: popl %es; \	265	2: popl %es
171	CFI_ADJUST_CFA_OFFSET -4;\	266	CFI_ADJUST_CFA_OFFSET -4
172	/CFI_RESTORE es;/\	267	/CFI_RESTORE es;/
173	3: popl %fs; \	268	3: popl %fs
174	CFI_ADJUST_CFA_OFFSET -4;\	269	CFI_ADJUST_CFA_OFFSET -4
175	/CFI_RESTORE fs;/\	270	/CFI_RESTORE fs;/
176	.pushsection .fixup,"ax"; \	271	POP_GS \pop
177	4: movl $0,(%esp); \	272	.pushsection .fixup, "ax"
178	jmp 1b; \	273	4: movl $0, (%esp)
179	5: movl $0,(%esp); \	274	jmp 1b
180	jmp 2b; \	275	5: movl $0, (%esp)
181	6: movl $0,(%esp); \	276	jmp 2b
182	jmp 3b; \	277	6: movl $0, (%esp)
183	.section __ex_table,"a";\	278	jmp 3b
184	.align 4; \	279	.section __ex_table, "a"
185	.long 1b,4b; \	280	.align 4
186	.long 2b,5b; \	281	.long 1b, 4b
187	.long 3b,6b; \	282	.long 2b, 5b
		283	.long 3b, 6b
188	.popsection	284	.popsection
		285	POP_GS_EX
		286	.endm
189		287
190	#define RING0_INT_FRAME \	288	.macro RING0_INT_FRAME
191	CFI_STARTPROC simple;\	289	CFI_STARTPROC simple
192	CFI_SIGNAL_FRAME;\	290	CFI_SIGNAL_FRAME
193	CFI_DEF_CFA esp, 3*4;\	291	CFI_DEF_CFA esp, 3*4
194	/CFI_OFFSET cs, -24;*/\	292	/CFI_OFFSET cs, -24;*/
195	CFI_OFFSET eip, -3*4	293	CFI_OFFSET eip, -3*4
		294	.endm
196		295
197	#define RING0_EC_FRAME \	296	.macro RING0_EC_FRAME
198	CFI_STARTPROC simple;\	297	CFI_STARTPROC simple
199	CFI_SIGNAL_FRAME;\	298	CFI_SIGNAL_FRAME
200	CFI_DEF_CFA esp, 4*4;\	299	CFI_DEF_CFA esp, 4*4
201	/CFI_OFFSET cs, -24;*/\	300	/CFI_OFFSET cs, -24;*/
202	CFI_OFFSET eip, -3*4	301	CFI_OFFSET eip, -3*4
		302	.endm
203		303
204	#define RING0_PTREGS_FRAME \	304	.macro RING0_PTREGS_FRAME
205	CFI_STARTPROC simple;\	305	CFI_STARTPROC simple
206	CFI_SIGNAL_FRAME;\	306	CFI_SIGNAL_FRAME
207	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\	307	CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
208	/CFI_OFFSET cs, PT_CS-PT_OLDESP;/\	308	/CFI_OFFSET cs, PT_CS-PT_OLDESP;/
209	CFI_OFFSET eip, PT_EIP-PT_OLDESP;\	309	CFI_OFFSET eip, PT_EIP-PT_OLDESP
210	/CFI_OFFSET es, PT_ES-PT_OLDESP;/\	310	/CFI_OFFSET es, PT_ES-PT_OLDESP;/
211	/CFI_OFFSET ds, PT_DS-PT_OLDESP;/\	311	/CFI_OFFSET ds, PT_DS-PT_OLDESP;/
212	CFI_OFFSET eax, PT_EAX-PT_OLDESP;\	312	CFI_OFFSET eax, PT_EAX-PT_OLDESP
213	CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\	313	CFI_OFFSET ebp, PT_EBP-PT_OLDESP
214	CFI_OFFSET edi, PT_EDI-PT_OLDESP;\	314	CFI_OFFSET edi, PT_EDI-PT_OLDESP
215	CFI_OFFSET esi, PT_ESI-PT_OLDESP;\	315	CFI_OFFSET esi, PT_ESI-PT_OLDESP
216	CFI_OFFSET edx, PT_EDX-PT_OLDESP;\	316	CFI_OFFSET edx, PT_EDX-PT_OLDESP
217	CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\	317	CFI_OFFSET ecx, PT_ECX-PT_OLDESP
218	CFI_OFFSET ebx, PT_EBX-PT_OLDESP	318	CFI_OFFSET ebx, PT_EBX-PT_OLDESP
		319	.endm
219		320
220	ENTRY(ret_from_fork)	321	ENTRY(ret_from_fork)
221	CFI_STARTPROC	322	CFI_STARTPROC
@@ -362,6 +463,7 @@ sysenter_exit:
362	xorl %ebp,%ebp	463	xorl %ebp,%ebp
363	TRACE_IRQS_ON	464	TRACE_IRQS_ON
364	1: mov PT_FS(%esp), %fs	465	1: mov PT_FS(%esp), %fs
		466	PTGS_TO_GS
365	ENABLE_INTERRUPTS_SYSEXIT	467	ENABLE_INTERRUPTS_SYSEXIT
366		468
367	#ifdef CONFIG_AUDITSYSCALL	469	#ifdef CONFIG_AUDITSYSCALL
@@ -410,6 +512,7 @@ sysexit_audit:
410	.align 4	512	.align 4
411	.long 1b,2b	513	.long 1b,2b
412	.popsection	514	.popsection
		515	PTGS_TO_GS_EX
413	ENDPROC(ia32_sysenter_target)	516	ENDPROC(ia32_sysenter_target)
414		517
415	# system call handler stub	518	# system call handler stub
@@ -452,8 +555,7 @@ restore_all:
452	restore_nocheck:	555	restore_nocheck:
453	TRACE_IRQS_IRET	556	TRACE_IRQS_IRET
454	restore_nocheck_notrace:	557	restore_nocheck_notrace:
455	RESTORE_REGS	558	RESTORE_REGS 4 # skip orig_eax/error_code
456	addl $4, %esp # skip orig_eax/error_code
457	CFI_ADJUST_CFA_OFFSET -4	559	CFI_ADJUST_CFA_OFFSET -4
458	irq_return:	560	irq_return:
459	INTERRUPT_RETURN	561	INTERRUPT_RETURN
@@ -595,28 +697,50 @@ syscall_badsys:
595	END(syscall_badsys)	697	END(syscall_badsys)
596	CFI_ENDPROC	698	CFI_ENDPROC
597		699
598	#define FIXUP_ESPFIX_STACK \	700	/*
599	/* since we are on a wrong stack, we cant make it a C code :( */ \	701	* System calls that need a pt_regs pointer.
600	PER_CPU(gdt_page, %ebx); \	702	*/
601	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \	703	#define PTREGSCALL(name) \
602	addl %esp, %eax; \	704	ALIGN; \
603	pushl $__KERNEL_DS; \	705	ptregs_##name: \
604	CFI_ADJUST_CFA_OFFSET 4; \	706	leal 4(%esp),%eax; \
605	pushl %eax; \	707	jmp sys_##name;
606	CFI_ADJUST_CFA_OFFSET 4; \	708
607	lss (%esp), %esp; \	709	PTREGSCALL(iopl)
608	CFI_ADJUST_CFA_OFFSET -8;	710	PTREGSCALL(fork)
609	#define UNWIND_ESPFIX_STACK \	711	PTREGSCALL(clone)
610	movl %ss, %eax; \	712	PTREGSCALL(vfork)
611	/* see if on espfix stack */ \	713	PTREGSCALL(execve)
612	cmpw $__ESPFIX_SS, %ax; \	714	PTREGSCALL(sigaltstack)
613	jne 27f; \	715	PTREGSCALL(sigreturn)
614	movl $__KERNEL_DS, %eax; \	716	PTREGSCALL(rt_sigreturn)
615	movl %eax, %ds; \	717	PTREGSCALL(vm86)
616	movl %eax, %es; \	718	PTREGSCALL(vm86old)
617	/* switch to normal stack */ \	719
618	FIXUP_ESPFIX_STACK; \	720	.macro FIXUP_ESPFIX_STACK
619	27:;	721	/* since we are on a wrong stack, we cant make it a C code :( */
		722	PER_CPU(gdt_page, %ebx)
		723	GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
		724	addl %esp, %eax
		725	pushl $__KERNEL_DS
		726	CFI_ADJUST_CFA_OFFSET 4
		727	pushl %eax
		728	CFI_ADJUST_CFA_OFFSET 4
		729	lss (%esp), %esp
		730	CFI_ADJUST_CFA_OFFSET -8
		731	.endm
		732	.macro UNWIND_ESPFIX_STACK
		733	movl %ss, %eax
		734	/* see if on espfix stack */
		735	cmpw $__ESPFIX_SS, %ax
		736	jne 27f
		737	movl $__KERNEL_DS, %eax
		738	movl %eax, %ds
		739	movl %eax, %es
		740	/* switch to normal stack */
		741	FIXUP_ESPFIX_STACK
		742	27:
		743	.endm
620		744
621	/*	745	/*
622	* Build the entry stubs and pointer table with some assembler magic.	746	* Build the entry stubs and pointer table with some assembler magic.
@@ -1070,7 +1194,10 @@ ENTRY(page_fault)
1070	CFI_ADJUST_CFA_OFFSET 4	1194	CFI_ADJUST_CFA_OFFSET 4
1071	ALIGN	1195	ALIGN
1072	error_code:	1196	error_code:
1073	/* the function address is in %fs's slot on the stack */	1197	/* the function address is in %gs's slot on the stack */
		1198	pushl %fs
		1199	CFI_ADJUST_CFA_OFFSET 4
		1200	/CFI_REL_OFFSET fs, 0/
1074	pushl %es	1201	pushl %es
1075	CFI_ADJUST_CFA_OFFSET 4	1202	CFI_ADJUST_CFA_OFFSET 4
1076	/CFI_REL_OFFSET es, 0/	1203	/CFI_REL_OFFSET es, 0/
@@ -1099,20 +1226,15 @@ error_code:
1099	CFI_ADJUST_CFA_OFFSET 4	1226	CFI_ADJUST_CFA_OFFSET 4
1100	CFI_REL_OFFSET ebx, 0	1227	CFI_REL_OFFSET ebx, 0
1101	cld	1228	cld
1102	pushl %fs
1103	CFI_ADJUST_CFA_OFFSET 4
1104	/CFI_REL_OFFSET fs, 0/
1105	movl $(__KERNEL_PERCPU), %ecx	1229	movl $(__KERNEL_PERCPU), %ecx
1106	movl %ecx, %fs	1230	movl %ecx, %fs
1107	UNWIND_ESPFIX_STACK	1231	UNWIND_ESPFIX_STACK
1108	popl %ecx	1232	GS_TO_REG %ecx
1109	CFI_ADJUST_CFA_OFFSET -4	1233	movl PT_GS(%esp), %edi # get the function address
1110	/CFI_REGISTER es, ecx/
1111	movl PT_FS(%esp), %edi # get the function address
1112	movl PT_ORIG_EAX(%esp), %edx # get the error code	1234	movl PT_ORIG_EAX(%esp), %edx # get the error code
1113	movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart	1235	movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1114	mov %ecx, PT_FS(%esp)	1236	REG_TO_PTGS %ecx
1115	/CFI_REL_OFFSET fs, ES/	1237	SET_KERNEL_GS %ecx
1116	movl $(__USER_DS), %ecx	1238	movl $(__USER_DS), %ecx
1117	movl %ecx, %ds	1239	movl %ecx, %ds
1118	movl %ecx, %es	1240	movl %ecx, %es
@@ -1136,26 +1258,27 @@ END(page_fault)
1136	* by hand onto the new stack - while updating the return eip past	1258	* by hand onto the new stack - while updating the return eip past
1137	* the instruction that would have done it for sysenter.	1259	* the instruction that would have done it for sysenter.
1138	*/	1260	*/
1139	#define FIX_STACK(offset, ok, label) \	1261	.macro FIX_STACK offset ok label
1140	cmpw $__KERNEL_CS,4(%esp); \	1262	cmpw $__KERNEL_CS, 4(%esp)
1141	jne ok; \	1263	jne \ok
1142	label: \	1264	\label:
1143	movl TSS_sysenter_sp0+offset(%esp),%esp; \	1265	movl TSS_sysenter_sp0 + \offset(%esp), %esp
1144	CFI_DEF_CFA esp, 0; \	1266	CFI_DEF_CFA esp, 0
1145	CFI_UNDEFINED eip; \	1267	CFI_UNDEFINED eip
1146	pushfl; \	1268	pushfl
1147	CFI_ADJUST_CFA_OFFSET 4; \	1269	CFI_ADJUST_CFA_OFFSET 4
1148	pushl $__KERNEL_CS; \	1270	pushl $__KERNEL_CS
1149	CFI_ADJUST_CFA_OFFSET 4; \	1271	CFI_ADJUST_CFA_OFFSET 4
1150	pushl $sysenter_past_esp; \	1272	pushl $sysenter_past_esp
1151	CFI_ADJUST_CFA_OFFSET 4; \	1273	CFI_ADJUST_CFA_OFFSET 4
1152	CFI_REL_OFFSET eip, 0	1274	CFI_REL_OFFSET eip, 0
		1275	.endm
1153		1276
1154	ENTRY(debug)	1277	ENTRY(debug)
1155	RING0_INT_FRAME	1278	RING0_INT_FRAME
1156	cmpl $ia32_sysenter_target,(%esp)	1279	cmpl $ia32_sysenter_target,(%esp)
1157	jne debug_stack_correct	1280	jne debug_stack_correct
1158	FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)	1281	FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
1159	debug_stack_correct:	1282	debug_stack_correct:
1160	pushl $-1 # mark this as an int	1283	pushl $-1 # mark this as an int
1161	CFI_ADJUST_CFA_OFFSET 4	1284	CFI_ADJUST_CFA_OFFSET 4
@@ -1213,7 +1336,7 @@ nmi_stack_correct:
1213		1336
1214	nmi_stack_fixup:	1337	nmi_stack_fixup:
1215	RING0_INT_FRAME	1338	RING0_INT_FRAME
1216	FIX_STACK(12,nmi_stack_correct, 1)	1339	FIX_STACK 12, nmi_stack_correct, 1
1217	jmp nmi_stack_correct	1340	jmp nmi_stack_correct
1218		1341
1219	nmi_debug_stack_check:	1342	nmi_debug_stack_check:
@@ -1224,7 +1347,7 @@ nmi_debug_stack_check:
1224	jb nmi_stack_correct	1347	jb nmi_stack_correct
1225	cmpl $debug_esp_fix_insn,(%esp)	1348	cmpl $debug_esp_fix_insn,(%esp)
1226	ja nmi_stack_correct	1349	ja nmi_stack_correct
1227	FIX_STACK(24,nmi_stack_correct, 1)	1350	FIX_STACK 24, nmi_stack_correct, 1
1228	jmp nmi_stack_correct	1351	jmp nmi_stack_correct
1229		1352
1230	nmi_espfix_stack:	1353	nmi_espfix_stack: