1 files changed, 290 insertions, 165 deletions

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 46469029e9d3..899e8938e79f 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -30,12 +30,13 @@
  *      1C(%esp) - %ds
  *      20(%esp) - %es
  *      24(%esp) - %fs
- *      28(%esp) - orig_eax
- *      2C(%esp) - %eip
- *      30(%esp) - %cs
- *      34(%esp) - %eflags
- *      38(%esp) - %oldesp
- *      3C(%esp) - %oldss
+ *      28(%esp) - %gs          saved iff !CONFIG_X86_32_LAZY_GS
+ *      2C(%esp) - orig_eax
+ *      30(%esp) - %eip
+ *      34(%esp) - %cs
+ *      38(%esp) - %eflags
+ *      3C(%esp) - %oldesp
+ *      40(%esp) - %oldss
  *
  * "current" is in register %ebx during any slow entries.
  */
@@ -46,7 +47,7 @@
 #include <asm/errno.h>
 #include <asm/segment.h>
 #include <asm/smp.h>
-#include <asm/page.h>
+#include <asm/page_types.h>
 #include <asm/desc.h>
 #include <asm/percpu.h>
 #include <asm/dwarf2.h>
@@ -101,121 +102,221 @@
 #define resume_userspace_sig    resume_userspace
 #endif
 
-#define SAVE_ALL \
-        cld; \
-        pushl %fs; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        /*CFI_REL_OFFSET fs, 0;*/\
-        pushl %es; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        /*CFI_REL_OFFSET es, 0;*/\
-        pushl %ds; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        /*CFI_REL_OFFSET ds, 0;*/\
-        pushl %eax; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET eax, 0;\
-        pushl %ebp; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET ebp, 0;\
-        pushl %edi; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET edi, 0;\
-        pushl %esi; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET esi, 0;\
-        pushl %edx; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET edx, 0;\
-        pushl %ecx; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET ecx, 0;\
-        pushl %ebx; \
-        CFI_ADJUST_CFA_OFFSET 4;\
-        CFI_REL_OFFSET ebx, 0;\
-        movl $(__USER_DS), %edx; \
-        movl %edx, %ds; \
-        movl %edx, %es; \
-        movl $(__KERNEL_PERCPU), %edx; \
+/*
+ * User gs save/restore
+ *
+ * %gs is used for userland TLS and kernel only uses it for stack
+ * canary which is required to be at %gs:20 by gcc.  Read the comment
+ * at the top of stackprotector.h for more info.
+ *
+ * Local labels 98 and 99 are used.
+ */
+#ifdef CONFIG_X86_32_LAZY_GS
+
+ /* unfortunately push/pop can't be no-op */
+.macro PUSH_GS
+        pushl $0
+        CFI_ADJUST_CFA_OFFSET 4
+.endm
+.macro POP_GS pop=0
+        addl $(4 + \pop), %esp
+        CFI_ADJUST_CFA_OFFSET -(4 + \pop)
+.endm
+.macro POP_GS_EX
+.endm
+
+ /* all the rest are no-op */
+.macro PTGS_TO_GS
+.endm
+.macro PTGS_TO_GS_EX
+.endm
+.macro GS_TO_REG reg
+.endm
+.macro REG_TO_PTGS reg
+.endm
+.macro SET_KERNEL_GS reg
+.endm
+
+#else   /* CONFIG_X86_32_LAZY_GS */
+
+.macro PUSH_GS
+        pushl %gs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET gs, 0*/
+.endm
+
+.macro POP_GS pop=0
+98:     popl %gs
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_RESTORE gs*/
+  .if \pop <> 0
+        add $\pop, %esp
+        CFI_ADJUST_CFA_OFFSET -\pop
+  .endif
+.endm
+.macro POP_GS_EX
+.pushsection .fixup, "ax"
+99:     movl $0, (%esp)
+        jmp 98b
+.section __ex_table, "a"
+        .align 4
+        .long 98b, 99b
+.popsection
+.endm
+
+.macro PTGS_TO_GS
+98:     mov PT_GS(%esp), %gs
+.endm
+.macro PTGS_TO_GS_EX
+.pushsection .fixup, "ax"
+99:     movl $0, PT_GS(%esp)
+        jmp 98b
+.section __ex_table, "a"
+        .align 4
+        .long 98b, 99b
+.popsection
+.endm
+
+.macro GS_TO_REG reg
+        movl %gs, \reg
+        /*CFI_REGISTER gs, \reg*/
+.endm
+.macro REG_TO_PTGS reg
+        movl \reg, PT_GS(%esp)
+        /*CFI_REL_OFFSET gs, PT_GS*/
+.endm
+.macro SET_KERNEL_GS reg
+        movl $(__KERNEL_STACK_CANARY), \reg
+        movl \reg, %gs
+.endm
+
+#endif  /* CONFIG_X86_32_LAZY_GS */
+
+.macro SAVE_ALL
+        cld
+        PUSH_GS
+        pushl %fs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET fs, 0;*/
+        pushl %es
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET es, 0;*/
+        pushl %ds
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET ds, 0;*/
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET eax, 0
+        pushl %ebp
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebp, 0
+        pushl %edi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edi, 0
+        pushl %esi
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET esi, 0
+        pushl %edx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET edx, 0
+        pushl %ecx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ecx, 0
+        pushl %ebx
+        CFI_ADJUST_CFA_OFFSET 4
+        CFI_REL_OFFSET ebx, 0
+        movl $(__USER_DS), %edx
+        movl %edx, %ds
+        movl %edx, %es
+        movl $(__KERNEL_PERCPU), %edx
         movl %edx, %fs
+        SET_KERNEL_GS %edx
+.endm
 
-#define RESTORE_INT_REGS \
-        popl %ebx;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE ebx;\
-        popl %ecx;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE ecx;\
-        popl %edx;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE edx;\
-        popl %esi;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE esi;\
-        popl %edi;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE edi;\
-        popl %ebp;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        CFI_RESTORE ebp;\
-        popl %eax;      \
-        CFI_ADJUST_CFA_OFFSET -4;\
+.macro RESTORE_INT_REGS
+        popl %ebx
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE ebx
+        popl %ecx
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE ecx
+        popl %edx
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE edx
+        popl %esi
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE esi
+        popl %edi
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE edi
+        popl %ebp
+        CFI_ADJUST_CFA_OFFSET -4
+        CFI_RESTORE ebp
+        popl %eax
+        CFI_ADJUST_CFA_OFFSET -4
         CFI_RESTORE eax
+.endm
 
-#define RESTORE_REGS    \
-        RESTORE_INT_REGS; \
-1:      popl %ds;       \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        /*CFI_RESTORE ds;*/\
-2:      popl %es;       \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        /*CFI_RESTORE es;*/\
-3:      popl %fs;       \
-        CFI_ADJUST_CFA_OFFSET -4;\
-        /*CFI_RESTORE fs;*/\
-.pushsection .fixup,"ax";       \
-4:      movl $0,(%esp); \
-        jmp 1b;         \
-5:      movl $0,(%esp); \
-        jmp 2b;         \
-6:      movl $0,(%esp); \
-        jmp 3b;         \
-.section __ex_table,"a";\
-        .align 4;       \
-        .long 1b,4b;    \
-        .long 2b,5b;    \
-        .long 3b,6b;    \
+.macro RESTORE_REGS pop=0
+        RESTORE_INT_REGS
+1:      popl %ds
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_RESTORE ds;*/
+2:      popl %es
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_RESTORE es;*/
+3:      popl %fs
+        CFI_ADJUST_CFA_OFFSET -4
+        /*CFI_RESTORE fs;*/
+        POP_GS \pop
+.pushsection .fixup, "ax"
+4:      movl $0, (%esp)
+        jmp 1b
+5:      movl $0, (%esp)
+        jmp 2b
+6:      movl $0, (%esp)
+        jmp 3b
+.section __ex_table, "a"
+        .align 4
+        .long 1b, 4b
+        .long 2b, 5b
+        .long 3b, 6b
 .popsection
+        POP_GS_EX
+.endm
 
-#define RING0_INT_FRAME \
-        CFI_STARTPROC simple;\
-        CFI_SIGNAL_FRAME;\
-        CFI_DEF_CFA esp, 3*4;\
-        /*CFI_OFFSET cs, -2*4;*/\
+.macro RING0_INT_FRAME
+        CFI_STARTPROC simple
+        CFI_SIGNAL_FRAME
+        CFI_DEF_CFA esp, 3*4
+        /*CFI_OFFSET cs, -2*4;*/
         CFI_OFFSET eip, -3*4
+.endm
 
-#define RING0_EC_FRAME \
-        CFI_STARTPROC simple;\
-        CFI_SIGNAL_FRAME;\
-        CFI_DEF_CFA esp, 4*4;\
-        /*CFI_OFFSET cs, -2*4;*/\
+.macro RING0_EC_FRAME
+        CFI_STARTPROC simple
+        CFI_SIGNAL_FRAME
+        CFI_DEF_CFA esp, 4*4
+        /*CFI_OFFSET cs, -2*4;*/
         CFI_OFFSET eip, -3*4
+.endm
 
-#define RING0_PTREGS_FRAME \
-        CFI_STARTPROC simple;\
-        CFI_SIGNAL_FRAME;\
-        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
-        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
-        CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
-        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
-        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
-        CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
-        CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
-        CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
-        CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
-        CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
-        CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
+.macro RING0_PTREGS_FRAME
+        CFI_STARTPROC simple
+        CFI_SIGNAL_FRAME
+        CFI_DEF_CFA esp, PT_OLDESP-PT_EBX
+        /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/
+        CFI_OFFSET eip, PT_EIP-PT_OLDESP
+        /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/
+        /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/
+        CFI_OFFSET eax, PT_EAX-PT_OLDESP
+        CFI_OFFSET ebp, PT_EBP-PT_OLDESP
+        CFI_OFFSET edi, PT_EDI-PT_OLDESP
+        CFI_OFFSET esi, PT_ESI-PT_OLDESP
+        CFI_OFFSET edx, PT_EDX-PT_OLDESP
+        CFI_OFFSET ecx, PT_ECX-PT_OLDESP
         CFI_OFFSET ebx, PT_EBX-PT_OLDESP
+.endm
 
 ENTRY(ret_from_fork)
         CFI_STARTPROC
@@ -362,6 +463,7 @@ sysenter_exit:
         xorl %ebp,%ebp
         TRACE_IRQS_ON
 1:      mov  PT_FS(%esp), %fs
+        PTGS_TO_GS
         ENABLE_INTERRUPTS_SYSEXIT
 
 #ifdef CONFIG_AUDITSYSCALL
@@ -410,6 +512,7 @@ sysexit_audit:
         .align 4
         .long 1b,2b
 .popsection
+        PTGS_TO_GS_EX
 ENDPROC(ia32_sysenter_target)
 
         # system call handler stub
@@ -452,8 +555,7 @@ restore_all:
 restore_nocheck:
         TRACE_IRQS_IRET
 restore_nocheck_notrace:
-        RESTORE_REGS
-        addl $4, %esp                   # skip orig_eax/error_code
+        RESTORE_REGS 4                  # skip orig_eax/error_code
         CFI_ADJUST_CFA_OFFSET -4
 irq_return:
         INTERRUPT_RETURN
@@ -595,28 +697,50 @@ syscall_badsys:
 END(syscall_badsys)
         CFI_ENDPROC
 
-#define FIXUP_ESPFIX_STACK \
-        /* since we are on a wrong stack, we cant make it a C code :( */ \
-        PER_CPU(gdt_page, %ebx); \
-        GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
-        addl %esp, %eax; \
-        pushl $__KERNEL_DS; \
-        CFI_ADJUST_CFA_OFFSET 4; \
-        pushl %eax; \
-        CFI_ADJUST_CFA_OFFSET 4; \
-        lss (%esp), %esp; \
-        CFI_ADJUST_CFA_OFFSET -8;
-#define UNWIND_ESPFIX_STACK \
-        movl %ss, %eax; \
-        /* see if on espfix stack */ \
-        cmpw $__ESPFIX_SS, %ax; \
-        jne 27f; \
-        movl $__KERNEL_DS, %eax; \
-        movl %eax, %ds; \
-        movl %eax, %es; \
-        /* switch to normal stack */ \
-        FIXUP_ESPFIX_STACK; \
-27:;
+/*
+ * System calls that need a pt_regs pointer.
+ */
+#define PTREGSCALL(name) \
+        ALIGN; \
+ptregs_##name: \
+        leal 4(%esp),%eax; \
+        jmp sys_##name;
+
+PTREGSCALL(iopl)
+PTREGSCALL(fork)
+PTREGSCALL(clone)
+PTREGSCALL(vfork)
+PTREGSCALL(execve)
+PTREGSCALL(sigaltstack)
+PTREGSCALL(sigreturn)
+PTREGSCALL(rt_sigreturn)
+PTREGSCALL(vm86)
+PTREGSCALL(vm86old)
+
+.macro FIXUP_ESPFIX_STACK
+        /* since we are on a wrong stack, we cant make it a C code :( */
+        PER_CPU(gdt_page, %ebx)
+        GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah)
+        addl %esp, %eax
+        pushl $__KERNEL_DS
+        CFI_ADJUST_CFA_OFFSET 4
+        pushl %eax
+        CFI_ADJUST_CFA_OFFSET 4
+        lss (%esp), %esp
+        CFI_ADJUST_CFA_OFFSET -8
+.endm
+.macro UNWIND_ESPFIX_STACK
+        movl %ss, %eax
+        /* see if on espfix stack */
+        cmpw $__ESPFIX_SS, %ax
+        jne 27f
+        movl $__KERNEL_DS, %eax
+        movl %eax, %ds
+        movl %eax, %es
+        /* switch to normal stack */
+        FIXUP_ESPFIX_STACK
+27:
+.endm
 
 /*
  * Build the entry stubs and pointer table with some assembler magic.
@@ -672,7 +796,7 @@ common_interrupt:
 ENDPROC(common_interrupt)
         CFI_ENDPROC
 
-#define BUILD_INTERRUPT(name, nr)       \
+#define BUILD_INTERRUPT3(name, nr, fn)  \
 ENTRY(name)                             \
         RING0_INT_FRAME;                \
         pushl $~(nr);                   \
@@ -680,13 +804,15 @@ ENTRY(name)				\
         SAVE_ALL;                       \
         TRACE_IRQS_OFF                  \
         movl %esp,%eax;                 \
-        call smp_##name;                \
+        call fn;                        \
         jmp ret_from_intr;              \
         CFI_ENDPROC;                    \
 ENDPROC(name)
 
+#define BUILD_INTERRUPT(name, nr)       BUILD_INTERRUPT3(name, nr, smp_##name)
+
 /* The include is where all of the SMP etc. interrupts come from */
-#include "entry_arch.h"
+#include <asm/entry_arch.h>
 
 ENTRY(coprocessor_error)
         RING0_INT_FRAME
@@ -1068,7 +1194,10 @@ ENTRY(page_fault)
         CFI_ADJUST_CFA_OFFSET 4
         ALIGN
 error_code:
-        /* the function address is in %fs's slot on the stack */
+        /* the function address is in %gs's slot on the stack */
+        pushl %fs
+        CFI_ADJUST_CFA_OFFSET 4
+        /*CFI_REL_OFFSET fs, 0*/
         pushl %es
         CFI_ADJUST_CFA_OFFSET 4
         /*CFI_REL_OFFSET es, 0*/
@@ -1097,20 +1226,15 @@ error_code:
         CFI_ADJUST_CFA_OFFSET 4
         CFI_REL_OFFSET ebx, 0
         cld
-        pushl %fs
-        CFI_ADJUST_CFA_OFFSET 4
-        /*CFI_REL_OFFSET fs, 0*/
         movl $(__KERNEL_PERCPU), %ecx
         movl %ecx, %fs
         UNWIND_ESPFIX_STACK
-        popl %ecx
-        CFI_ADJUST_CFA_OFFSET -4
-        /*CFI_REGISTER es, ecx*/
-        movl PT_FS(%esp), %edi          # get the function address
+        GS_TO_REG %ecx
+        movl PT_GS(%esp), %edi          # get the function address
         movl PT_ORIG_EAX(%esp), %edx    # get the error code
         movl $-1, PT_ORIG_EAX(%esp)     # no syscall to restart
-        mov  %ecx, PT_FS(%esp)
-        /*CFI_REL_OFFSET fs, ES*/
+        REG_TO_PTGS %ecx
+        SET_KERNEL_GS %ecx
         movl $(__USER_DS), %ecx
         movl %ecx, %ds
         movl %ecx, %es
@@ -1134,26 +1258,27 @@ END(page_fault)
  * by hand onto the new stack - while updating the return eip past
  * the instruction that would have done it for sysenter.
  */
-#define FIX_STACK(offset, ok, label)            \
-        cmpw $__KERNEL_CS,4(%esp);              \
-        jne ok;                                 \
-label:                                          \
-        movl TSS_sysenter_sp0+offset(%esp),%esp;        \
-        CFI_DEF_CFA esp, 0;                     \
-        CFI_UNDEFINED eip;                      \
-        pushfl;                                 \
-        CFI_ADJUST_CFA_OFFSET 4;                \
-        pushl $__KERNEL_CS;                     \
-        CFI_ADJUST_CFA_OFFSET 4;                \
-        pushl $sysenter_past_esp;               \
-        CFI_ADJUST_CFA_OFFSET 4;                \
+.macro FIX_STACK offset ok label
+        cmpw $__KERNEL_CS, 4(%esp)
+        jne \ok
+\label:
+        movl TSS_sysenter_sp0 + \offset(%esp), %esp
+        CFI_DEF_CFA esp, 0
+        CFI_UNDEFINED eip
+        pushfl
+        CFI_ADJUST_CFA_OFFSET 4
+        pushl $__KERNEL_CS
+        CFI_ADJUST_CFA_OFFSET 4
+        pushl $sysenter_past_esp
+        CFI_ADJUST_CFA_OFFSET 4
         CFI_REL_OFFSET eip, 0
+.endm
 
 ENTRY(debug)
         RING0_INT_FRAME
         cmpl $ia32_sysenter_target,(%esp)
         jne debug_stack_correct
-        FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
+        FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
 debug_stack_correct:
         pushl $-1                       # mark this as an int
         CFI_ADJUST_CFA_OFFSET 4
@@ -1211,7 +1336,7 @@ nmi_stack_correct:
 
 nmi_stack_fixup:
         RING0_INT_FRAME
-        FIX_STACK(12,nmi_stack_correct, 1)
+        FIX_STACK 12, nmi_stack_correct, 1
         jmp nmi_stack_correct
 
 nmi_debug_stack_check:
@@ -1222,7 +1347,7 @@ nmi_debug_stack_check:
         jb nmi_stack_correct
         cmpl $debug_esp_fix_insn,(%esp)
         ja nmi_stack_correct
-        FIX_STACK(24,nmi_stack_correct, 1)
+        FIX_STACK 24, nmi_stack_correct, 1
         jmp nmi_stack_correct
 
 nmi_espfix_stack:
@@ -1234,7 +1359,7 @@ nmi_espfix_stack:
         CFI_ADJUST_CFA_OFFSET 4
         pushl %esp
         CFI_ADJUST_CFA_OFFSET 4
-        addw $4, (%esp)
+        addl $4, (%esp)
         /* copy the iret frame of 12 bytes */
         .rept 3
         pushl 16(%esp)