1 files changed, 35 insertions, 37 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index a1346217e43c..83d1836b9467 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -48,10 +48,11 @@
 #include <asm/unistd.h>
 #include <asm/thread_info.h>
 #include <asm/hw_irq.h>
-#include <asm/page.h>
+#include <asm/page_types.h>
 #include <asm/irqflags.h>
 #include <asm/paravirt.h>
 #include <asm/ftrace.h>
+#include <asm/percpu.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -76,20 +77,17 @@ ENTRY(ftrace_caller)
         movq 8(%rbp), %rsi
         subq $MCOUNT_INSN_SIZE, %rdi
 
-.globl ftrace_call
-ftrace_call:
+GLOBAL(ftrace_call)
         call ftrace_stub
 
         MCOUNT_RESTORE_FRAME
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
-.globl ftrace_graph_call
-ftrace_graph_call:
+GLOBAL(ftrace_graph_call)
         jmp ftrace_stub
 #endif
 
-.globl ftrace_stub
-ftrace_stub:
+GLOBAL(ftrace_stub)
         retq
 END(ftrace_caller)
 
@@ -109,8 +107,7 @@ ENTRY(mcount)
         jnz ftrace_graph_caller
 #endif
 
-.globl ftrace_stub
-ftrace_stub:
+GLOBAL(ftrace_stub)
         retq
 
 trace:
@@ -147,9 +144,7 @@ ENTRY(ftrace_graph_caller)
         retq
 END(ftrace_graph_caller)
 
-
-.globl return_to_handler
-return_to_handler:
+GLOBAL(return_to_handler)
         subq  $80, %rsp
 
         movq %rax, (%rsp)
@@ -187,6 +182,7 @@ return_to_handler:
 ENTRY(native_usergs_sysret64)
         swapgs
         sysretq
+ENDPROC(native_usergs_sysret64)
 #endif /* CONFIG_PARAVIRT */
 
 
@@ -209,7 +205,7 @@ ENTRY(native_usergs_sysret64)
 
         /* %rsp:at FRAMEEND */
         .macro FIXUP_TOP_OF_STACK tmp offset=0
-        movq %gs:pda_oldrsp,\tmp
+        movq PER_CPU_VAR(old_rsp),\tmp
         movq \tmp,RSP+\offset(%rsp)
         movq $__USER_DS,SS+\offset(%rsp)
         movq $__USER_CS,CS+\offset(%rsp)
@@ -220,7 +216,7 @@ ENTRY(native_usergs_sysret64)
 
         .macro RESTORE_TOP_OF_STACK tmp offset=0
         movq RSP+\offset(%rsp),\tmp
-        movq \tmp,%gs:pda_oldrsp
+        movq \tmp,PER_CPU_VAR(old_rsp)
         movq EFLAGS+\offset(%rsp),\tmp
         movq \tmp,R11+\offset(%rsp)
         .endm
@@ -336,15 +332,15 @@ ENTRY(save_args)
         je 1f
         SWAPGS
         /*
-         * irqcount is used to check if a CPU is already on an interrupt stack
+         * irq_count is used to check if a CPU is already on an interrupt stack
          * or not. While this is essentially redundant with preempt_count it is
          * a little cheaper to use a separate counter in the PDA (short of
          * moving irq_enter into assembly, which would be too much work)
          */
-1:      incl %gs:pda_irqcount
+1:      incl PER_CPU_VAR(irq_count)
         jne 2f
         popq_cfi %rax                   /* move return address... */
-        mov %gs:pda_irqstackptr,%rsp
+        mov PER_CPU_VAR(irq_stack_ptr),%rsp
         EMPTY_FRAME 0
         pushq_cfi %rbp                  /* backlink for unwinder */
         pushq_cfi %rax                  /* ... to the new stack */
@@ -409,6 +405,8 @@ END(save_paranoid)
 ENTRY(ret_from_fork)
         DEFAULT_FRAME
 
+        LOCK ; btr $TIF_FORK,TI_flags(%r8)
+
         push kernel_eflags(%rip)
         CFI_ADJUST_CFA_OFFSET 8
         popf                                    # reset kernel eflags
@@ -468,7 +466,7 @@ END(ret_from_fork)
 ENTRY(system_call)
         CFI_STARTPROC   simple
         CFI_SIGNAL_FRAME
-        CFI_DEF_CFA     rsp,PDA_STACKOFFSET
+        CFI_DEF_CFA     rsp,KERNEL_STACK_OFFSET
         CFI_REGISTER    rip,rcx
         /*CFI_REGISTER  rflags,r11*/
         SWAPGS_UNSAFE_STACK
@@ -479,8 +477,8 @@ ENTRY(system_call)
          */
 ENTRY(system_call_after_swapgs)
 
-        movq    %rsp,%gs:pda_oldrsp
-        movq    %gs:pda_kernelstack,%rsp
+        movq    %rsp,PER_CPU_VAR(old_rsp)
+        movq    PER_CPU_VAR(kernel_stack),%rsp
         /*
          * No need to follow this irqs off/on section - it's straight
          * and short:
@@ -523,7 +521,7 @@ sysret_check:
         CFI_REGISTER    rip,rcx
         RESTORE_ARGS 0,-ARG_SKIP,1
         /*CFI_REGISTER  rflags,r11*/
-        movq    %gs:pda_oldrsp, %rsp
+        movq    PER_CPU_VAR(old_rsp), %rsp
         USERGS_SYSRET64
 
         CFI_RESTORE_STATE
@@ -630,16 +628,14 @@ tracesys:
  * Syscall return path ending with IRET.
  * Has correct top of stack, but partial stack frame.
  */
-        .globl int_ret_from_sys_call
-        .globl int_with_check
-int_ret_from_sys_call:
+GLOBAL(int_ret_from_sys_call)
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
         testl $3,CS-ARGOFFSET(%rsp)
         je retint_restore_args
         movl $_TIF_ALLWORK_MASK,%edi
         /* edi: mask to check */
-int_with_check:
+GLOBAL(int_with_check)
         LOCKDEP_SYS_EXIT_IRQ
         GET_THREAD_INFO(%rcx)
         movl TI_flags(%rcx),%edx
@@ -833,11 +829,11 @@ common_interrupt:
         XCPT_FRAME
         addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
         interrupt do_IRQ
-        /* 0(%rsp): oldrsp-ARGOFFSET */
+        /* 0(%rsp): old_rsp-ARGOFFSET */
 ret_from_intr:
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
         leaveq
         CFI_DEF_CFA_REGISTER    rsp
         CFI_ADJUST_CFA_OFFSET   -8
@@ -982,8 +978,10 @@ apicinterrupt IRQ_MOVE_CLEANUP_VECTOR \
         irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt
 #endif
 
+#ifdef CONFIG_X86_UV
 apicinterrupt UV_BAU_MESSAGE \
         uv_bau_message_intr1 uv_bau_message_interrupt
+#endif
 apicinterrupt LOCAL_TIMER_VECTOR \
         apic_timer_interrupt smp_apic_timer_interrupt
 
@@ -1073,10 +1071,10 @@ ENTRY(\sym)
         TRACE_IRQS_OFF
         movq %rsp,%rdi          /* pt_regs pointer */
         xorl %esi,%esi          /* no error code */
-        movq %gs:pda_data_offset, %rbp
-        subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        PER_CPU(init_tss, %rbp)
+        subq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
         call \do_sym
-        addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
+        addq $EXCEPTION_STKSZ, TSS_ist + (\ist - 1) * 8(%rbp)
         jmp paranoid_exit       /* %ebx: no swapgs flag */
         CFI_ENDPROC
 END(\sym)
@@ -1138,7 +1136,7 @@ ENTRY(native_load_gs_index)
         CFI_STARTPROC
         pushf
         CFI_ADJUST_CFA_OFFSET 8
-        DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
+        DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI)
         SWAPGS
 gs_change:
         movl %edi,%gs
@@ -1260,14 +1258,14 @@ ENTRY(call_softirq)
         CFI_REL_OFFSET rbp,0
         mov  %rsp,%rbp
         CFI_DEF_CFA_REGISTER rbp
-        incl %gs:pda_irqcount
-        cmove %gs:pda_irqstackptr,%rsp
+        incl PER_CPU_VAR(irq_count)
+        cmove PER_CPU_VAR(irq_stack_ptr),%rsp
         push  %rbp                      # backlink for old unwinder
         call __do_softirq
         leaveq
         CFI_DEF_CFA_REGISTER    rsp
         CFI_ADJUST_CFA_OFFSET   -8
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
         ret
         CFI_ENDPROC
 END(call_softirq)
@@ -1297,15 +1295,15 @@ ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
         movq %rdi, %rsp            # we don't return, adjust the stack frame
         CFI_ENDPROC
         DEFAULT_FRAME
-11:     incl %gs:pda_irqcount
+11:     incl PER_CPU_VAR(irq_count)
         movq %rsp,%rbp
         CFI_DEF_CFA_REGISTER rbp
-        cmovzq %gs:pda_irqstackptr,%rsp
+        cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
         pushq %rbp                      # backlink for old unwinder
         call xen_evtchn_do_upcall
         popq %rsp
         CFI_DEF_CFA_REGISTER rsp
-        decl %gs:pda_irqcount
+        decl PER_CPU_VAR(irq_count)
         jmp  error_exit
         CFI_ENDPROC
 END(do_hypervisor_callback)