x86-64: Move oldrsp from PDA to per-cpu.

tj: * in asm-offsets_64.c, pda.h inclusion shouldn't be removed as pda
      is still referenced in the file
    * s/oldrsp/old_rsp/

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

5 files changed, 15 insertions, 14 deletions

diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
index 4d28ffba6e1b..ae23deb99559 100644
--- a/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@ -14,7 +14,7 @@ struct x8664_pda {
         unsigned long unused1;
         unsigned long unused2;
         unsigned long unused3;
-        unsigned long oldrsp;           /* 24 user rsp for system call */
+        unsigned long unused4;
         int irqcount;                   /* 32 Irq nesting counter. Starts -1 */
         unsigned int unused6;           /* 36 was cpunumber */
 #ifdef CONFIG_CC_STACKPROTECTOR
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index cafff5f4a031..afda6deb8515 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -49,7 +49,6 @@ int main(void)
         BLANK();
 #undef ENTRY
 #define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
-        ENTRY(oldrsp); 
         ENTRY(irqcount);
         DEFINE(pda_size, sizeof(struct x8664_pda));
         BLANK();
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 0dd45859a7a8..7c27da407da7 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -210,7 +210,7 @@ ENTRY(native_usergs_sysret64)
 
         /* %rsp:at FRAMEEND */
         .macro FIXUP_TOP_OF_STACK tmp offset=0
-        movq %gs:pda_oldrsp,\tmp
+        movq PER_CPU_VAR(old_rsp),\tmp
         movq \tmp,RSP+\offset(%rsp)
         movq $__USER_DS,SS+\offset(%rsp)
         movq $__USER_CS,CS+\offset(%rsp)
@@ -221,7 +221,7 @@ ENTRY(native_usergs_sysret64)
 
         .macro RESTORE_TOP_OF_STACK tmp offset=0
         movq RSP+\offset(%rsp),\tmp
-        movq \tmp,%gs:pda_oldrsp
+        movq \tmp,PER_CPU_VAR(old_rsp)
         movq EFLAGS+\offset(%rsp),\tmp
         movq \tmp,R11+\offset(%rsp)
         .endm
@@ -479,7 +479,7 @@ ENTRY(system_call)
          */
 ENTRY(system_call_after_swapgs)
 
-        movq    %rsp,%gs:pda_oldrsp
+        movq    %rsp,PER_CPU_VAR(old_rsp)
         movq    PER_CPU_VAR(kernel_stack),%rsp
         /*
          * No need to follow this irqs off/on section - it's straight
@@ -523,7 +523,7 @@ sysret_check:
         CFI_REGISTER    rip,rcx
         RESTORE_ARGS 0,-ARG_SKIP,1
         /*CFI_REGISTER  rflags,r11*/
-        movq    %gs:pda_oldrsp, %rsp
+        movq    PER_CPU_VAR(old_rsp), %rsp
         USERGS_SYSRET64
 
         CFI_RESTORE_STATE
@@ -833,7 +833,7 @@ common_interrupt:
         XCPT_FRAME
         addq $-0x80,(%rsp)              /* Adjust vector to [-256,-1] range */
         interrupt do_IRQ
-        /* 0(%rsp): oldrsp-ARGOFFSET */
+        /* 0(%rsp): old_rsp-ARGOFFSET */
 ret_from_intr:
         DISABLE_INTERRUPTS(CLBR_NONE)
         TRACE_IRQS_OFF
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 6c5f57602108..480128918926 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -60,6 +60,8 @@ asmlinkage extern void ret_from_fork(void);
 DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
+DEFINE_PER_CPU(unsigned long, old_rsp);
+
 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
 
 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
@@ -395,7 +397,7 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
         load_gs_index(0);
         regs->ip                = new_ip;
         regs->sp                = new_sp;
-        write_pda(oldrsp, new_sp);
+        percpu_write(old_rsp, new_sp);
         regs->cs                = __USER_CS;
         regs->ss                = __USER_DS;
         regs->flags             = 0x200;
@@ -616,8 +618,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         /*
          * Switch the PDA and FPU contexts.
          */
-        prev->usersp = read_pda(oldrsp);
-        write_pda(oldrsp, next->usersp);
+        prev->usersp = percpu_read(old_rsp);
+        percpu_write(old_rsp, next->usersp);
         percpu_write(current_task, next_p);
 
         percpu_write(kernel_stack,
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 5a23e8993678..d6fc51f4ce85 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -194,11 +194,11 @@ RELOC(xen_sysexit, 1b+1)
 ENTRY(xen_sysret64)
         /* We're already on the usermode stack at this point, but still
            with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
         movq PER_CPU_VAR(kernel_stack),%rsp
 
         pushq $__USER_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
         pushq %r11
         pushq $__USER_CS
         pushq %rcx
@@ -211,11 +211,11 @@ RELOC(xen_sysret64, 1b+1)
 ENTRY(xen_sysret32)
         /* We're already on the usermode stack at this point, but still
            with the kernel gs, so we can easily switch back */
-        movq %rsp, %gs:pda_oldrsp
+        movq %rsp, PER_CPU_VAR(old_rsp)
         movq PER_CPU_VAR(kernel_stack), %rsp
 
         pushq $__USER32_DS
-        pushq %gs:pda_oldrsp
+        pushq PER_CPU_VAR(old_rsp)
         pushq %r11
         pushq $__USER32_CS
         pushq %rcx