x86: move stack_canary into irq_stack

Impact: x86_64 percpu area layout change, irq_stack now at the beginning

Now that the PDA is empty except for the stack canary, it can be removed.
The irqstack is moved to the start of the per-cpu section.  If the stack
protector is enabled, the canary overlaps the bottom 48 bytes of the irqstack.

tj: * updated subject
    * dropped asm relocation of irq_stack_ptr
    * updated comments a bit
    * rebased on top of stack canary changes

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

5 files changed, 27 insertions, 15 deletions

diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
index b473e952439a..ba46416634f0 100644
--- a/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@ -17,9 +17,6 @@ struct x8664_pda {
         unsigned long unused4;
         int unused5;
         unsigned int unused6;           /* 36 was cpunumber */
-        unsigned long stack_canary;     /* 40 stack canary value */
-                                        /* gcc-ABI: this canary MUST be at
-                                           offset 40!!! */
         short in_bootmem;               /* pda lives in bootmem */
 } ____cacheline_aligned_in_smp;
 
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 165d5272ece1..ce980db5e59d 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -133,12 +133,6 @@ do {							\
 /* We can use this directly for local CPU (faster). */
 DECLARE_PER_CPU(unsigned long, this_cpu_off);
 
-#ifdef CONFIG_X86_64
-extern void load_pda_offset(int cpu);
-#else
-static inline void load_pda_offset(int cpu) { }
-#endif
-
 #endif /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_SMP
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index f511246fa6cd..48676b943b92 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -379,8 +379,29 @@ union thread_xstate {
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(struct orig_ist, orig_ist);
 
-DECLARE_PER_CPU(char[IRQ_STACK_SIZE], irq_stack);
+union irq_stack_union {
+        char irq_stack[IRQ_STACK_SIZE];
+        /*
+         * GCC hardcodes the stack canary as %gs:40.  Since the
+         * irq_stack is the object at %gs:0, we reserve the bottom
+         * 48 bytes of the irq stack for the canary.
+         */
+        struct {
+                char gs_base[40];
+                unsigned long stack_canary;
+        };
+};
+
+DECLARE_PER_CPU(union irq_stack_union, irq_stack_union);
 DECLARE_PER_CPU(char *, irq_stack_ptr);
+
+static inline void load_gs_base(int cpu)
+{
+        /* Memory clobbers used to order pda/percpu accesses */
+        mb();
+        wrmsrl(MSR_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
+        mb();
+}
 #endif
 
 extern void print_cpu_info(struct cpuinfo_x86 *);
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 2383e5bb475c..36a700acaf2b 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -2,7 +2,7 @@
 #define _ASM_STACKPROTECTOR_H 1
 
 #include <asm/tsc.h>
-#include <asm/pda.h>
+#include <asm/processor.h>
 
 /*
  * Initialize the stackprotector canary value.
@@ -19,7 +19,7 @@ static __always_inline void boot_init_stack_canary(void)
          * Build time only check to make sure the stack_canary is at
          * offset 40 in the pda; this is a gcc ABI requirement
          */
-        BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
+        BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
 
         /*
          * We both use the random pool and the current TSC as a source
@@ -32,7 +32,7 @@ static __always_inline void boot_init_stack_canary(void)
         canary += tsc + (tsc << 32UL);
 
         current->stack_canary = canary;
-        write_pda(stack_canary, canary);
+        percpu_write(irq_stack_union.stack_canary, canary);
 }
 
 #endif
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index b77bd8bd3cc2..52eb748a68af 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -89,10 +89,10 @@ do {									\
 #ifdef CONFIG_CC_STACKPROTECTOR
 #define __switch_canary                                                   \
         "movq %P[task_canary](%%rsi),%%r8\n\t"                            \
-        "movq %%r8,%%gs:%P[pda_canary]\n\t"
+        "movq %%r8,%%gs:%P[gs_canary]\n\t"
 #define __switch_canary_param                                             \
         , [task_canary] "i" (offsetof(struct task_struct, stack_canary))  \
-        , [pda_canary] "i" (offsetof(struct x8664_pda, stack_canary))
+        , [gs_canary] "i" (offsetof(union irq_stack_union, stack_canary))
 #else   /* CC_STACKPROTECTOR */
 #define __switch_canary
 #define __switch_canary_param