x86-64: Move TLB state from PDA to per-cpu and consolidate with 32-bit.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

3 files changed, 9 insertions, 16 deletions

diff --git a/arch/x86/include/asm/mmu_context_64.h b/arch/x86/include/asm/mmu_context_64.h
index 677d36e9540a..c4572505ab3e 100644
--- a/arch/x86/include/asm/mmu_context_64.h
+++ b/arch/x86/include/asm/mmu_context_64.h
@@ -1,13 +1,11 @@
 #ifndef _ASM_X86_MMU_CONTEXT_64_H
 #define _ASM_X86_MMU_CONTEXT_64_H
 
-#include <asm/pda.h>
-
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 #ifdef CONFIG_SMP
-        if (read_pda(mmu_state) == TLBSTATE_OK)
-                write_pda(mmu_state, TLBSTATE_LAZY);
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+                percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
 #endif
 }
 
@@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                 /* stop flush ipis for the previous mm */
                 cpu_clear(cpu, prev->cpu_vm_mask);
 #ifdef CONFIG_SMP
-                write_pda(mmu_state, TLBSTATE_OK);
-                write_pda(active_mm, next);
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                percpu_write(cpu_tlbstate.active_mm, next);
 #endif
                 cpu_set(cpu, next->cpu_vm_mask);
                 load_cr3(next->pgd);
@@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
         }
 #ifdef CONFIG_SMP
         else {
-                write_pda(mmu_state, TLBSTATE_OK);
-                if (read_pda(active_mm) != next)
-                        BUG();
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+
                 if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
                         /* We were in lazy tlb mode and leave_mm disabled
                          * tlb flush IPI delivery. We must reload CR3
diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
index 69a40757e217..8ee835ed10e1 100644
--- a/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@ -25,9 +25,7 @@ struct x8664_pda {
         char *irqstackptr;
         short nodenumber;               /* number of current node (32k max) */
         short in_bootmem;               /* pda lives in bootmem */
-        short mmu_state;
         short isidle;
-        struct mm_struct *active_mm;
 } ____cacheline_aligned_in_smp;
 
 DECLARE_PER_CPU(struct x8664_pda, __pda);
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 17feaa9c7e76..d3539f998f88 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 #define TLBSTATE_OK     1
 #define TLBSTATE_LAZY   2
 
-#ifdef CONFIG_X86_32
 struct tlb_state {
         struct mm_struct *active_mm;
         int state;
-        char __cacheline_padding[L1_CACHE_BYTES-8];
 };
 DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
 
-void reset_lazy_tlbstate(void);
-#else
 static inline void reset_lazy_tlbstate(void)
 {
+        percpu_write(cpu_tlbstate.state, 0);
+        percpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
-#endif
 
 #endif  /* SMP */