Merge branch 'tj-percpu' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/misc into core/percpu

12 files changed, 68 insertions, 96 deletions

diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 0728480f5c56..c68c361697e1 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -1,39 +1,21 @@
 #ifndef _ASM_X86_CURRENT_H
 #define _ASM_X86_CURRENT_H
 
-#ifdef CONFIG_X86_32
 #include <linux/compiler.h>
 #include <asm/percpu.h>
 
+#ifndef __ASSEMBLY__
 struct task_struct;
 
 DECLARE_PER_CPU(struct task_struct *, current_task);
-static __always_inline struct task_struct *get_current(void)
-{
-        return percpu_read(current_task);
-}
-
-#else /* X86_32 */
-
-#ifndef __ASSEMBLY__
-#include <asm/pda.h>
-
-struct task_struct;
 
 static __always_inline struct task_struct *get_current(void)
 {
-        return read_pda(pcurrent);
+        return percpu_read(current_task);
 }
 
-#else /* __ASSEMBLY__ */
-
-#include <asm/asm-offsets.h>
-#define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg
+#define current get_current()
 
 #endif /* __ASSEMBLY__ */
 
-#endif /* X86_32 */
-
-#define current get_current()
-
 #endif /* _ASM_X86_CURRENT_H */
diff --git a/arch/x86/include/asm/hardirq_64.h b/arch/x86/include/asm/hardirq_64.h
index b5a6b5d56704..a65bab20f6ce 100644
--- a/arch/x86/include/asm/hardirq_64.h
+++ b/arch/x86/include/asm/hardirq_64.h
@@ -3,22 +3,36 @@
 
 #include <linux/threads.h>
 #include <linux/irq.h>
-#include <asm/pda.h>
 #include <asm/apic.h>
 
+typedef struct {
+        unsigned int __softirq_pending;
+        unsigned int __nmi_count;       /* arch dependent */
+        unsigned int apic_timer_irqs;   /* arch dependent */
+        unsigned int irq0_irqs;
+        unsigned int irq_resched_count;
+        unsigned int irq_call_count;
+        unsigned int irq_tlb_count;
+        unsigned int irq_thermal_count;
+        unsigned int irq_spurious_count;
+        unsigned int irq_threshold_count;
+} ____cacheline_aligned irq_cpustat_t;
+
+DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
+
 /* We can have at most NR_VECTORS irqs routed to a cpu at a time */
 #define MAX_HARDIRQS_PER_CPU NR_VECTORS
 
 #define __ARCH_IRQ_STAT 1
 
-#define inc_irq_stat(member)    add_pda(member, 1)
+#define inc_irq_stat(member)    percpu_add(irq_stat.member, 1)
 
-#define local_softirq_pending() read_pda(__softirq_pending)
+#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
 
 #define __ARCH_SET_SOFTIRQ_PENDING 1
 
-#define set_softirq_pending(x) write_pda(__softirq_pending, (x))
-#define or_softirq_pending(x)  or_pda(__softirq_pending, (x))
+#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x)  percpu_or(irq_stat.__softirq_pending, (x))
 
 extern void ack_bad_irq(unsigned int irq);
 
diff --git a/arch/x86/include/asm/mmu_context_64.h b/arch/x86/include/asm/mmu_context_64.h
index 677d36e9540a..c4572505ab3e 100644
--- a/arch/x86/include/asm/mmu_context_64.h
+++ b/arch/x86/include/asm/mmu_context_64.h
@@ -1,13 +1,11 @@
 #ifndef _ASM_X86_MMU_CONTEXT_64_H
 #define _ASM_X86_MMU_CONTEXT_64_H
 
-#include <asm/pda.h>
-
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 #ifdef CONFIG_SMP
-        if (read_pda(mmu_state) == TLBSTATE_OK)
-                write_pda(mmu_state, TLBSTATE_LAZY);
+        if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+                percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
 #endif
 }
 
@@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                 /* stop flush ipis for the previous mm */
                 cpu_clear(cpu, prev->cpu_vm_mask);
 #ifdef CONFIG_SMP
-                write_pda(mmu_state, TLBSTATE_OK);
-                write_pda(active_mm, next);
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                percpu_write(cpu_tlbstate.active_mm, next);
 #endif
                 cpu_set(cpu, next->cpu_vm_mask);
                 load_cr3(next->pgd);
@@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
         }
 #ifdef CONFIG_SMP
         else {
-                write_pda(mmu_state, TLBSTATE_OK);
-                if (read_pda(active_mm) != next)
-                        BUG();
+                percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+                BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+
                 if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
                         /* We were in lazy tlb mode and leave_mm disabled
                          * tlb flush IPI delivery. We must reload CR3
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 5ebca29f44f0..e27fdbe5f9e4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -13,8 +13,8 @@
 #define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1)
 #define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
 
-#define IRQSTACK_ORDER 2
-#define IRQSTACKSIZE (PAGE_SIZE << IRQSTACK_ORDER)
+#define IRQ_STACK_ORDER 2
+#define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
 
 #define STACKFAULT_STACK 1
 #define DOUBLEFAULT_STACK 2
diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
index 47f274fe6953..c31ca048a901 100644
--- a/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@ -11,33 +11,18 @@
 
 /* Per processor datastructure. %gs points to it while the kernel runs */
 struct x8664_pda {
-        struct task_struct *pcurrent;   /* 0  Current process */
-        unsigned long dummy;
-        unsigned long kernelstack;      /* 16 top of kernel stack for current */
-        unsigned long oldrsp;           /* 24 user rsp for system call */
-        int irqcount;                   /* 32 Irq nesting counter. Starts -1 */
-        unsigned int cpunumber;         /* 36 Logical CPU number */
+        unsigned long unused1;
+        unsigned long unused2;
+        unsigned long unused3;
+        unsigned long unused4;
+        int unused5;
+        unsigned int unused6;           /* 36 was cpunumber */
 #ifdef CONFIG_CC_STACKPROTECTOR
         unsigned long stack_canary;     /* 40 stack canary value */
                                         /* gcc-ABI: this canary MUST be at
                                            offset 40!!! */
 #endif
-        char *irqstackptr;
-        short nodenumber;               /* number of current node (32k max) */
         short in_bootmem;               /* pda lives in bootmem */
-        unsigned int __softirq_pending;
-        unsigned int __nmi_count;       /* number of NMI on this CPUs */
-        short mmu_state;
-        short isidle;
-        struct mm_struct *active_mm;
-        unsigned apic_timer_irqs;
-        unsigned irq0_irqs;
-        unsigned irq_resched_count;
-        unsigned irq_call_count;
-        unsigned irq_tlb_count;
-        unsigned irq_thermal_count;
-        unsigned irq_threshold_count;
-        unsigned irq_spurious_count;
 } ____cacheline_aligned_in_smp;
 
 DECLARE_PER_CPU(struct x8664_pda, __pda);
@@ -57,6 +42,4 @@ extern void pda_init(int);
 
 #endif
 
-#define PDA_STACKOFFSET (5*8)
-
 #endif /* _ASM_X86_PDA_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 03aa4b00a1c3..165d5272ece1 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -39,10 +39,10 @@
 #include <linux/stringify.h>
 
 #ifdef CONFIG_SMP
-#define __percpu_seg_str        "%%"__stringify(__percpu_seg)":"
+#define __percpu_arg(x)         "%%"__stringify(__percpu_seg)":%P" #x
 #define __my_cpu_offset         percpu_read(this_cpu_off)
 #else
-#define __percpu_seg_str
+#define __percpu_arg(x)         "%" #x
 #endif
 
 /* For arch-specific code, we can use direct single-insn ops (they
@@ -58,22 +58,22 @@ do {							\
         }                                               \
         switch (sizeof(var)) {                          \
         case 1:                                         \
-                asm(op "b %1,"__percpu_seg_str"%0"      \
+                asm(op "b %1,"__percpu_arg(0)           \
                     : "+m" (var)                        \
                     : "ri" ((T__)val));                 \
                 break;                                  \
         case 2:                                         \
-                asm(op "w %1,"__percpu_seg_str"%0"      \
+                asm(op "w %1,"__percpu_arg(0)           \
                     : "+m" (var)                        \
                     : "ri" ((T__)val));                 \
                 break;                                  \
         case 4:                                         \
-                asm(op "l %1,"__percpu_seg_str"%0"      \
+                asm(op "l %1,"__percpu_arg(0)           \
                     : "+m" (var)                        \
                     : "ri" ((T__)val));                 \
                 break;                                  \
         case 8:                                         \
-                asm(op "q %1,"__percpu_seg_str"%0"      \
+                asm(op "q %1,"__percpu_arg(0)           \
                     : "+m" (var)                        \
                     : "r" ((T__)val));                  \
                 break;                                  \
@@ -86,22 +86,22 @@ do {							\
         typeof(var) ret__;                              \
         switch (sizeof(var)) {                          \
         case 1:                                         \
-                asm(op "b "__percpu_seg_str"%1,%0"      \
+                asm(op "b "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
                     : "m" (var));                       \
                 break;                                  \
         case 2:                                         \
-                asm(op "w "__percpu_seg_str"%1,%0"      \
+                asm(op "w "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
                     : "m" (var));                       \
                 break;                                  \
         case 4:                                         \
-                asm(op "l "__percpu_seg_str"%1,%0"      \
+                asm(op "l "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
                     : "m" (var));                       \
                 break;                                  \
         case 8:                                         \
-                asm(op "q "__percpu_seg_str"%1,%0"      \
+                asm(op "q "__percpu_arg(1)",%0"         \
                     : "=r" (ret__)                      \
                     : "m" (var));                       \
                 break;                                  \
@@ -122,9 +122,9 @@ do {							\
 #define x86_test_and_clear_bit_percpu(bit, var)                         \
 ({                                                                      \
         int old__;                                                      \
-        asm volatile("btr %1,"__percpu_seg_str"%c2\n\tsbbl %0,%0"       \
-                     : "=r" (old__)                                     \
-                     : "dIr" (bit), "i" (&per_cpu__##var) : "memory");  \
+        asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0"           \
+                     : "=r" (old__), "+m" (per_cpu__##var)              \
+                     : "dIr" (bit));                                    \
         old__;                                                          \
 })
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 091cd8855f2e..f511246fa6cd 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -378,6 +378,9 @@ union thread_xstate {
 
 #ifdef CONFIG_X86_64
 DECLARE_PER_CPU(struct orig_ist, orig_ist);
+
+DECLARE_PER_CPU(char[IRQ_STACK_SIZE], irq_stack);
+DECLARE_PER_CPU(char *, irq_stack_ptr);
 #endif
 
 extern void print_cpu_info(struct cpuinfo_x86 *);
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index c7bbbbe65d3f..68636e767a91 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -25,9 +25,7 @@ extern unsigned int num_processors;
 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
 DECLARE_PER_CPU(cpumask_t, cpu_core_map);
 DECLARE_PER_CPU(u16, cpu_llc_id);
-#ifdef CONFIG_X86_32
 DECLARE_PER_CPU(int, cpu_number);
-#endif
 
 static inline struct cpumask *cpu_sibling_mask(int cpu)
 {
@@ -164,7 +162,7 @@ extern unsigned disabled_cpus __cpuinitdata;
 extern int safe_smp_processor_id(void);
 
 #elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id()  read_pda(cpunumber)
+#define raw_smp_processor_id() (percpu_read(cpu_number))
 
 #define stack_smp_processor_id()                                        \
 ({                                                              \
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 8e626ea33a1a..d1dc27dba36d 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -94,7 +94,7 @@ do {									\
              "call __switch_to\n\t"                                       \
              ".globl thread_return\n"                                     \
              "thread_return:\n\t"                                         \
-             "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"                       \
+             "movq "__percpu_arg([current_task])",%%rsi\n\t"              \
              "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
              LOCK_PREFIX "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"        \
              "movq %%rax,%%rdi\n\t"                                       \
@@ -106,7 +106,7 @@ do {									\
                [ti_flags] "i" (offsetof(struct thread_info, flags)),      \
                [tif_fork] "i" (TIF_FORK),                                 \
                [thread_info] "i" (offsetof(struct task_struct, stack)),   \
-               [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent))  \
+               [current_task] "m" (per_cpu_var(current_task))             \
              : "memory", "cc" __EXTRA_CLOBBER)
 #endif
 
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 98789647baa9..b46f8ca007b5 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -194,25 +194,21 @@ static inline struct thread_info *current_thread_info(void)
 
 #else /* X86_32 */
 
-#include <asm/pda.h>
+#include <asm/percpu.h>
+#define KERNEL_STACK_OFFSET (5*8)
 
 /*
  * macros/functions for gaining access to the thread information structure
  * preempt_count needs to be 1 initially, until the scheduler is functional.
  */
 #ifndef __ASSEMBLY__
-static inline struct thread_info *current_thread_info(void)
-{
-        struct thread_info *ti;
-        ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
-        return ti;
-}
+DECLARE_PER_CPU(unsigned long, kernel_stack);
 
-/* do not use in interrupt context */
-static inline struct thread_info *stack_thread_info(void)
+static inline struct thread_info *current_thread_info(void)
 {
         struct thread_info *ti;
-        asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1)));
+        ti = (void *)(percpu_read(kernel_stack) +
+                      KERNEL_STACK_OFFSET - THREAD_SIZE);
         return ti;
 }
 
@@ -220,8 +216,8 @@ static inline struct thread_info *stack_thread_info(void)
 
 /* how to get the thread information struct from ASM */
 #define GET_THREAD_INFO(reg) \
-        movq %gs:pda_kernelstack,reg ; \
-        subq $(THREAD_SIZE-PDA_STACKOFFSET),reg
+        movq PER_CPU_VAR(kernel_stack),reg ; \
+        subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
 
 #endif
 
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 17feaa9c7e76..d3539f998f88 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
 #define TLBSTATE_OK     1
 #define TLBSTATE_LAZY   2
 
-#ifdef CONFIG_X86_32
 struct tlb_state {
         struct mm_struct *active_mm;
         int state;
-        char __cacheline_padding[L1_CACHE_BYTES-8];
 };
 DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
 
-void reset_lazy_tlbstate(void);
-#else
 static inline void reset_lazy_tlbstate(void)
 {
+        percpu_write(cpu_tlbstate.state, 0);
+        percpu_write(cpu_tlbstate.active_mm, &init_mm);
 }
-#endif
 
 #endif  /* SMP */
 
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 87ca3fd86e88..ffea1fe03a99 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -83,7 +83,8 @@ extern cpumask_t *node_to_cpumask_map;
 DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
 
 /* Returns the number of the current Node. */
-#define numa_node_id()          read_pda(nodenumber)
+DECLARE_PER_CPU(int, node_number);
+#define numa_node_id()          percpu_read(node_number)
 
 #ifdef CONFIG_DEBUG_PER_CPU_MAPS
 extern int cpu_to_node(int cpu);