aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/ia32/ia32entry.S8
-rw-r--r--arch/x86/include/asm/current.h24
-rw-r--r--arch/x86/include/asm/hardirq_64.h24
-rw-r--r--arch/x86/include/asm/mmu_context_64.h16
-rw-r--r--arch/x86/include/asm/page_64.h4
-rw-r--r--arch/x86/include/asm/pda.h29
-rw-r--r--arch/x86/include/asm/percpu.h26
-rw-r--r--arch/x86/include/asm/processor.h3
-rw-r--r--arch/x86/include/asm/smp.h4
-rw-r--r--arch/x86/include/asm/system.h4
-rw-r--r--arch/x86/include/asm/thread_info.h20
-rw-r--r--arch/x86/include/asm/tlbflush.h7
-rw-r--r--arch/x86/include/asm/topology.h3
-rw-r--r--arch/x86/kernel/asm-offsets_64.c6
-rw-r--r--arch/x86/kernel/cpu/common.c71
-rw-r--r--arch/x86/kernel/dumpstack_64.c35
-rw-r--r--arch/x86/kernel/entry_64.S34
-rw-r--r--arch/x86/kernel/irq.c6
-rw-r--r--arch/x86/kernel/irq_64.c3
-rw-r--r--arch/x86/kernel/nmi.c10
-rw-r--r--arch/x86/kernel/process_32.c3
-rw-r--r--arch/x86/kernel/process_64.c22
-rw-r--r--arch/x86/kernel/setup_percpu.c18
-rw-r--r--arch/x86/kernel/smpboot.c6
-rw-r--r--arch/x86/kernel/smpcommon.c2
-rw-r--r--arch/x86/kernel/tlb_32.c12
-rw-r--r--arch/x86/kernel/tlb_64.c13
-rw-r--r--arch/x86/xen/mmu.c6
-rw-r--r--arch/x86/xen/smp.c21
-rw-r--r--arch/x86/xen/xen-asm_64.S31
30 files changed, 203 insertions, 268 deletions
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 256b00b61892..9c79b2477008 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -112,8 +112,8 @@ ENTRY(ia32_sysenter_target)
112 CFI_DEF_CFA rsp,0 112 CFI_DEF_CFA rsp,0
113 CFI_REGISTER rsp,rbp 113 CFI_REGISTER rsp,rbp
114 SWAPGS_UNSAFE_STACK 114 SWAPGS_UNSAFE_STACK
115 movq %gs:pda_kernelstack, %rsp 115 movq PER_CPU_VAR(kernel_stack), %rsp
116 addq $(PDA_STACKOFFSET),%rsp 116 addq $(KERNEL_STACK_OFFSET),%rsp
117 /* 117 /*
118 * No need to follow this irqs on/off section: the syscall 118 * No need to follow this irqs on/off section: the syscall
119 * disabled irqs, here we enable it straight after entry: 119 * disabled irqs, here we enable it straight after entry:
@@ -273,13 +273,13 @@ ENDPROC(ia32_sysenter_target)
273ENTRY(ia32_cstar_target) 273ENTRY(ia32_cstar_target)
274 CFI_STARTPROC32 simple 274 CFI_STARTPROC32 simple
275 CFI_SIGNAL_FRAME 275 CFI_SIGNAL_FRAME
276 CFI_DEF_CFA rsp,PDA_STACKOFFSET 276 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
277 CFI_REGISTER rip,rcx 277 CFI_REGISTER rip,rcx
278 /*CFI_REGISTER rflags,r11*/ 278 /*CFI_REGISTER rflags,r11*/
279 SWAPGS_UNSAFE_STACK 279 SWAPGS_UNSAFE_STACK
280 movl %esp,%r8d 280 movl %esp,%r8d
281 CFI_REGISTER rsp,r8 281 CFI_REGISTER rsp,r8
282 movq %gs:pda_kernelstack,%rsp 282 movq PER_CPU_VAR(kernel_stack),%rsp
283 /* 283 /*
284 * No need to follow this irqs on/off section: the syscall 284 * No need to follow this irqs on/off section: the syscall
285 * disabled irqs and here we enable it straight after entry: 285 * disabled irqs and here we enable it straight after entry:
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 0728480f5c56..c68c361697e1 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -1,39 +1,21 @@
1#ifndef _ASM_X86_CURRENT_H 1#ifndef _ASM_X86_CURRENT_H
2#define _ASM_X86_CURRENT_H 2#define _ASM_X86_CURRENT_H
3 3
4#ifdef CONFIG_X86_32
5#include <linux/compiler.h> 4#include <linux/compiler.h>
6#include <asm/percpu.h> 5#include <asm/percpu.h>
7 6
7#ifndef __ASSEMBLY__
8struct task_struct; 8struct task_struct;
9 9
10DECLARE_PER_CPU(struct task_struct *, current_task); 10DECLARE_PER_CPU(struct task_struct *, current_task);
11static __always_inline struct task_struct *get_current(void)
12{
13 return percpu_read(current_task);
14}
15
16#else /* X86_32 */
17
18#ifndef __ASSEMBLY__
19#include <asm/pda.h>
20
21struct task_struct;
22 11
23static __always_inline struct task_struct *get_current(void) 12static __always_inline struct task_struct *get_current(void)
24{ 13{
25 return read_pda(pcurrent); 14 return percpu_read(current_task);
26} 15}
27 16
28#else /* __ASSEMBLY__ */ 17#define current get_current()
29
30#include <asm/asm-offsets.h>
31#define GET_CURRENT(reg) movq %gs:(pda_pcurrent),reg
32 18
33#endif /* __ASSEMBLY__ */ 19#endif /* __ASSEMBLY__ */
34 20
35#endif /* X86_32 */
36
37#define current get_current()
38
39#endif /* _ASM_X86_CURRENT_H */ 21#endif /* _ASM_X86_CURRENT_H */
diff --git a/arch/x86/include/asm/hardirq_64.h b/arch/x86/include/asm/hardirq_64.h
index b5a6b5d56704..a65bab20f6ce 100644
--- a/arch/x86/include/asm/hardirq_64.h
+++ b/arch/x86/include/asm/hardirq_64.h
@@ -3,22 +3,36 @@
3 3
4#include <linux/threads.h> 4#include <linux/threads.h>
5#include <linux/irq.h> 5#include <linux/irq.h>
6#include <asm/pda.h>
7#include <asm/apic.h> 6#include <asm/apic.h>
8 7
8typedef struct {
9 unsigned int __softirq_pending;
10 unsigned int __nmi_count; /* arch dependent */
11 unsigned int apic_timer_irqs; /* arch dependent */
12 unsigned int irq0_irqs;
13 unsigned int irq_resched_count;
14 unsigned int irq_call_count;
15 unsigned int irq_tlb_count;
16 unsigned int irq_thermal_count;
17 unsigned int irq_spurious_count;
18 unsigned int irq_threshold_count;
19} ____cacheline_aligned irq_cpustat_t;
20
21DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
22
9/* We can have at most NR_VECTORS irqs routed to a cpu at a time */ 23/* We can have at most NR_VECTORS irqs routed to a cpu at a time */
10#define MAX_HARDIRQS_PER_CPU NR_VECTORS 24#define MAX_HARDIRQS_PER_CPU NR_VECTORS
11 25
12#define __ARCH_IRQ_STAT 1 26#define __ARCH_IRQ_STAT 1
13 27
14#define inc_irq_stat(member) add_pda(member, 1) 28#define inc_irq_stat(member) percpu_add(irq_stat.member, 1)
15 29
16#define local_softirq_pending() read_pda(__softirq_pending) 30#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
17 31
18#define __ARCH_SET_SOFTIRQ_PENDING 1 32#define __ARCH_SET_SOFTIRQ_PENDING 1
19 33
20#define set_softirq_pending(x) write_pda(__softirq_pending, (x)) 34#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
21#define or_softirq_pending(x) or_pda(__softirq_pending, (x)) 35#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
22 36
23extern void ack_bad_irq(unsigned int irq); 37extern void ack_bad_irq(unsigned int irq);
24 38
diff --git a/arch/x86/include/asm/mmu_context_64.h b/arch/x86/include/asm/mmu_context_64.h
index 677d36e9540a..c4572505ab3e 100644
--- a/arch/x86/include/asm/mmu_context_64.h
+++ b/arch/x86/include/asm/mmu_context_64.h
@@ -1,13 +1,11 @@
1#ifndef _ASM_X86_MMU_CONTEXT_64_H 1#ifndef _ASM_X86_MMU_CONTEXT_64_H
2#define _ASM_X86_MMU_CONTEXT_64_H 2#define _ASM_X86_MMU_CONTEXT_64_H
3 3
4#include <asm/pda.h>
5
6static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 4static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
7{ 5{
8#ifdef CONFIG_SMP 6#ifdef CONFIG_SMP
9 if (read_pda(mmu_state) == TLBSTATE_OK) 7 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
10 write_pda(mmu_state, TLBSTATE_LAZY); 8 percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
11#endif 9#endif
12} 10}
13 11
@@ -19,8 +17,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
19 /* stop flush ipis for the previous mm */ 17 /* stop flush ipis for the previous mm */
20 cpu_clear(cpu, prev->cpu_vm_mask); 18 cpu_clear(cpu, prev->cpu_vm_mask);
21#ifdef CONFIG_SMP 19#ifdef CONFIG_SMP
22 write_pda(mmu_state, TLBSTATE_OK); 20 percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
23 write_pda(active_mm, next); 21 percpu_write(cpu_tlbstate.active_mm, next);
24#endif 22#endif
25 cpu_set(cpu, next->cpu_vm_mask); 23 cpu_set(cpu, next->cpu_vm_mask);
26 load_cr3(next->pgd); 24 load_cr3(next->pgd);
@@ -30,9 +28,9 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
30 } 28 }
31#ifdef CONFIG_SMP 29#ifdef CONFIG_SMP
32 else { 30 else {
33 write_pda(mmu_state, TLBSTATE_OK); 31 percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
34 if (read_pda(active_mm) != next) 32 BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
35 BUG(); 33
36 if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) { 34 if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
37 /* We were in lazy tlb mode and leave_mm disabled 35 /* We were in lazy tlb mode and leave_mm disabled
38 * tlb flush IPI delivery. We must reload CR3 36 * tlb flush IPI delivery. We must reload CR3
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 5ebca29f44f0..e27fdbe5f9e4 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -13,8 +13,8 @@
13#define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1) 13#define DEBUG_STACK_ORDER (EXCEPTION_STACK_ORDER + 1)
14#define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER) 14#define DEBUG_STKSZ (PAGE_SIZE << DEBUG_STACK_ORDER)
15 15
16#define IRQSTACK_ORDER 2 16#define IRQ_STACK_ORDER 2
17#define IRQSTACKSIZE (PAGE_SIZE << IRQSTACK_ORDER) 17#define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
18 18
19#define STACKFAULT_STACK 1 19#define STACKFAULT_STACK 1
20#define DOUBLEFAULT_STACK 2 20#define DOUBLEFAULT_STACK 2
diff --git a/arch/x86/include/asm/pda.h b/arch/x86/include/asm/pda.h
index 47f274fe6953..c31ca048a901 100644
--- a/arch/x86/include/asm/pda.h
+++ b/arch/x86/include/asm/pda.h
@@ -11,33 +11,18 @@
11 11
12/* Per processor datastructure. %gs points to it while the kernel runs */ 12/* Per processor datastructure. %gs points to it while the kernel runs */
13struct x8664_pda { 13struct x8664_pda {
14 struct task_struct *pcurrent; /* 0 Current process */ 14 unsigned long unused1;
15 unsigned long dummy; 15 unsigned long unused2;
16 unsigned long kernelstack; /* 16 top of kernel stack for current */ 16 unsigned long unused3;
17 unsigned long oldrsp; /* 24 user rsp for system call */ 17 unsigned long unused4;
18 int irqcount; /* 32 Irq nesting counter. Starts -1 */ 18 int unused5;
19 unsigned int cpunumber; /* 36 Logical CPU number */ 19 unsigned int unused6; /* 36 was cpunumber */
20#ifdef CONFIG_CC_STACKPROTECTOR 20#ifdef CONFIG_CC_STACKPROTECTOR
21 unsigned long stack_canary; /* 40 stack canary value */ 21 unsigned long stack_canary; /* 40 stack canary value */
22 /* gcc-ABI: this canary MUST be at 22 /* gcc-ABI: this canary MUST be at
23 offset 40!!! */ 23 offset 40!!! */
24#endif 24#endif
25 char *irqstackptr;
26 short nodenumber; /* number of current node (32k max) */
27 short in_bootmem; /* pda lives in bootmem */ 25 short in_bootmem; /* pda lives in bootmem */
28 unsigned int __softirq_pending;
29 unsigned int __nmi_count; /* number of NMI on this CPUs */
30 short mmu_state;
31 short isidle;
32 struct mm_struct *active_mm;
33 unsigned apic_timer_irqs;
34 unsigned irq0_irqs;
35 unsigned irq_resched_count;
36 unsigned irq_call_count;
37 unsigned irq_tlb_count;
38 unsigned irq_thermal_count;
39 unsigned irq_threshold_count;
40 unsigned irq_spurious_count;
41} ____cacheline_aligned_in_smp; 26} ____cacheline_aligned_in_smp;
42 27
43DECLARE_PER_CPU(struct x8664_pda, __pda); 28DECLARE_PER_CPU(struct x8664_pda, __pda);
@@ -57,6 +42,4 @@ extern void pda_init(int);
57 42
58#endif 43#endif
59 44
60#define PDA_STACKOFFSET (5*8)
61
62#endif /* _ASM_X86_PDA_H */ 45#endif /* _ASM_X86_PDA_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 03aa4b00a1c3..165d5272ece1 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -39,10 +39,10 @@
39#include <linux/stringify.h> 39#include <linux/stringify.h>
40 40
41#ifdef CONFIG_SMP 41#ifdef CONFIG_SMP
42#define __percpu_seg_str "%%"__stringify(__percpu_seg)":" 42#define __percpu_arg(x) "%%"__stringify(__percpu_seg)":%P" #x
43#define __my_cpu_offset percpu_read(this_cpu_off) 43#define __my_cpu_offset percpu_read(this_cpu_off)
44#else 44#else
45#define __percpu_seg_str 45#define __percpu_arg(x) "%" #x
46#endif 46#endif
47 47
48/* For arch-specific code, we can use direct single-insn ops (they 48/* For arch-specific code, we can use direct single-insn ops (they
@@ -58,22 +58,22 @@ do { \
58 } \ 58 } \
59 switch (sizeof(var)) { \ 59 switch (sizeof(var)) { \
60 case 1: \ 60 case 1: \
61 asm(op "b %1,"__percpu_seg_str"%0" \ 61 asm(op "b %1,"__percpu_arg(0) \
62 : "+m" (var) \ 62 : "+m" (var) \
63 : "ri" ((T__)val)); \ 63 : "ri" ((T__)val)); \
64 break; \ 64 break; \
65 case 2: \ 65 case 2: \
66 asm(op "w %1,"__percpu_seg_str"%0" \ 66 asm(op "w %1,"__percpu_arg(0) \
67 : "+m" (var) \ 67 : "+m" (var) \
68 : "ri" ((T__)val)); \ 68 : "ri" ((T__)val)); \
69 break; \ 69 break; \
70 case 4: \ 70 case 4: \
71 asm(op "l %1,"__percpu_seg_str"%0" \ 71 asm(op "l %1,"__percpu_arg(0) \
72 : "+m" (var) \ 72 : "+m" (var) \
73 : "ri" ((T__)val)); \ 73 : "ri" ((T__)val)); \
74 break; \ 74 break; \
75 case 8: \ 75 case 8: \
76 asm(op "q %1,"__percpu_seg_str"%0" \ 76 asm(op "q %1,"__percpu_arg(0) \
77 : "+m" (var) \ 77 : "+m" (var) \
78 : "r" ((T__)val)); \ 78 : "r" ((T__)val)); \
79 break; \ 79 break; \
@@ -86,22 +86,22 @@ do { \
86 typeof(var) ret__; \ 86 typeof(var) ret__; \
87 switch (sizeof(var)) { \ 87 switch (sizeof(var)) { \
88 case 1: \ 88 case 1: \
89 asm(op "b "__percpu_seg_str"%1,%0" \ 89 asm(op "b "__percpu_arg(1)",%0" \
90 : "=r" (ret__) \ 90 : "=r" (ret__) \
91 : "m" (var)); \ 91 : "m" (var)); \
92 break; \ 92 break; \
93 case 2: \ 93 case 2: \
94 asm(op "w "__percpu_seg_str"%1,%0" \ 94 asm(op "w "__percpu_arg(1)",%0" \
95 : "=r" (ret__) \ 95 : "=r" (ret__) \
96 : "m" (var)); \ 96 : "m" (var)); \
97 break; \ 97 break; \
98 case 4: \ 98 case 4: \
99 asm(op "l "__percpu_seg_str"%1,%0" \ 99 asm(op "l "__percpu_arg(1)",%0" \
100 : "=r" (ret__) \ 100 : "=r" (ret__) \
101 : "m" (var)); \ 101 : "m" (var)); \
102 break; \ 102 break; \
103 case 8: \ 103 case 8: \
104 asm(op "q "__percpu_seg_str"%1,%0" \ 104 asm(op "q "__percpu_arg(1)",%0" \
105 : "=r" (ret__) \ 105 : "=r" (ret__) \
106 : "m" (var)); \ 106 : "m" (var)); \
107 break; \ 107 break; \
@@ -122,9 +122,9 @@ do { \
122#define x86_test_and_clear_bit_percpu(bit, var) \ 122#define x86_test_and_clear_bit_percpu(bit, var) \
123({ \ 123({ \
124 int old__; \ 124 int old__; \
125 asm volatile("btr %1,"__percpu_seg_str"%c2\n\tsbbl %0,%0" \ 125 asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \
126 : "=r" (old__) \ 126 : "=r" (old__), "+m" (per_cpu__##var) \
127 : "dIr" (bit), "i" (&per_cpu__##var) : "memory"); \ 127 : "dIr" (bit)); \
128 old__; \ 128 old__; \
129}) 129})
130 130
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 091cd8855f2e..f511246fa6cd 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -378,6 +378,9 @@ union thread_xstate {
378 378
379#ifdef CONFIG_X86_64 379#ifdef CONFIG_X86_64
380DECLARE_PER_CPU(struct orig_ist, orig_ist); 380DECLARE_PER_CPU(struct orig_ist, orig_ist);
381
382DECLARE_PER_CPU(char[IRQ_STACK_SIZE], irq_stack);
383DECLARE_PER_CPU(char *, irq_stack_ptr);
381#endif 384#endif
382 385
383extern void print_cpu_info(struct cpuinfo_x86 *); 386extern void print_cpu_info(struct cpuinfo_x86 *);
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index c7bbbbe65d3f..68636e767a91 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -25,9 +25,7 @@ extern unsigned int num_processors;
25DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); 25DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
26DECLARE_PER_CPU(cpumask_t, cpu_core_map); 26DECLARE_PER_CPU(cpumask_t, cpu_core_map);
27DECLARE_PER_CPU(u16, cpu_llc_id); 27DECLARE_PER_CPU(u16, cpu_llc_id);
28#ifdef CONFIG_X86_32
29DECLARE_PER_CPU(int, cpu_number); 28DECLARE_PER_CPU(int, cpu_number);
30#endif
31 29
32static inline struct cpumask *cpu_sibling_mask(int cpu) 30static inline struct cpumask *cpu_sibling_mask(int cpu)
33{ 31{
@@ -164,7 +162,7 @@ extern unsigned disabled_cpus __cpuinitdata;
164extern int safe_smp_processor_id(void); 162extern int safe_smp_processor_id(void);
165 163
166#elif defined(CONFIG_X86_64_SMP) 164#elif defined(CONFIG_X86_64_SMP)
167#define raw_smp_processor_id() read_pda(cpunumber) 165#define raw_smp_processor_id() (percpu_read(cpu_number))
168 166
169#define stack_smp_processor_id() \ 167#define stack_smp_processor_id() \
170({ \ 168({ \
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index 8e626ea33a1a..d1dc27dba36d 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -94,7 +94,7 @@ do { \
94 "call __switch_to\n\t" \ 94 "call __switch_to\n\t" \
95 ".globl thread_return\n" \ 95 ".globl thread_return\n" \
96 "thread_return:\n\t" \ 96 "thread_return:\n\t" \
97 "movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \ 97 "movq "__percpu_arg([current_task])",%%rsi\n\t" \
98 "movq %P[thread_info](%%rsi),%%r8\n\t" \ 98 "movq %P[thread_info](%%rsi),%%r8\n\t" \
99 LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \ 99 LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \
100 "movq %%rax,%%rdi\n\t" \ 100 "movq %%rax,%%rdi\n\t" \
@@ -106,7 +106,7 @@ do { \
106 [ti_flags] "i" (offsetof(struct thread_info, flags)), \ 106 [ti_flags] "i" (offsetof(struct thread_info, flags)), \
107 [tif_fork] "i" (TIF_FORK), \ 107 [tif_fork] "i" (TIF_FORK), \
108 [thread_info] "i" (offsetof(struct task_struct, stack)), \ 108 [thread_info] "i" (offsetof(struct task_struct, stack)), \
109 [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent)) \ 109 [current_task] "m" (per_cpu_var(current_task)) \
110 : "memory", "cc" __EXTRA_CLOBBER) 110 : "memory", "cc" __EXTRA_CLOBBER)
111#endif 111#endif
112 112
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 98789647baa9..b46f8ca007b5 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -194,25 +194,21 @@ static inline struct thread_info *current_thread_info(void)
194 194
195#else /* X86_32 */ 195#else /* X86_32 */
196 196
197#include <asm/pda.h> 197#include <asm/percpu.h>
198#define KERNEL_STACK_OFFSET (5*8)
198 199
199/* 200/*
200 * macros/functions for gaining access to the thread information structure 201 * macros/functions for gaining access to the thread information structure
201 * preempt_count needs to be 1 initially, until the scheduler is functional. 202 * preempt_count needs to be 1 initially, until the scheduler is functional.
202 */ 203 */
203#ifndef __ASSEMBLY__ 204#ifndef __ASSEMBLY__
204static inline struct thread_info *current_thread_info(void) 205DECLARE_PER_CPU(unsigned long, kernel_stack);
205{
206 struct thread_info *ti;
207 ti = (void *)(read_pda(kernelstack) + PDA_STACKOFFSET - THREAD_SIZE);
208 return ti;
209}
210 206
211/* do not use in interrupt context */ 207static inline struct thread_info *current_thread_info(void)
212static inline struct thread_info *stack_thread_info(void)
213{ 208{
214 struct thread_info *ti; 209 struct thread_info *ti;
215 asm("andq %%rsp,%0; " : "=r" (ti) : "0" (~(THREAD_SIZE - 1))); 210 ti = (void *)(percpu_read(kernel_stack) +
211 KERNEL_STACK_OFFSET - THREAD_SIZE);
216 return ti; 212 return ti;
217} 213}
218 214
@@ -220,8 +216,8 @@ static inline struct thread_info *stack_thread_info(void)
220 216
221/* how to get the thread information struct from ASM */ 217/* how to get the thread information struct from ASM */
222#define GET_THREAD_INFO(reg) \ 218#define GET_THREAD_INFO(reg) \
223 movq %gs:pda_kernelstack,reg ; \ 219 movq PER_CPU_VAR(kernel_stack),reg ; \
224 subq $(THREAD_SIZE-PDA_STACKOFFSET),reg 220 subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
225 221
226#endif 222#endif
227 223
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 17feaa9c7e76..d3539f998f88 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -148,20 +148,17 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
148#define TLBSTATE_OK 1 148#define TLBSTATE_OK 1
149#define TLBSTATE_LAZY 2 149#define TLBSTATE_LAZY 2
150 150
151#ifdef CONFIG_X86_32
152struct tlb_state { 151struct tlb_state {
153 struct mm_struct *active_mm; 152 struct mm_struct *active_mm;
154 int state; 153 int state;
155 char __cacheline_padding[L1_CACHE_BYTES-8];
156}; 154};
157DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate); 155DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
158 156
159void reset_lazy_tlbstate(void);
160#else
161static inline void reset_lazy_tlbstate(void) 157static inline void reset_lazy_tlbstate(void)
162{ 158{
159 percpu_write(cpu_tlbstate.state, 0);
160 percpu_write(cpu_tlbstate.active_mm, &init_mm);
163} 161}
164#endif
165 162
166#endif /* SMP */ 163#endif /* SMP */
167 164
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 87ca3fd86e88..ffea1fe03a99 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -83,7 +83,8 @@ extern cpumask_t *node_to_cpumask_map;
83DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map); 83DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
84 84
85/* Returns the number of the current Node. */ 85/* Returns the number of the current Node. */
86#define numa_node_id() read_pda(nodenumber) 86DECLARE_PER_CPU(int, node_number);
87#define numa_node_id() percpu_read(node_number)
87 88
88#ifdef CONFIG_DEBUG_PER_CPU_MAPS 89#ifdef CONFIG_DEBUG_PER_CPU_MAPS
89extern int cpu_to_node(int cpu); 90extern int cpu_to_node(int cpu);
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index f4cc81bfbf89..64c834a39aa8 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -49,12 +49,6 @@ int main(void)
49 BLANK(); 49 BLANK();
50#undef ENTRY 50#undef ENTRY
51#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry)) 51#define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))
52 ENTRY(kernelstack);
53 ENTRY(oldrsp);
54 ENTRY(pcurrent);
55 ENTRY(irqcount);
56 ENTRY(cpunumber);
57 ENTRY(irqstackptr);
58 DEFINE(pda_size, sizeof(struct x8664_pda)); 52 DEFINE(pda_size, sizeof(struct x8664_pda));
59 BLANK(); 53 BLANK();
60#undef ENTRY 54#undef ENTRY
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index c49498d40830..7976a6a0f65c 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -881,47 +881,32 @@ __setup("clearcpuid=", setup_disablecpuid);
881#ifdef CONFIG_X86_64 881#ifdef CONFIG_X86_64
882struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; 882struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
883 883
884static char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss; 884DEFINE_PER_CPU_PAGE_ALIGNED(char[IRQ_STACK_SIZE], irq_stack);
885#ifdef CONFIG_SMP
886DEFINE_PER_CPU(char *, irq_stack_ptr); /* will be set during per cpu init */
887#else
888DEFINE_PER_CPU(char *, irq_stack_ptr) =
889 per_cpu_var(irq_stack) + IRQ_STACK_SIZE - 64;
890#endif
891
892DEFINE_PER_CPU(unsigned long, kernel_stack) =
893 (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
894EXPORT_PER_CPU_SYMBOL(kernel_stack);
895
896DEFINE_PER_CPU(unsigned int, irq_count) = -1;
885 897
886void __cpuinit pda_init(int cpu) 898void __cpuinit pda_init(int cpu)
887{ 899{
888 struct x8664_pda *pda = cpu_pda(cpu);
889
890 /* Setup up data that may be needed in __get_free_pages early */ 900 /* Setup up data that may be needed in __get_free_pages early */
891 loadsegment(fs, 0); 901 loadsegment(fs, 0);
892 loadsegment(gs, 0); 902 loadsegment(gs, 0);
893 903
894 load_pda_offset(cpu); 904 load_pda_offset(cpu);
895
896 pda->cpunumber = cpu;
897 pda->irqcount = -1;
898 pda->kernelstack = (unsigned long)stack_thread_info() -
899 PDA_STACKOFFSET + THREAD_SIZE;
900 pda->active_mm = &init_mm;
901 pda->mmu_state = 0;
902
903 if (cpu == 0) {
904 /* others are initialized in smpboot.c */
905 pda->pcurrent = &init_task;
906 pda->irqstackptr = boot_cpu_stack;
907 pda->irqstackptr += IRQSTACKSIZE - 64;
908 } else {
909 if (!pda->irqstackptr) {
910 pda->irqstackptr = (char *)
911 __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER);
912 if (!pda->irqstackptr)
913 panic("cannot allocate irqstack for cpu %d",
914 cpu);
915 pda->irqstackptr += IRQSTACKSIZE - 64;
916 }
917
918 if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE)
919 pda->nodenumber = cpu_to_node(cpu);
920 }
921} 905}
922 906
923static char boot_exception_stacks[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + 907static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
924 DEBUG_STKSZ] __page_aligned_bss; 908 [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ])
909 __aligned(PAGE_SIZE);
925 910
926extern asmlinkage void ignore_sysret(void); 911extern asmlinkage void ignore_sysret(void);
927 912
@@ -979,15 +964,18 @@ void __cpuinit cpu_init(void)
979 struct tss_struct *t = &per_cpu(init_tss, cpu); 964 struct tss_struct *t = &per_cpu(init_tss, cpu);
980 struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu); 965 struct orig_ist *orig_ist = &per_cpu(orig_ist, cpu);
981 unsigned long v; 966 unsigned long v;
982 char *estacks = NULL;
983 struct task_struct *me; 967 struct task_struct *me;
984 int i; 968 int i;
985 969
986 /* CPU 0 is initialised in head64.c */ 970 /* CPU 0 is initialised in head64.c */
987 if (cpu != 0) 971 if (cpu != 0)
988 pda_init(cpu); 972 pda_init(cpu);
989 else 973
990 estacks = boot_exception_stacks; 974#ifdef CONFIG_NUMA
975 if (cpu != 0 && percpu_read(node_number) == 0 &&
976 cpu_to_node(cpu) != NUMA_NO_NODE)
977 percpu_write(node_number, cpu_to_node(cpu));
978#endif
991 979
992 me = current; 980 me = current;
993 981
@@ -1021,18 +1009,13 @@ void __cpuinit cpu_init(void)
1021 * set up and load the per-CPU TSS 1009 * set up and load the per-CPU TSS
1022 */ 1010 */
1023 if (!orig_ist->ist[0]) { 1011 if (!orig_ist->ist[0]) {
1024 static const unsigned int order[N_EXCEPTION_STACKS] = { 1012 static const unsigned int sizes[N_EXCEPTION_STACKS] = {
1025 [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STACK_ORDER, 1013 [0 ... N_EXCEPTION_STACKS - 1] = EXCEPTION_STKSZ,
1026 [DEBUG_STACK - 1] = DEBUG_STACK_ORDER 1014 [DEBUG_STACK - 1] = DEBUG_STKSZ
1027 }; 1015 };
1016 char *estacks = per_cpu(exception_stacks, cpu);
1028 for (v = 0; v < N_EXCEPTION_STACKS; v++) { 1017 for (v = 0; v < N_EXCEPTION_STACKS; v++) {
1029 if (cpu) { 1018 estacks += sizes[v];
1030 estacks = (char *)__get_free_pages(GFP_ATOMIC, order[v]);
1031 if (!estacks)
1032 panic("Cannot allocate exception "
1033 "stack %ld %d\n", v, cpu);
1034 }
1035 estacks += PAGE_SIZE << order[v];
1036 orig_ist->ist[v] = t->x86_tss.ist[v] = 1019 orig_ist->ist[v] = t->x86_tss.ist[v] =
1037 (unsigned long)estacks; 1020 (unsigned long)estacks;
1038 } 1021 }
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
index c302d0707048..d35db5993fd6 100644
--- a/arch/x86/kernel/dumpstack_64.c
+++ b/arch/x86/kernel/dumpstack_64.c
@@ -106,7 +106,8 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
106 const struct stacktrace_ops *ops, void *data) 106 const struct stacktrace_ops *ops, void *data)
107{ 107{
108 const unsigned cpu = get_cpu(); 108 const unsigned cpu = get_cpu();
109 unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr; 109 unsigned long *irq_stack_end =
110 (unsigned long *)per_cpu(irq_stack_ptr, cpu);
110 unsigned used = 0; 111 unsigned used = 0;
111 struct thread_info *tinfo; 112 struct thread_info *tinfo;
112 int graph = 0; 113 int graph = 0;
@@ -160,23 +161,23 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
160 stack = (unsigned long *) estack_end[-2]; 161 stack = (unsigned long *) estack_end[-2];
161 continue; 162 continue;
162 } 163 }
163 if (irqstack_end) { 164 if (irq_stack_end) {
164 unsigned long *irqstack; 165 unsigned long *irq_stack;
165 irqstack = irqstack_end - 166 irq_stack = irq_stack_end -
166 (IRQSTACKSIZE - 64) / sizeof(*irqstack); 167 (IRQ_STACK_SIZE - 64) / sizeof(*irq_stack);
167 168
168 if (stack >= irqstack && stack < irqstack_end) { 169 if (stack >= irq_stack && stack < irq_stack_end) {
169 if (ops->stack(data, "IRQ") < 0) 170 if (ops->stack(data, "IRQ") < 0)
170 break; 171 break;
171 bp = print_context_stack(tinfo, stack, bp, 172 bp = print_context_stack(tinfo, stack, bp,
172 ops, data, irqstack_end, &graph); 173 ops, data, irq_stack_end, &graph);
173 /* 174 /*
174 * We link to the next stack (which would be 175 * We link to the next stack (which would be
175 * the process stack normally) the last 176 * the process stack normally) the last
176 * pointer (index -1 to end) in the IRQ stack: 177 * pointer (index -1 to end) in the IRQ stack:
177 */ 178 */
178 stack = (unsigned long *) (irqstack_end[-1]); 179 stack = (unsigned long *) (irq_stack_end[-1]);
179 irqstack_end = NULL; 180 irq_stack_end = NULL;
180 ops->stack(data, "EOI"); 181 ops->stack(data, "EOI");
181 continue; 182 continue;
182 } 183 }
@@ -199,10 +200,10 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
199 unsigned long *stack; 200 unsigned long *stack;
200 int i; 201 int i;
201 const int cpu = smp_processor_id(); 202 const int cpu = smp_processor_id();
202 unsigned long *irqstack_end = 203 unsigned long *irq_stack_end =
203 (unsigned long *) (cpu_pda(cpu)->irqstackptr); 204 (unsigned long *)(per_cpu(irq_stack_ptr, cpu));
204 unsigned long *irqstack = 205 unsigned long *irq_stack =
205 (unsigned long *) (cpu_pda(cpu)->irqstackptr - IRQSTACKSIZE); 206 (unsigned long *)(per_cpu(irq_stack_ptr, cpu) - IRQ_STACK_SIZE);
206 207
207 /* 208 /*
208 * debugging aid: "show_stack(NULL, NULL);" prints the 209 * debugging aid: "show_stack(NULL, NULL);" prints the
@@ -218,9 +219,9 @@ show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
218 219
219 stack = sp; 220 stack = sp;
220 for (i = 0; i < kstack_depth_to_print; i++) { 221 for (i = 0; i < kstack_depth_to_print; i++) {
221 if (stack >= irqstack && stack <= irqstack_end) { 222 if (stack >= irq_stack && stack <= irq_stack_end) {
222 if (stack == irqstack_end) { 223 if (stack == irq_stack_end) {
223 stack = (unsigned long *) (irqstack_end[-1]); 224 stack = (unsigned long *) (irq_stack_end[-1]);
224 printk(" <EOI> "); 225 printk(" <EOI> ");
225 } 226 }
226 } else { 227 } else {
@@ -241,7 +242,7 @@ void show_registers(struct pt_regs *regs)
241 int i; 242 int i;
242 unsigned long sp; 243 unsigned long sp;
243 const int cpu = smp_processor_id(); 244 const int cpu = smp_processor_id();
244 struct task_struct *cur = cpu_pda(cpu)->pcurrent; 245 struct task_struct *cur = current;
245 246
246 sp = regs->sp; 247 sp = regs->sp;
247 printk("CPU %d ", cpu); 248 printk("CPU %d ", cpu);
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 4833f3a19650..c52b60919163 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -210,7 +210,7 @@ ENTRY(native_usergs_sysret64)
210 210
211 /* %rsp:at FRAMEEND */ 211 /* %rsp:at FRAMEEND */
212 .macro FIXUP_TOP_OF_STACK tmp offset=0 212 .macro FIXUP_TOP_OF_STACK tmp offset=0
213 movq %gs:pda_oldrsp,\tmp 213 movq PER_CPU_VAR(old_rsp),\tmp
214 movq \tmp,RSP+\offset(%rsp) 214 movq \tmp,RSP+\offset(%rsp)
215 movq $__USER_DS,SS+\offset(%rsp) 215 movq $__USER_DS,SS+\offset(%rsp)
216 movq $__USER_CS,CS+\offset(%rsp) 216 movq $__USER_CS,CS+\offset(%rsp)
@@ -221,7 +221,7 @@ ENTRY(native_usergs_sysret64)
221 221
222 .macro RESTORE_TOP_OF_STACK tmp offset=0 222 .macro RESTORE_TOP_OF_STACK tmp offset=0
223 movq RSP+\offset(%rsp),\tmp 223 movq RSP+\offset(%rsp),\tmp
224 movq \tmp,%gs:pda_oldrsp 224 movq \tmp,PER_CPU_VAR(old_rsp)
225 movq EFLAGS+\offset(%rsp),\tmp 225 movq EFLAGS+\offset(%rsp),\tmp
226 movq \tmp,R11+\offset(%rsp) 226 movq \tmp,R11+\offset(%rsp)
227 .endm 227 .endm
@@ -337,15 +337,15 @@ ENTRY(save_args)
337 je 1f 337 je 1f
338 SWAPGS 338 SWAPGS
339 /* 339 /*
340 * irqcount is used to check if a CPU is already on an interrupt stack 340 * irq_count is used to check if a CPU is already on an interrupt stack
341 * or not. While this is essentially redundant with preempt_count it is 341 * or not. While this is essentially redundant with preempt_count it is
342 * a little cheaper to use a separate counter in the PDA (short of 342 * a little cheaper to use a separate counter in the PDA (short of
343 * moving irq_enter into assembly, which would be too much work) 343 * moving irq_enter into assembly, which would be too much work)
344 */ 344 */
3451: incl %gs:pda_irqcount 3451: incl PER_CPU_VAR(irq_count)
346 jne 2f 346 jne 2f
347 popq_cfi %rax /* move return address... */ 347 popq_cfi %rax /* move return address... */
348 mov %gs:pda_irqstackptr,%rsp 348 mov PER_CPU_VAR(irq_stack_ptr),%rsp
349 EMPTY_FRAME 0 349 EMPTY_FRAME 0
350 pushq_cfi %rax /* ... to the new stack */ 350 pushq_cfi %rax /* ... to the new stack */
351 /* 351 /*
@@ -468,7 +468,7 @@ END(ret_from_fork)
468ENTRY(system_call) 468ENTRY(system_call)
469 CFI_STARTPROC simple 469 CFI_STARTPROC simple
470 CFI_SIGNAL_FRAME 470 CFI_SIGNAL_FRAME
471 CFI_DEF_CFA rsp,PDA_STACKOFFSET 471 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
472 CFI_REGISTER rip,rcx 472 CFI_REGISTER rip,rcx
473 /*CFI_REGISTER rflags,r11*/ 473 /*CFI_REGISTER rflags,r11*/
474 SWAPGS_UNSAFE_STACK 474 SWAPGS_UNSAFE_STACK
@@ -479,8 +479,8 @@ ENTRY(system_call)
479 */ 479 */
480ENTRY(system_call_after_swapgs) 480ENTRY(system_call_after_swapgs)
481 481
482 movq %rsp,%gs:pda_oldrsp 482 movq %rsp,PER_CPU_VAR(old_rsp)
483 movq %gs:pda_kernelstack,%rsp 483 movq PER_CPU_VAR(kernel_stack),%rsp
484 /* 484 /*
485 * No need to follow this irqs off/on section - it's straight 485 * No need to follow this irqs off/on section - it's straight
486 * and short: 486 * and short:
@@ -523,7 +523,7 @@ sysret_check:
523 CFI_REGISTER rip,rcx 523 CFI_REGISTER rip,rcx
524 RESTORE_ARGS 0,-ARG_SKIP,1 524 RESTORE_ARGS 0,-ARG_SKIP,1
525 /*CFI_REGISTER rflags,r11*/ 525 /*CFI_REGISTER rflags,r11*/
526 movq %gs:pda_oldrsp, %rsp 526 movq PER_CPU_VAR(old_rsp), %rsp
527 USERGS_SYSRET64 527 USERGS_SYSRET64
528 528
529 CFI_RESTORE_STATE 529 CFI_RESTORE_STATE
@@ -833,11 +833,11 @@ common_interrupt:
833 XCPT_FRAME 833 XCPT_FRAME
834 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ 834 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
835 interrupt do_IRQ 835 interrupt do_IRQ
836 /* 0(%rsp): oldrsp-ARGOFFSET */ 836 /* 0(%rsp): old_rsp-ARGOFFSET */
837ret_from_intr: 837ret_from_intr:
838 DISABLE_INTERRUPTS(CLBR_NONE) 838 DISABLE_INTERRUPTS(CLBR_NONE)
839 TRACE_IRQS_OFF 839 TRACE_IRQS_OFF
840 decl %gs:pda_irqcount 840 decl PER_CPU_VAR(irq_count)
841 leaveq 841 leaveq
842 CFI_DEF_CFA_REGISTER rsp 842 CFI_DEF_CFA_REGISTER rsp
843 CFI_ADJUST_CFA_OFFSET -8 843 CFI_ADJUST_CFA_OFFSET -8
@@ -1260,14 +1260,14 @@ ENTRY(call_softirq)
1260 CFI_REL_OFFSET rbp,0 1260 CFI_REL_OFFSET rbp,0
1261 mov %rsp,%rbp 1261 mov %rsp,%rbp
1262 CFI_DEF_CFA_REGISTER rbp 1262 CFI_DEF_CFA_REGISTER rbp
1263 incl %gs:pda_irqcount 1263 incl PER_CPU_VAR(irq_count)
1264 cmove %gs:pda_irqstackptr,%rsp 1264 cmove PER_CPU_VAR(irq_stack_ptr),%rsp
1265 push %rbp # backlink for old unwinder 1265 push %rbp # backlink for old unwinder
1266 call __do_softirq 1266 call __do_softirq
1267 leaveq 1267 leaveq
1268 CFI_DEF_CFA_REGISTER rsp 1268 CFI_DEF_CFA_REGISTER rsp
1269 CFI_ADJUST_CFA_OFFSET -8 1269 CFI_ADJUST_CFA_OFFSET -8
1270 decl %gs:pda_irqcount 1270 decl PER_CPU_VAR(irq_count)
1271 ret 1271 ret
1272 CFI_ENDPROC 1272 CFI_ENDPROC
1273END(call_softirq) 1273END(call_softirq)
@@ -1297,15 +1297,15 @@ ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs)
1297 movq %rdi, %rsp # we don't return, adjust the stack frame 1297 movq %rdi, %rsp # we don't return, adjust the stack frame
1298 CFI_ENDPROC 1298 CFI_ENDPROC
1299 DEFAULT_FRAME 1299 DEFAULT_FRAME
130011: incl %gs:pda_irqcount 130011: incl PER_CPU_VAR(irq_count)
1301 movq %rsp,%rbp 1301 movq %rsp,%rbp
1302 CFI_DEF_CFA_REGISTER rbp 1302 CFI_DEF_CFA_REGISTER rbp
1303 cmovzq %gs:pda_irqstackptr,%rsp 1303 cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp
1304 pushq %rbp # backlink for old unwinder 1304 pushq %rbp # backlink for old unwinder
1305 call xen_evtchn_do_upcall 1305 call xen_evtchn_do_upcall
1306 popq %rsp 1306 popq %rsp
1307 CFI_DEF_CFA_REGISTER rsp 1307 CFI_DEF_CFA_REGISTER rsp
1308 decl %gs:pda_irqcount 1308 decl PER_CPU_VAR(irq_count)
1309 jmp error_exit 1309 jmp error_exit
1310 CFI_ENDPROC 1310 CFI_ENDPROC
1311END(do_hypervisor_callback) 1311END(do_hypervisor_callback)
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 3973e2df7f87..8b30d0c2512c 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -36,11 +36,7 @@ void ack_bad_irq(unsigned int irq)
36#endif 36#endif
37} 37}
38 38
39#ifdef CONFIG_X86_32 39#define irq_stats(x) (&per_cpu(irq_stat, x))
40# define irq_stats(x) (&per_cpu(irq_stat, x))
41#else
42# define irq_stats(x) cpu_pda(x)
43#endif
44/* 40/*
45 * /proc/interrupts printing: 41 * /proc/interrupts printing:
46 */ 42 */
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 0b21cb1ea11f..1db05247b47f 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -19,6 +19,9 @@
19#include <asm/io_apic.h> 19#include <asm/io_apic.h>
20#include <asm/idle.h> 20#include <asm/idle.h>
21 21
22DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
23EXPORT_PER_CPU_SYMBOL(irq_stat);
24
22/* 25/*
23 * Probabilistic stack overflow check: 26 * Probabilistic stack overflow check:
24 * 27 *
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 7228979f1e7f..23b6d9e6e4f5 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -61,11 +61,7 @@ static int endflag __initdata;
61 61
62static inline unsigned int get_nmi_count(int cpu) 62static inline unsigned int get_nmi_count(int cpu)
63{ 63{
64#ifdef CONFIG_X86_64 64 return per_cpu(irq_stat, cpu).__nmi_count;
65 return cpu_pda(cpu)->__nmi_count;
66#else
67 return nmi_count(cpu);
68#endif
69} 65}
70 66
71static inline int mce_in_progress(void) 67static inline int mce_in_progress(void)
@@ -82,12 +78,8 @@ static inline int mce_in_progress(void)
82 */ 78 */
83static inline unsigned int get_timer_irqs(int cpu) 79static inline unsigned int get_timer_irqs(int cpu)
84{ 80{
85#ifdef CONFIG_X86_64
86 return read_pda(apic_timer_irqs) + read_pda(irq0_irqs);
87#else
88 return per_cpu(irq_stat, cpu).apic_timer_irqs + 81 return per_cpu(irq_stat, cpu).apic_timer_irqs +
89 per_cpu(irq_stat, cpu).irq0_irqs; 82 per_cpu(irq_stat, cpu).irq0_irqs;
90#endif
91} 83}
92 84
93#ifdef CONFIG_SMP 85#ifdef CONFIG_SMP
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 77d546817d94..2c00a57ccb90 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -66,9 +66,6 @@ asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
66DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; 66DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
67EXPORT_PER_CPU_SYMBOL(current_task); 67EXPORT_PER_CPU_SYMBOL(current_task);
68 68
69DEFINE_PER_CPU(int, cpu_number);
70EXPORT_PER_CPU_SYMBOL(cpu_number);
71
72/* 69/*
73 * Return saved PC of a blocked thread. 70 * Return saved PC of a blocked thread.
74 */ 71 */
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 416fb9282f4f..4523ff88a69d 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -57,6 +57,12 @@
57 57
58asmlinkage extern void ret_from_fork(void); 58asmlinkage extern void ret_from_fork(void);
59 59
60DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
61EXPORT_PER_CPU_SYMBOL(current_task);
62
63DEFINE_PER_CPU(unsigned long, old_rsp);
64static DEFINE_PER_CPU(unsigned char, is_idle);
65
60unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; 66unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
61 67
62static ATOMIC_NOTIFIER_HEAD(idle_notifier); 68static ATOMIC_NOTIFIER_HEAD(idle_notifier);
@@ -75,13 +81,13 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
75 81
76void enter_idle(void) 82void enter_idle(void)
77{ 83{
78 write_pda(isidle, 1); 84 percpu_write(is_idle, 1);
79 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); 85 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
80} 86}
81 87
82static void __exit_idle(void) 88static void __exit_idle(void)
83{ 89{
84 if (test_and_clear_bit_pda(0, isidle) == 0) 90 if (x86_test_and_clear_bit_percpu(0, is_idle) == 0)
85 return; 91 return;
86 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); 92 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
87} 93}
@@ -392,7 +398,7 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
392 load_gs_index(0); 398 load_gs_index(0);
393 regs->ip = new_ip; 399 regs->ip = new_ip;
394 regs->sp = new_sp; 400 regs->sp = new_sp;
395 write_pda(oldrsp, new_sp); 401 percpu_write(old_rsp, new_sp);
396 regs->cs = __USER_CS; 402 regs->cs = __USER_CS;
397 regs->ss = __USER_DS; 403 regs->ss = __USER_DS;
398 regs->flags = 0x200; 404 regs->flags = 0x200;
@@ -613,13 +619,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
613 /* 619 /*
614 * Switch the PDA and FPU contexts. 620 * Switch the PDA and FPU contexts.
615 */ 621 */
616 prev->usersp = read_pda(oldrsp); 622 prev->usersp = percpu_read(old_rsp);
617 write_pda(oldrsp, next->usersp); 623 percpu_write(old_rsp, next->usersp);
618 write_pda(pcurrent, next_p); 624 percpu_write(current_task, next_p);
619 625
620 write_pda(kernelstack, 626 percpu_write(kernel_stack,
621 (unsigned long)task_stack_page(next_p) + 627 (unsigned long)task_stack_page(next_p) +
622 THREAD_SIZE - PDA_STACKOFFSET); 628 THREAD_SIZE - KERNEL_STACK_OFFSET);
623#ifdef CONFIG_CC_STACKPROTECTOR 629#ifdef CONFIG_CC_STACKPROTECTOR
624 write_pda(stack_canary, next_p->stack_canary); 630 write_pda(stack_canary, next_p->stack_canary);
625 /* 631 /*
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index b5c35af2011d..efbafbbff584 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -22,6 +22,15 @@
22# define DBG(x...) 22# define DBG(x...)
23#endif 23#endif
24 24
25/*
26 * Could be inside CONFIG_HAVE_SETUP_PER_CPU_AREA with other stuff but
27 * voyager wants cpu_number too.
28 */
29#ifdef CONFIG_SMP
30DEFINE_PER_CPU(int, cpu_number);
31EXPORT_PER_CPU_SYMBOL(cpu_number);
32#endif
33
25#ifdef CONFIG_X86_LOCAL_APIC 34#ifdef CONFIG_X86_LOCAL_APIC
26unsigned int num_processors; 35unsigned int num_processors;
27unsigned disabled_cpus __cpuinitdata; 36unsigned disabled_cpus __cpuinitdata;
@@ -44,6 +53,8 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
44 53
45#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64) 54#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64)
46#define X86_64_NUMA 1 /* (used later) */ 55#define X86_64_NUMA 1 /* (used later) */
56DEFINE_PER_CPU(int, node_number) = 0;
57EXPORT_PER_CPU_SYMBOL(node_number);
47 58
48/* 59/*
49 * Map cpu index to node index 60 * Map cpu index to node index
@@ -192,7 +203,11 @@ void __init setup_per_cpu_areas(void)
192 203
193 memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start); 204 memcpy(ptr, __per_cpu_load, __per_cpu_end - __per_cpu_start);
194 per_cpu_offset(cpu) = ptr - __per_cpu_start; 205 per_cpu_offset(cpu) = ptr - __per_cpu_start;
206 per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
207 per_cpu(cpu_number, cpu) = cpu;
195#ifdef CONFIG_X86_64 208#ifdef CONFIG_X86_64
209 per_cpu(irq_stack_ptr, cpu) =
210 (char *)per_cpu(irq_stack, cpu) + IRQ_STACK_SIZE - 64;
196 /* 211 /*
197 * CPU0 modified pda in the init data area, reload pda 212 * CPU0 modified pda in the init data area, reload pda
198 * offset for CPU0 and clear the area for others. 213 * offset for CPU0 and clear the area for others.
@@ -202,7 +217,6 @@ void __init setup_per_cpu_areas(void)
202 else 217 else
203 memset(cpu_pda(cpu), 0, sizeof(*cpu_pda(cpu))); 218 memset(cpu_pda(cpu), 0, sizeof(*cpu_pda(cpu)));
204#endif 219#endif
205 per_cpu(this_cpu_off, cpu) = per_cpu_offset(cpu);
206 220
207 DBG("PERCPU: cpu %4d %p\n", cpu, ptr); 221 DBG("PERCPU: cpu %4d %p\n", cpu, ptr);
208 } 222 }
@@ -271,7 +285,7 @@ void __cpuinit numa_set_node(int cpu, int node)
271 per_cpu(x86_cpu_to_node_map, cpu) = node; 285 per_cpu(x86_cpu_to_node_map, cpu) = node;
272 286
273 if (node != NUMA_NO_NODE) 287 if (node != NUMA_NO_NODE)
274 cpu_pda(cpu)->nodenumber = node; 288 per_cpu(node_number, cpu) = node;
275} 289}
276 290
277void __cpuinit numa_clear_node(int cpu) 291void __cpuinit numa_clear_node(int cpu)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 2f0e0f1090f6..869b98840fd0 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -790,15 +790,17 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu)
790 790
791 set_idle_for_cpu(cpu, c_idle.idle); 791 set_idle_for_cpu(cpu, c_idle.idle);
792do_rest: 792do_rest:
793#ifdef CONFIG_X86_32
794 per_cpu(current_task, cpu) = c_idle.idle; 793 per_cpu(current_task, cpu) = c_idle.idle;
794#ifdef CONFIG_X86_32
795 init_gdt(cpu); 795 init_gdt(cpu);
796 /* Stack for startup_32 can be just as for start_secondary onwards */ 796 /* Stack for startup_32 can be just as for start_secondary onwards */
797 irq_ctx_init(cpu); 797 irq_ctx_init(cpu);
798#else 798#else
799 cpu_pda(cpu)->pcurrent = c_idle.idle;
800 clear_tsk_thread_flag(c_idle.idle, TIF_FORK); 799 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
801 initial_gs = per_cpu_offset(cpu); 800 initial_gs = per_cpu_offset(cpu);
801 per_cpu(kernel_stack, cpu) =
802 (unsigned long)task_stack_page(c_idle.idle) -
803 KERNEL_STACK_OFFSET + THREAD_SIZE;
802#endif 804#endif
803 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); 805 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
804 initial_code = (unsigned long)start_secondary; 806 initial_code = (unsigned long)start_secondary;
diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c
index 7e157810062f..add36b4e37c9 100644
--- a/arch/x86/kernel/smpcommon.c
+++ b/arch/x86/kernel/smpcommon.c
@@ -28,7 +28,5 @@ __cpuinit void init_gdt(int cpu)
28 28
29 write_gdt_entry(get_cpu_gdt_table(cpu), 29 write_gdt_entry(get_cpu_gdt_table(cpu),
30 GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S); 30 GDT_ENTRY_PERCPU, &gdt, DESCTYPE_S);
31
32 per_cpu(cpu_number, cpu) = cpu;
33} 31}
34#endif 32#endif
diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
index e65449d0f7d9..abf0808d6fc4 100644
--- a/arch/x86/kernel/tlb_32.c
+++ b/arch/x86/kernel/tlb_32.c
@@ -4,8 +4,8 @@
4 4
5#include <asm/tlbflush.h> 5#include <asm/tlbflush.h>
6 6
7DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) 7DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
8 ____cacheline_aligned = { &init_mm, 0, }; 8 = { &init_mm, 0, };
9 9
10/* must come after the send_IPI functions above for inlining */ 10/* must come after the send_IPI functions above for inlining */
11#include <mach_ipi.h> 11#include <mach_ipi.h>
@@ -231,14 +231,6 @@ void flush_tlb_all(void)
231 on_each_cpu(do_flush_tlb_all, NULL, 1); 231 on_each_cpu(do_flush_tlb_all, NULL, 1);
232} 232}
233 233
234void reset_lazy_tlbstate(void)
235{
236 int cpu = raw_smp_processor_id();
237
238 per_cpu(cpu_tlbstate, cpu).state = 0;
239 per_cpu(cpu_tlbstate, cpu).active_mm = &init_mm;
240}
241
242static int init_flush_cpumask(void) 234static int init_flush_cpumask(void)
243{ 235{
244 alloc_cpumask_var(&flush_cpumask, GFP_KERNEL); 236 alloc_cpumask_var(&flush_cpumask, GFP_KERNEL);
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
index 7f4141d3b661..e64a32c48825 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb_64.c
@@ -18,6 +18,9 @@
18#include <asm/uv/uv_hub.h> 18#include <asm/uv/uv_hub.h>
19#include <asm/uv/uv_bau.h> 19#include <asm/uv/uv_bau.h>
20 20
21DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
22 = { &init_mm, 0, };
23
21#include <mach_ipi.h> 24#include <mach_ipi.h>
22/* 25/*
23 * Smarter SMP flushing macros. 26 * Smarter SMP flushing macros.
@@ -62,9 +65,9 @@ static DEFINE_PER_CPU(union smp_flush_state, flush_state);
62 */ 65 */
63void leave_mm(int cpu) 66void leave_mm(int cpu)
64{ 67{
65 if (read_pda(mmu_state) == TLBSTATE_OK) 68 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
66 BUG(); 69 BUG();
67 cpu_clear(cpu, read_pda(active_mm)->cpu_vm_mask); 70 cpu_clear(cpu, percpu_read(cpu_tlbstate.active_mm)->cpu_vm_mask);
68 load_cr3(swapper_pg_dir); 71 load_cr3(swapper_pg_dir);
69} 72}
70EXPORT_SYMBOL_GPL(leave_mm); 73EXPORT_SYMBOL_GPL(leave_mm);
@@ -142,8 +145,8 @@ asmlinkage void smp_invalidate_interrupt(struct pt_regs *regs)
142 * BUG(); 145 * BUG();
143 */ 146 */
144 147
145 if (f->flush_mm == read_pda(active_mm)) { 148 if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
146 if (read_pda(mmu_state) == TLBSTATE_OK) { 149 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
147 if (f->flush_va == TLB_FLUSH_ALL) 150 if (f->flush_va == TLB_FLUSH_ALL)
148 local_flush_tlb(); 151 local_flush_tlb();
149 else 152 else
@@ -281,7 +284,7 @@ static void do_flush_tlb_all(void *info)
281 unsigned long cpu = smp_processor_id(); 284 unsigned long cpu = smp_processor_id();
282 285
283 __flush_tlb_all(); 286 __flush_tlb_all();
284 if (read_pda(mmu_state) == TLBSTATE_LAZY) 287 if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
285 leave_mm(cpu); 288 leave_mm(cpu);
286} 289}
287 290
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 7bc7852cc5c4..98cb9869eb24 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1063,11 +1063,7 @@ static void drop_other_mm_ref(void *info)
1063 struct mm_struct *mm = info; 1063 struct mm_struct *mm = info;
1064 struct mm_struct *active_mm; 1064 struct mm_struct *active_mm;
1065 1065
1066#ifdef CONFIG_X86_64 1066 active_mm = percpu_read(cpu_tlbstate.active_mm);
1067 active_mm = read_pda(active_mm);
1068#else
1069 active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
1070#endif
1071 1067
1072 if (active_mm == mm) 1068 if (active_mm == mm)
1073 leave_mm(smp_processor_id()); 1069 leave_mm(smp_processor_id());
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 3bfd6dd0b47c..72c2eb9b64cd 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -50,11 +50,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
50 */ 50 */
51static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id) 51static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
52{ 52{
53#ifdef CONFIG_X86_32 53 inc_irq_stat(irq_resched_count);
54 __get_cpu_var(irq_stat).irq_resched_count++;
55#else
56 add_pda(irq_resched_count, 1);
57#endif
58 54
59 return IRQ_HANDLED; 55 return IRQ_HANDLED;
60} 56}
@@ -283,12 +279,11 @@ static int __cpuinit xen_cpu_up(unsigned int cpu)
283 struct task_struct *idle = idle_task(cpu); 279 struct task_struct *idle = idle_task(cpu);
284 int rc; 280 int rc;
285 281
282 per_cpu(current_task, cpu) = idle;
286#ifdef CONFIG_X86_32 283#ifdef CONFIG_X86_32
287 init_gdt(cpu); 284 init_gdt(cpu);
288 per_cpu(current_task, cpu) = idle;
289 irq_ctx_init(cpu); 285 irq_ctx_init(cpu);
290#else 286#else
291 cpu_pda(cpu)->pcurrent = idle;
292 clear_tsk_thread_flag(idle, TIF_FORK); 287 clear_tsk_thread_flag(idle, TIF_FORK);
293#endif 288#endif
294 xen_setup_timer(cpu); 289 xen_setup_timer(cpu);
@@ -435,11 +430,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
435{ 430{
436 irq_enter(); 431 irq_enter();
437 generic_smp_call_function_interrupt(); 432 generic_smp_call_function_interrupt();
438#ifdef CONFIG_X86_32 433 inc_irq_stat(irq_call_count);
439 __get_cpu_var(irq_stat).irq_call_count++;
440#else
441 add_pda(irq_call_count, 1);
442#endif
443 irq_exit(); 434 irq_exit();
444 435
445 return IRQ_HANDLED; 436 return IRQ_HANDLED;
@@ -449,11 +440,7 @@ static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
449{ 440{
450 irq_enter(); 441 irq_enter();
451 generic_smp_call_function_single_interrupt(); 442 generic_smp_call_function_single_interrupt();
452#ifdef CONFIG_X86_32 443 inc_irq_stat(irq_call_count);
453 __get_cpu_var(irq_stat).irq_call_count++;
454#else
455 add_pda(irq_call_count, 1);
456#endif
457 irq_exit(); 444 irq_exit();
458 445
459 return IRQ_HANDLED; 446 return IRQ_HANDLED;
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index 05794c566e87..d6fc51f4ce85 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -17,6 +17,7 @@
17#include <asm/processor-flags.h> 17#include <asm/processor-flags.h>
18#include <asm/errno.h> 18#include <asm/errno.h>
19#include <asm/segment.h> 19#include <asm/segment.h>
20#include <asm/percpu.h>
20 21
21#include <xen/interface/xen.h> 22#include <xen/interface/xen.h>
22 23
@@ -28,12 +29,10 @@
28 29
29#if 1 30#if 1
30/* 31/*
31 x86-64 does not yet support direct access to percpu variables 32 FIXME: x86_64 now can support direct access to percpu variables
32 via a segment override, so we just need to make sure this code 33 via a segment override. Update xen accordingly.
33 never gets used
34 */ 34 */
35#define BUG ud2a 35#define BUG ud2a
36#define PER_CPU_VAR(var, off) 0xdeadbeef
37#endif 36#endif
38 37
39/* 38/*
@@ -45,14 +44,14 @@ ENTRY(xen_irq_enable_direct)
45 BUG 44 BUG
46 45
47 /* Unmask events */ 46 /* Unmask events */
48 movb $0, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) 47 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
49 48
50 /* Preempt here doesn't matter because that will deal with 49 /* Preempt here doesn't matter because that will deal with
51 any pending interrupts. The pending check may end up being 50 any pending interrupts. The pending check may end up being
52 run on the wrong CPU, but that doesn't hurt. */ 51 run on the wrong CPU, but that doesn't hurt. */
53 52
54 /* Test for pending */ 53 /* Test for pending */
55 testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending) 54 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
56 jz 1f 55 jz 1f
57 56
582: call check_events 572: call check_events
@@ -69,7 +68,7 @@ ENDPATCH(xen_irq_enable_direct)
69ENTRY(xen_irq_disable_direct) 68ENTRY(xen_irq_disable_direct)
70 BUG 69 BUG
71 70
72 movb $1, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) 71 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
73ENDPATCH(xen_irq_disable_direct) 72ENDPATCH(xen_irq_disable_direct)
74 ret 73 ret
75 ENDPROC(xen_irq_disable_direct) 74 ENDPROC(xen_irq_disable_direct)
@@ -87,7 +86,7 @@ ENDPATCH(xen_irq_disable_direct)
87ENTRY(xen_save_fl_direct) 86ENTRY(xen_save_fl_direct)
88 BUG 87 BUG
89 88
90 testb $0xff, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) 89 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
91 setz %ah 90 setz %ah
92 addb %ah,%ah 91 addb %ah,%ah
93ENDPATCH(xen_save_fl_direct) 92ENDPATCH(xen_save_fl_direct)
@@ -107,13 +106,13 @@ ENTRY(xen_restore_fl_direct)
107 BUG 106 BUG
108 107
109 testb $X86_EFLAGS_IF>>8, %ah 108 testb $X86_EFLAGS_IF>>8, %ah
110 setz PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_mask) 109 setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
111 /* Preempt here doesn't matter because that will deal with 110 /* Preempt here doesn't matter because that will deal with
112 any pending interrupts. The pending check may end up being 111 any pending interrupts. The pending check may end up being
113 run on the wrong CPU, but that doesn't hurt. */ 112 run on the wrong CPU, but that doesn't hurt. */
114 113
115 /* check for unmasked and pending */ 114 /* check for unmasked and pending */
116 cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info, XEN_vcpu_info_pending) 115 cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
117 jz 1f 116 jz 1f
1182: call check_events 1172: call check_events
1191: 1181:
@@ -195,11 +194,11 @@ RELOC(xen_sysexit, 1b+1)
195ENTRY(xen_sysret64) 194ENTRY(xen_sysret64)
196 /* We're already on the usermode stack at this point, but still 195 /* We're already on the usermode stack at this point, but still
197 with the kernel gs, so we can easily switch back */ 196 with the kernel gs, so we can easily switch back */
198 movq %rsp, %gs:pda_oldrsp 197 movq %rsp, PER_CPU_VAR(old_rsp)
199 movq %gs:pda_kernelstack,%rsp 198 movq PER_CPU_VAR(kernel_stack),%rsp
200 199
201 pushq $__USER_DS 200 pushq $__USER_DS
202 pushq %gs:pda_oldrsp 201 pushq PER_CPU_VAR(old_rsp)
203 pushq %r11 202 pushq %r11
204 pushq $__USER_CS 203 pushq $__USER_CS
205 pushq %rcx 204 pushq %rcx
@@ -212,11 +211,11 @@ RELOC(xen_sysret64, 1b+1)
212ENTRY(xen_sysret32) 211ENTRY(xen_sysret32)
213 /* We're already on the usermode stack at this point, but still 212 /* We're already on the usermode stack at this point, but still
214 with the kernel gs, so we can easily switch back */ 213 with the kernel gs, so we can easily switch back */
215 movq %rsp, %gs:pda_oldrsp 214 movq %rsp, PER_CPU_VAR(old_rsp)
216 movq %gs:pda_kernelstack, %rsp 215 movq PER_CPU_VAR(kernel_stack), %rsp
217 216
218 pushq $__USER32_DS 217 pushq $__USER32_DS
219 pushq %gs:pda_oldrsp 218 pushq PER_CPU_VAR(old_rsp)
220 pushq %r11 219 pushq %r11
221 pushq $__USER32_CS 220 pushq $__USER32_CS
222 pushq %rcx 221 pushq %rcx
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-20 09:53:29 -0500