aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/Kconfig17
-rw-r--r--arch/x86/ia32/ia32_signal.c5
-rw-r--r--arch/x86/include/asm/acpi.h1
-rw-r--r--arch/x86/include/asm/bitops.h10
-rw-r--r--arch/x86/include/asm/byteorder.h74
-rw-r--r--arch/x86/include/asm/iomap.h30
-rw-r--r--arch/x86/include/asm/iommu.h1
-rw-r--r--arch/x86/include/asm/irq.h4
-rw-r--r--arch/x86/include/asm/irq_regs_32.h2
-rw-r--r--arch/x86/include/asm/tsc.h8
-rw-r--r--arch/x86/kernel/acpi/boot.c1
-rw-r--r--arch/x86/kernel/ds.c25
-rw-r--r--arch/x86/kernel/early-quirks.c18
-rw-r--r--arch/x86/kernel/entry_64.S190
-rw-r--r--arch/x86/kernel/es7000_32.c9
-rw-r--r--arch/x86/kernel/kvmclock.c2
-rw-r--r--arch/x86/kernel/tsc_sync.c4
-rw-r--r--arch/x86/kernel/vsyscall_64.c9
-rw-r--r--arch/x86/kvm/Kconfig2
-rw-r--r--arch/x86/kvm/i8254.c4
-rw-r--r--arch/x86/kvm/mmu.c2
-rw-r--r--arch/x86/kvm/vmx.c3
-rw-r--r--arch/x86/kvm/vmx.h1
-rw-r--r--arch/x86/mach-voyager/voyager_smp.c16
-rw-r--r--arch/x86/mm/init_32.c2
25 files changed, 233 insertions, 207 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 93224b569187..3cf457f90e8e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -167,9 +167,12 @@ config GENERIC_PENDING_IRQ
167config X86_SMP 167config X86_SMP
168 bool 168 bool
169 depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64) 169 depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64)
170 select USE_GENERIC_SMP_HELPERS
171 default y 170 default y
172 171
172config USE_GENERIC_SMP_HELPERS
173 def_bool y
174 depends on SMP
175
173config X86_32_SMP 176config X86_32_SMP
174 def_bool y 177 def_bool y
175 depends on X86_32 && SMP 178 depends on X86_32 && SMP
@@ -239,21 +242,13 @@ config X86_FIND_SMP_CONFIG
239 def_bool y 242 def_bool y
240 depends on X86_MPPARSE || X86_VOYAGER 243 depends on X86_MPPARSE || X86_VOYAGER
241 244
242if ACPI
243config X86_MPPARSE 245config X86_MPPARSE
244 def_bool y 246 bool "Enable MPS table" if ACPI
245 bool "Enable MPS table" 247 default y
246 depends on X86_LOCAL_APIC 248 depends on X86_LOCAL_APIC
247 help 249 help
248 For old smp systems that do not have proper acpi support. Newer systems 250 For old smp systems that do not have proper acpi support. Newer systems
249 (esp with 64bit cpus) with acpi support, MADT and DSDT will override it 251 (esp with 64bit cpus) with acpi support, MADT and DSDT will override it
250endif
251
252if !ACPI
253config X86_MPPARSE
254 def_bool y
255 depends on X86_LOCAL_APIC
256endif
257 252
258choice 253choice
259 prompt "Subarchitecture Type" 254 prompt "Subarchitecture Type"
diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c
index 4bc02b23674b..e82ebd652263 100644
--- a/arch/x86/ia32/ia32_signal.c
+++ b/arch/x86/ia32/ia32_signal.c
@@ -572,11 +572,6 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
572 regs->dx = (unsigned long) &frame->info; 572 regs->dx = (unsigned long) &frame->info;
573 regs->cx = (unsigned long) &frame->uc; 573 regs->cx = (unsigned long) &frame->uc;
574 574
575 /* Make -mregparm=3 work */
576 regs->ax = sig;
577 regs->dx = (unsigned long) &frame->info;
578 regs->cx = (unsigned long) &frame->uc;
579
580 loadsegment(ds, __USER32_DS); 575 loadsegment(ds, __USER32_DS);
581 loadsegment(es, __USER32_DS); 576 loadsegment(es, __USER32_DS);
582 577
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 8d676d8ecde9..9830681446ad 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -113,7 +113,6 @@ static inline void acpi_disable_pci(void)
113 acpi_pci_disabled = 1; 113 acpi_pci_disabled = 1;
114 acpi_noirq_set(); 114 acpi_noirq_set();
115} 115}
116extern int acpi_irq_balance_set(char *str);
117 116
118/* routines for saving/restoring kernel state */ 117/* routines for saving/restoring kernel state */
119extern int acpi_save_state_mem(void); 118extern int acpi_save_state_mem(void);
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 360010322711..9fa9dcdf344b 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -168,7 +168,15 @@ static inline void __change_bit(int nr, volatile unsigned long *addr)
168 */ 168 */
169static inline void change_bit(int nr, volatile unsigned long *addr) 169static inline void change_bit(int nr, volatile unsigned long *addr)
170{ 170{
171 asm volatile(LOCK_PREFIX "btc %1,%0" : ADDR : "Ir" (nr)); 171 if (IS_IMMEDIATE(nr)) {
172 asm volatile(LOCK_PREFIX "xorb %1,%0"
173 : CONST_MASK_ADDR(nr, addr)
174 : "iq" ((u8)CONST_MASK(nr)));
175 } else {
176 asm volatile(LOCK_PREFIX "btc %1,%0"
177 : BITOP_ADDR(addr)
178 : "Ir" (nr));
179 }
172} 180}
173 181
174/** 182/**
diff --git a/arch/x86/include/asm/byteorder.h b/arch/x86/include/asm/byteorder.h
index e02ae2d89acf..f110ad417df3 100644
--- a/arch/x86/include/asm/byteorder.h
+++ b/arch/x86/include/asm/byteorder.h
@@ -4,26 +4,33 @@
4#include <asm/types.h> 4#include <asm/types.h>
5#include <linux/compiler.h> 5#include <linux/compiler.h>
6 6
7#ifdef __GNUC__ 7#define __LITTLE_ENDIAN
8 8
9#ifdef __i386__ 9static inline __attribute_const__ __u32 __arch_swab32(__u32 val)
10
11static inline __attribute_const__ __u32 ___arch__swab32(__u32 x)
12{ 10{
13#ifdef CONFIG_X86_BSWAP 11#ifdef __i386__
14 asm("bswap %0" : "=r" (x) : "0" (x)); 12# ifdef CONFIG_X86_BSWAP
15#else 13 asm("bswap %0" : "=r" (val) : "0" (val));
14# else
16 asm("xchgb %b0,%h0\n\t" /* swap lower bytes */ 15 asm("xchgb %b0,%h0\n\t" /* swap lower bytes */
17 "rorl $16,%0\n\t" /* swap words */ 16 "rorl $16,%0\n\t" /* swap words */
18 "xchgb %b0,%h0" /* swap higher bytes */ 17 "xchgb %b0,%h0" /* swap higher bytes */
19 : "=q" (x) 18 : "=q" (val)
20 : "0" (x)); 19 : "0" (val));
20# endif
21
22#else /* __i386__ */
23 asm("bswapl %0"
24 : "=r" (val)
25 : "0" (val));
21#endif 26#endif
22 return x; 27 return val;
23} 28}
29#define __arch_swab32 __arch_swab32
24 30
25static inline __attribute_const__ __u64 ___arch__swab64(__u64 val) 31static inline __attribute_const__ __u64 __arch_swab64(__u64 val)
26{ 32{
33#ifdef __i386__
27 union { 34 union {
28 struct { 35 struct {
29 __u32 a; 36 __u32 a;
@@ -32,50 +39,27 @@ static inline __attribute_const__ __u64 ___arch__swab64(__u64 val)
32 __u64 u; 39 __u64 u;
33 } v; 40 } v;
34 v.u = val; 41 v.u = val;
35#ifdef CONFIG_X86_BSWAP 42# ifdef CONFIG_X86_BSWAP
36 asm("bswapl %0 ; bswapl %1 ; xchgl %0,%1" 43 asm("bswapl %0 ; bswapl %1 ; xchgl %0,%1"
37 : "=r" (v.s.a), "=r" (v.s.b) 44 : "=r" (v.s.a), "=r" (v.s.b)
38 : "0" (v.s.a), "1" (v.s.b)); 45 : "0" (v.s.a), "1" (v.s.b));
39#else 46# else
40 v.s.a = ___arch__swab32(v.s.a); 47 v.s.a = __arch_swab32(v.s.a);
41 v.s.b = ___arch__swab32(v.s.b); 48 v.s.b = __arch_swab32(v.s.b);
42 asm("xchgl %0,%1" 49 asm("xchgl %0,%1"
43 : "=r" (v.s.a), "=r" (v.s.b) 50 : "=r" (v.s.a), "=r" (v.s.b)
44 : "0" (v.s.a), "1" (v.s.b)); 51 : "0" (v.s.a), "1" (v.s.b));
45#endif 52# endif
46 return v.u; 53 return v.u;
47}
48
49#else /* __i386__ */ 54#else /* __i386__ */
50
51static inline __attribute_const__ __u64 ___arch__swab64(__u64 x)
52{
53 asm("bswapq %0" 55 asm("bswapq %0"
54 : "=r" (x) 56 : "=r" (val)
55 : "0" (x)); 57 : "0" (val));
56 return x; 58 return val;
57}
58
59static inline __attribute_const__ __u32 ___arch__swab32(__u32 x)
60{
61 asm("bswapl %0"
62 : "=r" (x)
63 : "0" (x));
64 return x;
65}
66
67#endif 59#endif
60}
61#define __arch_swab64 __arch_swab64
68 62
69/* Do not define swab16. Gcc is smart enough to recognize "C" version and 63#include <linux/byteorder.h>
70 convert it into rotation or exhange. */
71
72#define __arch__swab64(x) ___arch__swab64(x)
73#define __arch__swab32(x) ___arch__swab32(x)
74
75#define __BYTEORDER_HAS_U64__
76
77#endif /* __GNUC__ */
78
79#include <linux/byteorder/little_endian.h>
80 64
81#endif /* _ASM_X86_BYTEORDER_H */ 65#endif /* _ASM_X86_BYTEORDER_H */
diff --git a/arch/x86/include/asm/iomap.h b/arch/x86/include/asm/iomap.h
new file mode 100644
index 000000000000..c1f06289b14b
--- /dev/null
+++ b/arch/x86/include/asm/iomap.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright © 2008 Ingo Molnar
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, write to the Free Software Foundation, Inc.,
16 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 */
18
19#include <linux/fs.h>
20#include <linux/mm.h>
21#include <linux/uaccess.h>
22#include <asm/cacheflush.h>
23#include <asm/pgtable.h>
24#include <asm/tlbflush.h>
25
26void *
27iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
28
29void
30iounmap_atomic(void *kvaddr, enum km_type type);
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index e4a552d44465..0b500c5b6446 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -6,7 +6,6 @@ extern void no_iommu_init(void);
6extern struct dma_mapping_ops nommu_dma_ops; 6extern struct dma_mapping_ops nommu_dma_ops;
7extern int force_iommu, no_iommu; 7extern int force_iommu, no_iommu;
8extern int iommu_detected; 8extern int iommu_detected;
9extern int dmar_disabled;
10 9
11extern unsigned long iommu_nr_pages(unsigned long addr, unsigned long len); 10extern unsigned long iommu_nr_pages(unsigned long addr, unsigned long len);
12 11
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index bae0eda95486..28e409fc73f3 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -31,10 +31,6 @@ static inline int irq_canonicalize(int irq)
31# endif 31# endif
32#endif 32#endif
33 33
34#ifdef CONFIG_IRQBALANCE
35extern int irqbalance_disable(char *str);
36#endif
37
38#ifdef CONFIG_HOTPLUG_CPU 34#ifdef CONFIG_HOTPLUG_CPU
39#include <linux/cpumask.h> 35#include <linux/cpumask.h>
40extern void fixup_irqs(cpumask_t map); 36extern void fixup_irqs(cpumask_t map);
diff --git a/arch/x86/include/asm/irq_regs_32.h b/arch/x86/include/asm/irq_regs_32.h
index af2f02d27fc7..86afd7473457 100644
--- a/arch/x86/include/asm/irq_regs_32.h
+++ b/arch/x86/include/asm/irq_regs_32.h
@@ -9,6 +9,8 @@
9 9
10#include <asm/percpu.h> 10#include <asm/percpu.h>
11 11
12#define ARCH_HAS_OWN_IRQ_REGS
13
12DECLARE_PER_CPU(struct pt_regs *, irq_regs); 14DECLARE_PER_CPU(struct pt_regs *, irq_regs);
13 15
14static inline struct pt_regs *get_irq_regs(void) 16static inline struct pt_regs *get_irq_regs(void)
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 9cd83a8e40d5..38ae163cc91b 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -34,8 +34,6 @@ static inline cycles_t get_cycles(void)
34 34
35static __always_inline cycles_t vget_cycles(void) 35static __always_inline cycles_t vget_cycles(void)
36{ 36{
37 cycles_t cycles;
38
39 /* 37 /*
40 * We only do VDSOs on TSC capable CPUs, so this shouldnt 38 * We only do VDSOs on TSC capable CPUs, so this shouldnt
41 * access boot_cpu_data (which is not VDSO-safe): 39 * access boot_cpu_data (which is not VDSO-safe):
@@ -44,11 +42,7 @@ static __always_inline cycles_t vget_cycles(void)
44 if (!cpu_has_tsc) 42 if (!cpu_has_tsc)
45 return 0; 43 return 0;
46#endif 44#endif
47 rdtsc_barrier(); 45 return (cycles_t)__native_read_tsc();
48 cycles = (cycles_t)__native_read_tsc();
49 rdtsc_barrier();
50
51 return cycles;
52} 46}
53 47
54extern void tsc_init(void); 48extern void tsc_init(void);
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 8c1f76abae9e..4c51a2f8fd31 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -1343,7 +1343,6 @@ static void __init acpi_process_madt(void)
1343 error = acpi_parse_madt_ioapic_entries(); 1343 error = acpi_parse_madt_ioapic_entries();
1344 if (!error) { 1344 if (!error) {
1345 acpi_irq_model = ACPI_IRQ_MODEL_IOAPIC; 1345 acpi_irq_model = ACPI_IRQ_MODEL_IOAPIC;
1346 acpi_irq_balance_set(NULL);
1347 acpi_ioapic = 1; 1346 acpi_ioapic = 1;
1348 1347
1349 smp_found_config = 1; 1348 smp_found_config = 1;
diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index 2b69994fd3a8..d1a121443bde 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -236,17 +236,33 @@ static inline struct ds_context *ds_alloc_context(struct task_struct *task)
236 struct ds_context *context = *p_context; 236 struct ds_context *context = *p_context;
237 237
238 if (!context) { 238 if (!context) {
239 spin_unlock(&ds_lock);
240
239 context = kzalloc(sizeof(*context), GFP_KERNEL); 241 context = kzalloc(sizeof(*context), GFP_KERNEL);
240 242
241 if (!context) 243 if (!context) {
244 spin_lock(&ds_lock);
242 return NULL; 245 return NULL;
246 }
243 247
244 context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL); 248 context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
245 if (!context->ds) { 249 if (!context->ds) {
246 kfree(context); 250 kfree(context);
251 spin_lock(&ds_lock);
247 return NULL; 252 return NULL;
248 } 253 }
249 254
255 spin_lock(&ds_lock);
256 /*
257 * Check for race - another CPU could have allocated
258 * it meanwhile:
259 */
260 if (*p_context) {
261 kfree(context->ds);
262 kfree(context);
263 return *p_context;
264 }
265
250 *p_context = context; 266 *p_context = context;
251 267
252 context->this = p_context; 268 context->this = p_context;
@@ -384,14 +400,15 @@ static int ds_request(struct task_struct *task, void *base, size_t size,
384 400
385 spin_lock(&ds_lock); 401 spin_lock(&ds_lock);
386 402
387 if (!check_tracer(task))
388 return -EPERM;
389
390 error = -ENOMEM; 403 error = -ENOMEM;
391 context = ds_alloc_context(task); 404 context = ds_alloc_context(task);
392 if (!context) 405 if (!context)
393 goto out_unlock; 406 goto out_unlock;
394 407
408 error = -EPERM;
409 if (!check_tracer(task))
410 goto out_unlock;
411
395 error = -EALREADY; 412 error = -EALREADY;
396 if (context->owner[qual] == current) 413 if (context->owner[qual] == current)
397 goto out_unlock; 414 goto out_unlock;
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 3ce029ffaa55..1b894b72c0f5 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -188,20 +188,6 @@ static void __init ati_bugs_contd(int num, int slot, int func)
188} 188}
189#endif 189#endif
190 190
191#ifdef CONFIG_DMAR
192static void __init intel_g33_dmar(int num, int slot, int func)
193{
194 struct acpi_table_header *dmar_tbl;
195 acpi_status status;
196
197 status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
198 if (ACPI_SUCCESS(status)) {
199 printk(KERN_INFO "BIOS BUG: DMAR advertised on Intel G31/G33 chipset -- ignoring\n");
200 dmar_disabled = 1;
201 }
202}
203#endif
204
205#define QFLAG_APPLY_ONCE 0x1 191#define QFLAG_APPLY_ONCE 0x1
206#define QFLAG_APPLIED 0x2 192#define QFLAG_APPLIED 0x2
207#define QFLAG_DONE (QFLAG_APPLY_ONCE|QFLAG_APPLIED) 193#define QFLAG_DONE (QFLAG_APPLY_ONCE|QFLAG_APPLIED)
@@ -225,10 +211,6 @@ static struct chipset early_qrk[] __initdata = {
225 PCI_CLASS_SERIAL_SMBUS, PCI_ANY_ID, 0, ati_bugs }, 211 PCI_CLASS_SERIAL_SMBUS, PCI_ANY_ID, 0, ati_bugs },
226 { PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_SBX00_SMBUS, 212 { PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_SBX00_SMBUS,
227 PCI_CLASS_SERIAL_SMBUS, PCI_ANY_ID, 0, ati_bugs_contd }, 213 PCI_CLASS_SERIAL_SMBUS, PCI_ANY_ID, 0, ati_bugs_contd },
228#ifdef CONFIG_DMAR
229 { PCI_VENDOR_ID_INTEL, 0x29c0,
230 PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, intel_g33_dmar },
231#endif
232 {} 214 {}
233}; 215};
234 216
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 369de6973c58..dbf06a0ef3d5 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -11,15 +11,15 @@
11 * 11 *
12 * NOTE: This code handles signal-recognition, which happens every time 12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call. 13 * after an interrupt and after each system call.
14 * 14 *
15 * Normal syscalls and interrupts don't save a full stack frame, this is 15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al. 16 * only done for syscall tracing, signals or fork/exec et.al.
17 * 17 *
18 * A note on terminology: 18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP 19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack. 20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11. 21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved. 22 * - full stack frame: Like partial stack frame, but all register saved.
23 * 23 *
24 * Some macro usage: 24 * Some macro usage:
25 * - CFI macros are used to generate dwarf2 unwind information for better 25 * - CFI macros are used to generate dwarf2 unwind information for better
@@ -142,7 +142,7 @@ END(mcount)
142 142
143#ifndef CONFIG_PREEMPT 143#ifndef CONFIG_PREEMPT
144#define retint_kernel retint_restore_args 144#define retint_kernel retint_restore_args
145#endif 145#endif
146 146
147#ifdef CONFIG_PARAVIRT 147#ifdef CONFIG_PARAVIRT
148ENTRY(native_usergs_sysret64) 148ENTRY(native_usergs_sysret64)
@@ -161,14 +161,14 @@ ENTRY(native_usergs_sysret64)
161.endm 161.endm
162 162
163/* 163/*
164 * C code is not supposed to know about undefined top of stack. Every time 164 * C code is not supposed to know about undefined top of stack. Every time
165 * a C function with an pt_regs argument is called from the SYSCALL based 165 * a C function with an pt_regs argument is called from the SYSCALL based
166 * fast path FIXUP_TOP_OF_STACK is needed. 166 * fast path FIXUP_TOP_OF_STACK is needed.
167 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs 167 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
168 * manipulation. 168 * manipulation.
169 */ 169 */
170 170
171 /* %rsp:at FRAMEEND */ 171 /* %rsp:at FRAMEEND */
172 .macro FIXUP_TOP_OF_STACK tmp 172 .macro FIXUP_TOP_OF_STACK tmp
173 movq %gs:pda_oldrsp,\tmp 173 movq %gs:pda_oldrsp,\tmp
174 movq \tmp,RSP(%rsp) 174 movq \tmp,RSP(%rsp)
@@ -244,8 +244,8 @@ ENTRY(native_usergs_sysret64)
244 .endm 244 .endm
245/* 245/*
246 * A newly forked process directly context switches into this. 246 * A newly forked process directly context switches into this.
247 */ 247 */
248/* rdi: prev */ 248/* rdi: prev */
249ENTRY(ret_from_fork) 249ENTRY(ret_from_fork)
250 CFI_DEFAULT_STACK 250 CFI_DEFAULT_STACK
251 push kernel_eflags(%rip) 251 push kernel_eflags(%rip)
@@ -256,7 +256,7 @@ ENTRY(ret_from_fork)
256 GET_THREAD_INFO(%rcx) 256 GET_THREAD_INFO(%rcx)
257 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx) 257 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
258 jnz rff_trace 258 jnz rff_trace
259rff_action: 259rff_action:
260 RESTORE_REST 260 RESTORE_REST
261 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread? 261 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
262 je int_ret_from_sys_call 262 je int_ret_from_sys_call
@@ -267,7 +267,7 @@ rff_action:
267rff_trace: 267rff_trace:
268 movq %rsp,%rdi 268 movq %rsp,%rdi
269 call syscall_trace_leave 269 call syscall_trace_leave
270 GET_THREAD_INFO(%rcx) 270 GET_THREAD_INFO(%rcx)
271 jmp rff_action 271 jmp rff_action
272 CFI_ENDPROC 272 CFI_ENDPROC
273END(ret_from_fork) 273END(ret_from_fork)
@@ -278,20 +278,20 @@ END(ret_from_fork)
278 * SYSCALL does not save anything on the stack and does not change the 278 * SYSCALL does not save anything on the stack and does not change the
279 * stack pointer. 279 * stack pointer.
280 */ 280 */
281 281
282/* 282/*
283 * Register setup: 283 * Register setup:
284 * rax system call number 284 * rax system call number
285 * rdi arg0 285 * rdi arg0
286 * rcx return address for syscall/sysret, C arg3 286 * rcx return address for syscall/sysret, C arg3
287 * rsi arg1 287 * rsi arg1
288 * rdx arg2 288 * rdx arg2
289 * r10 arg3 (--> moved to rcx for C) 289 * r10 arg3 (--> moved to rcx for C)
290 * r8 arg4 290 * r8 arg4
291 * r9 arg5 291 * r9 arg5
292 * r11 eflags for syscall/sysret, temporary for C 292 * r11 eflags for syscall/sysret, temporary for C
293 * r12-r15,rbp,rbx saved by C code, not touched. 293 * r12-r15,rbp,rbx saved by C code, not touched.
294 * 294 *
295 * Interrupts are off on entry. 295 * Interrupts are off on entry.
296 * Only called from user space. 296 * Only called from user space.
297 * 297 *
@@ -301,7 +301,7 @@ END(ret_from_fork)
301 * When user can change the frames always force IRET. That is because 301 * When user can change the frames always force IRET. That is because
302 * it deals with uncanonical addresses better. SYSRET has trouble 302 * it deals with uncanonical addresses better. SYSRET has trouble
303 * with them due to bugs in both AMD and Intel CPUs. 303 * with them due to bugs in both AMD and Intel CPUs.
304 */ 304 */
305 305
306ENTRY(system_call) 306ENTRY(system_call)
307 CFI_STARTPROC simple 307 CFI_STARTPROC simple
@@ -317,7 +317,7 @@ ENTRY(system_call)
317 */ 317 */
318ENTRY(system_call_after_swapgs) 318ENTRY(system_call_after_swapgs)
319 319
320 movq %rsp,%gs:pda_oldrsp 320 movq %rsp,%gs:pda_oldrsp
321 movq %gs:pda_kernelstack,%rsp 321 movq %gs:pda_kernelstack,%rsp
322 /* 322 /*
323 * No need to follow this irqs off/on section - it's straight 323 * No need to follow this irqs off/on section - it's straight
@@ -325,7 +325,7 @@ ENTRY(system_call_after_swapgs)
325 */ 325 */
326 ENABLE_INTERRUPTS(CLBR_NONE) 326 ENABLE_INTERRUPTS(CLBR_NONE)
327 SAVE_ARGS 8,1 327 SAVE_ARGS 8,1
328 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 328 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
329 movq %rcx,RIP-ARGOFFSET(%rsp) 329 movq %rcx,RIP-ARGOFFSET(%rsp)
330 CFI_REL_OFFSET rip,RIP-ARGOFFSET 330 CFI_REL_OFFSET rip,RIP-ARGOFFSET
331 GET_THREAD_INFO(%rcx) 331 GET_THREAD_INFO(%rcx)
@@ -339,19 +339,19 @@ system_call_fastpath:
339 movq %rax,RAX-ARGOFFSET(%rsp) 339 movq %rax,RAX-ARGOFFSET(%rsp)
340/* 340/*
341 * Syscall return path ending with SYSRET (fast path) 341 * Syscall return path ending with SYSRET (fast path)
342 * Has incomplete stack frame and undefined top of stack. 342 * Has incomplete stack frame and undefined top of stack.
343 */ 343 */
344ret_from_sys_call: 344ret_from_sys_call:
345 movl $_TIF_ALLWORK_MASK,%edi 345 movl $_TIF_ALLWORK_MASK,%edi
346 /* edi: flagmask */ 346 /* edi: flagmask */
347sysret_check: 347sysret_check:
348 LOCKDEP_SYS_EXIT 348 LOCKDEP_SYS_EXIT
349 GET_THREAD_INFO(%rcx) 349 GET_THREAD_INFO(%rcx)
350 DISABLE_INTERRUPTS(CLBR_NONE) 350 DISABLE_INTERRUPTS(CLBR_NONE)
351 TRACE_IRQS_OFF 351 TRACE_IRQS_OFF
352 movl TI_flags(%rcx),%edx 352 movl TI_flags(%rcx),%edx
353 andl %edi,%edx 353 andl %edi,%edx
354 jnz sysret_careful 354 jnz sysret_careful
355 CFI_REMEMBER_STATE 355 CFI_REMEMBER_STATE
356 /* 356 /*
357 * sysretq will re-enable interrupts: 357 * sysretq will re-enable interrupts:
@@ -366,7 +366,7 @@ sysret_check:
366 366
367 CFI_RESTORE_STATE 367 CFI_RESTORE_STATE
368 /* Handle reschedules */ 368 /* Handle reschedules */
369 /* edx: work, edi: workmask */ 369 /* edx: work, edi: workmask */
370sysret_careful: 370sysret_careful:
371 bt $TIF_NEED_RESCHED,%edx 371 bt $TIF_NEED_RESCHED,%edx
372 jnc sysret_signal 372 jnc sysret_signal
@@ -379,7 +379,7 @@ sysret_careful:
379 CFI_ADJUST_CFA_OFFSET -8 379 CFI_ADJUST_CFA_OFFSET -8
380 jmp sysret_check 380 jmp sysret_check
381 381
382 /* Handle a signal */ 382 /* Handle a signal */
383sysret_signal: 383sysret_signal:
384 TRACE_IRQS_ON 384 TRACE_IRQS_ON
385 ENABLE_INTERRUPTS(CLBR_NONE) 385 ENABLE_INTERRUPTS(CLBR_NONE)
@@ -398,7 +398,7 @@ sysret_signal:
398 DISABLE_INTERRUPTS(CLBR_NONE) 398 DISABLE_INTERRUPTS(CLBR_NONE)
399 TRACE_IRQS_OFF 399 TRACE_IRQS_OFF
400 jmp int_with_check 400 jmp int_with_check
401 401
402badsys: 402badsys:
403 movq $-ENOSYS,RAX-ARGOFFSET(%rsp) 403 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
404 jmp ret_from_sys_call 404 jmp ret_from_sys_call
@@ -437,7 +437,7 @@ sysret_audit:
437#endif /* CONFIG_AUDITSYSCALL */ 437#endif /* CONFIG_AUDITSYSCALL */
438 438
439 /* Do syscall tracing */ 439 /* Do syscall tracing */
440tracesys: 440tracesys:
441#ifdef CONFIG_AUDITSYSCALL 441#ifdef CONFIG_AUDITSYSCALL
442 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx) 442 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
443 jz auditsys 443 jz auditsys
@@ -460,8 +460,8 @@ tracesys:
460 call *sys_call_table(,%rax,8) 460 call *sys_call_table(,%rax,8)
461 movq %rax,RAX-ARGOFFSET(%rsp) 461 movq %rax,RAX-ARGOFFSET(%rsp)
462 /* Use IRET because user could have changed frame */ 462 /* Use IRET because user could have changed frame */
463 463
464/* 464/*
465 * Syscall return path ending with IRET. 465 * Syscall return path ending with IRET.
466 * Has correct top of stack, but partial stack frame. 466 * Has correct top of stack, but partial stack frame.
467 */ 467 */
@@ -505,18 +505,18 @@ int_very_careful:
505 TRACE_IRQS_ON 505 TRACE_IRQS_ON
506 ENABLE_INTERRUPTS(CLBR_NONE) 506 ENABLE_INTERRUPTS(CLBR_NONE)
507 SAVE_REST 507 SAVE_REST
508 /* Check for syscall exit trace */ 508 /* Check for syscall exit trace */
509 testl $_TIF_WORK_SYSCALL_EXIT,%edx 509 testl $_TIF_WORK_SYSCALL_EXIT,%edx
510 jz int_signal 510 jz int_signal
511 pushq %rdi 511 pushq %rdi
512 CFI_ADJUST_CFA_OFFSET 8 512 CFI_ADJUST_CFA_OFFSET 8
513 leaq 8(%rsp),%rdi # &ptregs -> arg1 513 leaq 8(%rsp),%rdi # &ptregs -> arg1
514 call syscall_trace_leave 514 call syscall_trace_leave
515 popq %rdi 515 popq %rdi
516 CFI_ADJUST_CFA_OFFSET -8 516 CFI_ADJUST_CFA_OFFSET -8
517 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi 517 andl $~(_TIF_WORK_SYSCALL_EXIT|_TIF_SYSCALL_EMU),%edi
518 jmp int_restore_rest 518 jmp int_restore_rest
519 519
520int_signal: 520int_signal:
521 testl $_TIF_DO_NOTIFY_MASK,%edx 521 testl $_TIF_DO_NOTIFY_MASK,%edx
522 jz 1f 522 jz 1f
@@ -531,11 +531,11 @@ int_restore_rest:
531 jmp int_with_check 531 jmp int_with_check
532 CFI_ENDPROC 532 CFI_ENDPROC
533END(system_call) 533END(system_call)
534 534
535/* 535/*
536 * Certain special system calls that need to save a complete full stack frame. 536 * Certain special system calls that need to save a complete full stack frame.
537 */ 537 */
538 538
539 .macro PTREGSCALL label,func,arg 539 .macro PTREGSCALL label,func,arg
540 .globl \label 540 .globl \label
541\label: 541\label:
@@ -572,7 +572,7 @@ ENTRY(ptregscall_common)
572 ret 572 ret
573 CFI_ENDPROC 573 CFI_ENDPROC
574END(ptregscall_common) 574END(ptregscall_common)
575 575
576ENTRY(stub_execve) 576ENTRY(stub_execve)
577 CFI_STARTPROC 577 CFI_STARTPROC
578 popq %r11 578 popq %r11
@@ -588,11 +588,11 @@ ENTRY(stub_execve)
588 jmp int_ret_from_sys_call 588 jmp int_ret_from_sys_call
589 CFI_ENDPROC 589 CFI_ENDPROC
590END(stub_execve) 590END(stub_execve)
591 591
592/* 592/*
593 * sigreturn is special because it needs to restore all registers on return. 593 * sigreturn is special because it needs to restore all registers on return.
594 * This cannot be done with SYSRET, so use the IRET return path instead. 594 * This cannot be done with SYSRET, so use the IRET return path instead.
595 */ 595 */
596ENTRY(stub_rt_sigreturn) 596ENTRY(stub_rt_sigreturn)
597 CFI_STARTPROC 597 CFI_STARTPROC
598 addq $8, %rsp 598 addq $8, %rsp
@@ -731,12 +731,12 @@ exit_intr:
731 GET_THREAD_INFO(%rcx) 731 GET_THREAD_INFO(%rcx)
732 testl $3,CS-ARGOFFSET(%rsp) 732 testl $3,CS-ARGOFFSET(%rsp)
733 je retint_kernel 733 je retint_kernel
734 734
735 /* Interrupt came from user space */ 735 /* Interrupt came from user space */
736 /* 736 /*
737 * Has a correct top of stack, but a partial stack frame 737 * Has a correct top of stack, but a partial stack frame
738 * %rcx: thread info. Interrupts off. 738 * %rcx: thread info. Interrupts off.
739 */ 739 */
740retint_with_reschedule: 740retint_with_reschedule:
741 movl $_TIF_WORK_MASK,%edi 741 movl $_TIF_WORK_MASK,%edi
742retint_check: 742retint_check:
@@ -809,20 +809,20 @@ retint_careful:
809 pushq %rdi 809 pushq %rdi
810 CFI_ADJUST_CFA_OFFSET 8 810 CFI_ADJUST_CFA_OFFSET 8
811 call schedule 811 call schedule
812 popq %rdi 812 popq %rdi
813 CFI_ADJUST_CFA_OFFSET -8 813 CFI_ADJUST_CFA_OFFSET -8
814 GET_THREAD_INFO(%rcx) 814 GET_THREAD_INFO(%rcx)
815 DISABLE_INTERRUPTS(CLBR_NONE) 815 DISABLE_INTERRUPTS(CLBR_NONE)
816 TRACE_IRQS_OFF 816 TRACE_IRQS_OFF
817 jmp retint_check 817 jmp retint_check
818 818
819retint_signal: 819retint_signal:
820 testl $_TIF_DO_NOTIFY_MASK,%edx 820 testl $_TIF_DO_NOTIFY_MASK,%edx
821 jz retint_swapgs 821 jz retint_swapgs
822 TRACE_IRQS_ON 822 TRACE_IRQS_ON
823 ENABLE_INTERRUPTS(CLBR_NONE) 823 ENABLE_INTERRUPTS(CLBR_NONE)
824 SAVE_REST 824 SAVE_REST
825 movq $-1,ORIG_RAX(%rsp) 825 movq $-1,ORIG_RAX(%rsp)
826 xorl %esi,%esi # oldset 826 xorl %esi,%esi # oldset
827 movq %rsp,%rdi # &pt_regs 827 movq %rsp,%rdi # &pt_regs
828 call do_notify_resume 828 call do_notify_resume
@@ -844,14 +844,14 @@ ENTRY(retint_kernel)
844 jnc retint_restore_args 844 jnc retint_restore_args
845 call preempt_schedule_irq 845 call preempt_schedule_irq
846 jmp exit_intr 846 jmp exit_intr
847#endif 847#endif
848 848
849 CFI_ENDPROC 849 CFI_ENDPROC
850END(common_interrupt) 850END(common_interrupt)
851 851
852/* 852/*
853 * APIC interrupts. 853 * APIC interrupts.
854 */ 854 */
855 .macro apicinterrupt num,func 855 .macro apicinterrupt num,func
856 INTR_FRAME 856 INTR_FRAME
857 pushq $~(\num) 857 pushq $~(\num)
@@ -869,14 +869,14 @@ ENTRY(threshold_interrupt)
869 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt 869 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
870END(threshold_interrupt) 870END(threshold_interrupt)
871 871
872#ifdef CONFIG_SMP 872#ifdef CONFIG_SMP
873ENTRY(reschedule_interrupt) 873ENTRY(reschedule_interrupt)
874 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt 874 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
875END(reschedule_interrupt) 875END(reschedule_interrupt)
876 876
877 .macro INVALIDATE_ENTRY num 877 .macro INVALIDATE_ENTRY num
878ENTRY(invalidate_interrupt\num) 878ENTRY(invalidate_interrupt\num)
879 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt 879 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
880END(invalidate_interrupt\num) 880END(invalidate_interrupt\num)
881 .endm 881 .endm
882 882
@@ -915,22 +915,22 @@ END(error_interrupt)
915ENTRY(spurious_interrupt) 915ENTRY(spurious_interrupt)
916 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt 916 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
917END(spurious_interrupt) 917END(spurious_interrupt)
918 918
919/* 919/*
920 * Exception entry points. 920 * Exception entry points.
921 */ 921 */
922 .macro zeroentry sym 922 .macro zeroentry sym
923 INTR_FRAME 923 INTR_FRAME
924 PARAVIRT_ADJUST_EXCEPTION_FRAME 924 PARAVIRT_ADJUST_EXCEPTION_FRAME
925 pushq $0 /* push error code/oldrax */ 925 pushq $0 /* push error code/oldrax */
926 CFI_ADJUST_CFA_OFFSET 8 926 CFI_ADJUST_CFA_OFFSET 8
927 pushq %rax /* push real oldrax to the rdi slot */ 927 pushq %rax /* push real oldrax to the rdi slot */
928 CFI_ADJUST_CFA_OFFSET 8 928 CFI_ADJUST_CFA_OFFSET 8
929 CFI_REL_OFFSET rax,0 929 CFI_REL_OFFSET rax,0
930 leaq \sym(%rip),%rax 930 leaq \sym(%rip),%rax
931 jmp error_entry 931 jmp error_entry
932 CFI_ENDPROC 932 CFI_ENDPROC
933 .endm 933 .endm
934 934
935 .macro errorentry sym 935 .macro errorentry sym
936 XCPT_FRAME 936 XCPT_FRAME
@@ -1044,13 +1044,13 @@ paranoid_schedule\trace:
1044 1044
1045/* 1045/*
1046 * Exception entry point. This expects an error code/orig_rax on the stack 1046 * Exception entry point. This expects an error code/orig_rax on the stack
1047 * and the exception handler in %rax. 1047 * and the exception handler in %rax.
1048 */ 1048 */
1049KPROBE_ENTRY(error_entry) 1049KPROBE_ENTRY(error_entry)
1050 _frame RDI 1050 _frame RDI
1051 CFI_REL_OFFSET rax,0 1051 CFI_REL_OFFSET rax,0
1052 /* rdi slot contains rax, oldrax contains error code */ 1052 /* rdi slot contains rax, oldrax contains error code */
1053 cld 1053 cld
1054 subq $14*8,%rsp 1054 subq $14*8,%rsp
1055 CFI_ADJUST_CFA_OFFSET (14*8) 1055 CFI_ADJUST_CFA_OFFSET (14*8)
1056 movq %rsi,13*8(%rsp) 1056 movq %rsi,13*8(%rsp)
@@ -1061,7 +1061,7 @@ KPROBE_ENTRY(error_entry)
1061 CFI_REL_OFFSET rdx,RDX 1061 CFI_REL_OFFSET rdx,RDX
1062 movq %rcx,11*8(%rsp) 1062 movq %rcx,11*8(%rsp)
1063 CFI_REL_OFFSET rcx,RCX 1063 CFI_REL_OFFSET rcx,RCX
1064 movq %rsi,10*8(%rsp) /* store rax */ 1064 movq %rsi,10*8(%rsp) /* store rax */
1065 CFI_REL_OFFSET rax,RAX 1065 CFI_REL_OFFSET rax,RAX
1066 movq %r8, 9*8(%rsp) 1066 movq %r8, 9*8(%rsp)
1067 CFI_REL_OFFSET r8,R8 1067 CFI_REL_OFFSET r8,R8
@@ -1071,29 +1071,29 @@ KPROBE_ENTRY(error_entry)
1071 CFI_REL_OFFSET r10,R10 1071 CFI_REL_OFFSET r10,R10
1072 movq %r11,6*8(%rsp) 1072 movq %r11,6*8(%rsp)
1073 CFI_REL_OFFSET r11,R11 1073 CFI_REL_OFFSET r11,R11
1074 movq %rbx,5*8(%rsp) 1074 movq %rbx,5*8(%rsp)
1075 CFI_REL_OFFSET rbx,RBX 1075 CFI_REL_OFFSET rbx,RBX
1076 movq %rbp,4*8(%rsp) 1076 movq %rbp,4*8(%rsp)
1077 CFI_REL_OFFSET rbp,RBP 1077 CFI_REL_OFFSET rbp,RBP
1078 movq %r12,3*8(%rsp) 1078 movq %r12,3*8(%rsp)
1079 CFI_REL_OFFSET r12,R12 1079 CFI_REL_OFFSET r12,R12
1080 movq %r13,2*8(%rsp) 1080 movq %r13,2*8(%rsp)
1081 CFI_REL_OFFSET r13,R13 1081 CFI_REL_OFFSET r13,R13
1082 movq %r14,1*8(%rsp) 1082 movq %r14,1*8(%rsp)
1083 CFI_REL_OFFSET r14,R14 1083 CFI_REL_OFFSET r14,R14
1084 movq %r15,(%rsp) 1084 movq %r15,(%rsp)
1085 CFI_REL_OFFSET r15,R15 1085 CFI_REL_OFFSET r15,R15
1086 xorl %ebx,%ebx 1086 xorl %ebx,%ebx
1087 testl $3,CS(%rsp) 1087 testl $3,CS(%rsp)
1088 je error_kernelspace 1088 je error_kernelspace
1089error_swapgs: 1089error_swapgs:
1090 SWAPGS 1090 SWAPGS
1091error_sti: 1091error_sti:
1092 TRACE_IRQS_OFF 1092 TRACE_IRQS_OFF
1093 movq %rdi,RDI(%rsp) 1093 movq %rdi,RDI(%rsp)
1094 CFI_REL_OFFSET rdi,RDI 1094 CFI_REL_OFFSET rdi,RDI
1095 movq %rsp,%rdi 1095 movq %rsp,%rdi
1096 movq ORIG_RAX(%rsp),%rsi /* get error code */ 1096 movq ORIG_RAX(%rsp),%rsi /* get error code */
1097 movq $-1,ORIG_RAX(%rsp) 1097 movq $-1,ORIG_RAX(%rsp)
1098 call *%rax 1098 call *%rax
1099 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */ 1099 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
@@ -1102,7 +1102,7 @@ error_exit:
1102 RESTORE_REST 1102 RESTORE_REST
1103 DISABLE_INTERRUPTS(CLBR_NONE) 1103 DISABLE_INTERRUPTS(CLBR_NONE)
1104 TRACE_IRQS_OFF 1104 TRACE_IRQS_OFF
1105 GET_THREAD_INFO(%rcx) 1105 GET_THREAD_INFO(%rcx)
1106 testl %eax,%eax 1106 testl %eax,%eax
1107 jne retint_kernel 1107 jne retint_kernel
1108 LOCKDEP_SYS_EXIT_IRQ 1108 LOCKDEP_SYS_EXIT_IRQ
@@ -1118,7 +1118,7 @@ error_kernelspace:
1118 /* There are two places in the kernel that can potentially fault with 1118 /* There are two places in the kernel that can potentially fault with
1119 usergs. Handle them here. The exception handlers after 1119 usergs. Handle them here. The exception handlers after
1120 iret run with kernel gs again, so don't set the user space flag. 1120 iret run with kernel gs again, so don't set the user space flag.
1121 B stepping K8s sometimes report an truncated RIP for IRET 1121 B stepping K8s sometimes report an truncated RIP for IRET
1122 exceptions returning to compat mode. Check for these here too. */ 1122 exceptions returning to compat mode. Check for these here too. */
1123 leaq irq_return(%rip),%rcx 1123 leaq irq_return(%rip),%rcx
1124 cmpq %rcx,RIP(%rsp) 1124 cmpq %rcx,RIP(%rsp)
@@ -1130,17 +1130,17 @@ error_kernelspace:
1130 je error_swapgs 1130 je error_swapgs
1131 jmp error_sti 1131 jmp error_sti
1132KPROBE_END(error_entry) 1132KPROBE_END(error_entry)
1133 1133
1134 /* Reload gs selector with exception handling */ 1134 /* Reload gs selector with exception handling */
1135 /* edi: new selector */ 1135 /* edi: new selector */
1136ENTRY(native_load_gs_index) 1136ENTRY(native_load_gs_index)
1137 CFI_STARTPROC 1137 CFI_STARTPROC
1138 pushf 1138 pushf
1139 CFI_ADJUST_CFA_OFFSET 8 1139 CFI_ADJUST_CFA_OFFSET 8
1140 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI)) 1140 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
1141 SWAPGS 1141 SWAPGS
1142gs_change: 1142gs_change:
1143 movl %edi,%gs 1143 movl %edi,%gs
11442: mfence /* workaround */ 11442: mfence /* workaround */
1145 SWAPGS 1145 SWAPGS
1146 popf 1146 popf
@@ -1148,20 +1148,20 @@ gs_change:
1148 ret 1148 ret
1149 CFI_ENDPROC 1149 CFI_ENDPROC
1150ENDPROC(native_load_gs_index) 1150ENDPROC(native_load_gs_index)
1151 1151
1152 .section __ex_table,"a" 1152 .section __ex_table,"a"
1153 .align 8 1153 .align 8
1154 .quad gs_change,bad_gs 1154 .quad gs_change,bad_gs
1155 .previous 1155 .previous
1156 .section .fixup,"ax" 1156 .section .fixup,"ax"
1157 /* running with kernelgs */ 1157 /* running with kernelgs */
1158bad_gs: 1158bad_gs:
1159 SWAPGS /* switch back to user gs */ 1159 SWAPGS /* switch back to user gs */
1160 xorl %eax,%eax 1160 xorl %eax,%eax
1161 movl %eax,%gs 1161 movl %eax,%gs
1162 jmp 2b 1162 jmp 2b
1163 .previous 1163 .previous
1164 1164
1165/* 1165/*
1166 * Create a kernel thread. 1166 * Create a kernel thread.
1167 * 1167 *
@@ -1184,7 +1184,7 @@ ENTRY(kernel_thread)
1184 1184
1185 xorl %r8d,%r8d 1185 xorl %r8d,%r8d
1186 xorl %r9d,%r9d 1186 xorl %r9d,%r9d
1187 1187
1188 # clone now 1188 # clone now
1189 call do_fork 1189 call do_fork
1190 movq %rax,RAX(%rsp) 1190 movq %rax,RAX(%rsp)
@@ -1195,14 +1195,14 @@ ENTRY(kernel_thread)
1195 * so internally to the x86_64 port you can rely on kernel_thread() 1195 * so internally to the x86_64 port you can rely on kernel_thread()
1196 * not to reschedule the child before returning, this avoids the need 1196 * not to reschedule the child before returning, this avoids the need
1197 * of hacks for example to fork off the per-CPU idle tasks. 1197 * of hacks for example to fork off the per-CPU idle tasks.
1198 * [Hopefully no generic code relies on the reschedule -AK] 1198 * [Hopefully no generic code relies on the reschedule -AK]
1199 */ 1199 */
1200 RESTORE_ALL 1200 RESTORE_ALL
1201 UNFAKE_STACK_FRAME 1201 UNFAKE_STACK_FRAME
1202 ret 1202 ret
1203 CFI_ENDPROC 1203 CFI_ENDPROC
1204ENDPROC(kernel_thread) 1204ENDPROC(kernel_thread)
1205 1205
1206child_rip: 1206child_rip:
1207 pushq $0 # fake return address 1207 pushq $0 # fake return address
1208 CFI_STARTPROC 1208 CFI_STARTPROC
@@ -1237,10 +1237,10 @@ ENDPROC(child_rip)
1237ENTRY(kernel_execve) 1237ENTRY(kernel_execve)
1238 CFI_STARTPROC 1238 CFI_STARTPROC
1239 FAKE_STACK_FRAME $0 1239 FAKE_STACK_FRAME $0
1240 SAVE_ALL 1240 SAVE_ALL
1241 movq %rsp,%rcx 1241 movq %rsp,%rcx
1242 call sys_execve 1242 call sys_execve
1243 movq %rax, RAX(%rsp) 1243 movq %rax, RAX(%rsp)
1244 RESTORE_REST 1244 RESTORE_REST
1245 testq %rax,%rax 1245 testq %rax,%rax
1246 je int_ret_from_sys_call 1246 je int_ret_from_sys_call
@@ -1259,7 +1259,7 @@ ENTRY(coprocessor_error)
1259END(coprocessor_error) 1259END(coprocessor_error)
1260 1260
1261ENTRY(simd_coprocessor_error) 1261ENTRY(simd_coprocessor_error)
1262 zeroentry do_simd_coprocessor_error 1262 zeroentry do_simd_coprocessor_error
1263END(simd_coprocessor_error) 1263END(simd_coprocessor_error)
1264 1264
1265ENTRY(device_not_available) 1265ENTRY(device_not_available)
@@ -1271,12 +1271,12 @@ KPROBE_ENTRY(debug)
1271 INTR_FRAME 1271 INTR_FRAME
1272 PARAVIRT_ADJUST_EXCEPTION_FRAME 1272 PARAVIRT_ADJUST_EXCEPTION_FRAME
1273 pushq $0 1273 pushq $0
1274 CFI_ADJUST_CFA_OFFSET 8 1274 CFI_ADJUST_CFA_OFFSET 8
1275 paranoidentry do_debug, DEBUG_STACK 1275 paranoidentry do_debug, DEBUG_STACK
1276 paranoidexit 1276 paranoidexit
1277KPROBE_END(debug) 1277KPROBE_END(debug)
1278 1278
1279 /* runs on exception stack */ 1279 /* runs on exception stack */
1280KPROBE_ENTRY(nmi) 1280KPROBE_ENTRY(nmi)
1281 INTR_FRAME 1281 INTR_FRAME
1282 PARAVIRT_ADJUST_EXCEPTION_FRAME 1282 PARAVIRT_ADJUST_EXCEPTION_FRAME
@@ -1310,7 +1310,7 @@ ENTRY(bounds)
1310END(bounds) 1310END(bounds)
1311 1311
1312ENTRY(invalid_op) 1312ENTRY(invalid_op)
1313 zeroentry do_invalid_op 1313 zeroentry do_invalid_op
1314END(invalid_op) 1314END(invalid_op)
1315 1315
1316ENTRY(coprocessor_segment_overrun) 1316ENTRY(coprocessor_segment_overrun)
@@ -1365,7 +1365,7 @@ ENTRY(machine_check)
1365 INTR_FRAME 1365 INTR_FRAME
1366 PARAVIRT_ADJUST_EXCEPTION_FRAME 1366 PARAVIRT_ADJUST_EXCEPTION_FRAME
1367 pushq $0 1367 pushq $0
1368 CFI_ADJUST_CFA_OFFSET 8 1368 CFI_ADJUST_CFA_OFFSET 8
1369 paranoidentry do_machine_check 1369 paranoidentry do_machine_check
1370 jmp paranoid_exit1 1370 jmp paranoid_exit1
1371 CFI_ENDPROC 1371 CFI_ENDPROC
diff --git a/arch/x86/kernel/es7000_32.c b/arch/x86/kernel/es7000_32.c
index f454c78fcef6..0aa2c443d600 100644
--- a/arch/x86/kernel/es7000_32.c
+++ b/arch/x86/kernel/es7000_32.c
@@ -250,31 +250,24 @@ int __init find_unisys_acpi_oem_table(unsigned long *oem_addr)
250{ 250{
251 struct acpi_table_header *header = NULL; 251 struct acpi_table_header *header = NULL;
252 int i = 0; 252 int i = 0;
253 acpi_size tbl_size;
254 253
255 while (ACPI_SUCCESS(acpi_get_table_with_size("OEM1", i++, &header, &tbl_size))) { 254 while (ACPI_SUCCESS(acpi_get_table("OEM1", i++, &header))) {
256 if (!memcmp((char *) &header->oem_id, "UNISYS", 6)) { 255 if (!memcmp((char *) &header->oem_id, "UNISYS", 6)) {
257 struct oem_table *t = (struct oem_table *)header; 256 struct oem_table *t = (struct oem_table *)header;
258 257
259 oem_addrX = t->OEMTableAddr; 258 oem_addrX = t->OEMTableAddr;
260 oem_size = t->OEMTableSize; 259 oem_size = t->OEMTableSize;
261 early_acpi_os_unmap_memory(header, tbl_size);
262 260
263 *oem_addr = (unsigned long)__acpi_map_table(oem_addrX, 261 *oem_addr = (unsigned long)__acpi_map_table(oem_addrX,
264 oem_size); 262 oem_size);
265 return 0; 263 return 0;
266 } 264 }
267 early_acpi_os_unmap_memory(header, tbl_size);
268 } 265 }
269 return -1; 266 return -1;
270} 267}
271 268
272void __init unmap_unisys_acpi_oem_table(unsigned long oem_addr) 269void __init unmap_unisys_acpi_oem_table(unsigned long oem_addr)
273{ 270{
274 if (!oem_addr)
275 return;
276
277 __acpi_unmap_table((char *)oem_addr, oem_size);
278} 271}
279#endif 272#endif
280 273
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 774ac4991568..1c9cc431ea4f 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -128,7 +128,7 @@ static int kvm_register_clock(char *txt)
128} 128}
129 129
130#ifdef CONFIG_X86_LOCAL_APIC 130#ifdef CONFIG_X86_LOCAL_APIC
131static void kvm_setup_secondary_clock(void) 131static void __devinit kvm_setup_secondary_clock(void)
132{ 132{
133 /* 133 /*
134 * Now that the first cpu already had this clocksource initialized, 134 * Now that the first cpu already had this clocksource initialized,
diff --git a/arch/x86/kernel/tsc_sync.c b/arch/x86/kernel/tsc_sync.c
index 9ffb01c31c40..1c0dfbca87c1 100644
--- a/arch/x86/kernel/tsc_sync.c
+++ b/arch/x86/kernel/tsc_sync.c
@@ -46,7 +46,9 @@ static __cpuinit void check_tsc_warp(void)
46 cycles_t start, now, prev, end; 46 cycles_t start, now, prev, end;
47 int i; 47 int i;
48 48
49 rdtsc_barrier();
49 start = get_cycles(); 50 start = get_cycles();
51 rdtsc_barrier();
50 /* 52 /*
51 * The measurement runs for 20 msecs: 53 * The measurement runs for 20 msecs:
52 */ 54 */
@@ -61,7 +63,9 @@ static __cpuinit void check_tsc_warp(void)
61 */ 63 */
62 __raw_spin_lock(&sync_lock); 64 __raw_spin_lock(&sync_lock);
63 prev = last_tsc; 65 prev = last_tsc;
66 rdtsc_barrier();
64 now = get_cycles(); 67 now = get_cycles();
68 rdtsc_barrier();
65 last_tsc = now; 69 last_tsc = now;
66 __raw_spin_unlock(&sync_lock); 70 __raw_spin_unlock(&sync_lock);
67 71
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 0b8b6690a86d..ebf2f12900f5 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -128,7 +128,16 @@ static __always_inline void do_vgettimeofday(struct timeval * tv)
128 gettimeofday(tv,NULL); 128 gettimeofday(tv,NULL);
129 return; 129 return;
130 } 130 }
131
132 /*
133 * Surround the RDTSC by barriers, to make sure it's not
134 * speculated to outside the seqlock critical section and
135 * does not cause time warps:
136 */
137 rdtsc_barrier();
131 now = vread(); 138 now = vread();
139 rdtsc_barrier();
140
132 base = __vsyscall_gtod_data.clock.cycle_last; 141 base = __vsyscall_gtod_data.clock.cycle_last;
133 mask = __vsyscall_gtod_data.clock.mask; 142 mask = __vsyscall_gtod_data.clock.mask;
134 mult = __vsyscall_gtod_data.clock.mult; 143 mult = __vsyscall_gtod_data.clock.mult;
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index ce3251ce5504..b81125f0bdee 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -20,6 +20,8 @@ if VIRTUALIZATION
20config KVM 20config KVM
21 tristate "Kernel-based Virtual Machine (KVM) support" 21 tristate "Kernel-based Virtual Machine (KVM) support"
22 depends on HAVE_KVM 22 depends on HAVE_KVM
23 # for device assignment:
24 depends on PCI
23 select PREEMPT_NOTIFIERS 25 select PREEMPT_NOTIFIERS
24 select MMU_NOTIFIER 26 select MMU_NOTIFIER
25 select ANON_INODES 27 select ANON_INODES
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 8772dc946823..59ebd37ad79e 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -548,8 +548,10 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm)
548 mutex_lock(&kvm->lock); 548 mutex_lock(&kvm->lock);
549 pit->irq_source_id = kvm_request_irq_source_id(kvm); 549 pit->irq_source_id = kvm_request_irq_source_id(kvm);
550 mutex_unlock(&kvm->lock); 550 mutex_unlock(&kvm->lock);
551 if (pit->irq_source_id < 0) 551 if (pit->irq_source_id < 0) {
552 kfree(pit);
552 return NULL; 553 return NULL;
554 }
553 555
554 mutex_init(&pit->pit_state.lock); 556 mutex_init(&pit->pit_state.lock);
555 mutex_lock(&pit->pit_state.lock); 557 mutex_lock(&pit->pit_state.lock);
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2a5e64881d9b..f1983d9477cd 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -314,7 +314,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
314 if (r) 314 if (r)
315 goto out; 315 goto out;
316 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, 316 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
317 rmap_desc_cache, 1); 317 rmap_desc_cache, 4);
318 if (r) 318 if (r)
319 goto out; 319 goto out;
320 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); 320 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 2643b430d83a..d06b4dc0e2ea 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -3564,7 +3564,8 @@ static int __init vmx_init(void)
3564 bypass_guest_pf = 0; 3564 bypass_guest_pf = 0;
3565 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK | 3565 kvm_mmu_set_base_ptes(VMX_EPT_READABLE_MASK |
3566 VMX_EPT_WRITABLE_MASK | 3566 VMX_EPT_WRITABLE_MASK |
3567 VMX_EPT_DEFAULT_MT << VMX_EPT_MT_EPTE_SHIFT); 3567 VMX_EPT_DEFAULT_MT << VMX_EPT_MT_EPTE_SHIFT |
3568 VMX_EPT_IGMT_BIT);
3568 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull, 3569 kvm_mmu_set_mask_ptes(0ull, 0ull, 0ull, 0ull,
3569 VMX_EPT_EXECUTABLE_MASK); 3570 VMX_EPT_EXECUTABLE_MASK);
3570 kvm_enable_tdp(); 3571 kvm_enable_tdp();
diff --git a/arch/x86/kvm/vmx.h b/arch/x86/kvm/vmx.h
index 3e010d21fdd7..ec5edc339da6 100644
--- a/arch/x86/kvm/vmx.h
+++ b/arch/x86/kvm/vmx.h
@@ -352,6 +352,7 @@ enum vmcs_field {
352#define VMX_EPT_READABLE_MASK 0x1ull 352#define VMX_EPT_READABLE_MASK 0x1ull
353#define VMX_EPT_WRITABLE_MASK 0x2ull 353#define VMX_EPT_WRITABLE_MASK 0x2ull
354#define VMX_EPT_EXECUTABLE_MASK 0x4ull 354#define VMX_EPT_EXECUTABLE_MASK 0x4ull
355#define VMX_EPT_IGMT_BIT (1ull << 6)
355 356
356#define VMX_EPT_IDENTITY_PAGETABLE_ADDR 0xfffbc000ul 357#define VMX_EPT_IDENTITY_PAGETABLE_ADDR 0xfffbc000ul
357 358
diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c
index 0e331652681e..52145007bd7e 100644
--- a/arch/x86/mach-voyager/voyager_smp.c
+++ b/arch/x86/mach-voyager/voyager_smp.c
@@ -7,6 +7,7 @@
7 * This file provides all the same external entries as smp.c but uses 7 * This file provides all the same external entries as smp.c but uses
8 * the voyager hal to provide the functionality 8 * the voyager hal to provide the functionality
9 */ 9 */
10#include <linux/cpu.h>
10#include <linux/module.h> 11#include <linux/module.h>
11#include <linux/mm.h> 12#include <linux/mm.h>
12#include <linux/kernel_stat.h> 13#include <linux/kernel_stat.h>
@@ -1790,6 +1791,17 @@ void __init smp_setup_processor_id(void)
1790 x86_write_percpu(cpu_number, hard_smp_processor_id()); 1791 x86_write_percpu(cpu_number, hard_smp_processor_id());
1791} 1792}
1792 1793
1794static void voyager_send_call_func(cpumask_t callmask)
1795{
1796 __u32 mask = cpus_addr(callmask)[0] & ~(1 << smp_processor_id());
1797 send_CPI(mask, VIC_CALL_FUNCTION_CPI);
1798}
1799
1800static void voyager_send_call_func_single(int cpu)
1801{
1802 send_CPI(1 << cpu, VIC_CALL_FUNCTION_SINGLE_CPI);
1803}
1804
1793struct smp_ops smp_ops = { 1805struct smp_ops smp_ops = {
1794 .smp_prepare_boot_cpu = voyager_smp_prepare_boot_cpu, 1806 .smp_prepare_boot_cpu = voyager_smp_prepare_boot_cpu,
1795 .smp_prepare_cpus = voyager_smp_prepare_cpus, 1807 .smp_prepare_cpus = voyager_smp_prepare_cpus,
@@ -1799,6 +1811,6 @@ struct smp_ops smp_ops = {
1799 .smp_send_stop = voyager_smp_send_stop, 1811 .smp_send_stop = voyager_smp_send_stop,
1800 .smp_send_reschedule = voyager_smp_send_reschedule, 1812 .smp_send_reschedule = voyager_smp_send_reschedule,
1801 1813
1802 .send_call_func_ipi = native_send_call_func_ipi, 1814 .send_call_func_ipi = voyager_send_call_func,
1803 .send_call_func_single_ipi = native_send_call_func_single_ipi, 1815 .send_call_func_single_ipi = voyager_send_call_func_single,
1804}; 1816};
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index c483f4242079..3ffed259883e 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -102,6 +102,8 @@ static pmd_t * __init one_md_table_init(pgd_t *pgd)
102 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT)); 102 set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
103 pud = pud_offset(pgd, 0); 103 pud = pud_offset(pgd, 0);
104 BUG_ON(pmd_table != pmd_offset(pud, 0)); 104 BUG_ON(pmd_table != pmd_offset(pud, 0));
105
106 return pmd_table;
105 } 107 }
106#endif 108#endif
107 pud = pud_offset(pgd, 0); 109 pud = pud_offset(pgd, 0);