aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include/asm
diff options
context:
space:
mode:
authorAndrea Bastoni <bastoni@cs.unc.edu>2010-10-23 01:01:49 -0400
committerAndrea Bastoni <bastoni@cs.unc.edu>2010-10-23 01:01:49 -0400
commit3dd41424090a0ca3a660218d06afe6ff4441bad3 (patch)
tree511ef1bb1799027fc5aad574adce49120ecadd87 /arch/x86/include/asm
parent5c5456402d467969b217d7fdd6670f8c8600f5a8 (diff)
parentf6f94e2ab1b33f0082ac22d71f66385a60d8157f (diff)
Merge commit 'v2.6.36' into wip-merge-2.6.36
Conflicts: Makefile arch/x86/include/asm/unistd_32.h arch/x86/kernel/syscall_table_32.S kernel/sched.c kernel/time/tick-sched.c Relevant API and functions changes (solved in this commit): - (API) .enqueue_task() (enqueue_task_litmus), dequeue_task() (dequeue_task_litmus), [litmus/sched_litmus.c] - (API) .select_task_rq() (select_task_rq_litmus) [litmus/sched_litmus.c] - (API) sysrq_dump_trace_buffer() and sysrq_handle_kill_rt_tasks() [litmus/sched_trace.c] - struct kfifo internal buffer name changed (buffer -> buf) [litmus/sched_trace.c] - add_wait_queue_exclusive_locked -> __add_wait_queue_tail_exclusive [litmus/fmlp.c] - syscall numbers for both x86_32 and x86_64
Diffstat (limited to 'arch/x86/include/asm')
-rw-r--r--arch/x86/include/asm/Kbuild29
-rw-r--r--arch/x86/include/asm/acpi.h4
-rw-r--r--arch/x86/include/asm/alternative-asm.h4
-rw-r--r--arch/x86/include/asm/alternative.h27
-rw-r--r--arch/x86/include/asm/amd_iommu_proto.h6
-rw-r--r--arch/x86/include/asm/amd_iommu_types.h46
-rw-r--r--arch/x86/include/asm/apb_timer.h1
-rw-r--r--arch/x86/include/asm/apic.h13
-rw-r--r--arch/x86/include/asm/arch_hweight.h61
-rw-r--r--arch/x86/include/asm/atomic.h25
-rw-r--r--arch/x86/include/asm/atomic64_32.h278
-rw-r--r--arch/x86/include/asm/atomic64_64.h25
-rw-r--r--arch/x86/include/asm/bitops.h6
-rw-r--r--arch/x86/include/asm/boot.h2
-rw-r--r--arch/x86/include/asm/bootparam.h11
-rw-r--r--arch/x86/include/asm/cache.h2
-rw-r--r--arch/x86/include/asm/cacheflush.h46
-rw-r--r--arch/x86/include/asm/cmpxchg_32.h201
-rw-r--r--arch/x86/include/asm/cmpxchg_64.h83
-rw-r--r--arch/x86/include/asm/compat.h2
-rw-r--r--arch/x86/include/asm/cpufeature.h97
-rw-r--r--arch/x86/include/asm/dma-mapping.h8
-rw-r--r--arch/x86/include/asm/ds.h302
-rw-r--r--arch/x86/include/asm/dwarf2.h12
-rw-r--r--arch/x86/include/asm/e820.h7
-rw-r--r--arch/x86/include/asm/hardirq.h2
-rw-r--r--arch/x86/include/asm/highmem.h2
-rw-r--r--arch/x86/include/asm/hw_breakpoint.h12
-rw-r--r--arch/x86/include/asm/hyperv.h11
-rw-r--r--arch/x86/include/asm/hypervisor.h28
-rw-r--r--arch/x86/include/asm/i387.h155
-rw-r--r--arch/x86/include/asm/i8253.h2
-rw-r--r--arch/x86/include/asm/insn.h2
-rw-r--r--arch/x86/include/asm/inst.h96
-rw-r--r--arch/x86/include/asm/intel_scu_ipc.h47
-rw-r--r--arch/x86/include/asm/io_apic.h13
-rw-r--r--arch/x86/include/asm/iomap.h4
-rw-r--r--arch/x86/include/asm/irq_vectors.h3
-rw-r--r--arch/x86/include/asm/kdebug.h6
-rw-r--r--arch/x86/include/asm/kgdb.h23
-rw-r--r--arch/x86/include/asm/kprobes.h2
-rw-r--r--arch/x86/include/asm/kvm.h39
-rw-r--r--arch/x86/include/asm/kvm_emulate.h77
-rw-r--r--arch/x86/include/asm/kvm_host.h174
-rw-r--r--arch/x86/include/asm/kvm_para.h13
-rw-r--r--arch/x86/include/asm/local64.h1
-rw-r--r--arch/x86/include/asm/mce.h12
-rw-r--r--arch/x86/include/asm/mpspec.h10
-rw-r--r--arch/x86/include/asm/mrst.h26
-rw-r--r--arch/x86/include/asm/mshyperv.h14
-rw-r--r--arch/x86/include/asm/msr-index.h49
-rw-r--r--arch/x86/include/asm/msr.h4
-rw-r--r--arch/x86/include/asm/nmi.h2
-rw-r--r--arch/x86/include/asm/olpc_ofw.h31
-rw-r--r--arch/x86/include/asm/page.h7
-rw-r--r--arch/x86/include/asm/pci.h6
-rw-r--r--arch/x86/include/asm/pci_x86.h5
-rw-r--r--arch/x86/include/asm/percpu.h28
-rw-r--r--arch/x86/include/asm/perf_event.h92
-rw-r--r--arch/x86/include/asm/perf_event_p4.h800
-rw-r--r--arch/x86/include/asm/pgtable_32.h1
-rw-r--r--arch/x86/include/asm/pgtable_32_types.h2
-rw-r--r--arch/x86/include/asm/pgtable_64.h4
-rw-r--r--arch/x86/include/asm/processor.h68
-rw-r--r--arch/x86/include/asm/ptrace-abi.h57
-rw-r--r--arch/x86/include/asm/ptrace.h6
-rw-r--r--arch/x86/include/asm/pvclock-abi.h4
-rw-r--r--arch/x86/include/asm/pvclock.h1
-rw-r--r--arch/x86/include/asm/rdc321x_defs.h12
-rw-r--r--arch/x86/include/asm/required-features.h2
-rw-r--r--arch/x86/include/asm/rwsem.h21
-rw-r--r--arch/x86/include/asm/scatterlist.h4
-rw-r--r--arch/x86/include/asm/setup.h2
-rw-r--r--arch/x86/include/asm/stacktrace.h49
-rw-r--r--arch/x86/include/asm/suspend_32.h2
-rw-r--r--arch/x86/include/asm/suspend_64.h2
-rw-r--r--arch/x86/include/asm/svm.h9
-rw-r--r--arch/x86/include/asm/sys_ia32.h15
-rw-r--r--arch/x86/include/asm/syscalls.h5
-rw-r--r--arch/x86/include/asm/system.h9
-rw-r--r--arch/x86/include/asm/thread_info.h15
-rw-r--r--arch/x86/include/asm/topology.h26
-rw-r--r--arch/x86/include/asm/trampoline.h5
-rw-r--r--arch/x86/include/asm/traps.h2
-rw-r--r--arch/x86/include/asm/tsc.h2
-rw-r--r--arch/x86/include/asm/unistd_32.h7
-rw-r--r--arch/x86/include/asm/unistd_64.h10
-rw-r--r--arch/x86/include/asm/uv/uv_bau.h304
-rw-r--r--arch/x86/include/asm/uv/uv_hub.h2
-rw-r--r--arch/x86/include/asm/uv/uv_mmrs.h528
-rw-r--r--arch/x86/include/asm/vmware.h27
-rw-r--r--arch/x86/include/asm/vmx.h17
-rw-r--r--arch/x86/include/asm/x86_init.h2
-rw-r--r--arch/x86/include/asm/xen/hypercall.h6
-rw-r--r--arch/x86/include/asm/xen/page.h8
-rw-r--r--arch/x86/include/asm/xen/swiotlb-xen.h14
-rw-r--r--arch/x86/include/asm/xsave.h45
97 files changed, 2794 insertions, 1597 deletions
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 493092efaa3b..6fa90a845e4c 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -3,24 +3,23 @@ include include/asm-generic/Kbuild.asm
3header-y += boot.h 3header-y += boot.h
4header-y += bootparam.h 4header-y += bootparam.h
5header-y += debugreg.h 5header-y += debugreg.h
6header-y += e820.h
7header-y += hw_breakpoint.h
8header-y += hyperv.h
9header-y += ist.h
6header-y += ldt.h 10header-y += ldt.h
11header-y += mce.h
7header-y += msr-index.h 12header-y += msr-index.h
13header-y += msr.h
14header-y += mtrr.h
15header-y += posix_types_32.h
16header-y += posix_types_64.h
8header-y += prctl.h 17header-y += prctl.h
18header-y += processor-flags.h
9header-y += ptrace-abi.h 19header-y += ptrace-abi.h
10header-y += sigcontext32.h 20header-y += sigcontext32.h
11header-y += ucontext.h 21header-y += ucontext.h
12header-y += processor-flags.h 22header-y += unistd_32.h
13header-y += hw_breakpoint.h 23header-y += unistd_64.h
14header-y += hyperv.h 24header-y += vm86.h
15 25header-y += vsyscall.h
16unifdef-y += e820.h
17unifdef-y += ist.h
18unifdef-y += mce.h
19unifdef-y += msr.h
20unifdef-y += mtrr.h
21unifdef-y += posix_types_32.h
22unifdef-y += posix_types_64.h
23unifdef-y += unistd_32.h
24unifdef-y += unistd_64.h
25unifdef-y += vm86.h
26unifdef-y += vsyscall.h
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 56f462cf22d2..92091de11113 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -85,7 +85,6 @@ extern int acpi_ioapic;
85extern int acpi_noirq; 85extern int acpi_noirq;
86extern int acpi_strict; 86extern int acpi_strict;
87extern int acpi_disabled; 87extern int acpi_disabled;
88extern int acpi_ht;
89extern int acpi_pci_disabled; 88extern int acpi_pci_disabled;
90extern int acpi_skip_timer_override; 89extern int acpi_skip_timer_override;
91extern int acpi_use_timer_override; 90extern int acpi_use_timer_override;
@@ -97,7 +96,6 @@ void acpi_pic_sci_set_trigger(unsigned int, u16);
97static inline void disable_acpi(void) 96static inline void disable_acpi(void)
98{ 97{
99 acpi_disabled = 1; 98 acpi_disabled = 1;
100 acpi_ht = 0;
101 acpi_pci_disabled = 1; 99 acpi_pci_disabled = 1;
102 acpi_noirq = 1; 100 acpi_noirq = 1;
103} 101}
@@ -136,7 +134,7 @@ static inline unsigned int acpi_processor_cstate_check(unsigned int max_cstate)
136 boot_cpu_data.x86_model <= 0x05 && 134 boot_cpu_data.x86_model <= 0x05 &&
137 boot_cpu_data.x86_mask < 0x0A) 135 boot_cpu_data.x86_mask < 0x0A)
138 return 1; 136 return 1;
139 else if (boot_cpu_has(X86_FEATURE_AMDC1E)) 137 else if (c1e_detected)
140 return 1; 138 return 1;
141 else 139 else
142 return max_cstate; 140 return max_cstate;
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index b97f786a48d5..a63a68be1cce 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -6,8 +6,8 @@
6 .macro LOCK_PREFIX 6 .macro LOCK_PREFIX
71: lock 71: lock
8 .section .smp_locks,"a" 8 .section .smp_locks,"a"
9 _ASM_ALIGN 9 .balign 4
10 _ASM_PTR 1b 10 .long 1b - .
11 .previous 11 .previous
12 .endm 12 .endm
13#else 13#else
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index b09ec55650b3..bc6abb7bc7ee 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -28,27 +28,26 @@
28 */ 28 */
29 29
30#ifdef CONFIG_SMP 30#ifdef CONFIG_SMP
31#define LOCK_PREFIX \ 31#define LOCK_PREFIX_HERE \
32 ".section .smp_locks,\"a\"\n" \ 32 ".section .smp_locks,\"a\"\n" \
33 _ASM_ALIGN "\n" \ 33 ".balign 4\n" \
34 _ASM_PTR "661f\n" /* address */ \ 34 ".long 671f - .\n" /* offset */ \
35 ".previous\n" \ 35 ".previous\n" \
36 "661:\n\tlock; " 36 "671:"
37
38#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
37 39
38#else /* ! CONFIG_SMP */ 40#else /* ! CONFIG_SMP */
41#define LOCK_PREFIX_HERE ""
39#define LOCK_PREFIX "" 42#define LOCK_PREFIX ""
40#endif 43#endif
41 44
42/* This must be included *after* the definition of LOCK_PREFIX */
43#include <asm/cpufeature.h>
44
45struct alt_instr { 45struct alt_instr {
46 u8 *instr; /* original instruction */ 46 u8 *instr; /* original instruction */
47 u8 *replacement; 47 u8 *replacement;
48 u8 cpuid; /* cpuid bit set for replacement */ 48 u16 cpuid; /* cpuid bit set for replacement */
49 u8 instrlen; /* length of original instruction */ 49 u8 instrlen; /* length of original instruction */
50 u8 replacementlen; /* length of new instruction, <= instrlen */ 50 u8 replacementlen; /* length of new instruction, <= instrlen */
51 u8 pad1;
52#ifdef CONFIG_X86_64 51#ifdef CONFIG_X86_64
53 u32 pad2; 52 u32 pad2;
54#endif 53#endif
@@ -86,9 +85,11 @@ static inline int alternatives_text_reserved(void *start, void *end)
86 _ASM_ALIGN "\n" \ 85 _ASM_ALIGN "\n" \
87 _ASM_PTR "661b\n" /* label */ \ 86 _ASM_PTR "661b\n" /* label */ \
88 _ASM_PTR "663f\n" /* new instruction */ \ 87 _ASM_PTR "663f\n" /* new instruction */ \
89 " .byte " __stringify(feature) "\n" /* feature bit */ \ 88 " .word " __stringify(feature) "\n" /* feature bit */ \
90 " .byte 662b-661b\n" /* sourcelen */ \ 89 " .byte 662b-661b\n" /* sourcelen */ \
91 " .byte 664f-663f\n" /* replacementlen */ \ 90 " .byte 664f-663f\n" /* replacementlen */ \
91 ".previous\n" \
92 ".section .discard,\"aw\",@progbits\n" \
92 " .byte 0xff + (664f-663f) - (662b-661b)\n" /* rlen <= slen */ \ 93 " .byte 0xff + (664f-663f) - (662b-661b)\n" /* rlen <= slen */ \
93 ".previous\n" \ 94 ".previous\n" \
94 ".section .altinstr_replacement, \"ax\"\n" \ 95 ".section .altinstr_replacement, \"ax\"\n" \
@@ -96,6 +97,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
96 ".previous" 97 ".previous"
97 98
98/* 99/*
100 * This must be included *after* the definition of ALTERNATIVE due to
101 * <asm/arch_hweight.h>
102 */
103#include <asm/cpufeature.h>
104
105/*
99 * Alternative instructions for different CPU types or capabilities. 106 * Alternative instructions for different CPU types or capabilities.
100 * 107 *
101 * This allows to use optimized instructions even on generic binary 108 * This allows to use optimized instructions even on generic binary
diff --git a/arch/x86/include/asm/amd_iommu_proto.h b/arch/x86/include/asm/amd_iommu_proto.h
index d2544f1d705d..cb030374b90a 100644
--- a/arch/x86/include/asm/amd_iommu_proto.h
+++ b/arch/x86/include/asm/amd_iommu_proto.h
@@ -38,4 +38,10 @@ static inline void amd_iommu_stats_init(void) { }
38 38
39#endif /* !CONFIG_AMD_IOMMU_STATS */ 39#endif /* !CONFIG_AMD_IOMMU_STATS */
40 40
41static inline bool is_rd890_iommu(struct pci_dev *pdev)
42{
43 return (pdev->vendor == PCI_VENDOR_ID_ATI) &&
44 (pdev->device == PCI_DEVICE_ID_RD890_IOMMU);
45}
46
41#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */ 47#endif /* _ASM_X86_AMD_IOMMU_PROTO_H */
diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h
index 86a0ff0aeac7..08616180deaf 100644
--- a/arch/x86/include/asm/amd_iommu_types.h
+++ b/arch/x86/include/asm/amd_iommu_types.h
@@ -174,6 +174,40 @@
174 (~((1ULL << (12 + ((lvl) * 9))) - 1))) 174 (~((1ULL << (12 + ((lvl) * 9))) - 1)))
175#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr)) 175#define PM_ALIGNED(lvl, addr) ((PM_MAP_MASK(lvl) & (addr)) == (addr))
176 176
177/*
178 * Returns the page table level to use for a given page size
179 * Pagesize is expected to be a power-of-two
180 */
181#define PAGE_SIZE_LEVEL(pagesize) \
182 ((__ffs(pagesize) - 12) / 9)
183/*
184 * Returns the number of ptes to use for a given page size
185 * Pagesize is expected to be a power-of-two
186 */
187#define PAGE_SIZE_PTE_COUNT(pagesize) \
188 (1ULL << ((__ffs(pagesize) - 12) % 9))
189
190/*
191 * Aligns a given io-virtual address to a given page size
192 * Pagesize is expected to be a power-of-two
193 */
194#define PAGE_SIZE_ALIGN(address, pagesize) \
195 ((address) & ~((pagesize) - 1))
196/*
197 * Creates an IOMMU PTE for an address an a given pagesize
198 * The PTE has no permission bits set
199 * Pagesize is expected to be a power-of-two larger than 4096
200 */
201#define PAGE_SIZE_PTE(address, pagesize) \
202 (((address) | ((pagesize) - 1)) & \
203 (~(pagesize >> 1)) & PM_ADDR_MASK)
204
205/*
206 * Takes a PTE value with mode=0x07 and returns the page size it maps
207 */
208#define PTE_PAGE_SIZE(pte) \
209 (1ULL << (1 + ffz(((pte) | 0xfffULL))))
210
177#define IOMMU_PTE_P (1ULL << 0) 211#define IOMMU_PTE_P (1ULL << 0)
178#define IOMMU_PTE_TV (1ULL << 1) 212#define IOMMU_PTE_TV (1ULL << 1)
179#define IOMMU_PTE_U (1ULL << 59) 213#define IOMMU_PTE_U (1ULL << 59)
@@ -334,6 +368,9 @@ struct amd_iommu {
334 /* capabilities of that IOMMU read from ACPI */ 368 /* capabilities of that IOMMU read from ACPI */
335 u32 cap; 369 u32 cap;
336 370
371 /* flags read from acpi table */
372 u8 acpi_flags;
373
337 /* 374 /*
338 * Capability pointer. There could be more than one IOMMU per PCI 375 * Capability pointer. There could be more than one IOMMU per PCI
339 * device function if there are more than one AMD IOMMU capability 376 * device function if there are more than one AMD IOMMU capability
@@ -377,6 +414,15 @@ struct amd_iommu {
377 414
378 /* default dma_ops domain for that IOMMU */ 415 /* default dma_ops domain for that IOMMU */
379 struct dma_ops_domain *default_dom; 416 struct dma_ops_domain *default_dom;
417
418 /*
419 * This array is required to work around a potential BIOS bug.
420 * The BIOS may miss to restore parts of the PCI configuration
421 * space when the system resumes from S3. The result is that the
422 * IOMMU does not execute commands anymore which leads to system
423 * failure.
424 */
425 u32 cache_cfg[4];
380}; 426};
381 427
382/* 428/*
diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h
index c74a2eebe570..a69b1ac9eaf8 100644
--- a/arch/x86/include/asm/apb_timer.h
+++ b/arch/x86/include/asm/apb_timer.h
@@ -55,7 +55,6 @@ extern unsigned long apbt_quick_calibrate(void);
55extern int arch_setup_apbt_irqs(int irq, int trigger, int mask, int cpu); 55extern int arch_setup_apbt_irqs(int irq, int trigger, int mask, int cpu);
56extern void apbt_setup_secondary_clock(void); 56extern void apbt_setup_secondary_clock(void);
57extern unsigned int boot_cpu_id; 57extern unsigned int boot_cpu_id;
58extern int disable_apbt_percpu;
59 58
60extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint); 59extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint);
61extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr); 60extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr);
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index b4ac2cdcb64f..1fa03e04ae44 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -373,6 +373,7 @@ extern atomic_t init_deasserted;
373extern int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip); 373extern int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip);
374#endif 374#endif
375 375
376#ifdef CONFIG_X86_LOCAL_APIC
376static inline u32 apic_read(u32 reg) 377static inline u32 apic_read(u32 reg)
377{ 378{
378 return apic->read(reg); 379 return apic->read(reg);
@@ -403,10 +404,19 @@ static inline u32 safe_apic_wait_icr_idle(void)
403 return apic->safe_wait_icr_idle(); 404 return apic->safe_wait_icr_idle();
404} 405}
405 406
407#else /* CONFIG_X86_LOCAL_APIC */
408
409static inline u32 apic_read(u32 reg) { return 0; }
410static inline void apic_write(u32 reg, u32 val) { }
411static inline u64 apic_icr_read(void) { return 0; }
412static inline void apic_icr_write(u32 low, u32 high) { }
413static inline void apic_wait_icr_idle(void) { }
414static inline u32 safe_apic_wait_icr_idle(void) { return 0; }
415
416#endif /* CONFIG_X86_LOCAL_APIC */
406 417
407static inline void ack_APIC_irq(void) 418static inline void ack_APIC_irq(void)
408{ 419{
409#ifdef CONFIG_X86_LOCAL_APIC
410 /* 420 /*
411 * ack_APIC_irq() actually gets compiled as a single instruction 421 * ack_APIC_irq() actually gets compiled as a single instruction
412 * ... yummie. 422 * ... yummie.
@@ -414,7 +424,6 @@ static inline void ack_APIC_irq(void)
414 424
415 /* Docs say use 0 for future compatibility */ 425 /* Docs say use 0 for future compatibility */
416 apic_write(APIC_EOI, 0); 426 apic_write(APIC_EOI, 0);
417#endif
418} 427}
419 428
420static inline unsigned default_get_apic_id(unsigned long x) 429static inline unsigned default_get_apic_id(unsigned long x)
diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h
new file mode 100644
index 000000000000..9686c3d9ff73
--- /dev/null
+++ b/arch/x86/include/asm/arch_hweight.h
@@ -0,0 +1,61 @@
1#ifndef _ASM_X86_HWEIGHT_H
2#define _ASM_X86_HWEIGHT_H
3
4#ifdef CONFIG_64BIT
5/* popcnt %edi, %eax -- redundant REX prefix for alignment */
6#define POPCNT32 ".byte 0xf3,0x40,0x0f,0xb8,0xc7"
7/* popcnt %rdi, %rax */
8#define POPCNT64 ".byte 0xf3,0x48,0x0f,0xb8,0xc7"
9#define REG_IN "D"
10#define REG_OUT "a"
11#else
12/* popcnt %eax, %eax */
13#define POPCNT32 ".byte 0xf3,0x0f,0xb8,0xc0"
14#define REG_IN "a"
15#define REG_OUT "a"
16#endif
17
18/*
19 * __sw_hweightXX are called from within the alternatives below
20 * and callee-clobbered registers need to be taken care of. See
21 * ARCH_HWEIGHT_CFLAGS in <arch/x86/Kconfig> for the respective
22 * compiler switches.
23 */
24static inline unsigned int __arch_hweight32(unsigned int w)
25{
26 unsigned int res = 0;
27
28 asm (ALTERNATIVE("call __sw_hweight32", POPCNT32, X86_FEATURE_POPCNT)
29 : "="REG_OUT (res)
30 : REG_IN (w));
31
32 return res;
33}
34
35static inline unsigned int __arch_hweight16(unsigned int w)
36{
37 return __arch_hweight32(w & 0xffff);
38}
39
40static inline unsigned int __arch_hweight8(unsigned int w)
41{
42 return __arch_hweight32(w & 0xff);
43}
44
45static inline unsigned long __arch_hweight64(__u64 w)
46{
47 unsigned long res = 0;
48
49#ifdef CONFIG_X86_32
50 return __arch_hweight32((u32)w) +
51 __arch_hweight32((u32)(w >> 32));
52#else
53 asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT)
54 : "="REG_OUT (res)
55 : REG_IN (w));
56#endif /* CONFIG_X86_32 */
57
58 return res;
59}
60
61#endif
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 8f8217b9bdac..952a826ac4e5 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -22,7 +22,7 @@
22 */ 22 */
23static inline int atomic_read(const atomic_t *v) 23static inline int atomic_read(const atomic_t *v)
24{ 24{
25 return v->counter; 25 return (*(volatile int *)&(v)->counter);
26} 26}
27 27
28/** 28/**
@@ -246,6 +246,29 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
246 246
247#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 247#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
248 248
249/*
250 * atomic_dec_if_positive - decrement by 1 if old value positive
251 * @v: pointer of type atomic_t
252 *
253 * The function returns the old value of *v minus 1, even if
254 * the atomic variable, v, was not decremented.
255 */
256static inline int atomic_dec_if_positive(atomic_t *v)
257{
258 int c, old, dec;
259 c = atomic_read(v);
260 for (;;) {
261 dec = c - 1;
262 if (unlikely(dec < 0))
263 break;
264 old = atomic_cmpxchg((v), c, dec);
265 if (likely(old == c))
266 break;
267 c = old;
268 }
269 return dec;
270}
271
249/** 272/**
250 * atomic_inc_short - increment of a short integer 273 * atomic_inc_short - increment of a short integer
251 * @v: pointer to type int 274 * @v: pointer to type int
diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h
index 03027bf28de5..2a934aa19a43 100644
--- a/arch/x86/include/asm/atomic64_32.h
+++ b/arch/x86/include/asm/atomic64_32.h
@@ -14,109 +14,193 @@ typedef struct {
14 14
15#define ATOMIC64_INIT(val) { (val) } 15#define ATOMIC64_INIT(val) { (val) }
16 16
17extern u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val); 17#ifdef CONFIG_X86_CMPXCHG64
18#define ATOMIC64_ALTERNATIVE_(f, g) "call atomic64_" #g "_cx8"
19#else
20#define ATOMIC64_ALTERNATIVE_(f, g) ALTERNATIVE("call atomic64_" #f "_386", "call atomic64_" #g "_cx8", X86_FEATURE_CX8)
21#endif
22
23#define ATOMIC64_ALTERNATIVE(f) ATOMIC64_ALTERNATIVE_(f, f)
24
25/**
26 * atomic64_cmpxchg - cmpxchg atomic64 variable
27 * @p: pointer to type atomic64_t
28 * @o: expected value
29 * @n: new value
30 *
31 * Atomically sets @v to @n if it was equal to @o and returns
32 * the old value.
33 */
34
35static inline long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
36{
37 return cmpxchg64(&v->counter, o, n);
38}
18 39
19/** 40/**
20 * atomic64_xchg - xchg atomic64 variable 41 * atomic64_xchg - xchg atomic64 variable
21 * @ptr: pointer to type atomic64_t 42 * @v: pointer to type atomic64_t
22 * @new_val: value to assign 43 * @n: value to assign
23 * 44 *
24 * Atomically xchgs the value of @ptr to @new_val and returns 45 * Atomically xchgs the value of @v to @n and returns
25 * the old value. 46 * the old value.
26 */ 47 */
27extern u64 atomic64_xchg(atomic64_t *ptr, u64 new_val); 48static inline long long atomic64_xchg(atomic64_t *v, long long n)
49{
50 long long o;
51 unsigned high = (unsigned)(n >> 32);
52 unsigned low = (unsigned)n;
53 asm volatile(ATOMIC64_ALTERNATIVE(xchg)
54 : "=A" (o), "+b" (low), "+c" (high)
55 : "S" (v)
56 : "memory"
57 );
58 return o;
59}
28 60
29/** 61/**
30 * atomic64_set - set atomic64 variable 62 * atomic64_set - set atomic64 variable
31 * @ptr: pointer to type atomic64_t 63 * @v: pointer to type atomic64_t
32 * @new_val: value to assign 64 * @n: value to assign
33 * 65 *
34 * Atomically sets the value of @ptr to @new_val. 66 * Atomically sets the value of @v to @n.
35 */ 67 */
36extern void atomic64_set(atomic64_t *ptr, u64 new_val); 68static inline void atomic64_set(atomic64_t *v, long long i)
69{
70 unsigned high = (unsigned)(i >> 32);
71 unsigned low = (unsigned)i;
72 asm volatile(ATOMIC64_ALTERNATIVE(set)
73 : "+b" (low), "+c" (high)
74 : "S" (v)
75 : "eax", "edx", "memory"
76 );
77}
37 78
38/** 79/**
39 * atomic64_read - read atomic64 variable 80 * atomic64_read - read atomic64 variable
40 * @ptr: pointer to type atomic64_t 81 * @v: pointer to type atomic64_t
41 * 82 *
42 * Atomically reads the value of @ptr and returns it. 83 * Atomically reads the value of @v and returns it.
43 */ 84 */
44static inline u64 atomic64_read(atomic64_t *ptr) 85static inline long long atomic64_read(atomic64_t *v)
45{ 86{
46 u64 res; 87 long long r;
47 88 asm volatile(ATOMIC64_ALTERNATIVE(read)
48 /* 89 : "=A" (r), "+c" (v)
49 * Note, we inline this atomic64_t primitive because 90 : : "memory"
50 * it only clobbers EAX/EDX and leaves the others 91 );
51 * untouched. We also (somewhat subtly) rely on the 92 return r;
52 * fact that cmpxchg8b returns the current 64-bit value 93 }
53 * of the memory location we are touching:
54 */
55 asm volatile(
56 "mov %%ebx, %%eax\n\t"
57 "mov %%ecx, %%edx\n\t"
58 LOCK_PREFIX "cmpxchg8b %1\n"
59 : "=&A" (res)
60 : "m" (*ptr)
61 );
62
63 return res;
64}
65
66extern u64 atomic64_read(atomic64_t *ptr);
67 94
68/** 95/**
69 * atomic64_add_return - add and return 96 * atomic64_add_return - add and return
70 * @delta: integer value to add 97 * @i: integer value to add
71 * @ptr: pointer to type atomic64_t 98 * @v: pointer to type atomic64_t
72 * 99 *
73 * Atomically adds @delta to @ptr and returns @delta + *@ptr 100 * Atomically adds @i to @v and returns @i + *@v
74 */ 101 */
75extern u64 atomic64_add_return(u64 delta, atomic64_t *ptr); 102static inline long long atomic64_add_return(long long i, atomic64_t *v)
103{
104 asm volatile(ATOMIC64_ALTERNATIVE(add_return)
105 : "+A" (i), "+c" (v)
106 : : "memory"
107 );
108 return i;
109}
76 110
77/* 111/*
78 * Other variants with different arithmetic operators: 112 * Other variants with different arithmetic operators:
79 */ 113 */
80extern u64 atomic64_sub_return(u64 delta, atomic64_t *ptr); 114static inline long long atomic64_sub_return(long long i, atomic64_t *v)
81extern u64 atomic64_inc_return(atomic64_t *ptr); 115{
82extern u64 atomic64_dec_return(atomic64_t *ptr); 116 asm volatile(ATOMIC64_ALTERNATIVE(sub_return)
117 : "+A" (i), "+c" (v)
118 : : "memory"
119 );
120 return i;
121}
122
123static inline long long atomic64_inc_return(atomic64_t *v)
124{
125 long long a;
126 asm volatile(ATOMIC64_ALTERNATIVE(inc_return)
127 : "=A" (a)
128 : "S" (v)
129 : "memory", "ecx"
130 );
131 return a;
132}
133
134static inline long long atomic64_dec_return(atomic64_t *v)
135{
136 long long a;
137 asm volatile(ATOMIC64_ALTERNATIVE(dec_return)
138 : "=A" (a)
139 : "S" (v)
140 : "memory", "ecx"
141 );
142 return a;
143}
83 144
84/** 145/**
85 * atomic64_add - add integer to atomic64 variable 146 * atomic64_add - add integer to atomic64 variable
86 * @delta: integer value to add 147 * @i: integer value to add
87 * @ptr: pointer to type atomic64_t 148 * @v: pointer to type atomic64_t
88 * 149 *
89 * Atomically adds @delta to @ptr. 150 * Atomically adds @i to @v.
90 */ 151 */
91extern void atomic64_add(u64 delta, atomic64_t *ptr); 152static inline long long atomic64_add(long long i, atomic64_t *v)
153{
154 asm volatile(ATOMIC64_ALTERNATIVE_(add, add_return)
155 : "+A" (i), "+c" (v)
156 : : "memory"
157 );
158 return i;
159}
92 160
93/** 161/**
94 * atomic64_sub - subtract the atomic64 variable 162 * atomic64_sub - subtract the atomic64 variable
95 * @delta: integer value to subtract 163 * @i: integer value to subtract
96 * @ptr: pointer to type atomic64_t 164 * @v: pointer to type atomic64_t
97 * 165 *
98 * Atomically subtracts @delta from @ptr. 166 * Atomically subtracts @i from @v.
99 */ 167 */
100extern void atomic64_sub(u64 delta, atomic64_t *ptr); 168static inline long long atomic64_sub(long long i, atomic64_t *v)
169{
170 asm volatile(ATOMIC64_ALTERNATIVE_(sub, sub_return)
171 : "+A" (i), "+c" (v)
172 : : "memory"
173 );
174 return i;
175}
101 176
102/** 177/**
103 * atomic64_sub_and_test - subtract value from variable and test result 178 * atomic64_sub_and_test - subtract value from variable and test result
104 * @delta: integer value to subtract 179 * @i: integer value to subtract
105 * @ptr: pointer to type atomic64_t 180 * @v: pointer to type atomic64_t
106 * 181 *
107 * Atomically subtracts @delta from @ptr and returns 182 * Atomically subtracts @i from @v and returns
108 * true if the result is zero, or false for all 183 * true if the result is zero, or false for all
109 * other cases. 184 * other cases.
110 */ 185 */
111extern int atomic64_sub_and_test(u64 delta, atomic64_t *ptr); 186static inline int atomic64_sub_and_test(long long i, atomic64_t *v)
187{
188 return atomic64_sub_return(i, v) == 0;
189}
112 190
113/** 191/**
114 * atomic64_inc - increment atomic64 variable 192 * atomic64_inc - increment atomic64 variable
115 * @ptr: pointer to type atomic64_t 193 * @v: pointer to type atomic64_t
116 * 194 *
117 * Atomically increments @ptr by 1. 195 * Atomically increments @v by 1.
118 */ 196 */
119extern void atomic64_inc(atomic64_t *ptr); 197static inline void atomic64_inc(atomic64_t *v)
198{
199 asm volatile(ATOMIC64_ALTERNATIVE_(inc, inc_return)
200 : : "S" (v)
201 : "memory", "eax", "ecx", "edx"
202 );
203}
120 204
121/** 205/**
122 * atomic64_dec - decrement atomic64 variable 206 * atomic64_dec - decrement atomic64 variable
@@ -124,37 +208,97 @@ extern void atomic64_inc(atomic64_t *ptr);
124 * 208 *
125 * Atomically decrements @ptr by 1. 209 * Atomically decrements @ptr by 1.
126 */ 210 */
127extern void atomic64_dec(atomic64_t *ptr); 211static inline void atomic64_dec(atomic64_t *v)
212{
213 asm volatile(ATOMIC64_ALTERNATIVE_(dec, dec_return)
214 : : "S" (v)
215 : "memory", "eax", "ecx", "edx"
216 );
217}
128 218
129/** 219/**
130 * atomic64_dec_and_test - decrement and test 220 * atomic64_dec_and_test - decrement and test
131 * @ptr: pointer to type atomic64_t 221 * @v: pointer to type atomic64_t
132 * 222 *
133 * Atomically decrements @ptr by 1 and 223 * Atomically decrements @v by 1 and
134 * returns true if the result is 0, or false for all other 224 * returns true if the result is 0, or false for all other
135 * cases. 225 * cases.
136 */ 226 */
137extern int atomic64_dec_and_test(atomic64_t *ptr); 227static inline int atomic64_dec_and_test(atomic64_t *v)
228{
229 return atomic64_dec_return(v) == 0;
230}
138 231
139/** 232/**
140 * atomic64_inc_and_test - increment and test 233 * atomic64_inc_and_test - increment and test
141 * @ptr: pointer to type atomic64_t 234 * @v: pointer to type atomic64_t
142 * 235 *
143 * Atomically increments @ptr by 1 236 * Atomically increments @v by 1
144 * and returns true if the result is zero, or false for all 237 * and returns true if the result is zero, or false for all
145 * other cases. 238 * other cases.
146 */ 239 */
147extern int atomic64_inc_and_test(atomic64_t *ptr); 240static inline int atomic64_inc_and_test(atomic64_t *v)
241{
242 return atomic64_inc_return(v) == 0;
243}
148 244
149/** 245/**
150 * atomic64_add_negative - add and test if negative 246 * atomic64_add_negative - add and test if negative
151 * @delta: integer value to add 247 * @i: integer value to add
152 * @ptr: pointer to type atomic64_t 248 * @v: pointer to type atomic64_t
153 * 249 *
154 * Atomically adds @delta to @ptr and returns true 250 * Atomically adds @i to @v and returns true
155 * if the result is negative, or false when 251 * if the result is negative, or false when
156 * result is greater than or equal to zero. 252 * result is greater than or equal to zero.
157 */ 253 */
158extern int atomic64_add_negative(u64 delta, atomic64_t *ptr); 254static inline int atomic64_add_negative(long long i, atomic64_t *v)
255{
256 return atomic64_add_return(i, v) < 0;
257}
258
259/**
260 * atomic64_add_unless - add unless the number is a given value
261 * @v: pointer of type atomic64_t
262 * @a: the amount to add to v...
263 * @u: ...unless v is equal to u.
264 *
265 * Atomically adds @a to @v, so long as it was not @u.
266 * Returns non-zero if @v was not @u, and zero otherwise.
267 */
268static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u)
269{
270 unsigned low = (unsigned)u;
271 unsigned high = (unsigned)(u >> 32);
272 asm volatile(ATOMIC64_ALTERNATIVE(add_unless) "\n\t"
273 : "+A" (a), "+c" (v), "+S" (low), "+D" (high)
274 : : "memory");
275 return (int)a;
276}
277
278
279static inline int atomic64_inc_not_zero(atomic64_t *v)
280{
281 int r;
282 asm volatile(ATOMIC64_ALTERNATIVE(inc_not_zero)
283 : "=a" (r)
284 : "S" (v)
285 : "ecx", "edx", "memory"
286 );
287 return r;
288}
289
290static inline long long atomic64_dec_if_positive(atomic64_t *v)
291{
292 long long r;
293 asm volatile(ATOMIC64_ALTERNATIVE(dec_if_positive)
294 : "=A" (r)
295 : "S" (v)
296 : "ecx", "memory"
297 );
298 return r;
299}
300
301#undef ATOMIC64_ALTERNATIVE
302#undef ATOMIC64_ALTERNATIVE_
159 303
160#endif /* _ASM_X86_ATOMIC64_32_H */ 304#endif /* _ASM_X86_ATOMIC64_32_H */
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 51c5b4056929..49fd1ea22951 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -18,7 +18,7 @@
18 */ 18 */
19static inline long atomic64_read(const atomic64_t *v) 19static inline long atomic64_read(const atomic64_t *v)
20{ 20{
21 return v->counter; 21 return (*(volatile long *)&(v)->counter);
22} 22}
23 23
24/** 24/**
@@ -221,4 +221,27 @@ static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
221 221
222#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 222#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
223 223
224/*
225 * atomic64_dec_if_positive - decrement by 1 if old value positive
226 * @v: pointer of type atomic_t
227 *
228 * The function returns the old value of *v minus 1, even if
229 * the atomic variable, v, was not decremented.
230 */
231static inline long atomic64_dec_if_positive(atomic64_t *v)
232{
233 long c, old, dec;
234 c = atomic64_read(v);
235 for (;;) {
236 dec = c - 1;
237 if (unlikely(dec < 0))
238 break;
239 old = atomic64_cmpxchg((v), c, dec);
240 if (likely(old == c))
241 break;
242 c = old;
243 }
244 return dec;
245}
246
224#endif /* _ASM_X86_ATOMIC64_64_H */ 247#endif /* _ASM_X86_ATOMIC64_64_H */
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 02b47a603fc8..bafd80defa43 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -309,7 +309,7 @@ static inline int test_and_change_bit(int nr, volatile unsigned long *addr)
309static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr) 309static __always_inline int constant_test_bit(unsigned int nr, const volatile unsigned long *addr)
310{ 310{
311 return ((1UL << (nr % BITS_PER_LONG)) & 311 return ((1UL << (nr % BITS_PER_LONG)) &
312 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0; 312 (addr[nr / BITS_PER_LONG])) != 0;
313} 313}
314 314
315static inline int variable_test_bit(int nr, volatile const unsigned long *addr) 315static inline int variable_test_bit(int nr, volatile const unsigned long *addr)
@@ -444,7 +444,9 @@ static inline int fls(int x)
444 444
445#define ARCH_HAS_FAST_MULTIPLIER 1 445#define ARCH_HAS_FAST_MULTIPLIER 1
446 446
447#include <asm-generic/bitops/hweight.h> 447#include <asm/arch_hweight.h>
448
449#include <asm-generic/bitops/const_hweight.h>
448 450
449#endif /* __KERNEL__ */ 451#endif /* __KERNEL__ */
450 452
diff --git a/arch/x86/include/asm/boot.h b/arch/x86/include/asm/boot.h
index 7a1065958ba9..3b62ab56c7a0 100644
--- a/arch/x86/include/asm/boot.h
+++ b/arch/x86/include/asm/boot.h
@@ -24,7 +24,7 @@
24#define MIN_KERNEL_ALIGN (_AC(1, UL) << MIN_KERNEL_ALIGN_LG2) 24#define MIN_KERNEL_ALIGN (_AC(1, UL) << MIN_KERNEL_ALIGN_LG2)
25 25
26#if (CONFIG_PHYSICAL_ALIGN & (CONFIG_PHYSICAL_ALIGN-1)) || \ 26#if (CONFIG_PHYSICAL_ALIGN & (CONFIG_PHYSICAL_ALIGN-1)) || \
27 (CONFIG_PHYSICAL_ALIGN < (_AC(1, UL) << MIN_KERNEL_ALIGN_LG2)) 27 (CONFIG_PHYSICAL_ALIGN < MIN_KERNEL_ALIGN)
28#error "Invalid value for CONFIG_PHYSICAL_ALIGN" 28#error "Invalid value for CONFIG_PHYSICAL_ALIGN"
29#endif 29#endif
30 30
diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h
index 6be33d83c716..8e6218550e77 100644
--- a/arch/x86/include/asm/bootparam.h
+++ b/arch/x86/include/asm/bootparam.h
@@ -70,6 +70,14 @@ struct sys_desc_table {
70 __u8 table[14]; 70 __u8 table[14];
71}; 71};
72 72
73/* Gleaned from OFW's set-parameters in cpu/x86/pc/linux.fth */
74struct olpc_ofw_header {
75 __u32 ofw_magic; /* OFW signature */
76 __u32 ofw_version;
77 __u32 cif_handler; /* callback into OFW */
78 __u32 irq_desc_table;
79} __attribute__((packed));
80
73struct efi_info { 81struct efi_info {
74 __u32 efi_loader_signature; 82 __u32 efi_loader_signature;
75 __u32 efi_systab; 83 __u32 efi_systab;
@@ -92,7 +100,8 @@ struct boot_params {
92 __u8 hd0_info[16]; /* obsolete! */ /* 0x080 */ 100 __u8 hd0_info[16]; /* obsolete! */ /* 0x080 */
93 __u8 hd1_info[16]; /* obsolete! */ /* 0x090 */ 101 __u8 hd1_info[16]; /* obsolete! */ /* 0x090 */
94 struct sys_desc_table sys_desc_table; /* 0x0a0 */ 102 struct sys_desc_table sys_desc_table; /* 0x0a0 */
95 __u8 _pad4[144]; /* 0x0b0 */ 103 struct olpc_ofw_header olpc_ofw_header; /* 0x0b0 */
104 __u8 _pad4[128]; /* 0x0c0 */
96 struct edid_info edid_info; /* 0x140 */ 105 struct edid_info edid_info; /* 0x140 */
97 struct efi_info efi_info; /* 0x1c0 */ 106 struct efi_info efi_info; /* 0x1c0 */
98 __u32 alt_mem_k; /* 0x1e0 */ 107 __u32 alt_mem_k; /* 0x1e0 */
diff --git a/arch/x86/include/asm/cache.h b/arch/x86/include/asm/cache.h
index 2f9047cfaaca..48f99f15452e 100644
--- a/arch/x86/include/asm/cache.h
+++ b/arch/x86/include/asm/cache.h
@@ -7,7 +7,7 @@
7#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT) 7#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)
8#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) 8#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
9 9
10#define __read_mostly __attribute__((__section__(".data.read_mostly"))) 10#define __read_mostly __attribute__((__section__(".data..read_mostly")))
11 11
12#define INTERNODE_CACHE_SHIFT CONFIG_X86_INTERNODE_CACHE_SHIFT 12#define INTERNODE_CACHE_SHIFT CONFIG_X86_INTERNODE_CACHE_SHIFT
13#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT) 13#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT)
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h
index 634c40a739a6..63e35ec9075c 100644
--- a/arch/x86/include/asm/cacheflush.h
+++ b/arch/x86/include/asm/cacheflush.h
@@ -44,9 +44,6 @@ static inline void copy_from_user_page(struct vm_area_struct *vma,
44 memcpy(dst, src, len); 44 memcpy(dst, src, len);
45} 45}
46 46
47#define PG_WC PG_arch_1
48PAGEFLAG(WC, WC)
49
50#ifdef CONFIG_X86_PAT 47#ifdef CONFIG_X86_PAT
51/* 48/*
52 * X86 PAT uses page flags WC and Uncached together to keep track of 49 * X86 PAT uses page flags WC and Uncached together to keep track of
@@ -55,16 +52,24 @@ PAGEFLAG(WC, WC)
55 * _PAGE_CACHE_UC_MINUS and fourth state where page's memory type has not 52 * _PAGE_CACHE_UC_MINUS and fourth state where page's memory type has not
56 * been changed from its default (value of -1 used to denote this). 53 * been changed from its default (value of -1 used to denote this).
57 * Note we do not support _PAGE_CACHE_UC here. 54 * Note we do not support _PAGE_CACHE_UC here.
58 *
59 * Caller must hold memtype_lock for atomicity.
60 */ 55 */
56
57#define _PGMT_DEFAULT 0
58#define _PGMT_WC (1UL << PG_arch_1)
59#define _PGMT_UC_MINUS (1UL << PG_uncached)
60#define _PGMT_WB (1UL << PG_uncached | 1UL << PG_arch_1)
61#define _PGMT_MASK (1UL << PG_uncached | 1UL << PG_arch_1)
62#define _PGMT_CLEAR_MASK (~_PGMT_MASK)
63
61static inline unsigned long get_page_memtype(struct page *pg) 64static inline unsigned long get_page_memtype(struct page *pg)
62{ 65{
63 if (!PageUncached(pg) && !PageWC(pg)) 66 unsigned long pg_flags = pg->flags & _PGMT_MASK;
67
68 if (pg_flags == _PGMT_DEFAULT)
64 return -1; 69 return -1;
65 else if (!PageUncached(pg) && PageWC(pg)) 70 else if (pg_flags == _PGMT_WC)
66 return _PAGE_CACHE_WC; 71 return _PAGE_CACHE_WC;
67 else if (PageUncached(pg) && !PageWC(pg)) 72 else if (pg_flags == _PGMT_UC_MINUS)
68 return _PAGE_CACHE_UC_MINUS; 73 return _PAGE_CACHE_UC_MINUS;
69 else 74 else
70 return _PAGE_CACHE_WB; 75 return _PAGE_CACHE_WB;
@@ -72,25 +77,26 @@ static inline unsigned long get_page_memtype(struct page *pg)
72 77
73static inline void set_page_memtype(struct page *pg, unsigned long memtype) 78static inline void set_page_memtype(struct page *pg, unsigned long memtype)
74{ 79{
80 unsigned long memtype_flags = _PGMT_DEFAULT;
81 unsigned long old_flags;
82 unsigned long new_flags;
83
75 switch (memtype) { 84 switch (memtype) {
76 case _PAGE_CACHE_WC: 85 case _PAGE_CACHE_WC:
77 ClearPageUncached(pg); 86 memtype_flags = _PGMT_WC;
78 SetPageWC(pg);
79 break; 87 break;
80 case _PAGE_CACHE_UC_MINUS: 88 case _PAGE_CACHE_UC_MINUS:
81 SetPageUncached(pg); 89 memtype_flags = _PGMT_UC_MINUS;
82 ClearPageWC(pg);
83 break; 90 break;
84 case _PAGE_CACHE_WB: 91 case _PAGE_CACHE_WB:
85 SetPageUncached(pg); 92 memtype_flags = _PGMT_WB;
86 SetPageWC(pg);
87 break;
88 default:
89 case -1:
90 ClearPageUncached(pg);
91 ClearPageWC(pg);
92 break; 93 break;
93 } 94 }
95
96 do {
97 old_flags = pg->flags;
98 new_flags = (old_flags & _PGMT_CLEAR_MASK) | memtype_flags;
99 } while (cmpxchg(&pg->flags, old_flags, new_flags) != old_flags);
94} 100}
95#else 101#else
96static inline unsigned long get_page_memtype(struct page *pg) { return -1; } 102static inline unsigned long get_page_memtype(struct page *pg) { return -1; }
@@ -139,9 +145,11 @@ int set_memory_np(unsigned long addr, int numpages);
139int set_memory_4k(unsigned long addr, int numpages); 145int set_memory_4k(unsigned long addr, int numpages);
140 146
141int set_memory_array_uc(unsigned long *addr, int addrinarray); 147int set_memory_array_uc(unsigned long *addr, int addrinarray);
148int set_memory_array_wc(unsigned long *addr, int addrinarray);
142int set_memory_array_wb(unsigned long *addr, int addrinarray); 149int set_memory_array_wb(unsigned long *addr, int addrinarray);
143 150
144int set_pages_array_uc(struct page **pages, int addrinarray); 151int set_pages_array_uc(struct page **pages, int addrinarray);
152int set_pages_array_wc(struct page **pages, int addrinarray);
145int set_pages_array_wb(struct page **pages, int addrinarray); 153int set_pages_array_wb(struct page **pages, int addrinarray);
146 154
147/* 155/*
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index ffb9bb6b6c37..284a6e8f7ce1 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -11,38 +11,42 @@
11extern void __xchg_wrong_size(void); 11extern void __xchg_wrong_size(void);
12 12
13/* 13/*
14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway 14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
15 * Note 2: xchg has side effect, so that attribute volatile is necessary, 15 * Since this is generally used to protect other memory information, we
16 * but generally the primitive is invalid, *ptr is output argument. --ANK 16 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
17 * information around.
17 */ 18 */
18
19struct __xchg_dummy {
20 unsigned long a[100];
21};
22#define __xg(x) ((struct __xchg_dummy *)(x))
23
24#define __xchg(x, ptr, size) \ 19#define __xchg(x, ptr, size) \
25({ \ 20({ \
26 __typeof(*(ptr)) __x = (x); \ 21 __typeof(*(ptr)) __x = (x); \
27 switch (size) { \ 22 switch (size) { \
28 case 1: \ 23 case 1: \
29 asm volatile("xchgb %b0,%1" \ 24 { \
30 : "=q" (__x) \ 25 volatile u8 *__ptr = (volatile u8 *)(ptr); \
31 : "m" (*__xg(ptr)), "0" (__x) \ 26 asm volatile("xchgb %0,%1" \
27 : "=q" (__x), "+m" (*__ptr) \
28 : "0" (__x) \
32 : "memory"); \ 29 : "memory"); \
33 break; \ 30 break; \
31 } \
34 case 2: \ 32 case 2: \
35 asm volatile("xchgw %w0,%1" \ 33 { \
36 : "=r" (__x) \ 34 volatile u16 *__ptr = (volatile u16 *)(ptr); \
37 : "m" (*__xg(ptr)), "0" (__x) \ 35 asm volatile("xchgw %0,%1" \
36 : "=r" (__x), "+m" (*__ptr) \
37 : "0" (__x) \
38 : "memory"); \ 38 : "memory"); \
39 break; \ 39 break; \
40 } \
40 case 4: \ 41 case 4: \
42 { \
43 volatile u32 *__ptr = (volatile u32 *)(ptr); \
41 asm volatile("xchgl %0,%1" \ 44 asm volatile("xchgl %0,%1" \
42 : "=r" (__x) \ 45 : "=r" (__x), "+m" (*__ptr) \
43 : "m" (*__xg(ptr)), "0" (__x) \ 46 : "0" (__x) \
44 : "memory"); \ 47 : "memory"); \
45 break; \ 48 break; \
49 } \
46 default: \ 50 default: \
47 __xchg_wrong_size(); \ 51 __xchg_wrong_size(); \
48 } \ 52 } \
@@ -53,60 +57,33 @@ struct __xchg_dummy {
53 __xchg((v), (ptr), sizeof(*ptr)) 57 __xchg((v), (ptr), sizeof(*ptr))
54 58
55/* 59/*
56 * The semantics of XCHGCMP8B are a bit strange, this is why 60 * CMPXCHG8B only writes to the target if we had the previous
57 * there is a loop and the loading of %%eax and %%edx has to 61 * value in registers, otherwise it acts as a read and gives us the
58 * be inside. This inlines well in most cases, the cached 62 * "new previous" value. That is why there is a loop. Preloading
59 * cost is around ~38 cycles. (in the future we might want 63 * EDX:EAX is a performance optimization: in the common case it means
60 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that 64 * we need only one locked operation.
61 * might have an implicit FPU-save as a cost, so it's not 65 *
62 * clear which path to go.) 66 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very
67 * least an FPU save and/or %cr0.ts manipulation.
63 * 68 *
64 * cmpxchg8b must be used with the lock prefix here to allow 69 * cmpxchg8b must be used with the lock prefix here to allow the
65 * the instruction to be executed atomically, see page 3-102 70 * instruction to be executed atomically. We need to have the reader
66 * of the instruction set reference 24319102.pdf. We need 71 * side to see the coherent 64bit value.
67 * the reader side to see the coherent 64bit value.
68 */ 72 */
69static inline void __set_64bit(unsigned long long *ptr, 73static inline void set_64bit(volatile u64 *ptr, u64 value)
70 unsigned int low, unsigned int high)
71{ 74{
75 u32 low = value;
76 u32 high = value >> 32;
77 u64 prev = *ptr;
78
72 asm volatile("\n1:\t" 79 asm volatile("\n1:\t"
73 "movl (%0), %%eax\n\t" 80 LOCK_PREFIX "cmpxchg8b %0\n\t"
74 "movl 4(%0), %%edx\n\t"
75 LOCK_PREFIX "cmpxchg8b (%0)\n\t"
76 "jnz 1b" 81 "jnz 1b"
77 : /* no outputs */ 82 : "=m" (*ptr), "+A" (prev)
78 : "D"(ptr), 83 : "b" (low), "c" (high)
79 "b"(low), 84 : "memory");
80 "c"(high)
81 : "ax", "dx", "memory");
82}
83
84static inline void __set_64bit_constant(unsigned long long *ptr,
85 unsigned long long value)
86{
87 __set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32));
88}
89
90#define ll_low(x) *(((unsigned int *)&(x)) + 0)
91#define ll_high(x) *(((unsigned int *)&(x)) + 1)
92
93static inline void __set_64bit_var(unsigned long long *ptr,
94 unsigned long long value)
95{
96 __set_64bit(ptr, ll_low(value), ll_high(value));
97} 85}
98 86
99#define set_64bit(ptr, value) \
100 (__builtin_constant_p((value)) \
101 ? __set_64bit_constant((ptr), (value)) \
102 : __set_64bit_var((ptr), (value)))
103
104#define _set_64bit(ptr, value) \
105 (__builtin_constant_p(value) \
106 ? __set_64bit(ptr, (unsigned int)(value), \
107 (unsigned int)((value) >> 32)) \
108 : __set_64bit(ptr, ll_low((value)), ll_high((value))))
109
110extern void __cmpxchg_wrong_size(void); 87extern void __cmpxchg_wrong_size(void);
111 88
112/* 89/*
@@ -121,23 +98,32 @@ extern void __cmpxchg_wrong_size(void);
121 __typeof__(*(ptr)) __new = (new); \ 98 __typeof__(*(ptr)) __new = (new); \
122 switch (size) { \ 99 switch (size) { \
123 case 1: \ 100 case 1: \
124 asm volatile(lock "cmpxchgb %b1,%2" \ 101 { \
125 : "=a"(__ret) \ 102 volatile u8 *__ptr = (volatile u8 *)(ptr); \
126 : "q"(__new), "m"(*__xg(ptr)), "0"(__old) \ 103 asm volatile(lock "cmpxchgb %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "q" (__new), "0" (__old) \
127 : "memory"); \ 106 : "memory"); \
128 break; \ 107 break; \
108 } \
129 case 2: \ 109 case 2: \
130 asm volatile(lock "cmpxchgw %w1,%2" \ 110 { \
131 : "=a"(__ret) \ 111 volatile u16 *__ptr = (volatile u16 *)(ptr); \
132 : "r"(__new), "m"(*__xg(ptr)), "0"(__old) \ 112 asm volatile(lock "cmpxchgw %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
133 : "memory"); \ 115 : "memory"); \
134 break; \ 116 break; \
117 } \
135 case 4: \ 118 case 4: \
136 asm volatile(lock "cmpxchgl %1,%2" \ 119 { \
137 : "=a"(__ret) \ 120 volatile u32 *__ptr = (volatile u32 *)(ptr); \
138 : "r"(__new), "m"(*__xg(ptr)), "0"(__old) \ 121 asm volatile(lock "cmpxchgl %2,%1" \
122 : "=a" (__ret), "+m" (*__ptr) \
123 : "r" (__new), "0" (__old) \
139 : "memory"); \ 124 : "memory"); \
140 break; \ 125 break; \
126 } \
141 default: \ 127 default: \
142 __cmpxchg_wrong_size(); \ 128 __cmpxchg_wrong_size(); \
143 } \ 129 } \
@@ -175,32 +161,28 @@ extern void __cmpxchg_wrong_size(void);
175 (unsigned long long)(n))) 161 (unsigned long long)(n)))
176#endif 162#endif
177 163
178static inline unsigned long long __cmpxchg64(volatile void *ptr, 164static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
179 unsigned long long old,
180 unsigned long long new)
181{ 165{
182 unsigned long long prev; 166 u64 prev;
183 asm volatile(LOCK_PREFIX "cmpxchg8b %3" 167 asm volatile(LOCK_PREFIX "cmpxchg8b %1"
184 : "=A"(prev) 168 : "=A" (prev),
185 : "b"((unsigned long)new), 169 "+m" (*ptr)
186 "c"((unsigned long)(new >> 32)), 170 : "b" ((u32)new),
187 "m"(*__xg(ptr)), 171 "c" ((u32)(new >> 32)),
188 "0"(old) 172 "0" (old)
189 : "memory"); 173 : "memory");
190 return prev; 174 return prev;
191} 175}
192 176
193static inline unsigned long long __cmpxchg64_local(volatile void *ptr, 177static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
194 unsigned long long old,
195 unsigned long long new)
196{ 178{
197 unsigned long long prev; 179 u64 prev;
198 asm volatile("cmpxchg8b %3" 180 asm volatile("cmpxchg8b %1"
199 : "=A"(prev) 181 : "=A" (prev),
200 : "b"((unsigned long)new), 182 "+m" (*ptr)
201 "c"((unsigned long)(new >> 32)), 183 : "b" ((u32)new),
202 "m"(*__xg(ptr)), 184 "c" ((u32)(new >> 32)),
203 "0"(old) 185 "0" (old)
204 : "memory"); 186 : "memory");
205 return prev; 187 return prev;
206} 188}
@@ -264,14 +246,13 @@ static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
264 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 246 * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
265 */ 247 */
266 248
267extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
268
269#define cmpxchg64(ptr, o, n) \ 249#define cmpxchg64(ptr, o, n) \
270({ \ 250({ \
271 __typeof__(*(ptr)) __ret; \ 251 __typeof__(*(ptr)) __ret; \
272 __typeof__(*(ptr)) __old = (o); \ 252 __typeof__(*(ptr)) __old = (o); \
273 __typeof__(*(ptr)) __new = (n); \ 253 __typeof__(*(ptr)) __new = (n); \
274 alternative_io("call cmpxchg8b_emu", \ 254 alternative_io(LOCK_PREFIX_HERE \
255 "call cmpxchg8b_emu", \
275 "lock; cmpxchg8b (%%esi)" , \ 256 "lock; cmpxchg8b (%%esi)" , \
276 X86_FEATURE_CX8, \ 257 X86_FEATURE_CX8, \
277 "=A" (__ret), \ 258 "=A" (__ret), \
@@ -282,20 +263,20 @@ extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
282 __ret; }) 263 __ret; })
283 264
284 265
285 266#define cmpxchg64_local(ptr, o, n) \
286#define cmpxchg64_local(ptr, o, n) \ 267({ \
287({ \ 268 __typeof__(*(ptr)) __ret; \
288 __typeof__(*(ptr)) __ret; \ 269 __typeof__(*(ptr)) __old = (o); \
289 if (likely(boot_cpu_data.x86 > 4)) \ 270 __typeof__(*(ptr)) __new = (n); \
290 __ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr), \ 271 alternative_io("call cmpxchg8b_emu", \
291 (unsigned long long)(o), \ 272 "cmpxchg8b (%%esi)" , \
292 (unsigned long long)(n)); \ 273 X86_FEATURE_CX8, \
293 else \ 274 "=A" (__ret), \
294 __ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \ 275 "S" ((ptr)), "0" (__old), \
295 (unsigned long long)(o), \ 276 "b" ((unsigned int)__new), \
296 (unsigned long long)(n)); \ 277 "c" ((unsigned int)(__new>>32)) \
297 __ret; \ 278 : "memory"); \
298}) 279 __ret; })
299 280
300#endif 281#endif
301 282
diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h
index 485ae415faec..423ae58aa020 100644
--- a/arch/x86/include/asm/cmpxchg_64.h
+++ b/arch/x86/include/asm/cmpxchg_64.h
@@ -3,51 +3,60 @@
3 3
4#include <asm/alternative.h> /* Provides LOCK_PREFIX */ 4#include <asm/alternative.h> /* Provides LOCK_PREFIX */
5 5
6#define __xg(x) ((volatile long *)(x)) 6static inline void set_64bit(volatile u64 *ptr, u64 val)
7
8static inline void set_64bit(volatile unsigned long *ptr, unsigned long val)
9{ 7{
10 *ptr = val; 8 *ptr = val;
11} 9}
12 10
13#define _set_64bit set_64bit
14
15extern void __xchg_wrong_size(void); 11extern void __xchg_wrong_size(void);
16extern void __cmpxchg_wrong_size(void); 12extern void __cmpxchg_wrong_size(void);
17 13
18/* 14/*
19 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway 15 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
20 * Note 2: xchg has side effect, so that attribute volatile is necessary, 16 * Since this is generally used to protect other memory information, we
21 * but generally the primitive is invalid, *ptr is output argument. --ANK 17 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
18 * information around.
22 */ 19 */
23#define __xchg(x, ptr, size) \ 20#define __xchg(x, ptr, size) \
24({ \ 21({ \
25 __typeof(*(ptr)) __x = (x); \ 22 __typeof(*(ptr)) __x = (x); \
26 switch (size) { \ 23 switch (size) { \
27 case 1: \ 24 case 1: \
28 asm volatile("xchgb %b0,%1" \ 25 { \
29 : "=q" (__x) \ 26 volatile u8 *__ptr = (volatile u8 *)(ptr); \
30 : "m" (*__xg(ptr)), "0" (__x) \ 27 asm volatile("xchgb %0,%1" \
28 : "=q" (__x), "+m" (*__ptr) \
29 : "0" (__x) \
31 : "memory"); \ 30 : "memory"); \
32 break; \ 31 break; \
32 } \
33 case 2: \ 33 case 2: \
34 asm volatile("xchgw %w0,%1" \ 34 { \
35 : "=r" (__x) \ 35 volatile u16 *__ptr = (volatile u16 *)(ptr); \
36 : "m" (*__xg(ptr)), "0" (__x) \ 36 asm volatile("xchgw %0,%1" \
37 : "=r" (__x), "+m" (*__ptr) \
38 : "0" (__x) \
37 : "memory"); \ 39 : "memory"); \
38 break; \ 40 break; \
41 } \
39 case 4: \ 42 case 4: \
40 asm volatile("xchgl %k0,%1" \ 43 { \
41 : "=r" (__x) \ 44 volatile u32 *__ptr = (volatile u32 *)(ptr); \
42 : "m" (*__xg(ptr)), "0" (__x) \ 45 asm volatile("xchgl %0,%1" \
46 : "=r" (__x), "+m" (*__ptr) \
47 : "0" (__x) \
43 : "memory"); \ 48 : "memory"); \
44 break; \ 49 break; \
50 } \
45 case 8: \ 51 case 8: \
52 { \
53 volatile u64 *__ptr = (volatile u64 *)(ptr); \
46 asm volatile("xchgq %0,%1" \ 54 asm volatile("xchgq %0,%1" \
47 : "=r" (__x) \ 55 : "=r" (__x), "+m" (*__ptr) \
48 : "m" (*__xg(ptr)), "0" (__x) \ 56 : "0" (__x) \
49 : "memory"); \ 57 : "memory"); \
50 break; \ 58 break; \
59 } \
51 default: \ 60 default: \
52 __xchg_wrong_size(); \ 61 __xchg_wrong_size(); \
53 } \ 62 } \
@@ -71,29 +80,41 @@ extern void __cmpxchg_wrong_size(void);
71 __typeof__(*(ptr)) __new = (new); \ 80 __typeof__(*(ptr)) __new = (new); \
72 switch (size) { \ 81 switch (size) { \
73 case 1: \ 82 case 1: \
74 asm volatile(lock "cmpxchgb %b1,%2" \ 83 { \
75 : "=a"(__ret) \ 84 volatile u8 *__ptr = (volatile u8 *)(ptr); \
76 : "q"(__new), "m"(*__xg(ptr)), "0"(__old) \ 85 asm volatile(lock "cmpxchgb %2,%1" \
86 : "=a" (__ret), "+m" (*__ptr) \
87 : "q" (__new), "0" (__old) \
77 : "memory"); \ 88 : "memory"); \
78 break; \ 89 break; \
90 } \
79 case 2: \ 91 case 2: \
80 asm volatile(lock "cmpxchgw %w1,%2" \ 92 { \
81 : "=a"(__ret) \ 93 volatile u16 *__ptr = (volatile u16 *)(ptr); \
82 : "r"(__new), "m"(*__xg(ptr)), "0"(__old) \ 94 asm volatile(lock "cmpxchgw %2,%1" \
95 : "=a" (__ret), "+m" (*__ptr) \
96 : "r" (__new), "0" (__old) \
83 : "memory"); \ 97 : "memory"); \
84 break; \ 98 break; \
99 } \
85 case 4: \ 100 case 4: \
86 asm volatile(lock "cmpxchgl %k1,%2" \ 101 { \
87 : "=a"(__ret) \ 102 volatile u32 *__ptr = (volatile u32 *)(ptr); \
88 : "r"(__new), "m"(*__xg(ptr)), "0"(__old) \ 103 asm volatile(lock "cmpxchgl %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "r" (__new), "0" (__old) \
89 : "memory"); \ 106 : "memory"); \
90 break; \ 107 break; \
108 } \
91 case 8: \ 109 case 8: \
92 asm volatile(lock "cmpxchgq %1,%2" \ 110 { \
93 : "=a"(__ret) \ 111 volatile u64 *__ptr = (volatile u64 *)(ptr); \
94 : "r"(__new), "m"(*__xg(ptr)), "0"(__old) \ 112 asm volatile(lock "cmpxchgq %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
95 : "memory"); \ 115 : "memory"); \
96 break; \ 116 break; \
117 } \
97 default: \ 118 default: \
98 __cmpxchg_wrong_size(); \ 119 __cmpxchg_wrong_size(); \
99 } \ 120 } \
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index 306160e58b48..1d9cd27c2920 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -205,7 +205,7 @@ static inline compat_uptr_t ptr_to_compat(void __user *uptr)
205 return (u32)(unsigned long)uptr; 205 return (u32)(unsigned long)uptr;
206} 206}
207 207
208static inline void __user *compat_alloc_user_space(long len) 208static inline void __user *arch_compat_alloc_user_space(long len)
209{ 209{
210 struct pt_regs *regs = task_pt_regs(current); 210 struct pt_regs *regs = task_pt_regs(current);
211 return (void __user *)regs->sp - len; 211 return (void __user *)regs->sp - len;
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 0cd82d068613..3f76523589af 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -6,7 +6,7 @@
6 6
7#include <asm/required-features.h> 7#include <asm/required-features.h>
8 8
9#define NCAPINTS 9 /* N 32-bit words worth of info */ 9#define NCAPINTS 10 /* N 32-bit words worth of info */
10 10
11/* 11/*
12 * Note: If the comment begins with a quoted string, that string is used 12 * Note: If the comment begins with a quoted string, that string is used
@@ -89,7 +89,7 @@
89#define X86_FEATURE_LFENCE_RDTSC (3*32+18) /* "" Lfence synchronizes RDTSC */ 89#define X86_FEATURE_LFENCE_RDTSC (3*32+18) /* "" Lfence synchronizes RDTSC */
90#define X86_FEATURE_11AP (3*32+19) /* "" Bad local APIC aka 11AP */ 90#define X86_FEATURE_11AP (3*32+19) /* "" Bad local APIC aka 11AP */
91#define X86_FEATURE_NOPL (3*32+20) /* The NOPL (0F 1F) instructions */ 91#define X86_FEATURE_NOPL (3*32+20) /* The NOPL (0F 1F) instructions */
92#define X86_FEATURE_AMDC1E (3*32+21) /* AMD C1E detected */ 92 /* 21 available, was AMD_C1E */
93#define X86_FEATURE_XTOPOLOGY (3*32+22) /* cpu topology enum extensions */ 93#define X86_FEATURE_XTOPOLOGY (3*32+22) /* cpu topology enum extensions */
94#define X86_FEATURE_TSC_RELIABLE (3*32+23) /* TSC is known to be reliable */ 94#define X86_FEATURE_TSC_RELIABLE (3*32+23) /* TSC is known to be reliable */
95#define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */ 95#define X86_FEATURE_NONSTOP_TSC (3*32+24) /* TSC does not stop in C states */
@@ -124,6 +124,8 @@
124#define X86_FEATURE_XSAVE (4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */ 124#define X86_FEATURE_XSAVE (4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
125#define X86_FEATURE_OSXSAVE (4*32+27) /* "" XSAVE enabled in the OS */ 125#define X86_FEATURE_OSXSAVE (4*32+27) /* "" XSAVE enabled in the OS */
126#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ 126#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */
127#define X86_FEATURE_F16C (4*32+29) /* 16-bit fp conversions */
128#define X86_FEATURE_RDRND (4*32+30) /* The RDRAND instruction */
127#define X86_FEATURE_HYPERVISOR (4*32+31) /* Running on a hypervisor */ 129#define X86_FEATURE_HYPERVISOR (4*32+31) /* Running on a hypervisor */
128 130
129/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ 131/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
@@ -157,24 +159,34 @@
157 159
158/* 160/*
159 * Auxiliary flags: Linux defined - For features scattered in various 161 * Auxiliary flags: Linux defined - For features scattered in various
160 * CPUID levels like 0x6, 0xA etc 162 * CPUID levels like 0x6, 0xA etc, word 7
161 */ 163 */
162#define X86_FEATURE_IDA (7*32+ 0) /* Intel Dynamic Acceleration */ 164#define X86_FEATURE_IDA (7*32+ 0) /* Intel Dynamic Acceleration */
163#define X86_FEATURE_ARAT (7*32+ 1) /* Always Running APIC Timer */ 165#define X86_FEATURE_ARAT (7*32+ 1) /* Always Running APIC Timer */
166#define X86_FEATURE_CPB (7*32+ 2) /* AMD Core Performance Boost */
167#define X86_FEATURE_EPB (7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */
168#define X86_FEATURE_XSAVEOPT (7*32+ 4) /* Optimized Xsave */
169#define X86_FEATURE_PLN (7*32+ 5) /* Intel Power Limit Notification */
170#define X86_FEATURE_PTS (7*32+ 6) /* Intel Package Thermal Status */
171#define X86_FEATURE_DTS (7*32+ 7) /* Digital Thermal Sensor */
164 172
165/* Virtualization flags: Linux defined */ 173/* Virtualization flags: Linux defined, word 8 */
166#define X86_FEATURE_TPR_SHADOW (8*32+ 0) /* Intel TPR Shadow */ 174#define X86_FEATURE_TPR_SHADOW (8*32+ 0) /* Intel TPR Shadow */
167#define X86_FEATURE_VNMI (8*32+ 1) /* Intel Virtual NMI */ 175#define X86_FEATURE_VNMI (8*32+ 1) /* Intel Virtual NMI */
168#define X86_FEATURE_FLEXPRIORITY (8*32+ 2) /* Intel FlexPriority */ 176#define X86_FEATURE_FLEXPRIORITY (8*32+ 2) /* Intel FlexPriority */
169#define X86_FEATURE_EPT (8*32+ 3) /* Intel Extended Page Table */ 177#define X86_FEATURE_EPT (8*32+ 3) /* Intel Extended Page Table */
170#define X86_FEATURE_VPID (8*32+ 4) /* Intel Virtual Processor ID */ 178#define X86_FEATURE_VPID (8*32+ 4) /* Intel Virtual Processor ID */
171#define X86_FEATURE_NPT (8*32+5) /* AMD Nested Page Table support */ 179#define X86_FEATURE_NPT (8*32+ 5) /* AMD Nested Page Table support */
172#define X86_FEATURE_LBRV (8*32+6) /* AMD LBR Virtualization support */ 180#define X86_FEATURE_LBRV (8*32+ 6) /* AMD LBR Virtualization support */
173#define X86_FEATURE_SVML (8*32+7) /* "svm_lock" AMD SVM locking MSR */ 181#define X86_FEATURE_SVML (8*32+ 7) /* "svm_lock" AMD SVM locking MSR */
174#define X86_FEATURE_NRIPS (8*32+8) /* "nrip_save" AMD SVM next_rip save */ 182#define X86_FEATURE_NRIPS (8*32+ 8) /* "nrip_save" AMD SVM next_rip save */
183
184/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
185#define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
175 186
176#if defined(__KERNEL__) && !defined(__ASSEMBLY__) 187#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
177 188
189#include <asm/asm.h>
178#include <linux/bitops.h> 190#include <linux/bitops.h>
179 191
180extern const char * const x86_cap_flags[NCAPINTS*32]; 192extern const char * const x86_cap_flags[NCAPINTS*32];
@@ -192,7 +204,9 @@ extern const char * const x86_power_flags[32];
192 (((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4)) || \ 204 (((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4)) || \
193 (((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5)) || \ 205 (((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5)) || \
194 (((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ 206 (((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \
195 (((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) ) \ 207 (((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) || \
208 (((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8)) || \
209 (((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) \
196 ? 1 : \ 210 ? 1 : \
197 test_cpu_cap(c, bit)) 211 test_cpu_cap(c, bit))
198 212
@@ -283,6 +297,71 @@ extern const char * const x86_power_flags[32];
283 297
284#endif /* CONFIG_X86_64 */ 298#endif /* CONFIG_X86_64 */
285 299
300#if __GNUC__ >= 4
301/*
302 * Static testing of CPU features. Used the same as boot_cpu_has().
303 * These are only valid after alternatives have run, but will statically
304 * patch the target code for additional performance.
305 *
306 */
307static __always_inline __pure bool __static_cpu_has(u16 bit)
308{
309#if __GNUC__ > 4 || __GNUC_MINOR__ >= 5
310 asm goto("1: jmp %l[t_no]\n"
311 "2:\n"
312 ".section .altinstructions,\"a\"\n"
313 _ASM_ALIGN "\n"
314 _ASM_PTR "1b\n"
315 _ASM_PTR "0\n" /* no replacement */
316 " .word %P0\n" /* feature bit */
317 " .byte 2b - 1b\n" /* source len */
318 " .byte 0\n" /* replacement len */
319 ".previous\n"
320 /* skipping size check since replacement size = 0 */
321 : : "i" (bit) : : t_no);
322 return true;
323 t_no:
324 return false;
325#else
326 u8 flag;
327 /* Open-coded due to __stringify() in ALTERNATIVE() */
328 asm volatile("1: movb $0,%0\n"
329 "2:\n"
330 ".section .altinstructions,\"a\"\n"
331 _ASM_ALIGN "\n"
332 _ASM_PTR "1b\n"
333 _ASM_PTR "3f\n"
334 " .word %P1\n" /* feature bit */
335 " .byte 2b - 1b\n" /* source len */
336 " .byte 4f - 3f\n" /* replacement len */
337 ".previous\n"
338 ".section .discard,\"aw\",@progbits\n"
339 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
340 ".previous\n"
341 ".section .altinstr_replacement,\"ax\"\n"
342 "3: movb $1,%0\n"
343 "4:\n"
344 ".previous\n"
345 : "=qm" (flag) : "i" (bit));
346 return flag;
347#endif
348}
349
350#define static_cpu_has(bit) \
351( \
352 __builtin_constant_p(boot_cpu_has(bit)) ? \
353 boot_cpu_has(bit) : \
354 __builtin_constant_p(bit) ? \
355 __static_cpu_has(bit) : \
356 boot_cpu_has(bit) \
357)
358#else
359/*
360 * gcc 3.x is too stupid to do the static test; fall back to dynamic.
361 */
362#define static_cpu_has(bit) boot_cpu_has(bit)
363#endif
364
286#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ 365#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
287 366
288#endif /* _ASM_X86_CPUFEATURE_H */ 367#endif /* _ASM_X86_CPUFEATURE_H */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index ac91eed21061..d4c419f883a0 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -54,7 +54,6 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
54 54
55#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) 55#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
56#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) 56#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
57#define dma_is_consistent(d, h) (1)
58 57
59extern int dma_supported(struct device *hwdev, u64 mask); 58extern int dma_supported(struct device *hwdev, u64 mask);
60extern int dma_set_mask(struct device *dev, u64 mask); 59extern int dma_set_mask(struct device *dev, u64 mask);
@@ -87,13 +86,6 @@ dma_cache_sync(struct device *dev, void *vaddr, size_t size,
87 flush_write_buffers(); 86 flush_write_buffers();
88} 87}
89 88
90static inline int dma_get_cache_alignment(void)
91{
92 /* no easy way to get cache size on all x86, so return the
93 * maximum possible, to be safe */
94 return boot_cpu_data.x86_clflush_size;
95}
96
97static inline unsigned long dma_alloc_coherent_mask(struct device *dev, 89static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
98 gfp_t gfp) 90 gfp_t gfp)
99{ 91{
diff --git a/arch/x86/include/asm/ds.h b/arch/x86/include/asm/ds.h
deleted file mode 100644
index 70dac199b093..000000000000
--- a/arch/x86/include/asm/ds.h
+++ /dev/null
@@ -1,302 +0,0 @@
1/*
2 * Debug Store (DS) support
3 *
4 * This provides a low-level interface to the hardware's Debug Store
5 * feature that is used for branch trace store (BTS) and
6 * precise-event based sampling (PEBS).
7 *
8 * It manages:
9 * - DS and BTS hardware configuration
10 * - buffer overflow handling (to be done)
11 * - buffer access
12 *
13 * It does not do:
14 * - security checking (is the caller allowed to trace the task)
15 * - buffer allocation (memory accounting)
16 *
17 *
18 * Copyright (C) 2007-2009 Intel Corporation.
19 * Markus Metzger <markus.t.metzger@intel.com>, 2007-2009
20 */
21
22#ifndef _ASM_X86_DS_H
23#define _ASM_X86_DS_H
24
25
26#include <linux/types.h>
27#include <linux/init.h>
28#include <linux/err.h>
29
30
31#ifdef CONFIG_X86_DS
32
33struct task_struct;
34struct ds_context;
35struct ds_tracer;
36struct bts_tracer;
37struct pebs_tracer;
38
39typedef void (*bts_ovfl_callback_t)(struct bts_tracer *);
40typedef void (*pebs_ovfl_callback_t)(struct pebs_tracer *);
41
42
43/*
44 * A list of features plus corresponding macros to talk about them in
45 * the ds_request function's flags parameter.
46 *
47 * We use the enum to index an array of corresponding control bits;
48 * we use the macro to index a flags bit-vector.
49 */
50enum ds_feature {
51 dsf_bts = 0,
52 dsf_bts_kernel,
53#define BTS_KERNEL (1 << dsf_bts_kernel)
54 /* trace kernel-mode branches */
55
56 dsf_bts_user,
57#define BTS_USER (1 << dsf_bts_user)
58 /* trace user-mode branches */
59
60 dsf_bts_overflow,
61 dsf_bts_max,
62 dsf_pebs = dsf_bts_max,
63
64 dsf_pebs_max,
65 dsf_ctl_max = dsf_pebs_max,
66 dsf_bts_timestamps = dsf_ctl_max,
67#define BTS_TIMESTAMPS (1 << dsf_bts_timestamps)
68 /* add timestamps into BTS trace */
69
70#define BTS_USER_FLAGS (BTS_KERNEL | BTS_USER | BTS_TIMESTAMPS)
71};
72
73
74/*
75 * Request BTS or PEBS
76 *
77 * Due to alignement constraints, the actual buffer may be slightly
78 * smaller than the requested or provided buffer.
79 *
80 * Returns a pointer to a tracer structure on success, or
81 * ERR_PTR(errcode) on failure.
82 *
83 * The interrupt threshold is independent from the overflow callback
84 * to allow users to use their own overflow interrupt handling mechanism.
85 *
86 * The function might sleep.
87 *
88 * task: the task to request recording for
89 * cpu: the cpu to request recording for
90 * base: the base pointer for the (non-pageable) buffer;
91 * size: the size of the provided buffer in bytes
92 * ovfl: pointer to a function to be called on buffer overflow;
93 * NULL if cyclic buffer requested
94 * th: the interrupt threshold in records from the end of the buffer;
95 * -1 if no interrupt threshold is requested.
96 * flags: a bit-mask of the above flags
97 */
98extern struct bts_tracer *ds_request_bts_task(struct task_struct *task,
99 void *base, size_t size,
100 bts_ovfl_callback_t ovfl,
101 size_t th, unsigned int flags);
102extern struct bts_tracer *ds_request_bts_cpu(int cpu, void *base, size_t size,
103 bts_ovfl_callback_t ovfl,
104 size_t th, unsigned int flags);
105extern struct pebs_tracer *ds_request_pebs_task(struct task_struct *task,
106 void *base, size_t size,
107 pebs_ovfl_callback_t ovfl,
108 size_t th, unsigned int flags);
109extern struct pebs_tracer *ds_request_pebs_cpu(int cpu,
110 void *base, size_t size,
111 pebs_ovfl_callback_t ovfl,
112 size_t th, unsigned int flags);
113
114/*
115 * Release BTS or PEBS resources
116 * Suspend and resume BTS or PEBS tracing
117 *
118 * Must be called with irq's enabled.
119 *
120 * tracer: the tracer handle returned from ds_request_~()
121 */
122extern void ds_release_bts(struct bts_tracer *tracer);
123extern void ds_suspend_bts(struct bts_tracer *tracer);
124extern void ds_resume_bts(struct bts_tracer *tracer);
125extern void ds_release_pebs(struct pebs_tracer *tracer);
126extern void ds_suspend_pebs(struct pebs_tracer *tracer);
127extern void ds_resume_pebs(struct pebs_tracer *tracer);
128
129/*
130 * Release BTS or PEBS resources
131 * Suspend and resume BTS or PEBS tracing
132 *
133 * Cpu tracers must call this on the traced cpu.
134 * Task tracers must call ds_release_~_noirq() for themselves.
135 *
136 * May be called with irq's disabled.
137 *
138 * Returns 0 if successful;
139 * -EPERM if the cpu tracer does not trace the current cpu.
140 * -EPERM if the task tracer does not trace itself.
141 *
142 * tracer: the tracer handle returned from ds_request_~()
143 */
144extern int ds_release_bts_noirq(struct bts_tracer *tracer);
145extern int ds_suspend_bts_noirq(struct bts_tracer *tracer);
146extern int ds_resume_bts_noirq(struct bts_tracer *tracer);
147extern int ds_release_pebs_noirq(struct pebs_tracer *tracer);
148extern int ds_suspend_pebs_noirq(struct pebs_tracer *tracer);
149extern int ds_resume_pebs_noirq(struct pebs_tracer *tracer);
150
151
152/*
153 * The raw DS buffer state as it is used for BTS and PEBS recording.
154 *
155 * This is the low-level, arch-dependent interface for working
156 * directly on the raw trace data.
157 */
158struct ds_trace {
159 /* the number of bts/pebs records */
160 size_t n;
161 /* the size of a bts/pebs record in bytes */
162 size_t size;
163 /* pointers into the raw buffer:
164 - to the first entry */
165 void *begin;
166 /* - one beyond the last entry */
167 void *end;
168 /* - one beyond the newest entry */
169 void *top;
170 /* - the interrupt threshold */
171 void *ith;
172 /* flags given on ds_request() */
173 unsigned int flags;
174};
175
176/*
177 * An arch-independent view on branch trace data.
178 */
179enum bts_qualifier {
180 bts_invalid,
181#define BTS_INVALID bts_invalid
182
183 bts_branch,
184#define BTS_BRANCH bts_branch
185
186 bts_task_arrives,
187#define BTS_TASK_ARRIVES bts_task_arrives
188
189 bts_task_departs,
190#define BTS_TASK_DEPARTS bts_task_departs
191
192 bts_qual_bit_size = 4,
193 bts_qual_max = (1 << bts_qual_bit_size),
194};
195
196struct bts_struct {
197 __u64 qualifier;
198 union {
199 /* BTS_BRANCH */
200 struct {
201 __u64 from;
202 __u64 to;
203 } lbr;
204 /* BTS_TASK_ARRIVES or BTS_TASK_DEPARTS */
205 struct {
206 __u64 clock;
207 pid_t pid;
208 } event;
209 } variant;
210};
211
212
213/*
214 * The BTS state.
215 *
216 * This gives access to the raw DS state and adds functions to provide
217 * an arch-independent view of the BTS data.
218 */
219struct bts_trace {
220 struct ds_trace ds;
221
222 int (*read)(struct bts_tracer *tracer, const void *at,
223 struct bts_struct *out);
224 int (*write)(struct bts_tracer *tracer, const struct bts_struct *in);
225};
226
227
228/*
229 * The PEBS state.
230 *
231 * This gives access to the raw DS state and the PEBS-specific counter
232 * reset value.
233 */
234struct pebs_trace {
235 struct ds_trace ds;
236
237 /* the number of valid counters in the below array */
238 unsigned int counters;
239
240#define MAX_PEBS_COUNTERS 4
241 /* the counter reset value */
242 unsigned long long counter_reset[MAX_PEBS_COUNTERS];
243};
244
245
246/*
247 * Read the BTS or PEBS trace.
248 *
249 * Returns a view on the trace collected for the parameter tracer.
250 *
251 * The view remains valid as long as the traced task is not running or
252 * the tracer is suspended.
253 * Writes into the trace buffer are not reflected.
254 *
255 * tracer: the tracer handle returned from ds_request_~()
256 */
257extern const struct bts_trace *ds_read_bts(struct bts_tracer *tracer);
258extern const struct pebs_trace *ds_read_pebs(struct pebs_tracer *tracer);
259
260
261/*
262 * Reset the write pointer of the BTS/PEBS buffer.
263 *
264 * Returns 0 on success; -Eerrno on error
265 *
266 * tracer: the tracer handle returned from ds_request_~()
267 */
268extern int ds_reset_bts(struct bts_tracer *tracer);
269extern int ds_reset_pebs(struct pebs_tracer *tracer);
270
271/*
272 * Set the PEBS counter reset value.
273 *
274 * Returns 0 on success; -Eerrno on error
275 *
276 * tracer: the tracer handle returned from ds_request_pebs()
277 * counter: the index of the counter
278 * value: the new counter reset value
279 */
280extern int ds_set_pebs_reset(struct pebs_tracer *tracer,
281 unsigned int counter, u64 value);
282
283/*
284 * Initialization
285 */
286struct cpuinfo_x86;
287extern void __cpuinit ds_init_intel(struct cpuinfo_x86 *);
288
289/*
290 * Context switch work
291 */
292extern void ds_switch_to(struct task_struct *prev, struct task_struct *next);
293
294#else /* CONFIG_X86_DS */
295
296struct cpuinfo_x86;
297static inline void __cpuinit ds_init_intel(struct cpuinfo_x86 *ignored) {}
298static inline void ds_switch_to(struct task_struct *prev,
299 struct task_struct *next) {}
300
301#endif /* CONFIG_X86_DS */
302#endif /* _ASM_X86_DS_H */
diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h
index ae6253ab9029..733f7e91e7a9 100644
--- a/arch/x86/include/asm/dwarf2.h
+++ b/arch/x86/include/asm/dwarf2.h
@@ -34,6 +34,18 @@
34#define CFI_SIGNAL_FRAME 34#define CFI_SIGNAL_FRAME
35#endif 35#endif
36 36
37#if defined(CONFIG_AS_CFI_SECTIONS) && defined(__ASSEMBLY__)
38 /*
39 * Emit CFI data in .debug_frame sections, not .eh_frame sections.
40 * The latter we currently just discard since we don't do DWARF
41 * unwinding at runtime. So only the offline DWARF information is
42 * useful to anyone. Note we should not use this directive if this
43 * file is used in the vDSO assembly, or if vmlinux.lds.S gets
44 * changed so it doesn't discard .eh_frame.
45 */
46 .cfi_sections .debug_frame
47#endif
48
37#else 49#else
38 50
39/* 51/*
diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h
index 0e22296790d3..ec8a52d14ab1 100644
--- a/arch/x86/include/asm/e820.h
+++ b/arch/x86/include/asm/e820.h
@@ -45,7 +45,12 @@
45#define E820_NVS 4 45#define E820_NVS 4
46#define E820_UNUSABLE 5 46#define E820_UNUSABLE 5
47 47
48/* reserved RAM used by kernel itself */ 48/*
49 * reserved RAM used by kernel itself
50 * if CONFIG_INTEL_TXT is enabled, memory of this type will be
51 * included in the S3 integrity calculation and so should not include
52 * any memory that BIOS might alter over the S3 transition
53 */
49#define E820_RESERVED_KERN 128 54#define E820_RESERVED_KERN 128
50 55
51#ifndef __ASSEMBLY__ 56#ifndef __ASSEMBLY__
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 0f8576427cfe..aeab29aee617 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,7 +35,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
35 35
36#define __ARCH_IRQ_STAT 36#define __ARCH_IRQ_STAT
37 37
38#define inc_irq_stat(member) percpu_add(irq_stat.member, 1) 38#define inc_irq_stat(member) percpu_inc(irq_stat.member)
39 39
40#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending) 40#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
41 41
diff --git a/arch/x86/include/asm/highmem.h b/arch/x86/include/asm/highmem.h
index a726650fc80f..8caac76ac324 100644
--- a/arch/x86/include/asm/highmem.h
+++ b/arch/x86/include/asm/highmem.h
@@ -61,7 +61,7 @@ void *kmap(struct page *page);
61void kunmap(struct page *page); 61void kunmap(struct page *page);
62void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot); 62void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot);
63void *kmap_atomic(struct page *page, enum km_type type); 63void *kmap_atomic(struct page *page, enum km_type type);
64void kunmap_atomic(void *kvaddr, enum km_type type); 64void kunmap_atomic_notypecheck(void *kvaddr, enum km_type type);
65void *kmap_atomic_pfn(unsigned long pfn, enum km_type type); 65void *kmap_atomic_pfn(unsigned long pfn, enum km_type type);
66void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot); 66void *kmap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
67struct page *kmap_atomic_to_page(void *ptr); 67struct page *kmap_atomic_to_page(void *ptr);
diff --git a/arch/x86/include/asm/hw_breakpoint.h b/arch/x86/include/asm/hw_breakpoint.h
index 2a1bd8f4f23a..824ca07860d0 100644
--- a/arch/x86/include/asm/hw_breakpoint.h
+++ b/arch/x86/include/asm/hw_breakpoint.h
@@ -20,10 +20,10 @@ struct arch_hw_breakpoint {
20#include <linux/list.h> 20#include <linux/list.h>
21 21
22/* Available HW breakpoint length encodings */ 22/* Available HW breakpoint length encodings */
23#define X86_BREAKPOINT_LEN_X 0x40
23#define X86_BREAKPOINT_LEN_1 0x40 24#define X86_BREAKPOINT_LEN_1 0x40
24#define X86_BREAKPOINT_LEN_2 0x44 25#define X86_BREAKPOINT_LEN_2 0x44
25#define X86_BREAKPOINT_LEN_4 0x4c 26#define X86_BREAKPOINT_LEN_4 0x4c
26#define X86_BREAKPOINT_LEN_EXECUTE 0x40
27 27
28#ifdef CONFIG_X86_64 28#ifdef CONFIG_X86_64
29#define X86_BREAKPOINT_LEN_8 0x48 29#define X86_BREAKPOINT_LEN_8 0x48
@@ -41,12 +41,16 @@ struct arch_hw_breakpoint {
41/* Total number of available HW breakpoint registers */ 41/* Total number of available HW breakpoint registers */
42#define HBP_NUM 4 42#define HBP_NUM 4
43 43
44static inline int hw_breakpoint_slots(int type)
45{
46 return HBP_NUM;
47}
48
44struct perf_event; 49struct perf_event;
45struct pmu; 50struct pmu;
46 51
47extern int arch_check_va_in_userspace(unsigned long va, u8 hbp_len); 52extern int arch_check_bp_in_kernelspace(struct perf_event *bp);
48extern int arch_validate_hwbkpt_settings(struct perf_event *bp, 53extern int arch_validate_hwbkpt_settings(struct perf_event *bp);
49 struct task_struct *tsk);
50extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused, 54extern int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
51 unsigned long val, void *data); 55 unsigned long val, void *data);
52 56
diff --git a/arch/x86/include/asm/hyperv.h b/arch/x86/include/asm/hyperv.h
index e153a2b3889a..5df477ac3af7 100644
--- a/arch/x86/include/asm/hyperv.h
+++ b/arch/x86/include/asm/hyperv.h
@@ -1,5 +1,5 @@
1#ifndef _ASM_X86_KVM_HYPERV_H 1#ifndef _ASM_X86_HYPERV_H
2#define _ASM_X86_KVM_HYPERV_H 2#define _ASM_X86_HYPERV_H
3 3
4#include <linux/types.h> 4#include <linux/types.h>
5 5
@@ -14,6 +14,10 @@
14#define HYPERV_CPUID_ENLIGHTMENT_INFO 0x40000004 14#define HYPERV_CPUID_ENLIGHTMENT_INFO 0x40000004
15#define HYPERV_CPUID_IMPLEMENT_LIMITS 0x40000005 15#define HYPERV_CPUID_IMPLEMENT_LIMITS 0x40000005
16 16
17#define HYPERV_HYPERVISOR_PRESENT_BIT 0x80000000
18#define HYPERV_CPUID_MIN 0x40000005
19#define HYPERV_CPUID_MAX 0x4000ffff
20
17/* 21/*
18 * Feature identification. EAX indicates which features are available 22 * Feature identification. EAX indicates which features are available
19 * to the partition based upon the current partition privileges. 23 * to the partition based upon the current partition privileges.
@@ -129,6 +133,9 @@
129/* MSR used to provide vcpu index */ 133/* MSR used to provide vcpu index */
130#define HV_X64_MSR_VP_INDEX 0x40000002 134#define HV_X64_MSR_VP_INDEX 0x40000002
131 135
136/* MSR used to read the per-partition time reference counter */
137#define HV_X64_MSR_TIME_REF_COUNT 0x40000020
138
132/* Define the virtual APIC registers */ 139/* Define the virtual APIC registers */
133#define HV_X64_MSR_EOI 0x40000070 140#define HV_X64_MSR_EOI 0x40000070
134#define HV_X64_MSR_ICR 0x40000071 141#define HV_X64_MSR_ICR 0x40000071
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index b78c0941e422..ff2546ce7178 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -17,10 +17,34 @@
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 * 18 *
19 */ 19 */
20#ifndef ASM_X86__HYPERVISOR_H 20#ifndef _ASM_X86_HYPERVISOR_H
21#define ASM_X86__HYPERVISOR_H 21#define _ASM_X86_HYPERVISOR_H
22 22
23extern void init_hypervisor(struct cpuinfo_x86 *c); 23extern void init_hypervisor(struct cpuinfo_x86 *c);
24extern void init_hypervisor_platform(void); 24extern void init_hypervisor_platform(void);
25 25
26/*
27 * x86 hypervisor information
28 */
29struct hypervisor_x86 {
30 /* Hypervisor name */
31 const char *name;
32
33 /* Detection routine */
34 bool (*detect)(void);
35
36 /* Adjust CPU feature bits (run once per CPU) */
37 void (*set_cpu_features)(struct cpuinfo_x86 *);
38
39 /* Platform setup (run once per boot) */
40 void (*init_platform)(void);
41};
42
43extern const struct hypervisor_x86 *x86_hyper;
44
45/* Recognized hypervisors */
46extern const struct hypervisor_x86 x86_hyper_vmware;
47extern const struct hypervisor_x86 x86_hyper_ms_hyperv;
48extern const struct hypervisor_x86 x86_hyper_xen_hvm;
49
26#endif 50#endif
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index da2930924501..a73a8d5a5e69 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -16,7 +16,9 @@
16#include <linux/kernel_stat.h> 16#include <linux/kernel_stat.h>
17#include <linux/regset.h> 17#include <linux/regset.h>
18#include <linux/hardirq.h> 18#include <linux/hardirq.h>
19#include <linux/slab.h>
19#include <asm/asm.h> 20#include <asm/asm.h>
21#include <asm/cpufeature.h>
20#include <asm/processor.h> 22#include <asm/processor.h>
21#include <asm/sigcontext.h> 23#include <asm/sigcontext.h>
22#include <asm/user.h> 24#include <asm/user.h>
@@ -29,7 +31,6 @@ extern void mxcsr_feature_mask_init(void);
29extern int init_fpu(struct task_struct *child); 31extern int init_fpu(struct task_struct *child);
30extern asmlinkage void math_state_restore(void); 32extern asmlinkage void math_state_restore(void);
31extern void __math_state_restore(void); 33extern void __math_state_restore(void);
32extern void init_thread_xstate(void);
33extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); 34extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
34 35
35extern user_regset_active_fn fpregs_active, xfpregs_active; 36extern user_regset_active_fn fpregs_active, xfpregs_active;
@@ -56,6 +57,25 @@ extern int restore_i387_xstate_ia32(void __user *buf);
56 57
57#define X87_FSW_ES (1 << 7) /* Exception Summary */ 58#define X87_FSW_ES (1 << 7) /* Exception Summary */
58 59
60static __always_inline __pure bool use_xsaveopt(void)
61{
62 return static_cpu_has(X86_FEATURE_XSAVEOPT);
63}
64
65static __always_inline __pure bool use_xsave(void)
66{
67 return static_cpu_has(X86_FEATURE_XSAVE);
68}
69
70extern void __sanitize_i387_state(struct task_struct *);
71
72static inline void sanitize_i387_state(struct task_struct *tsk)
73{
74 if (!use_xsaveopt())
75 return;
76 __sanitize_i387_state(tsk);
77}
78
59#ifdef CONFIG_X86_64 79#ifdef CONFIG_X86_64
60 80
61/* Ignore delayed exceptions from user space */ 81/* Ignore delayed exceptions from user space */
@@ -91,15 +111,15 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
91 values. The kernel data segment can be sometimes 0 and sometimes 111 values. The kernel data segment can be sometimes 0 and sometimes
92 new user value. Both should be ok. 112 new user value. Both should be ok.
93 Use the PDA as safe address because it should be already in L1. */ 113 Use the PDA as safe address because it should be already in L1. */
94static inline void clear_fpu_state(struct task_struct *tsk) 114static inline void fpu_clear(struct fpu *fpu)
95{ 115{
96 struct xsave_struct *xstate = &tsk->thread.xstate->xsave; 116 struct xsave_struct *xstate = &fpu->state->xsave;
97 struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; 117 struct i387_fxsave_struct *fx = &fpu->state->fxsave;
98 118
99 /* 119 /*
100 * xsave header may indicate the init state of the FP. 120 * xsave header may indicate the init state of the FP.
101 */ 121 */
102 if ((task_thread_info(tsk)->status & TS_XSAVE) && 122 if (use_xsave() &&
103 !(xstate->xsave_hdr.xstate_bv & XSTATE_FP)) 123 !(xstate->xsave_hdr.xstate_bv & XSTATE_FP))
104 return; 124 return;
105 125
@@ -111,10 +131,24 @@ static inline void clear_fpu_state(struct task_struct *tsk)
111 X86_FEATURE_FXSAVE_LEAK); 131 X86_FEATURE_FXSAVE_LEAK);
112} 132}
113 133
134static inline void clear_fpu_state(struct task_struct *tsk)
135{
136 fpu_clear(&tsk->thread.fpu);
137}
138
114static inline int fxsave_user(struct i387_fxsave_struct __user *fx) 139static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
115{ 140{
116 int err; 141 int err;
117 142
143 /*
144 * Clear the bytes not touched by the fxsave and reserved
145 * for the SW usage.
146 */
147 err = __clear_user(&fx->sw_reserved,
148 sizeof(struct _fpx_sw_bytes));
149 if (unlikely(err))
150 return -EFAULT;
151
118 asm volatile("1: rex64/fxsave (%[fx])\n\t" 152 asm volatile("1: rex64/fxsave (%[fx])\n\t"
119 "2:\n" 153 "2:\n"
120 ".section .fixup,\"ax\"\n" 154 ".section .fixup,\"ax\"\n"
@@ -135,7 +169,7 @@ static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
135 return err; 169 return err;
136} 170}
137 171
138static inline void fxsave(struct task_struct *tsk) 172static inline void fpu_fxsave(struct fpu *fpu)
139{ 173{
140 /* Using "rex64; fxsave %0" is broken because, if the memory operand 174 /* Using "rex64; fxsave %0" is broken because, if the memory operand
141 uses any extended registers for addressing, a second REX prefix 175 uses any extended registers for addressing, a second REX prefix
@@ -145,42 +179,45 @@ static inline void fxsave(struct task_struct *tsk)
145 /* Using "fxsaveq %0" would be the ideal choice, but is only supported 179 /* Using "fxsaveq %0" would be the ideal choice, but is only supported
146 starting with gas 2.16. */ 180 starting with gas 2.16. */
147 __asm__ __volatile__("fxsaveq %0" 181 __asm__ __volatile__("fxsaveq %0"
148 : "=m" (tsk->thread.xstate->fxsave)); 182 : "=m" (fpu->state->fxsave));
149#elif 0 183#elif 0
150 /* Using, as a workaround, the properly prefixed form below isn't 184 /* Using, as a workaround, the properly prefixed form below isn't
151 accepted by any binutils version so far released, complaining that 185 accepted by any binutils version so far released, complaining that
152 the same type of prefix is used twice if an extended register is 186 the same type of prefix is used twice if an extended register is
153 needed for addressing (fix submitted to mainline 2005-11-21). */ 187 needed for addressing (fix submitted to mainline 2005-11-21). */
154 __asm__ __volatile__("rex64/fxsave %0" 188 __asm__ __volatile__("rex64/fxsave %0"
155 : "=m" (tsk->thread.xstate->fxsave)); 189 : "=m" (fpu->state->fxsave));
156#else 190#else
157 /* This, however, we can work around by forcing the compiler to select 191 /* This, however, we can work around by forcing the compiler to select
158 an addressing mode that doesn't require extended registers. */ 192 an addressing mode that doesn't require extended registers. */
159 __asm__ __volatile__("rex64/fxsave (%1)" 193 __asm__ __volatile__("rex64/fxsave (%1)"
160 : "=m" (tsk->thread.xstate->fxsave) 194 : "=m" (fpu->state->fxsave)
161 : "cdaSDb" (&tsk->thread.xstate->fxsave)); 195 : "cdaSDb" (&fpu->state->fxsave));
162#endif 196#endif
163} 197}
164 198
165static inline void __save_init_fpu(struct task_struct *tsk) 199static inline void fpu_save_init(struct fpu *fpu)
166{ 200{
167 if (task_thread_info(tsk)->status & TS_XSAVE) 201 if (use_xsave())
168 xsave(tsk); 202 fpu_xsave(fpu);
169 else 203 else
170 fxsave(tsk); 204 fpu_fxsave(fpu);
205
206 fpu_clear(fpu);
207}
171 208
172 clear_fpu_state(tsk); 209static inline void __save_init_fpu(struct task_struct *tsk)
210{
211 fpu_save_init(&tsk->thread.fpu);
173 task_thread_info(tsk)->status &= ~TS_USEDFPU; 212 task_thread_info(tsk)->status &= ~TS_USEDFPU;
174} 213}
175 214
176#else /* CONFIG_X86_32 */ 215#else /* CONFIG_X86_32 */
177 216
178#ifdef CONFIG_MATH_EMULATION 217#ifdef CONFIG_MATH_EMULATION
179extern void finit_task(struct task_struct *tsk); 218extern void finit_soft_fpu(struct i387_soft_struct *soft);
180#else 219#else
181static inline void finit_task(struct task_struct *tsk) 220static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
182{
183}
184#endif 221#endif
185 222
186static inline void tolerant_fwait(void) 223static inline void tolerant_fwait(void)
@@ -216,13 +253,13 @@ static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
216/* 253/*
217 * These must be called with preempt disabled 254 * These must be called with preempt disabled
218 */ 255 */
219static inline void __save_init_fpu(struct task_struct *tsk) 256static inline void fpu_save_init(struct fpu *fpu)
220{ 257{
221 if (task_thread_info(tsk)->status & TS_XSAVE) { 258 if (use_xsave()) {
222 struct xsave_struct *xstate = &tsk->thread.xstate->xsave; 259 struct xsave_struct *xstate = &fpu->state->xsave;
223 struct i387_fxsave_struct *fx = &tsk->thread.xstate->fxsave; 260 struct i387_fxsave_struct *fx = &fpu->state->fxsave;
224 261
225 xsave(tsk); 262 fpu_xsave(fpu);
226 263
227 /* 264 /*
228 * xsave header may indicate the init state of the FP. 265 * xsave header may indicate the init state of the FP.
@@ -246,8 +283,8 @@ static inline void __save_init_fpu(struct task_struct *tsk)
246 "fxsave %[fx]\n" 283 "fxsave %[fx]\n"
247 "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", 284 "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
248 X86_FEATURE_FXSR, 285 X86_FEATURE_FXSR,
249 [fx] "m" (tsk->thread.xstate->fxsave), 286 [fx] "m" (fpu->state->fxsave),
250 [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); 287 [fsw] "m" (fpu->state->fxsave.swd) : "memory");
251clear_state: 288clear_state:
252 /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception 289 /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
253 is pending. Clear the x87 state here by setting it to fixed 290 is pending. Clear the x87 state here by setting it to fixed
@@ -259,17 +296,34 @@ clear_state:
259 X86_FEATURE_FXSAVE_LEAK, 296 X86_FEATURE_FXSAVE_LEAK,
260 [addr] "m" (safe_address)); 297 [addr] "m" (safe_address));
261end: 298end:
299 ;
300}
301
302static inline void __save_init_fpu(struct task_struct *tsk)
303{
304 fpu_save_init(&tsk->thread.fpu);
262 task_thread_info(tsk)->status &= ~TS_USEDFPU; 305 task_thread_info(tsk)->status &= ~TS_USEDFPU;
263} 306}
264 307
308
265#endif /* CONFIG_X86_64 */ 309#endif /* CONFIG_X86_64 */
266 310
267static inline int restore_fpu_checking(struct task_struct *tsk) 311static inline int fpu_fxrstor_checking(struct fpu *fpu)
312{
313 return fxrstor_checking(&fpu->state->fxsave);
314}
315
316static inline int fpu_restore_checking(struct fpu *fpu)
268{ 317{
269 if (task_thread_info(tsk)->status & TS_XSAVE) 318 if (use_xsave())
270 return xrstor_checking(&tsk->thread.xstate->xsave); 319 return fpu_xrstor_checking(fpu);
271 else 320 else
272 return fxrstor_checking(&tsk->thread.xstate->fxsave); 321 return fpu_fxrstor_checking(fpu);
322}
323
324static inline int restore_fpu_checking(struct task_struct *tsk)
325{
326 return fpu_restore_checking(&tsk->thread.fpu);
273} 327}
274 328
275/* 329/*
@@ -397,30 +451,61 @@ static inline void clear_fpu(struct task_struct *tsk)
397static inline unsigned short get_fpu_cwd(struct task_struct *tsk) 451static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
398{ 452{
399 if (cpu_has_fxsr) { 453 if (cpu_has_fxsr) {
400 return tsk->thread.xstate->fxsave.cwd; 454 return tsk->thread.fpu.state->fxsave.cwd;
401 } else { 455 } else {
402 return (unsigned short)tsk->thread.xstate->fsave.cwd; 456 return (unsigned short)tsk->thread.fpu.state->fsave.cwd;
403 } 457 }
404} 458}
405 459
406static inline unsigned short get_fpu_swd(struct task_struct *tsk) 460static inline unsigned short get_fpu_swd(struct task_struct *tsk)
407{ 461{
408 if (cpu_has_fxsr) { 462 if (cpu_has_fxsr) {
409 return tsk->thread.xstate->fxsave.swd; 463 return tsk->thread.fpu.state->fxsave.swd;
410 } else { 464 } else {
411 return (unsigned short)tsk->thread.xstate->fsave.swd; 465 return (unsigned short)tsk->thread.fpu.state->fsave.swd;
412 } 466 }
413} 467}
414 468
415static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) 469static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
416{ 470{
417 if (cpu_has_xmm) { 471 if (cpu_has_xmm) {
418 return tsk->thread.xstate->fxsave.mxcsr; 472 return tsk->thread.fpu.state->fxsave.mxcsr;
419 } else { 473 } else {
420 return MXCSR_DEFAULT; 474 return MXCSR_DEFAULT;
421 } 475 }
422} 476}
423 477
478static bool fpu_allocated(struct fpu *fpu)
479{
480 return fpu->state != NULL;
481}
482
483static inline int fpu_alloc(struct fpu *fpu)
484{
485 if (fpu_allocated(fpu))
486 return 0;
487 fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL);
488 if (!fpu->state)
489 return -ENOMEM;
490 WARN_ON((unsigned long)fpu->state & 15);
491 return 0;
492}
493
494static inline void fpu_free(struct fpu *fpu)
495{
496 if (fpu->state) {
497 kmem_cache_free(task_xstate_cachep, fpu->state);
498 fpu->state = NULL;
499 }
500}
501
502static inline void fpu_copy(struct fpu *dst, struct fpu *src)
503{
504 memcpy(dst->state, src->state, xstate_size);
505}
506
507extern void fpu_finit(struct fpu *fpu);
508
424#endif /* __ASSEMBLY__ */ 509#endif /* __ASSEMBLY__ */
425 510
426#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5 511#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5
diff --git a/arch/x86/include/asm/i8253.h b/arch/x86/include/asm/i8253.h
index 1edbf89680fd..fc1f579fb965 100644
--- a/arch/x86/include/asm/i8253.h
+++ b/arch/x86/include/asm/i8253.h
@@ -6,7 +6,7 @@
6#define PIT_CH0 0x40 6#define PIT_CH0 0x40
7#define PIT_CH2 0x42 7#define PIT_CH2 0x42
8 8
9extern spinlock_t i8253_lock; 9extern raw_spinlock_t i8253_lock;
10 10
11extern struct clock_event_device *global_clock_event; 11extern struct clock_event_device *global_clock_event;
12 12
diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h
index 96c2e0ad04ca..88c765e16410 100644
--- a/arch/x86/include/asm/insn.h
+++ b/arch/x86/include/asm/insn.h
@@ -68,6 +68,8 @@ struct insn {
68 const insn_byte_t *next_byte; 68 const insn_byte_t *next_byte;
69}; 69};
70 70
71#define MAX_INSN_SIZE 16
72
71#define X86_MODRM_MOD(modrm) (((modrm) & 0xc0) >> 6) 73#define X86_MODRM_MOD(modrm) (((modrm) & 0xc0) >> 6)
72#define X86_MODRM_REG(modrm) (((modrm) & 0x38) >> 3) 74#define X86_MODRM_REG(modrm) (((modrm) & 0x38) >> 3)
73#define X86_MODRM_RM(modrm) ((modrm) & 0x07) 75#define X86_MODRM_RM(modrm) ((modrm) & 0x07)
diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h
index 14cf526091f9..280bf7fb6aba 100644
--- a/arch/x86/include/asm/inst.h
+++ b/arch/x86/include/asm/inst.h
@@ -7,7 +7,66 @@
7 7
8#ifdef __ASSEMBLY__ 8#ifdef __ASSEMBLY__
9 9
10#define REG_NUM_INVALID 100
11
12#define REG_TYPE_R64 0
13#define REG_TYPE_XMM 1
14#define REG_TYPE_INVALID 100
15
16 .macro R64_NUM opd r64
17 \opd = REG_NUM_INVALID
18 .ifc \r64,%rax
19 \opd = 0
20 .endif
21 .ifc \r64,%rcx
22 \opd = 1
23 .endif
24 .ifc \r64,%rdx
25 \opd = 2
26 .endif
27 .ifc \r64,%rbx
28 \opd = 3
29 .endif
30 .ifc \r64,%rsp
31 \opd = 4
32 .endif
33 .ifc \r64,%rbp
34 \opd = 5
35 .endif
36 .ifc \r64,%rsi
37 \opd = 6
38 .endif
39 .ifc \r64,%rdi
40 \opd = 7
41 .endif
42 .ifc \r64,%r8
43 \opd = 8
44 .endif
45 .ifc \r64,%r9
46 \opd = 9
47 .endif
48 .ifc \r64,%r10
49 \opd = 10
50 .endif
51 .ifc \r64,%r11
52 \opd = 11
53 .endif
54 .ifc \r64,%r12
55 \opd = 12
56 .endif
57 .ifc \r64,%r13
58 \opd = 13
59 .endif
60 .ifc \r64,%r14
61 \opd = 14
62 .endif
63 .ifc \r64,%r15
64 \opd = 15
65 .endif
66 .endm
67
10 .macro XMM_NUM opd xmm 68 .macro XMM_NUM opd xmm
69 \opd = REG_NUM_INVALID
11 .ifc \xmm,%xmm0 70 .ifc \xmm,%xmm0
12 \opd = 0 71 \opd = 0
13 .endif 72 .endif
@@ -58,13 +117,25 @@
58 .endif 117 .endif
59 .endm 118 .endm
60 119
120 .macro REG_TYPE type reg
121 R64_NUM reg_type_r64 \reg
122 XMM_NUM reg_type_xmm \reg
123 .if reg_type_r64 <> REG_NUM_INVALID
124 \type = REG_TYPE_R64
125 .elseif reg_type_xmm <> REG_NUM_INVALID
126 \type = REG_TYPE_XMM
127 .else
128 \type = REG_TYPE_INVALID
129 .endif
130 .endm
131
61 .macro PFX_OPD_SIZE 132 .macro PFX_OPD_SIZE
62 .byte 0x66 133 .byte 0x66
63 .endm 134 .endm
64 135
65 .macro PFX_REX opd1 opd2 136 .macro PFX_REX opd1 opd2 W=0
66 .if (\opd1 | \opd2) & 8 137 .if ((\opd1 | \opd2) & 8) || \W
67 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) 138 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
68 .endif 139 .endif
69 .endm 140 .endm
70 141
@@ -145,6 +216,25 @@
145 .byte 0x0f, 0x38, 0xdf 216 .byte 0x0f, 0x38, 0xdf
146 MODRM 0xc0 aesdeclast_opd1 aesdeclast_opd2 217 MODRM 0xc0 aesdeclast_opd1 aesdeclast_opd2
147 .endm 218 .endm
219
220 .macro MOVQ_R64_XMM opd1 opd2
221 REG_TYPE movq_r64_xmm_opd1_type \opd1
222 .if movq_r64_xmm_opd1_type == REG_TYPE_XMM
223 XMM_NUM movq_r64_xmm_opd1 \opd1
224 R64_NUM movq_r64_xmm_opd2 \opd2
225 .else
226 R64_NUM movq_r64_xmm_opd1 \opd1
227 XMM_NUM movq_r64_xmm_opd2 \opd2
228 .endif
229 PFX_OPD_SIZE
230 PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
231 .if movq_r64_xmm_opd1_type == REG_TYPE_XMM
232 .byte 0x0f, 0x7e
233 .else
234 .byte 0x0f, 0x6e
235 .endif
236 MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
237 .endm
148#endif 238#endif
149 239
150#endif 240#endif
diff --git a/arch/x86/include/asm/intel_scu_ipc.h b/arch/x86/include/asm/intel_scu_ipc.h
new file mode 100644
index 000000000000..29f66793cc55
--- /dev/null
+++ b/arch/x86/include/asm/intel_scu_ipc.h
@@ -0,0 +1,47 @@
1#ifndef _ASM_X86_INTEL_SCU_IPC_H_
2#define _ASM_X86_INTEL_SCU_IPC_H_
3
4#define IPCMSG_VRTC 0xFA /* Set vRTC device */
5
6/* Command id associated with message IPCMSG_VRTC */
7#define IPC_CMD_VRTC_SETTIME 1 /* Set time */
8#define IPC_CMD_VRTC_SETALARM 2 /* Set alarm */
9
10/* Read single register */
11int intel_scu_ipc_ioread8(u16 addr, u8 *data);
12
13/* Read two sequential registers */
14int intel_scu_ipc_ioread16(u16 addr, u16 *data);
15
16/* Read four sequential registers */
17int intel_scu_ipc_ioread32(u16 addr, u32 *data);
18
19/* Read a vector */
20int intel_scu_ipc_readv(u16 *addr, u8 *data, int len);
21
22/* Write single register */
23int intel_scu_ipc_iowrite8(u16 addr, u8 data);
24
25/* Write two sequential registers */
26int intel_scu_ipc_iowrite16(u16 addr, u16 data);
27
28/* Write four sequential registers */
29int intel_scu_ipc_iowrite32(u16 addr, u32 data);
30
31/* Write a vector */
32int intel_scu_ipc_writev(u16 *addr, u8 *data, int len);
33
34/* Update single register based on the mask */
35int intel_scu_ipc_update_register(u16 addr, u8 data, u8 mask);
36
37/* Issue commands to the SCU with or without data */
38int intel_scu_ipc_simple_command(int cmd, int sub);
39int intel_scu_ipc_command(int cmd, int sub, u32 *in, int inlen,
40 u32 *out, int outlen);
41/* I2C control api */
42int intel_scu_ipc_i2c_cntrl(u32 addr, u32 *data);
43
44/* Update FW version */
45int intel_scu_ipc_fw_update(u8 *buffer, u32 length);
46
47#endif
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 35832a03a515..9cb2edb87c2f 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -159,7 +159,6 @@ struct io_apic_irq_attr;
159extern int io_apic_set_pci_routing(struct device *dev, int irq, 159extern int io_apic_set_pci_routing(struct device *dev, int irq,
160 struct io_apic_irq_attr *irq_attr); 160 struct io_apic_irq_attr *irq_attr);
161void setup_IO_APIC_irq_extra(u32 gsi); 161void setup_IO_APIC_irq_extra(u32 gsi);
162extern int (*ioapic_renumber_irq)(int ioapic, int irq);
163extern void ioapic_init_mappings(void); 162extern void ioapic_init_mappings(void);
164extern void ioapic_insert_resources(void); 163extern void ioapic_insert_resources(void);
165 164
@@ -180,12 +179,13 @@ extern void ioapic_write_entry(int apic, int pin,
180extern void setup_ioapic_ids_from_mpc(void); 179extern void setup_ioapic_ids_from_mpc(void);
181 180
182struct mp_ioapic_gsi{ 181struct mp_ioapic_gsi{
183 int gsi_base; 182 u32 gsi_base;
184 int gsi_end; 183 u32 gsi_end;
185}; 184};
186extern struct mp_ioapic_gsi mp_gsi_routing[]; 185extern struct mp_ioapic_gsi mp_gsi_routing[];
187int mp_find_ioapic(int gsi); 186extern u32 gsi_top;
188int mp_find_ioapic_pin(int ioapic, int gsi); 187int mp_find_ioapic(u32 gsi);
188int mp_find_ioapic_pin(int ioapic, u32 gsi);
189void __init mp_register_ioapic(int id, u32 address, u32 gsi_base); 189void __init mp_register_ioapic(int id, u32 address, u32 gsi_base);
190extern void __init pre_init_apic_IRQ0(void); 190extern void __init pre_init_apic_IRQ0(void);
191 191
@@ -197,7 +197,8 @@ static const int timer_through_8259 = 0;
197static inline void ioapic_init_mappings(void) { } 197static inline void ioapic_init_mappings(void) { }
198static inline void ioapic_insert_resources(void) { } 198static inline void ioapic_insert_resources(void) { }
199static inline void probe_nr_irqs_gsi(void) { } 199static inline void probe_nr_irqs_gsi(void) { }
200static inline int mp_find_ioapic(int gsi) { return 0; } 200#define gsi_top (NR_IRQS_LEGACY)
201static inline int mp_find_ioapic(u32 gsi) { return 0; }
201 202
202struct io_apic_irq_attr; 203struct io_apic_irq_attr;
203static inline int io_apic_set_pci_routing(struct device *dev, int irq, 204static inline int io_apic_set_pci_routing(struct device *dev, int irq,
diff --git a/arch/x86/include/asm/iomap.h b/arch/x86/include/asm/iomap.h
index f35eb45d6576..c4191b3b7056 100644
--- a/arch/x86/include/asm/iomap.h
+++ b/arch/x86/include/asm/iomap.h
@@ -26,11 +26,11 @@
26#include <asm/pgtable.h> 26#include <asm/pgtable.h>
27#include <asm/tlbflush.h> 27#include <asm/tlbflush.h>
28 28
29void * 29void __iomem *
30iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot); 30iomap_atomic_prot_pfn(unsigned long pfn, enum km_type type, pgprot_t prot);
31 31
32void 32void
33iounmap_atomic(void *kvaddr, enum km_type type); 33iounmap_atomic(void __iomem *kvaddr, enum km_type type);
34 34
35int 35int
36iomap_create_wc(resource_size_t base, unsigned long size, pgprot_t *prot); 36iomap_create_wc(resource_size_t base, unsigned long size, pgprot_t *prot);
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index bb5318bbe0e4..6143ebeeebfa 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -130,6 +130,9 @@
130 */ 130 */
131#define MCE_SELF_VECTOR 0xeb 131#define MCE_SELF_VECTOR 0xeb
132 132
133/* Xen vector callback to receive events in a HVM domain */
134#define XEN_HVM_EVTCHN_CALLBACK 0xe9
135
133#define NR_VECTORS 256 136#define NR_VECTORS 256
134 137
135#define FPU_IRQ 13 138#define FPU_IRQ 13
diff --git a/arch/x86/include/asm/kdebug.h b/arch/x86/include/asm/kdebug.h
index fa7c0b974761..5bdfca86581b 100644
--- a/arch/x86/include/asm/kdebug.h
+++ b/arch/x86/include/asm/kdebug.h
@@ -33,5 +33,11 @@ extern void __show_regs(struct pt_regs *regs, int all);
33extern void show_regs(struct pt_regs *regs); 33extern void show_regs(struct pt_regs *regs);
34extern unsigned long oops_begin(void); 34extern unsigned long oops_begin(void);
35extern void oops_end(unsigned long, struct pt_regs *, int signr); 35extern void oops_end(unsigned long, struct pt_regs *, int signr);
36#ifdef CONFIG_KEXEC
37extern int in_crash_kexec;
38#else
39/* no crash dump is ever in progress if no crash kernel can be kexec'd */
40#define in_crash_kexec 0
41#endif
36 42
37#endif /* _ASM_X86_KDEBUG_H */ 43#endif /* _ASM_X86_KDEBUG_H */
diff --git a/arch/x86/include/asm/kgdb.h b/arch/x86/include/asm/kgdb.h
index e6c6c808489f..396f5b5fc4d7 100644
--- a/arch/x86/include/asm/kgdb.h
+++ b/arch/x86/include/asm/kgdb.h
@@ -39,9 +39,11 @@ enum regnames {
39 GDB_FS, /* 14 */ 39 GDB_FS, /* 14 */
40 GDB_GS, /* 15 */ 40 GDB_GS, /* 15 */
41}; 41};
42#define GDB_ORIG_AX 41
43#define DBG_MAX_REG_NUM 16
42#define NUMREGBYTES ((GDB_GS+1)*4) 44#define NUMREGBYTES ((GDB_GS+1)*4)
43#else /* ! CONFIG_X86_32 */ 45#else /* ! CONFIG_X86_32 */
44enum regnames64 { 46enum regnames {
45 GDB_AX, /* 0 */ 47 GDB_AX, /* 0 */
46 GDB_BX, /* 1 */ 48 GDB_BX, /* 1 */
47 GDB_CX, /* 2 */ 49 GDB_CX, /* 2 */
@@ -59,15 +61,15 @@ enum regnames64 {
59 GDB_R14, /* 14 */ 61 GDB_R14, /* 14 */
60 GDB_R15, /* 15 */ 62 GDB_R15, /* 15 */
61 GDB_PC, /* 16 */ 63 GDB_PC, /* 16 */
64 GDB_PS, /* 17 */
65 GDB_CS, /* 18 */
66 GDB_SS, /* 19 */
62}; 67};
63 68#define GDB_ORIG_AX 57
64enum regnames32 { 69#define DBG_MAX_REG_NUM 20
65 GDB_PS = 34, 70/* 17 64 bit regs and 3 32 bit regs */
66 GDB_CS, 71#define NUMREGBYTES ((17 * 8) + (3 * 4))
67 GDB_SS, 72#endif /* ! CONFIG_X86_32 */
68};
69#define NUMREGBYTES ((GDB_SS+1)*4)
70#endif /* CONFIG_X86_32 */
71 73
72static inline void arch_kgdb_breakpoint(void) 74static inline void arch_kgdb_breakpoint(void)
73{ 75{
@@ -76,4 +78,7 @@ static inline void arch_kgdb_breakpoint(void)
76#define BREAK_INSTR_SIZE 1 78#define BREAK_INSTR_SIZE 1
77#define CACHE_FLUSH_IS_SAFE 1 79#define CACHE_FLUSH_IS_SAFE 1
78 80
81extern int kgdb_ll_trap(int cmd, const char *str,
82 struct pt_regs *regs, long err, int trap, int sig);
83
79#endif /* _ASM_X86_KGDB_H */ 84#endif /* _ASM_X86_KGDB_H */
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index 4ffa345a8ccb..547882539157 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -24,6 +24,7 @@
24#include <linux/types.h> 24#include <linux/types.h>
25#include <linux/ptrace.h> 25#include <linux/ptrace.h>
26#include <linux/percpu.h> 26#include <linux/percpu.h>
27#include <asm/insn.h>
27 28
28#define __ARCH_WANT_KPROBES_INSN_SLOT 29#define __ARCH_WANT_KPROBES_INSN_SLOT
29 30
@@ -36,7 +37,6 @@ typedef u8 kprobe_opcode_t;
36#define RELATIVEJUMP_SIZE 5 37#define RELATIVEJUMP_SIZE 5
37#define RELATIVECALL_OPCODE 0xe8 38#define RELATIVECALL_OPCODE 0xe8
38#define RELATIVE_ADDR_SIZE 4 39#define RELATIVE_ADDR_SIZE 4
39#define MAX_INSN_SIZE 16
40#define MAX_STACK_SIZE 64 40#define MAX_STACK_SIZE 64
41#define MIN_STACK_SIZE(ADDR) \ 41#define MIN_STACK_SIZE(ADDR) \
42 (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \ 42 (((MAX_STACK_SIZE) < (((unsigned long)current_thread_info()) + \
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index f46b79f6c16c..4d8dcbdfc120 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -21,6 +21,9 @@
21#define __KVM_HAVE_PIT_STATE2 21#define __KVM_HAVE_PIT_STATE2
22#define __KVM_HAVE_XEN_HVM 22#define __KVM_HAVE_XEN_HVM
23#define __KVM_HAVE_VCPU_EVENTS 23#define __KVM_HAVE_VCPU_EVENTS
24#define __KVM_HAVE_DEBUGREGS
25#define __KVM_HAVE_XSAVE
26#define __KVM_HAVE_XCRS
24 27
25/* Architectural interrupt line count. */ 28/* Architectural interrupt line count. */
26#define KVM_NR_INTERRUPTS 256 29#define KVM_NR_INTERRUPTS 256
@@ -257,6 +260,11 @@ struct kvm_reinject_control {
257/* When set in flags, include corresponding fields on KVM_SET_VCPU_EVENTS */ 260/* When set in flags, include corresponding fields on KVM_SET_VCPU_EVENTS */
258#define KVM_VCPUEVENT_VALID_NMI_PENDING 0x00000001 261#define KVM_VCPUEVENT_VALID_NMI_PENDING 0x00000001
259#define KVM_VCPUEVENT_VALID_SIPI_VECTOR 0x00000002 262#define KVM_VCPUEVENT_VALID_SIPI_VECTOR 0x00000002
263#define KVM_VCPUEVENT_VALID_SHADOW 0x00000004
264
265/* Interrupt shadow states */
266#define KVM_X86_SHADOW_INT_MOV_SS 0x01
267#define KVM_X86_SHADOW_INT_STI 0x02
260 268
261/* for KVM_GET/SET_VCPU_EVENTS */ 269/* for KVM_GET/SET_VCPU_EVENTS */
262struct kvm_vcpu_events { 270struct kvm_vcpu_events {
@@ -271,7 +279,7 @@ struct kvm_vcpu_events {
271 __u8 injected; 279 __u8 injected;
272 __u8 nr; 280 __u8 nr;
273 __u8 soft; 281 __u8 soft;
274 __u8 pad; 282 __u8 shadow;
275 } interrupt; 283 } interrupt;
276 struct { 284 struct {
277 __u8 injected; 285 __u8 injected;
@@ -284,4 +292,33 @@ struct kvm_vcpu_events {
284 __u32 reserved[10]; 292 __u32 reserved[10];
285}; 293};
286 294
295/* for KVM_GET/SET_DEBUGREGS */
296struct kvm_debugregs {
297 __u64 db[4];
298 __u64 dr6;
299 __u64 dr7;
300 __u64 flags;
301 __u64 reserved[9];
302};
303
304/* for KVM_CAP_XSAVE */
305struct kvm_xsave {
306 __u32 region[1024];
307};
308
309#define KVM_MAX_XCRS 16
310
311struct kvm_xcr {
312 __u32 xcr;
313 __u32 reserved;
314 __u64 value;
315};
316
317struct kvm_xcrs {
318 __u32 nr_xcrs;
319 __u32 flags;
320 struct kvm_xcr xcrs[KVM_MAX_XCRS];
321 __u64 padding[16];
322};
323
287#endif /* _ASM_X86_KVM_H */ 324#endif /* _ASM_X86_KVM_H */
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index 7a6f54fa13ba..1f99ecfc48e1 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -11,6 +11,8 @@
11#ifndef _ASM_X86_KVM_X86_EMULATE_H 11#ifndef _ASM_X86_KVM_X86_EMULATE_H
12#define _ASM_X86_KVM_X86_EMULATE_H 12#define _ASM_X86_KVM_X86_EMULATE_H
13 13
14#include <asm/desc_defs.h>
15
14struct x86_emulate_ctxt; 16struct x86_emulate_ctxt;
15 17
16/* 18/*
@@ -49,8 +51,10 @@ struct x86_emulate_ctxt;
49#define X86EMUL_UNHANDLEABLE 1 51#define X86EMUL_UNHANDLEABLE 1
50/* Terminate emulation but return success to the caller. */ 52/* Terminate emulation but return success to the caller. */
51#define X86EMUL_PROPAGATE_FAULT 2 /* propagate a generated fault to guest */ 53#define X86EMUL_PROPAGATE_FAULT 2 /* propagate a generated fault to guest */
52#define X86EMUL_RETRY_INSTR 2 /* retry the instruction for some reason */ 54#define X86EMUL_RETRY_INSTR 3 /* retry the instruction for some reason */
53#define X86EMUL_CMPXCHG_FAILED 2 /* cmpxchg did not see expected value */ 55#define X86EMUL_CMPXCHG_FAILED 4 /* cmpxchg did not see expected value */
56#define X86EMUL_IO_NEEDED 5 /* IO is needed to complete emulation */
57
54struct x86_emulate_ops { 58struct x86_emulate_ops {
55 /* 59 /*
56 * read_std: Read bytes of standard (non-emulated/special) memory. 60 * read_std: Read bytes of standard (non-emulated/special) memory.
@@ -63,6 +67,15 @@ struct x86_emulate_ops {
63 unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error); 67 unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
64 68
65 /* 69 /*
70 * write_std: Write bytes of standard (non-emulated/special) memory.
71 * Used for descriptor writing.
72 * @addr: [IN ] Linear address to which to write.
73 * @val: [OUT] Value write to memory, zero-extended to 'u_long'.
74 * @bytes: [IN ] Number of bytes to write to memory.
75 */
76 int (*write_std)(unsigned long addr, void *val,
77 unsigned int bytes, struct kvm_vcpu *vcpu, u32 *error);
78 /*
66 * fetch: Read bytes of standard (non-emulated/special) memory. 79 * fetch: Read bytes of standard (non-emulated/special) memory.
67 * Used for instruction fetch. 80 * Used for instruction fetch.
68 * @addr: [IN ] Linear address from which to read. 81 * @addr: [IN ] Linear address from which to read.
@@ -81,6 +94,7 @@ struct x86_emulate_ops {
81 int (*read_emulated)(unsigned long addr, 94 int (*read_emulated)(unsigned long addr,
82 void *val, 95 void *val,
83 unsigned int bytes, 96 unsigned int bytes,
97 unsigned int *error,
84 struct kvm_vcpu *vcpu); 98 struct kvm_vcpu *vcpu);
85 99
86 /* 100 /*
@@ -93,6 +107,7 @@ struct x86_emulate_ops {
93 int (*write_emulated)(unsigned long addr, 107 int (*write_emulated)(unsigned long addr,
94 const void *val, 108 const void *val,
95 unsigned int bytes, 109 unsigned int bytes,
110 unsigned int *error,
96 struct kvm_vcpu *vcpu); 111 struct kvm_vcpu *vcpu);
97 112
98 /* 113 /*
@@ -107,15 +122,46 @@ struct x86_emulate_ops {
107 const void *old, 122 const void *old,
108 const void *new, 123 const void *new,
109 unsigned int bytes, 124 unsigned int bytes,
125 unsigned int *error,
110 struct kvm_vcpu *vcpu); 126 struct kvm_vcpu *vcpu);
111 127
128 int (*pio_in_emulated)(int size, unsigned short port, void *val,
129 unsigned int count, struct kvm_vcpu *vcpu);
130
131 int (*pio_out_emulated)(int size, unsigned short port, const void *val,
132 unsigned int count, struct kvm_vcpu *vcpu);
133
134 bool (*get_cached_descriptor)(struct desc_struct *desc,
135 int seg, struct kvm_vcpu *vcpu);
136 void (*set_cached_descriptor)(struct desc_struct *desc,
137 int seg, struct kvm_vcpu *vcpu);
138 u16 (*get_segment_selector)(int seg, struct kvm_vcpu *vcpu);
139 void (*set_segment_selector)(u16 sel, int seg, struct kvm_vcpu *vcpu);
140 unsigned long (*get_cached_segment_base)(int seg, struct kvm_vcpu *vcpu);
141 void (*get_gdt)(struct desc_ptr *dt, struct kvm_vcpu *vcpu);
142 ulong (*get_cr)(int cr, struct kvm_vcpu *vcpu);
143 int (*set_cr)(int cr, ulong val, struct kvm_vcpu *vcpu);
144 int (*cpl)(struct kvm_vcpu *vcpu);
145 int (*get_dr)(int dr, unsigned long *dest, struct kvm_vcpu *vcpu);
146 int (*set_dr)(int dr, unsigned long value, struct kvm_vcpu *vcpu);
147 int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data);
148 int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata);
112}; 149};
113 150
114/* Type, address-of, and value of an instruction's operand. */ 151/* Type, address-of, and value of an instruction's operand. */
115struct operand { 152struct operand {
116 enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type; 153 enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type;
117 unsigned int bytes; 154 unsigned int bytes;
118 unsigned long val, orig_val, *ptr; 155 union {
156 unsigned long orig_val;
157 u64 orig_val64;
158 };
159 unsigned long *ptr;
160 union {
161 unsigned long val;
162 u64 val64;
163 char valptr[sizeof(unsigned long) + 2];
164 };
119}; 165};
120 166
121struct fetch_cache { 167struct fetch_cache {
@@ -124,6 +170,12 @@ struct fetch_cache {
124 unsigned long end; 170 unsigned long end;
125}; 171};
126 172
173struct read_cache {
174 u8 data[1024];
175 unsigned long pos;
176 unsigned long end;
177};
178
127struct decode_cache { 179struct decode_cache {
128 u8 twobyte; 180 u8 twobyte;
129 u8 b; 181 u8 b;
@@ -139,7 +191,7 @@ struct decode_cache {
139 u8 seg_override; 191 u8 seg_override;
140 unsigned int d; 192 unsigned int d;
141 unsigned long regs[NR_VCPU_REGS]; 193 unsigned long regs[NR_VCPU_REGS];
142 unsigned long eip, eip_orig; 194 unsigned long eip;
143 /* modrm */ 195 /* modrm */
144 u8 modrm; 196 u8 modrm;
145 u8 modrm_mod; 197 u8 modrm_mod;
@@ -151,16 +203,16 @@ struct decode_cache {
151 void *modrm_ptr; 203 void *modrm_ptr;
152 unsigned long modrm_val; 204 unsigned long modrm_val;
153 struct fetch_cache fetch; 205 struct fetch_cache fetch;
206 struct read_cache io_read;
207 struct read_cache mem_read;
154}; 208};
155 209
156#define X86_SHADOW_INT_MOV_SS 1
157#define X86_SHADOW_INT_STI 2
158
159struct x86_emulate_ctxt { 210struct x86_emulate_ctxt {
160 /* Register state before/after emulation. */ 211 /* Register state before/after emulation. */
161 struct kvm_vcpu *vcpu; 212 struct kvm_vcpu *vcpu;
162 213
163 unsigned long eflags; 214 unsigned long eflags;
215 unsigned long eip; /* eip before instruction emulation */
164 /* Emulated execution mode, represented by an X86EMUL_MODE value. */ 216 /* Emulated execution mode, represented by an X86EMUL_MODE value. */
165 int mode; 217 int mode;
166 u32 cs_base; 218 u32 cs_base;
@@ -168,6 +220,13 @@ struct x86_emulate_ctxt {
168 /* interruptibility state, as a result of execution of STI or MOV SS */ 220 /* interruptibility state, as a result of execution of STI or MOV SS */
169 int interruptibility; 221 int interruptibility;
170 222
223 bool restart; /* restart string instruction after writeback */
224
225 int exception; /* exception that happens during emulation or -1 */
226 u32 error_code; /* error code for exception */
227 bool error_code_valid;
228 unsigned long cr2; /* faulted address in case of #PF */
229
171 /* decode cache */ 230 /* decode cache */
172 struct decode_cache decode; 231 struct decode_cache decode;
173}; 232};
@@ -194,5 +253,9 @@ int x86_decode_insn(struct x86_emulate_ctxt *ctxt,
194 struct x86_emulate_ops *ops); 253 struct x86_emulate_ops *ops);
195int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, 254int x86_emulate_insn(struct x86_emulate_ctxt *ctxt,
196 struct x86_emulate_ops *ops); 255 struct x86_emulate_ops *ops);
256int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
257 struct x86_emulate_ops *ops,
258 u16 tss_selector, int reason,
259 bool has_error_code, u32 error_code);
197 260
198#endif /* _ASM_X86_KVM_X86_EMULATE_H */ 261#endif /* _ASM_X86_KVM_X86_EMULATE_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 06d9e79ca37d..c52e2eb40a1e 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -15,6 +15,7 @@
15#include <linux/mm.h> 15#include <linux/mm.h>
16#include <linux/mmu_notifier.h> 16#include <linux/mmu_notifier.h>
17#include <linux/tracepoint.h> 17#include <linux/tracepoint.h>
18#include <linux/cpumask.h>
18 19
19#include <linux/kvm.h> 20#include <linux/kvm.h>
20#include <linux/kvm_para.h> 21#include <linux/kvm_para.h>
@@ -39,11 +40,14 @@
39 0xFFFFFF0000000000ULL) 40 0xFFFFFF0000000000ULL)
40 41
41#define INVALID_PAGE (~(hpa_t)0) 42#define INVALID_PAGE (~(hpa_t)0)
43#define VALID_PAGE(x) ((x) != INVALID_PAGE)
44
42#define UNMAPPED_GVA (~(gpa_t)0) 45#define UNMAPPED_GVA (~(gpa_t)0)
43 46
44/* KVM Hugepage definitions for x86 */ 47/* KVM Hugepage definitions for x86 */
45#define KVM_NR_PAGE_SIZES 3 48#define KVM_NR_PAGE_SIZES 3
46#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + (((x) - 1) * 9)) 49#define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9)
50#define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x))
47#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) 51#define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x))
48#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 52#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
49#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 53#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
@@ -69,8 +73,6 @@
69 73
70#define IOPL_SHIFT 12 74#define IOPL_SHIFT 12
71 75
72#define KVM_ALIAS_SLOTS 4
73
74#define KVM_PERMILLE_MMU_PAGES 20 76#define KVM_PERMILLE_MMU_PAGES 20
75#define KVM_MIN_ALLOC_MMU_PAGES 64 77#define KVM_MIN_ALLOC_MMU_PAGES 64
76#define KVM_MMU_HASH_SHIFT 10 78#define KVM_MMU_HASH_SHIFT 10
@@ -171,15 +173,15 @@ struct kvm_pte_chain {
171union kvm_mmu_page_role { 173union kvm_mmu_page_role {
172 unsigned word; 174 unsigned word;
173 struct { 175 struct {
174 unsigned glevels:4;
175 unsigned level:4; 176 unsigned level:4;
177 unsigned cr4_pae:1;
176 unsigned quadrant:2; 178 unsigned quadrant:2;
177 unsigned pad_for_nice_hex_output:6; 179 unsigned pad_for_nice_hex_output:6;
178 unsigned direct:1; 180 unsigned direct:1;
179 unsigned access:3; 181 unsigned access:3;
180 unsigned invalid:1; 182 unsigned invalid:1;
181 unsigned cr4_pge:1;
182 unsigned nxe:1; 183 unsigned nxe:1;
184 unsigned cr0_wp:1;
183 }; 185 };
184}; 186};
185 187
@@ -187,8 +189,6 @@ struct kvm_mmu_page {
187 struct list_head link; 189 struct list_head link;
188 struct hlist_node hash_link; 190 struct hlist_node hash_link;
189 191
190 struct list_head oos_link;
191
192 /* 192 /*
193 * The following two entries are used to key the shadow page in the 193 * The following two entries are used to key the shadow page in the
194 * hash table. 194 * hash table.
@@ -204,9 +204,9 @@ struct kvm_mmu_page {
204 * in this shadow page. 204 * in this shadow page.
205 */ 205 */
206 DECLARE_BITMAP(slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS); 206 DECLARE_BITMAP(slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS);
207 int multimapped; /* More than one parent_pte? */ 207 bool multimapped; /* More than one parent_pte? */
208 int root_count; /* Currently serving as active root */
209 bool unsync; 208 bool unsync;
209 int root_count; /* Currently serving as active root */
210 unsigned int unsync_children; 210 unsigned int unsync_children;
211 union { 211 union {
212 u64 *parent_pte; /* !multimapped */ 212 u64 *parent_pte; /* !multimapped */
@@ -224,14 +224,9 @@ struct kvm_pv_mmu_op_buffer {
224 224
225struct kvm_pio_request { 225struct kvm_pio_request {
226 unsigned long count; 226 unsigned long count;
227 int cur_count;
228 gva_t guest_gva;
229 int in; 227 int in;
230 int port; 228 int port;
231 int size; 229 int size;
232 int string;
233 int down;
234 int rep;
235}; 230};
236 231
237/* 232/*
@@ -248,7 +243,7 @@ struct kvm_mmu {
248 void (*prefetch_page)(struct kvm_vcpu *vcpu, 243 void (*prefetch_page)(struct kvm_vcpu *vcpu,
249 struct kvm_mmu_page *page); 244 struct kvm_mmu_page *page);
250 int (*sync_page)(struct kvm_vcpu *vcpu, 245 int (*sync_page)(struct kvm_vcpu *vcpu,
251 struct kvm_mmu_page *sp); 246 struct kvm_mmu_page *sp, bool clear_unsync);
252 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva); 247 void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva);
253 hpa_t root_hpa; 248 hpa_t root_hpa;
254 int root_level; 249 int root_level;
@@ -308,8 +303,8 @@ struct kvm_vcpu_arch {
308 unsigned long mmu_seq; 303 unsigned long mmu_seq;
309 } update_pte; 304 } update_pte;
310 305
311 struct i387_fxsave_struct host_fx_image; 306 struct fpu guest_fpu;
312 struct i387_fxsave_struct guest_fx_image; 307 u64 xcr0;
313 308
314 gva_t mmio_fault_cr2; 309 gva_t mmio_fault_cr2;
315 struct kvm_pio_request pio; 310 struct kvm_pio_request pio;
@@ -320,6 +315,7 @@ struct kvm_vcpu_arch {
320 struct kvm_queued_exception { 315 struct kvm_queued_exception {
321 bool pending; 316 bool pending;
322 bool has_error_code; 317 bool has_error_code;
318 bool reinject;
323 u8 nr; 319 u8 nr;
324 u32 error_code; 320 u32 error_code;
325 } exception; 321 } exception;
@@ -362,33 +358,19 @@ struct kvm_vcpu_arch {
362 u64 *mce_banks; 358 u64 *mce_banks;
363 359
364 /* used for guest single stepping over the given code position */ 360 /* used for guest single stepping over the given code position */
365 u16 singlestep_cs;
366 unsigned long singlestep_rip; 361 unsigned long singlestep_rip;
362
367 /* fields used by HYPER-V emulation */ 363 /* fields used by HYPER-V emulation */
368 u64 hv_vapic; 364 u64 hv_vapic;
369};
370 365
371struct kvm_mem_alias { 366 cpumask_var_t wbinvd_dirty_mask;
372 gfn_t base_gfn;
373 unsigned long npages;
374 gfn_t target_gfn;
375#define KVM_ALIAS_INVALID 1UL
376 unsigned long flags;
377};
378
379#define KVM_ARCH_HAS_UNALIAS_INSTANTIATION
380
381struct kvm_mem_aliases {
382 struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
383 int naliases;
384}; 367};
385 368
386struct kvm_arch { 369struct kvm_arch {
387 struct kvm_mem_aliases *aliases;
388
389 unsigned int n_free_mmu_pages; 370 unsigned int n_free_mmu_pages;
390 unsigned int n_requested_mmu_pages; 371 unsigned int n_requested_mmu_pages;
391 unsigned int n_alloc_mmu_pages; 372 unsigned int n_alloc_mmu_pages;
373 atomic_t invlpg_counter;
392 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; 374 struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES];
393 /* 375 /*
394 * Hash table of struct kvm_mmu_page. 376 * Hash table of struct kvm_mmu_page.
@@ -461,11 +443,6 @@ struct kvm_vcpu_stat {
461 u32 nmi_injections; 443 u32 nmi_injections;
462}; 444};
463 445
464struct descriptor_table {
465 u16 limit;
466 unsigned long base;
467} __attribute__((packed));
468
469struct kvm_x86_ops { 446struct kvm_x86_ops {
470 int (*cpu_has_kvm_support)(void); /* __init */ 447 int (*cpu_has_kvm_support)(void); /* __init */
471 int (*disabled_by_bios)(void); /* __init */ 448 int (*disabled_by_bios)(void); /* __init */
@@ -503,12 +480,11 @@ struct kvm_x86_ops {
503 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); 480 void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
504 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); 481 void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
505 void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); 482 void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
506 void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); 483 void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
507 void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); 484 void (*set_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
508 void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); 485 void (*get_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
509 void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); 486 void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
510 int (*get_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long *dest); 487 void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
511 int (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value);
512 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg); 488 void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
513 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); 489 unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
514 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); 490 void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
@@ -527,7 +503,8 @@ struct kvm_x86_ops {
527 void (*set_irq)(struct kvm_vcpu *vcpu); 503 void (*set_irq)(struct kvm_vcpu *vcpu);
528 void (*set_nmi)(struct kvm_vcpu *vcpu); 504 void (*set_nmi)(struct kvm_vcpu *vcpu);
529 void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr, 505 void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr,
530 bool has_error_code, u32 error_code); 506 bool has_error_code, u32 error_code,
507 bool reinject);
531 int (*interrupt_allowed)(struct kvm_vcpu *vcpu); 508 int (*interrupt_allowed)(struct kvm_vcpu *vcpu);
532 int (*nmi_allowed)(struct kvm_vcpu *vcpu); 509 int (*nmi_allowed)(struct kvm_vcpu *vcpu);
533 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); 510 bool (*get_nmi_mask)(struct kvm_vcpu *vcpu);
@@ -541,6 +518,10 @@ struct kvm_x86_ops {
541 int (*get_lpage_level)(void); 518 int (*get_lpage_level)(void);
542 bool (*rdtscp_supported)(void); 519 bool (*rdtscp_supported)(void);
543 520
521 void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry);
522
523 bool (*has_wbinvd_exit)(void);
524
544 const struct trace_print_flags *exit_reasons_str; 525 const struct trace_print_flags *exit_reasons_str;
545}; 526};
546 527
@@ -584,47 +565,38 @@ enum emulation_result {
584#define EMULTYPE_SKIP (1 << 2) 565#define EMULTYPE_SKIP (1 << 2)
585int emulate_instruction(struct kvm_vcpu *vcpu, 566int emulate_instruction(struct kvm_vcpu *vcpu,
586 unsigned long cr2, u16 error_code, int emulation_type); 567 unsigned long cr2, u16 error_code, int emulation_type);
587void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context);
588void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); 568void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
589void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); 569void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address);
590void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
591 unsigned long *rflags);
592 570
593unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr);
594void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value,
595 unsigned long *rflags);
596void kvm_enable_efer_bits(u64); 571void kvm_enable_efer_bits(u64);
597int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); 572int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data);
598int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); 573int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data);
599 574
600struct x86_emulate_ctxt; 575struct x86_emulate_ctxt;
601 576
602int kvm_emulate_pio(struct kvm_vcpu *vcpu, int in, 577int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port);
603 int size, unsigned port);
604int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, int in,
605 int size, unsigned long count, int down,
606 gva_t address, int rep, unsigned port);
607void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); 578void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
608int kvm_emulate_halt(struct kvm_vcpu *vcpu); 579int kvm_emulate_halt(struct kvm_vcpu *vcpu);
609int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); 580int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address);
610int emulate_clts(struct kvm_vcpu *vcpu); 581int emulate_clts(struct kvm_vcpu *vcpu);
611int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, 582int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
612 unsigned long *dest);
613int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr,
614 unsigned long value);
615 583
616void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); 584void kvm_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
617int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg); 585int kvm_load_segment_descriptor(struct kvm_vcpu *vcpu, u16 selector, int seg);
618 586
619int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason); 587int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int reason,
588 bool has_error_code, u32 error_code);
620 589
621void kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); 590int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
622void kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3); 591int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3);
623void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); 592int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
624void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); 593void kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8);
594int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val);
595int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val);
625unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); 596unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu);
626void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); 597void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw);
627void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); 598void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
599int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr);
628 600
629int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); 601int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
630int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); 602int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data);
@@ -634,6 +606,8 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
634 606
635void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); 607void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
636void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); 608void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
609void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
610void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
637void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, 611void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2,
638 u32 error_code); 612 u32 error_code);
639bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); 613bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
@@ -642,14 +616,7 @@ int kvm_pic_set_irq(void *opaque, int irq, int level);
642 616
643void kvm_inject_nmi(struct kvm_vcpu *vcpu); 617void kvm_inject_nmi(struct kvm_vcpu *vcpu);
644 618
645void fx_init(struct kvm_vcpu *vcpu); 619int fx_init(struct kvm_vcpu *vcpu);
646
647int emulator_write_emulated(unsigned long addr,
648 const void *val,
649 unsigned int bytes,
650 struct kvm_vcpu *vcpu);
651
652unsigned long segment_base(u16 selector);
653 620
654void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu); 621void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu);
655void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, 622void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
@@ -675,12 +642,9 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
675void kvm_enable_tdp(void); 642void kvm_enable_tdp(void);
676void kvm_disable_tdp(void); 643void kvm_disable_tdp(void);
677 644
678int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
679int complete_pio(struct kvm_vcpu *vcpu); 645int complete_pio(struct kvm_vcpu *vcpu);
680bool kvm_check_iopl(struct kvm_vcpu *vcpu); 646bool kvm_check_iopl(struct kvm_vcpu *vcpu);
681 647
682struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn);
683
684static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) 648static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
685{ 649{
686 struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); 650 struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT);
@@ -688,20 +652,6 @@ static inline struct kvm_mmu_page *page_header(hpa_t shadow_page)
688 return (struct kvm_mmu_page *)page_private(page); 652 return (struct kvm_mmu_page *)page_private(page);
689} 653}
690 654
691static inline u16 kvm_read_fs(void)
692{
693 u16 seg;
694 asm("mov %%fs, %0" : "=g"(seg));
695 return seg;
696}
697
698static inline u16 kvm_read_gs(void)
699{
700 u16 seg;
701 asm("mov %%gs, %0" : "=g"(seg));
702 return seg;
703}
704
705static inline u16 kvm_read_ldt(void) 655static inline u16 kvm_read_ldt(void)
706{ 656{
707 u16 ldt; 657 u16 ldt;
@@ -709,38 +659,11 @@ static inline u16 kvm_read_ldt(void)
709 return ldt; 659 return ldt;
710} 660}
711 661
712static inline void kvm_load_fs(u16 sel)
713{
714 asm("mov %0, %%fs" : : "rm"(sel));
715}
716
717static inline void kvm_load_gs(u16 sel)
718{
719 asm("mov %0, %%gs" : : "rm"(sel));
720}
721
722static inline void kvm_load_ldt(u16 sel) 662static inline void kvm_load_ldt(u16 sel)
723{ 663{
724 asm("lldt %0" : : "rm"(sel)); 664 asm("lldt %0" : : "rm"(sel));
725} 665}
726 666
727static inline void kvm_get_idt(struct descriptor_table *table)
728{
729 asm("sidt %0" : "=m"(*table));
730}
731
732static inline void kvm_get_gdt(struct descriptor_table *table)
733{
734 asm("sgdt %0" : "=m"(*table));
735}
736
737static inline unsigned long kvm_read_tr_base(void)
738{
739 u16 tr;
740 asm("str %0" : "=g"(tr));
741 return segment_base(tr);
742}
743
744#ifdef CONFIG_X86_64 667#ifdef CONFIG_X86_64
745static inline unsigned long read_msr(unsigned long msr) 668static inline unsigned long read_msr(unsigned long msr)
746{ 669{
@@ -751,21 +674,6 @@ static inline unsigned long read_msr(unsigned long msr)
751} 674}
752#endif 675#endif
753 676
754static inline void kvm_fx_save(struct i387_fxsave_struct *image)
755{
756 asm("fxsave (%0)":: "r" (image));
757}
758
759static inline void kvm_fx_restore(struct i387_fxsave_struct *image)
760{
761 asm("fxrstor (%0)":: "r" (image));
762}
763
764static inline void kvm_fx_finit(void)
765{
766 asm("finit");
767}
768
769static inline u32 get_rdx_init_val(void) 677static inline u32 get_rdx_init_val(void)
770{ 678{
771 return 0x600; /* P6 family */ 679 return 0x600; /* P6 family */
@@ -826,4 +734,6 @@ int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
826void kvm_define_shared_msr(unsigned index, u32 msr); 734void kvm_define_shared_msr(unsigned index, u32 msr);
827void kvm_set_shared_msr(unsigned index, u64 val, u64 mask); 735void kvm_set_shared_msr(unsigned index, u64 val, u64 mask);
828 736
737bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip);
738
829#endif /* _ASM_X86_KVM_HOST_H */ 739#endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index ffae1420e7d7..05eba5e9a8e8 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -16,10 +16,23 @@
16#define KVM_FEATURE_CLOCKSOURCE 0 16#define KVM_FEATURE_CLOCKSOURCE 0
17#define KVM_FEATURE_NOP_IO_DELAY 1 17#define KVM_FEATURE_NOP_IO_DELAY 1
18#define KVM_FEATURE_MMU_OP 2 18#define KVM_FEATURE_MMU_OP 2
19/* This indicates that the new set of kvmclock msrs
20 * are available. The use of 0x11 and 0x12 is deprecated
21 */
22#define KVM_FEATURE_CLOCKSOURCE2 3
23
24/* The last 8 bits are used to indicate how to interpret the flags field
25 * in pvclock structure. If no bits are set, all flags are ignored.
26 */
27#define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24
19 28
20#define MSR_KVM_WALL_CLOCK 0x11 29#define MSR_KVM_WALL_CLOCK 0x11
21#define MSR_KVM_SYSTEM_TIME 0x12 30#define MSR_KVM_SYSTEM_TIME 0x12
22 31
32/* Custom MSRs falls in the range 0x4b564d00-0x4b564dff */
33#define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00
34#define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01
35
23#define KVM_MAX_MMU_OP_BATCH 32 36#define KVM_MAX_MMU_OP_BATCH 32
24 37
25/* Operations for KVM_HC_MMU_OP */ 38/* Operations for KVM_HC_MMU_OP */
diff --git a/arch/x86/include/asm/local64.h b/arch/x86/include/asm/local64.h
new file mode 100644
index 000000000000..36c93b5cc239
--- /dev/null
+++ b/arch/x86/include/asm/local64.h
@@ -0,0 +1 @@
#include <asm-generic/local64.h>
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 6c3fdd631ed3..c62c13cb9788 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -38,6 +38,10 @@
38#define MCM_ADDR_MEM 3 /* memory address */ 38#define MCM_ADDR_MEM 3 /* memory address */
39#define MCM_ADDR_GENERIC 7 /* generic */ 39#define MCM_ADDR_GENERIC 7 /* generic */
40 40
41/* CTL2 register defines */
42#define MCI_CTL2_CMCI_EN (1ULL << 30)
43#define MCI_CTL2_CMCI_THRESHOLD_MASK 0x7fffULL
44
41#define MCJ_CTX_MASK 3 45#define MCJ_CTX_MASK 3
42#define MCJ_CTX(flags) ((flags) & MCJ_CTX_MASK) 46#define MCJ_CTX(flags) ((flags) & MCJ_CTX_MASK)
43#define MCJ_CTX_RANDOM 0 /* inject context: random */ 47#define MCJ_CTX_RANDOM 0 /* inject context: random */
@@ -225,5 +229,13 @@ extern void mcheck_intel_therm_init(void);
225static inline void mcheck_intel_therm_init(void) { } 229static inline void mcheck_intel_therm_init(void) { }
226#endif 230#endif
227 231
232/*
233 * Used by APEI to report memory error via /dev/mcelog
234 */
235
236struct cper_sec_mem_err;
237extern void apei_mce_report_mem_error(int corrected,
238 struct cper_sec_mem_err *mem_err);
239
228#endif /* __KERNEL__ */ 240#endif /* __KERNEL__ */
229#endif /* _ASM_X86_MCE_H */ 241#endif /* _ASM_X86_MCE_H */
diff --git a/arch/x86/include/asm/mpspec.h b/arch/x86/include/asm/mpspec.h
index d8bf23a88d05..c82868e9f905 100644
--- a/arch/x86/include/asm/mpspec.h
+++ b/arch/x86/include/asm/mpspec.h
@@ -105,16 +105,6 @@ extern void mp_config_acpi_legacy_irqs(void);
105struct device; 105struct device;
106extern int mp_register_gsi(struct device *dev, u32 gsi, int edge_level, 106extern int mp_register_gsi(struct device *dev, u32 gsi, int edge_level,
107 int active_high_low); 107 int active_high_low);
108extern int acpi_probe_gsi(void);
109#ifdef CONFIG_X86_IO_APIC
110extern int mp_find_ioapic(int gsi);
111extern int mp_find_ioapic_pin(int ioapic, int gsi);
112#endif
113#else /* !CONFIG_ACPI: */
114static inline int acpi_probe_gsi(void)
115{
116 return 0;
117}
118#endif /* CONFIG_ACPI */ 108#endif /* CONFIG_ACPI */
119 109
120#define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_APICS) 110#define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_APICS)
diff --git a/arch/x86/include/asm/mrst.h b/arch/x86/include/asm/mrst.h
index 451d30e7f62d..16350740edf6 100644
--- a/arch/x86/include/asm/mrst.h
+++ b/arch/x86/include/asm/mrst.h
@@ -13,6 +13,32 @@
13extern int pci_mrst_init(void); 13extern int pci_mrst_init(void);
14int __init sfi_parse_mrtc(struct sfi_table_header *table); 14int __init sfi_parse_mrtc(struct sfi_table_header *table);
15 15
16/*
17 * Medfield is the follow-up of Moorestown, it combines two chip solution into
18 * one. Other than that it also added always-on and constant tsc and lapic
19 * timers. Medfield is the platform name, and the chip name is called Penwell
20 * we treat Medfield/Penwell as a variant of Moorestown. Penwell can be
21 * identified via MSRs.
22 */
23enum mrst_cpu_type {
24 MRST_CPU_CHIP_LINCROFT = 1,
25 MRST_CPU_CHIP_PENWELL,
26};
27
28extern enum mrst_cpu_type __mrst_cpu_chip;
29static enum mrst_cpu_type mrst_identify_cpu(void)
30{
31 return __mrst_cpu_chip;
32}
33
34enum mrst_timer_options {
35 MRST_TIMER_DEFAULT,
36 MRST_TIMER_APBT_ONLY,
37 MRST_TIMER_LAPIC_APBT,
38};
39
40extern enum mrst_timer_options mrst_timer_options;
41
16#define SFI_MTMR_MAX_NUM 8 42#define SFI_MTMR_MAX_NUM 8
17#define SFI_MRTC_MAX 8 43#define SFI_MRTC_MAX 8
18 44
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
new file mode 100644
index 000000000000..79ce5685ab64
--- /dev/null
+++ b/arch/x86/include/asm/mshyperv.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_X86_MSHYPER_H
2#define _ASM_X86_MSHYPER_H
3
4#include <linux/types.h>
5#include <asm/hyperv.h>
6
7struct ms_hyperv_info {
8 u32 features;
9 u32 hints;
10};
11
12extern struct ms_hyperv_info ms_hyperv;
13
14#endif
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 4604e6a54d36..986f7790fdb2 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -20,6 +20,7 @@
20#define _EFER_LMA 10 /* Long mode active (read-only) */ 20#define _EFER_LMA 10 /* Long mode active (read-only) */
21#define _EFER_NX 11 /* No execute enable */ 21#define _EFER_NX 11 /* No execute enable */
22#define _EFER_SVME 12 /* Enable virtualization */ 22#define _EFER_SVME 12 /* Enable virtualization */
23#define _EFER_LMSLE 13 /* Long Mode Segment Limit Enable */
23#define _EFER_FFXSR 14 /* Enable Fast FXSAVE/FXRSTOR */ 24#define _EFER_FFXSR 14 /* Enable Fast FXSAVE/FXRSTOR */
24 25
25#define EFER_SCE (1<<_EFER_SCE) 26#define EFER_SCE (1<<_EFER_SCE)
@@ -27,6 +28,7 @@
27#define EFER_LMA (1<<_EFER_LMA) 28#define EFER_LMA (1<<_EFER_LMA)
28#define EFER_NX (1<<_EFER_NX) 29#define EFER_NX (1<<_EFER_NX)
29#define EFER_SVME (1<<_EFER_SVME) 30#define EFER_SVME (1<<_EFER_SVME)
31#define EFER_LMSLE (1<<_EFER_LMSLE)
30#define EFER_FFXSR (1<<_EFER_FFXSR) 32#define EFER_FFXSR (1<<_EFER_FFXSR)
31 33
32/* Intel MSRs. Some also available on other CPUs */ 34/* Intel MSRs. Some also available on other CPUs */
@@ -71,11 +73,14 @@
71#define MSR_IA32_LASTINTTOIP 0x000001de 73#define MSR_IA32_LASTINTTOIP 0x000001de
72 74
73/* DEBUGCTLMSR bits (others vary by model): */ 75/* DEBUGCTLMSR bits (others vary by model): */
74#define _DEBUGCTLMSR_LBR 0 /* last branch recording */ 76#define DEBUGCTLMSR_LBR (1UL << 0) /* last branch recording */
75#define _DEBUGCTLMSR_BTF 1 /* single-step on branches */ 77#define DEBUGCTLMSR_BTF (1UL << 1) /* single-step on branches */
76 78#define DEBUGCTLMSR_TR (1UL << 6)
77#define DEBUGCTLMSR_LBR (1UL << _DEBUGCTLMSR_LBR) 79#define DEBUGCTLMSR_BTS (1UL << 7)
78#define DEBUGCTLMSR_BTF (1UL << _DEBUGCTLMSR_BTF) 80#define DEBUGCTLMSR_BTINT (1UL << 8)
81#define DEBUGCTLMSR_BTS_OFF_OS (1UL << 9)
82#define DEBUGCTLMSR_BTS_OFF_USR (1UL << 10)
83#define DEBUGCTLMSR_FREEZE_LBRS_ON_PMI (1UL << 11)
79 84
80#define MSR_IA32_MC0_CTL 0x00000400 85#define MSR_IA32_MC0_CTL 0x00000400
81#define MSR_IA32_MC0_STATUS 0x00000401 86#define MSR_IA32_MC0_STATUS 0x00000401
@@ -91,9 +96,6 @@
91#define MSR_IA32_MC0_CTL2 0x00000280 96#define MSR_IA32_MC0_CTL2 0x00000280
92#define MSR_IA32_MCx_CTL2(x) (MSR_IA32_MC0_CTL2 + (x)) 97#define MSR_IA32_MCx_CTL2(x) (MSR_IA32_MC0_CTL2 + (x))
93 98
94#define CMCI_EN (1ULL << 30)
95#define CMCI_THRESHOLD_MASK 0xffffULL
96
97#define MSR_P6_PERFCTR0 0x000000c1 99#define MSR_P6_PERFCTR0 0x000000c1
98#define MSR_P6_PERFCTR1 0x000000c2 100#define MSR_P6_PERFCTR1 0x000000c2
99#define MSR_P6_EVNTSEL0 0x00000186 101#define MSR_P6_EVNTSEL0 0x00000186
@@ -107,6 +109,7 @@
107#define MSR_AMD64_PATCH_LOADER 0xc0010020 109#define MSR_AMD64_PATCH_LOADER 0xc0010020
108#define MSR_AMD64_OSVW_ID_LENGTH 0xc0010140 110#define MSR_AMD64_OSVW_ID_LENGTH 0xc0010140
109#define MSR_AMD64_OSVW_STATUS 0xc0010141 111#define MSR_AMD64_OSVW_STATUS 0xc0010141
112#define MSR_AMD64_DC_CFG 0xc0011022
110#define MSR_AMD64_IBSFETCHCTL 0xc0011030 113#define MSR_AMD64_IBSFETCHCTL 0xc0011030
111#define MSR_AMD64_IBSFETCHLINAD 0xc0011031 114#define MSR_AMD64_IBSFETCHLINAD 0xc0011031
112#define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032 115#define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032
@@ -155,8 +158,6 @@
155#define MSR_K7_FID_VID_STATUS 0xc0010042 158#define MSR_K7_FID_VID_STATUS 0xc0010042
156 159
157/* K6 MSRs */ 160/* K6 MSRs */
158#define MSR_K6_EFER 0xc0000080
159#define MSR_K6_STAR 0xc0000081
160#define MSR_K6_WHCR 0xc0000082 161#define MSR_K6_WHCR 0xc0000082
161#define MSR_K6_UWCCR 0xc0000085 162#define MSR_K6_UWCCR 0xc0000085
162#define MSR_K6_EPMR 0xc0000086 163#define MSR_K6_EPMR 0xc0000086
@@ -199,8 +200,9 @@
199#define MSR_IA32_EBL_CR_POWERON 0x0000002a 200#define MSR_IA32_EBL_CR_POWERON 0x0000002a
200#define MSR_IA32_FEATURE_CONTROL 0x0000003a 201#define MSR_IA32_FEATURE_CONTROL 0x0000003a
201 202
202#define FEATURE_CONTROL_LOCKED (1<<0) 203#define FEATURE_CONTROL_LOCKED (1<<0)
203#define FEATURE_CONTROL_VMXON_ENABLED (1<<2) 204#define FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX (1<<1)
205#define FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX (1<<2)
204 206
205#define MSR_IA32_APICBASE 0x0000001b 207#define MSR_IA32_APICBASE 0x0000001b
206#define MSR_IA32_APICBASE_BSP (1<<8) 208#define MSR_IA32_APICBASE_BSP (1<<8)
@@ -219,12 +221,14 @@
219#define MSR_IA32_THERM_CONTROL 0x0000019a 221#define MSR_IA32_THERM_CONTROL 0x0000019a
220#define MSR_IA32_THERM_INTERRUPT 0x0000019b 222#define MSR_IA32_THERM_INTERRUPT 0x0000019b
221 223
222#define THERM_INT_LOW_ENABLE (1 << 0) 224#define THERM_INT_HIGH_ENABLE (1 << 0)
223#define THERM_INT_HIGH_ENABLE (1 << 1) 225#define THERM_INT_LOW_ENABLE (1 << 1)
226#define THERM_INT_PLN_ENABLE (1 << 24)
224 227
225#define MSR_IA32_THERM_STATUS 0x0000019c 228#define MSR_IA32_THERM_STATUS 0x0000019c
226 229
227#define THERM_STATUS_PROCHOT (1 << 0) 230#define THERM_STATUS_PROCHOT (1 << 0)
231#define THERM_STATUS_POWER_LIMIT (1 << 10)
228 232
229#define MSR_THERM2_CTL 0x0000019d 233#define MSR_THERM2_CTL 0x0000019d
230 234
@@ -232,6 +236,21 @@
232 236
233#define MSR_IA32_MISC_ENABLE 0x000001a0 237#define MSR_IA32_MISC_ENABLE 0x000001a0
234 238
239#define MSR_IA32_TEMPERATURE_TARGET 0x000001a2
240
241#define MSR_IA32_ENERGY_PERF_BIAS 0x000001b0
242
243#define MSR_IA32_PACKAGE_THERM_STATUS 0x000001b1
244
245#define PACKAGE_THERM_STATUS_PROCHOT (1 << 0)
246#define PACKAGE_THERM_STATUS_POWER_LIMIT (1 << 10)
247
248#define MSR_IA32_PACKAGE_THERM_INTERRUPT 0x000001b2
249
250#define PACKAGE_THERM_INT_HIGH_ENABLE (1 << 0)
251#define PACKAGE_THERM_INT_LOW_ENABLE (1 << 1)
252#define PACKAGE_THERM_INT_PLN_ENABLE (1 << 24)
253
235/* MISC_ENABLE bits: architectural */ 254/* MISC_ENABLE bits: architectural */
236#define MSR_IA32_MISC_ENABLE_FAST_STRING (1ULL << 0) 255#define MSR_IA32_MISC_ENABLE_FAST_STRING (1ULL << 0)
237#define MSR_IA32_MISC_ENABLE_TCC (1ULL << 1) 256#define MSR_IA32_MISC_ENABLE_TCC (1ULL << 1)
@@ -359,6 +378,8 @@
359#define MSR_P4_U2L_ESCR0 0x000003b0 378#define MSR_P4_U2L_ESCR0 0x000003b0
360#define MSR_P4_U2L_ESCR1 0x000003b1 379#define MSR_P4_U2L_ESCR1 0x000003b1
361 380
381#define MSR_P4_PEBS_MATRIX_VERT 0x000003f2
382
362/* Intel Core-based CPU performance counters */ 383/* Intel Core-based CPU performance counters */
363#define MSR_CORE_PERF_FIXED_CTR0 0x00000309 384#define MSR_CORE_PERF_FIXED_CTR0 0x00000309
364#define MSR_CORE_PERF_FIXED_CTR1 0x0000030a 385#define MSR_CORE_PERF_FIXED_CTR1 0x0000030a
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index c5bc4c2d33f5..084ef95274cd 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -148,8 +148,8 @@ static inline unsigned long long native_read_pmc(int counter)
148#define rdmsr(msr, val1, val2) \ 148#define rdmsr(msr, val1, val2) \
149do { \ 149do { \
150 u64 __val = native_read_msr((msr)); \ 150 u64 __val = native_read_msr((msr)); \
151 (val1) = (u32)__val; \ 151 (void)((val1) = (u32)__val); \
152 (val2) = (u32)(__val >> 32); \ 152 (void)((val2) = (u32)(__val >> 32)); \
153} while (0) 153} while (0)
154 154
155static inline void wrmsr(unsigned msr, unsigned low, unsigned high) 155static inline void wrmsr(unsigned msr, unsigned low, unsigned high)
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
index 93da9c3f3341..932f0f86b4b7 100644
--- a/arch/x86/include/asm/nmi.h
+++ b/arch/x86/include/asm/nmi.h
@@ -17,7 +17,9 @@ int do_nmi_callback(struct pt_regs *regs, int cpu);
17 17
18extern void die_nmi(char *str, struct pt_regs *regs, int do_panic); 18extern void die_nmi(char *str, struct pt_regs *regs, int do_panic);
19extern int check_nmi_watchdog(void); 19extern int check_nmi_watchdog(void);
20#if !defined(CONFIG_LOCKUP_DETECTOR)
20extern int nmi_watchdog_enabled; 21extern int nmi_watchdog_enabled;
22#endif
21extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); 23extern int avail_to_resrv_perfctr_nmi_bit(unsigned int);
22extern int reserve_perfctr_nmi(unsigned int); 24extern int reserve_perfctr_nmi(unsigned int);
23extern void release_perfctr_nmi(unsigned int); 25extern void release_perfctr_nmi(unsigned int);
diff --git a/arch/x86/include/asm/olpc_ofw.h b/arch/x86/include/asm/olpc_ofw.h
new file mode 100644
index 000000000000..08fde475cb3b
--- /dev/null
+++ b/arch/x86/include/asm/olpc_ofw.h
@@ -0,0 +1,31 @@
1#ifndef _ASM_X86_OLPC_OFW_H
2#define _ASM_X86_OLPC_OFW_H
3
4/* index into the page table containing the entry OFW occupies */
5#define OLPC_OFW_PDE_NR 1022
6
7#define OLPC_OFW_SIG 0x2057464F /* aka "OFW " */
8
9#ifdef CONFIG_OLPC_OPENFIRMWARE
10
11/* run an OFW command by calling into the firmware */
12#define olpc_ofw(name, args, res) \
13 __olpc_ofw((name), ARRAY_SIZE(args), args, ARRAY_SIZE(res), res)
14
15extern int __olpc_ofw(const char *name, int nr_args, const void **args, int nr_res,
16 void **res);
17
18/* determine whether OFW is available and lives in the proper memory */
19extern void olpc_ofw_detect(void);
20
21/* install OFW's pde permanently into the kernel's pgtable */
22extern void setup_olpc_ofw_pgd(void);
23
24#else /* !CONFIG_OLPC_OPENFIRMWARE */
25
26static inline void olpc_ofw_detect(void) { }
27static inline void setup_olpc_ofw_pgd(void) { }
28
29#endif /* !CONFIG_OLPC_OPENFIRMWARE */
30
31#endif /* _ASM_X86_OLPC_OFW_H */
diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h
index 625c3f0e741a..8ca82839288a 100644
--- a/arch/x86/include/asm/page.h
+++ b/arch/x86/include/asm/page.h
@@ -37,6 +37,13 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr,
37#define __pa_nodebug(x) __phys_addr_nodebug((unsigned long)(x)) 37#define __pa_nodebug(x) __phys_addr_nodebug((unsigned long)(x))
38/* __pa_symbol should be used for C visible symbols. 38/* __pa_symbol should be used for C visible symbols.
39 This seems to be the official gcc blessed way to do such arithmetic. */ 39 This seems to be the official gcc blessed way to do such arithmetic. */
40/*
41 * We need __phys_reloc_hide() here because gcc may assume that there is no
42 * overflow during __pa() calculation and can optimize it unexpectedly.
43 * Newer versions of gcc provide -fno-strict-overflow switch to handle this
44 * case properly. Once all supported versions of gcc understand it, we can
45 * remove this Voodoo magic stuff. (i.e. once gcc3.x is deprecated)
46 */
40#define __pa_symbol(x) __pa(__phys_reloc_hide((unsigned long)(x))) 47#define __pa_symbol(x) __pa(__phys_reloc_hide((unsigned long)(x)))
41 48
42#define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET)) 49#define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET))
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index 404a880ea325..d395540ff894 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -27,6 +27,9 @@ extern struct pci_bus *pci_scan_bus_on_node(int busno, struct pci_ops *ops,
27 int node); 27 int node);
28extern struct pci_bus *pci_scan_bus_with_sysdata(int busno); 28extern struct pci_bus *pci_scan_bus_with_sysdata(int busno);
29 29
30#ifdef CONFIG_PCI
31
32#ifdef CONFIG_PCI_DOMAINS
30static inline int pci_domain_nr(struct pci_bus *bus) 33static inline int pci_domain_nr(struct pci_bus *bus)
31{ 34{
32 struct pci_sysdata *sd = bus->sysdata; 35 struct pci_sysdata *sd = bus->sysdata;
@@ -37,13 +40,12 @@ static inline int pci_proc_domain(struct pci_bus *bus)
37{ 40{
38 return pci_domain_nr(bus); 41 return pci_domain_nr(bus);
39} 42}
40 43#endif
41 44
42/* Can be used to override the logic in pci_scan_bus for skipping 45/* Can be used to override the logic in pci_scan_bus for skipping
43 already-configured bus numbers - to be used for buggy BIOSes 46 already-configured bus numbers - to be used for buggy BIOSes
44 or architectures with incomplete PCI setup by the loader */ 47 or architectures with incomplete PCI setup by the loader */
45 48
46#ifdef CONFIG_PCI
47extern unsigned int pcibios_assign_all_busses(void); 49extern unsigned int pcibios_assign_all_busses(void);
48extern int pci_legacy_init(void); 50extern int pci_legacy_init(void);
49# ifdef CONFIG_ACPI 51# ifdef CONFIG_ACPI
diff --git a/arch/x86/include/asm/pci_x86.h b/arch/x86/include/asm/pci_x86.h
index 1a0422348d6d..49c7219826f9 100644
--- a/arch/x86/include/asm/pci_x86.h
+++ b/arch/x86/include/asm/pci_x86.h
@@ -30,6 +30,7 @@
30#define PCI_HAS_IO_ECS 0x40000 30#define PCI_HAS_IO_ECS 0x40000
31#define PCI_NOASSIGN_ROMS 0x80000 31#define PCI_NOASSIGN_ROMS 0x80000
32#define PCI_ROOT_NO_CRS 0x100000 32#define PCI_ROOT_NO_CRS 0x100000
33#define PCI_NOASSIGN_BARS 0x200000
33 34
34extern unsigned int pci_probe; 35extern unsigned int pci_probe;
35extern unsigned long pirq_table_addr; 36extern unsigned long pirq_table_addr;
@@ -53,6 +54,8 @@ extern int pcibios_last_bus;
53extern struct pci_bus *pci_root_bus; 54extern struct pci_bus *pci_root_bus;
54extern struct pci_ops pci_root_ops; 55extern struct pci_ops pci_root_ops;
55 56
57void pcibios_scan_specific_bus(int busn);
58
56/* pci-irq.c */ 59/* pci-irq.c */
57 60
58struct irq_info { 61struct irq_info {
@@ -83,7 +86,7 @@ struct irq_routing_table {
83 86
84extern unsigned int pcibios_irq_mask; 87extern unsigned int pcibios_irq_mask;
85 88
86extern spinlock_t pci_config_lock; 89extern raw_spinlock_t pci_config_lock;
87 90
88extern int (*pcibios_enable_irq)(struct pci_dev *dev); 91extern int (*pcibios_enable_irq)(struct pci_dev *dev);
89extern void (*pcibios_disable_irq)(struct pci_dev *dev); 92extern void (*pcibios_disable_irq)(struct pci_dev *dev);
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 66a272dfd8b8..cd28f9ad910d 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -77,6 +77,7 @@ do { \
77 if (0) { \ 77 if (0) { \
78 pto_T__ pto_tmp__; \ 78 pto_T__ pto_tmp__; \
79 pto_tmp__ = (val); \ 79 pto_tmp__ = (val); \
80 (void)pto_tmp__; \
80 } \ 81 } \
81 switch (sizeof(var)) { \ 82 switch (sizeof(var)) { \
82 case 1: \ 83 case 1: \
@@ -105,7 +106,7 @@ do { \
105 106
106/* 107/*
107 * Generate a percpu add to memory instruction and optimize code 108 * Generate a percpu add to memory instruction and optimize code
108 * if a one is added or subtracted. 109 * if one is added or subtracted.
109 */ 110 */
110#define percpu_add_op(var, val) \ 111#define percpu_add_op(var, val) \
111do { \ 112do { \
@@ -115,6 +116,7 @@ do { \
115 if (0) { \ 116 if (0) { \
116 pao_T__ pao_tmp__; \ 117 pao_T__ pao_tmp__; \
117 pao_tmp__ = (val); \ 118 pao_tmp__ = (val); \
119 (void)pao_tmp__; \
118 } \ 120 } \
119 switch (sizeof(var)) { \ 121 switch (sizeof(var)) { \
120 case 1: \ 122 case 1: \
@@ -190,6 +192,29 @@ do { \
190 pfo_ret__; \ 192 pfo_ret__; \
191}) 193})
192 194
195#define percpu_unary_op(op, var) \
196({ \
197 switch (sizeof(var)) { \
198 case 1: \
199 asm(op "b "__percpu_arg(0) \
200 : "+m" (var)); \
201 break; \
202 case 2: \
203 asm(op "w "__percpu_arg(0) \
204 : "+m" (var)); \
205 break; \
206 case 4: \
207 asm(op "l "__percpu_arg(0) \
208 : "+m" (var)); \
209 break; \
210 case 8: \
211 asm(op "q "__percpu_arg(0) \
212 : "+m" (var)); \
213 break; \
214 default: __bad_percpu_size(); \
215 } \
216})
217
193/* 218/*
194 * percpu_read() makes gcc load the percpu variable every time it is 219 * percpu_read() makes gcc load the percpu variable every time it is
195 * accessed while percpu_read_stable() allows the value to be cached. 220 * accessed while percpu_read_stable() allows the value to be cached.
@@ -207,6 +232,7 @@ do { \
207#define percpu_and(var, val) percpu_to_op("and", var, val) 232#define percpu_and(var, val) percpu_to_op("and", var, val)
208#define percpu_or(var, val) percpu_to_op("or", var, val) 233#define percpu_or(var, val) percpu_to_op("or", var, val)
209#define percpu_xor(var, val) percpu_to_op("xor", var, val) 234#define percpu_xor(var, val) percpu_to_op("xor", var, val)
235#define percpu_inc(var) percpu_unary_op("inc", var)
210 236
211#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 237#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
212#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp)) 238#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index db6109a885a7..6e742cc4251b 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -5,7 +5,7 @@
5 * Performance event hw details: 5 * Performance event hw details:
6 */ 6 */
7 7
8#define X86_PMC_MAX_GENERIC 8 8#define X86_PMC_MAX_GENERIC 32
9#define X86_PMC_MAX_FIXED 3 9#define X86_PMC_MAX_FIXED 3
10 10
11#define X86_PMC_IDX_GENERIC 0 11#define X86_PMC_IDX_GENERIC 0
@@ -18,39 +18,31 @@
18#define MSR_ARCH_PERFMON_EVENTSEL0 0x186 18#define MSR_ARCH_PERFMON_EVENTSEL0 0x186
19#define MSR_ARCH_PERFMON_EVENTSEL1 0x187 19#define MSR_ARCH_PERFMON_EVENTSEL1 0x187
20 20
21#define ARCH_PERFMON_EVENTSEL_ENABLE (1 << 22) 21#define ARCH_PERFMON_EVENTSEL_EVENT 0x000000FFULL
22#define ARCH_PERFMON_EVENTSEL_ANY (1 << 21) 22#define ARCH_PERFMON_EVENTSEL_UMASK 0x0000FF00ULL
23#define ARCH_PERFMON_EVENTSEL_INT (1 << 20) 23#define ARCH_PERFMON_EVENTSEL_USR (1ULL << 16)
24#define ARCH_PERFMON_EVENTSEL_OS (1 << 17) 24#define ARCH_PERFMON_EVENTSEL_OS (1ULL << 17)
25#define ARCH_PERFMON_EVENTSEL_USR (1 << 16) 25#define ARCH_PERFMON_EVENTSEL_EDGE (1ULL << 18)
26 26#define ARCH_PERFMON_EVENTSEL_INT (1ULL << 20)
27/* 27#define ARCH_PERFMON_EVENTSEL_ANY (1ULL << 21)
28 * Includes eventsel and unit mask as well: 28#define ARCH_PERFMON_EVENTSEL_ENABLE (1ULL << 22)
29 */ 29#define ARCH_PERFMON_EVENTSEL_INV (1ULL << 23)
30 30#define ARCH_PERFMON_EVENTSEL_CMASK 0xFF000000ULL
31 31
32#define INTEL_ARCH_EVTSEL_MASK 0x000000FFULL 32#define AMD64_EVENTSEL_EVENT \
33#define INTEL_ARCH_UNIT_MASK 0x0000FF00ULL 33 (ARCH_PERFMON_EVENTSEL_EVENT | (0x0FULL << 32))
34#define INTEL_ARCH_EDGE_MASK 0x00040000ULL 34#define INTEL_ARCH_EVENT_MASK \
35#define INTEL_ARCH_INV_MASK 0x00800000ULL 35 (ARCH_PERFMON_EVENTSEL_UMASK | ARCH_PERFMON_EVENTSEL_EVENT)
36#define INTEL_ARCH_CNT_MASK 0xFF000000ULL 36
37#define INTEL_ARCH_EVENT_MASK (INTEL_ARCH_UNIT_MASK|INTEL_ARCH_EVTSEL_MASK) 37#define X86_RAW_EVENT_MASK \
38 38 (ARCH_PERFMON_EVENTSEL_EVENT | \
39/* 39 ARCH_PERFMON_EVENTSEL_UMASK | \
40 * filter mask to validate fixed counter events. 40 ARCH_PERFMON_EVENTSEL_EDGE | \
41 * the following filters disqualify for fixed counters: 41 ARCH_PERFMON_EVENTSEL_INV | \
42 * - inv 42 ARCH_PERFMON_EVENTSEL_CMASK)
43 * - edge 43#define AMD64_RAW_EVENT_MASK \
44 * - cnt-mask 44 (X86_RAW_EVENT_MASK | \
45 * The other filters are supported by fixed counters. 45 AMD64_EVENTSEL_EVENT)
46 * The any-thread option is supported starting with v3.
47 */
48#define INTEL_ARCH_FIXED_MASK \
49 (INTEL_ARCH_CNT_MASK| \
50 INTEL_ARCH_INV_MASK| \
51 INTEL_ARCH_EDGE_MASK|\
52 INTEL_ARCH_UNIT_MASK|\
53 INTEL_ARCH_EVENT_MASK)
54 46
55#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c 47#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c
56#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) 48#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)
@@ -67,7 +59,7 @@
67union cpuid10_eax { 59union cpuid10_eax {
68 struct { 60 struct {
69 unsigned int version_id:8; 61 unsigned int version_id:8;
70 unsigned int num_events:8; 62 unsigned int num_counters:8;
71 unsigned int bit_width:8; 63 unsigned int bit_width:8;
72 unsigned int mask_length:8; 64 unsigned int mask_length:8;
73 } split; 65 } split;
@@ -76,8 +68,9 @@ union cpuid10_eax {
76 68
77union cpuid10_edx { 69union cpuid10_edx {
78 struct { 70 struct {
79 unsigned int num_events_fixed:4; 71 unsigned int num_counters_fixed:5;
80 unsigned int reserved:28; 72 unsigned int bit_width_fixed:8;
73 unsigned int reserved:19;
81 } split; 74 } split;
82 unsigned int full; 75 unsigned int full;
83}; 76};
@@ -136,6 +129,31 @@ extern void perf_events_lapic_init(void);
136 129
137#define PERF_EVENT_INDEX_OFFSET 0 130#define PERF_EVENT_INDEX_OFFSET 0
138 131
132/*
133 * Abuse bit 3 of the cpu eflags register to indicate proper PEBS IP fixups.
134 * This flag is otherwise unused and ABI specified to be 0, so nobody should
135 * care what we do with it.
136 */
137#define PERF_EFLAGS_EXACT (1UL << 3)
138
139struct pt_regs;
140extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
141extern unsigned long perf_misc_flags(struct pt_regs *regs);
142#define perf_misc_flags(regs) perf_misc_flags(regs)
143
144#include <asm/stacktrace.h>
145
146/*
147 * We abuse bit 3 from flags to pass exact information, see perf_misc_flags
148 * and the comment with PERF_EFLAGS_EXACT.
149 */
150#define perf_arch_fetch_caller_regs(regs, __ip) { \
151 (regs)->ip = (__ip); \
152 (regs)->bp = caller_frame_pointer(); \
153 (regs)->cs = __KERNEL_CS; \
154 regs->flags = 0; \
155}
156
139#else 157#else
140static inline void init_hw_perf_events(void) { } 158static inline void init_hw_perf_events(void) { }
141static inline void perf_events_lapic_init(void) { } 159static inline void perf_events_lapic_init(void) { }
diff --git a/arch/x86/include/asm/perf_event_p4.h b/arch/x86/include/asm/perf_event_p4.h
new file mode 100644
index 000000000000..def500776b16
--- /dev/null
+++ b/arch/x86/include/asm/perf_event_p4.h
@@ -0,0 +1,800 @@
1/*
2 * Netburst Perfomance Events (P4, old Xeon)
3 */
4
5#ifndef PERF_EVENT_P4_H
6#define PERF_EVENT_P4_H
7
8#include <linux/cpu.h>
9#include <linux/bitops.h>
10
11/*
12 * NetBurst has perfomance MSRs shared between
13 * threads if HT is turned on, ie for both logical
14 * processors (mem: in turn in Atom with HT support
15 * perf-MSRs are not shared and every thread has its
16 * own perf-MSRs set)
17 */
18#define ARCH_P4_TOTAL_ESCR (46)
19#define ARCH_P4_RESERVED_ESCR (2) /* IQ_ESCR(0,1) not always present */
20#define ARCH_P4_MAX_ESCR (ARCH_P4_TOTAL_ESCR - ARCH_P4_RESERVED_ESCR)
21#define ARCH_P4_MAX_CCCR (18)
22
23#define P4_ESCR_EVENT_MASK 0x7e000000U
24#define P4_ESCR_EVENT_SHIFT 25
25#define P4_ESCR_EVENTMASK_MASK 0x01fffe00U
26#define P4_ESCR_EVENTMASK_SHIFT 9
27#define P4_ESCR_TAG_MASK 0x000001e0U
28#define P4_ESCR_TAG_SHIFT 5
29#define P4_ESCR_TAG_ENABLE 0x00000010U
30#define P4_ESCR_T0_OS 0x00000008U
31#define P4_ESCR_T0_USR 0x00000004U
32#define P4_ESCR_T1_OS 0x00000002U
33#define P4_ESCR_T1_USR 0x00000001U
34
35#define P4_ESCR_EVENT(v) ((v) << P4_ESCR_EVENT_SHIFT)
36#define P4_ESCR_EMASK(v) ((v) << P4_ESCR_EVENTMASK_SHIFT)
37#define P4_ESCR_TAG(v) ((v) << P4_ESCR_TAG_SHIFT)
38
39/* Non HT mask */
40#define P4_ESCR_MASK \
41 (P4_ESCR_EVENT_MASK | \
42 P4_ESCR_EVENTMASK_MASK | \
43 P4_ESCR_TAG_MASK | \
44 P4_ESCR_TAG_ENABLE | \
45 P4_ESCR_T0_OS | \
46 P4_ESCR_T0_USR)
47
48/* HT mask */
49#define P4_ESCR_MASK_HT \
50 (P4_ESCR_MASK | P4_ESCR_T1_OS | P4_ESCR_T1_USR)
51
52#define P4_CCCR_OVF 0x80000000U
53#define P4_CCCR_CASCADE 0x40000000U
54#define P4_CCCR_OVF_PMI_T0 0x04000000U
55#define P4_CCCR_OVF_PMI_T1 0x08000000U
56#define P4_CCCR_FORCE_OVF 0x02000000U
57#define P4_CCCR_EDGE 0x01000000U
58#define P4_CCCR_THRESHOLD_MASK 0x00f00000U
59#define P4_CCCR_THRESHOLD_SHIFT 20
60#define P4_CCCR_COMPLEMENT 0x00080000U
61#define P4_CCCR_COMPARE 0x00040000U
62#define P4_CCCR_ESCR_SELECT_MASK 0x0000e000U
63#define P4_CCCR_ESCR_SELECT_SHIFT 13
64#define P4_CCCR_ENABLE 0x00001000U
65#define P4_CCCR_THREAD_SINGLE 0x00010000U
66#define P4_CCCR_THREAD_BOTH 0x00020000U
67#define P4_CCCR_THREAD_ANY 0x00030000U
68#define P4_CCCR_RESERVED 0x00000fffU
69
70#define P4_CCCR_THRESHOLD(v) ((v) << P4_CCCR_THRESHOLD_SHIFT)
71#define P4_CCCR_ESEL(v) ((v) << P4_CCCR_ESCR_SELECT_SHIFT)
72
73/* Non HT mask */
74#define P4_CCCR_MASK \
75 (P4_CCCR_OVF | \
76 P4_CCCR_CASCADE | \
77 P4_CCCR_OVF_PMI_T0 | \
78 P4_CCCR_FORCE_OVF | \
79 P4_CCCR_EDGE | \
80 P4_CCCR_THRESHOLD_MASK | \
81 P4_CCCR_COMPLEMENT | \
82 P4_CCCR_COMPARE | \
83 P4_CCCR_ESCR_SELECT_MASK | \
84 P4_CCCR_ENABLE)
85
86/* HT mask */
87#define P4_CCCR_MASK_HT \
88 (P4_CCCR_MASK | P4_CCCR_OVF_PMI_T1 | P4_CCCR_THREAD_ANY)
89
90#define P4_GEN_ESCR_EMASK(class, name, bit) \
91 class##__##name = ((1 << bit) << P4_ESCR_EVENTMASK_SHIFT)
92#define P4_ESCR_EMASK_BIT(class, name) class##__##name
93
94/*
95 * config field is 64bit width and consists of
96 * HT << 63 | ESCR << 32 | CCCR
97 * where HT is HyperThreading bit (since ESCR
98 * has it reserved we may use it for own purpose)
99 *
100 * note that this is NOT the addresses of respective
101 * ESCR and CCCR but rather an only packed value should
102 * be unpacked and written to a proper addresses
103 *
104 * the base idea is to pack as much info as possible
105 */
106#define p4_config_pack_escr(v) (((u64)(v)) << 32)
107#define p4_config_pack_cccr(v) (((u64)(v)) & 0xffffffffULL)
108#define p4_config_unpack_escr(v) (((u64)(v)) >> 32)
109#define p4_config_unpack_cccr(v) (((u64)(v)) & 0xffffffffULL)
110
111#define p4_config_unpack_emask(v) \
112 ({ \
113 u32 t = p4_config_unpack_escr((v)); \
114 t = t & P4_ESCR_EVENTMASK_MASK; \
115 t = t >> P4_ESCR_EVENTMASK_SHIFT; \
116 t; \
117 })
118
119#define p4_config_unpack_event(v) \
120 ({ \
121 u32 t = p4_config_unpack_escr((v)); \
122 t = t & P4_ESCR_EVENT_MASK; \
123 t = t >> P4_ESCR_EVENT_SHIFT; \
124 t; \
125 })
126
127#define P4_CONFIG_HT_SHIFT 63
128#define P4_CONFIG_HT (1ULL << P4_CONFIG_HT_SHIFT)
129
130static inline bool p4_is_event_cascaded(u64 config)
131{
132 u32 cccr = p4_config_unpack_cccr(config);
133 return !!(cccr & P4_CCCR_CASCADE);
134}
135
136static inline int p4_ht_config_thread(u64 config)
137{
138 return !!(config & P4_CONFIG_HT);
139}
140
141static inline u64 p4_set_ht_bit(u64 config)
142{
143 return config | P4_CONFIG_HT;
144}
145
146static inline u64 p4_clear_ht_bit(u64 config)
147{
148 return config & ~P4_CONFIG_HT;
149}
150
151static inline int p4_ht_active(void)
152{
153#ifdef CONFIG_SMP
154 return smp_num_siblings > 1;
155#endif
156 return 0;
157}
158
159static inline int p4_ht_thread(int cpu)
160{
161#ifdef CONFIG_SMP
162 if (smp_num_siblings == 2)
163 return cpu != cpumask_first(__get_cpu_var(cpu_sibling_map));
164#endif
165 return 0;
166}
167
168static inline int p4_should_swap_ts(u64 config, int cpu)
169{
170 return p4_ht_config_thread(config) ^ p4_ht_thread(cpu);
171}
172
173static inline u32 p4_default_cccr_conf(int cpu)
174{
175 /*
176 * Note that P4_CCCR_THREAD_ANY is "required" on
177 * non-HT machines (on HT machines we count TS events
178 * regardless the state of second logical processor
179 */
180 u32 cccr = P4_CCCR_THREAD_ANY;
181
182 if (!p4_ht_thread(cpu))
183 cccr |= P4_CCCR_OVF_PMI_T0;
184 else
185 cccr |= P4_CCCR_OVF_PMI_T1;
186
187 return cccr;
188}
189
190static inline u32 p4_default_escr_conf(int cpu, int exclude_os, int exclude_usr)
191{
192 u32 escr = 0;
193
194 if (!p4_ht_thread(cpu)) {
195 if (!exclude_os)
196 escr |= P4_ESCR_T0_OS;
197 if (!exclude_usr)
198 escr |= P4_ESCR_T0_USR;
199 } else {
200 if (!exclude_os)
201 escr |= P4_ESCR_T1_OS;
202 if (!exclude_usr)
203 escr |= P4_ESCR_T1_USR;
204 }
205
206 return escr;
207}
208
209/*
210 * This are the events which should be used in "Event Select"
211 * field of ESCR register, they are like unique keys which allow
212 * the kernel to determinate which CCCR and COUNTER should be
213 * used to track an event
214 */
215enum P4_EVENTS {
216 P4_EVENT_TC_DELIVER_MODE,
217 P4_EVENT_BPU_FETCH_REQUEST,
218 P4_EVENT_ITLB_REFERENCE,
219 P4_EVENT_MEMORY_CANCEL,
220 P4_EVENT_MEMORY_COMPLETE,
221 P4_EVENT_LOAD_PORT_REPLAY,
222 P4_EVENT_STORE_PORT_REPLAY,
223 P4_EVENT_MOB_LOAD_REPLAY,
224 P4_EVENT_PAGE_WALK_TYPE,
225 P4_EVENT_BSQ_CACHE_REFERENCE,
226 P4_EVENT_IOQ_ALLOCATION,
227 P4_EVENT_IOQ_ACTIVE_ENTRIES,
228 P4_EVENT_FSB_DATA_ACTIVITY,
229 P4_EVENT_BSQ_ALLOCATION,
230 P4_EVENT_BSQ_ACTIVE_ENTRIES,
231 P4_EVENT_SSE_INPUT_ASSIST,
232 P4_EVENT_PACKED_SP_UOP,
233 P4_EVENT_PACKED_DP_UOP,
234 P4_EVENT_SCALAR_SP_UOP,
235 P4_EVENT_SCALAR_DP_UOP,
236 P4_EVENT_64BIT_MMX_UOP,
237 P4_EVENT_128BIT_MMX_UOP,
238 P4_EVENT_X87_FP_UOP,
239 P4_EVENT_TC_MISC,
240 P4_EVENT_GLOBAL_POWER_EVENTS,
241 P4_EVENT_TC_MS_XFER,
242 P4_EVENT_UOP_QUEUE_WRITES,
243 P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE,
244 P4_EVENT_RETIRED_BRANCH_TYPE,
245 P4_EVENT_RESOURCE_STALL,
246 P4_EVENT_WC_BUFFER,
247 P4_EVENT_B2B_CYCLES,
248 P4_EVENT_BNR,
249 P4_EVENT_SNOOP,
250 P4_EVENT_RESPONSE,
251 P4_EVENT_FRONT_END_EVENT,
252 P4_EVENT_EXECUTION_EVENT,
253 P4_EVENT_REPLAY_EVENT,
254 P4_EVENT_INSTR_RETIRED,
255 P4_EVENT_UOPS_RETIRED,
256 P4_EVENT_UOP_TYPE,
257 P4_EVENT_BRANCH_RETIRED,
258 P4_EVENT_MISPRED_BRANCH_RETIRED,
259 P4_EVENT_X87_ASSIST,
260 P4_EVENT_MACHINE_CLEAR,
261 P4_EVENT_INSTR_COMPLETED,
262};
263
264#define P4_OPCODE(event) event##_OPCODE
265#define P4_OPCODE_ESEL(opcode) ((opcode & 0x00ff) >> 0)
266#define P4_OPCODE_EVNT(opcode) ((opcode & 0xff00) >> 8)
267#define P4_OPCODE_PACK(event, sel) (((event) << 8) | sel)
268
269/*
270 * Comments below the event represent ESCR restriction
271 * for this event and counter index per ESCR
272 *
273 * MSR_P4_IQ_ESCR0 and MSR_P4_IQ_ESCR1 are available only on early
274 * processor builds (family 0FH, models 01H-02H). These MSRs
275 * are not available on later versions, so that we don't use
276 * them completely
277 *
278 * Also note that CCCR1 do not have P4_CCCR_ENABLE bit properly
279 * working so that we should not use this CCCR and respective
280 * counter as result
281 */
282enum P4_EVENT_OPCODES {
283 P4_OPCODE(P4_EVENT_TC_DELIVER_MODE) = P4_OPCODE_PACK(0x01, 0x01),
284 /*
285 * MSR_P4_TC_ESCR0: 4, 5
286 * MSR_P4_TC_ESCR1: 6, 7
287 */
288
289 P4_OPCODE(P4_EVENT_BPU_FETCH_REQUEST) = P4_OPCODE_PACK(0x03, 0x00),
290 /*
291 * MSR_P4_BPU_ESCR0: 0, 1
292 * MSR_P4_BPU_ESCR1: 2, 3
293 */
294
295 P4_OPCODE(P4_EVENT_ITLB_REFERENCE) = P4_OPCODE_PACK(0x18, 0x03),
296 /*
297 * MSR_P4_ITLB_ESCR0: 0, 1
298 * MSR_P4_ITLB_ESCR1: 2, 3
299 */
300
301 P4_OPCODE(P4_EVENT_MEMORY_CANCEL) = P4_OPCODE_PACK(0x02, 0x05),
302 /*
303 * MSR_P4_DAC_ESCR0: 8, 9
304 * MSR_P4_DAC_ESCR1: 10, 11
305 */
306
307 P4_OPCODE(P4_EVENT_MEMORY_COMPLETE) = P4_OPCODE_PACK(0x08, 0x02),
308 /*
309 * MSR_P4_SAAT_ESCR0: 8, 9
310 * MSR_P4_SAAT_ESCR1: 10, 11
311 */
312
313 P4_OPCODE(P4_EVENT_LOAD_PORT_REPLAY) = P4_OPCODE_PACK(0x04, 0x02),
314 /*
315 * MSR_P4_SAAT_ESCR0: 8, 9
316 * MSR_P4_SAAT_ESCR1: 10, 11
317 */
318
319 P4_OPCODE(P4_EVENT_STORE_PORT_REPLAY) = P4_OPCODE_PACK(0x05, 0x02),
320 /*
321 * MSR_P4_SAAT_ESCR0: 8, 9
322 * MSR_P4_SAAT_ESCR1: 10, 11
323 */
324
325 P4_OPCODE(P4_EVENT_MOB_LOAD_REPLAY) = P4_OPCODE_PACK(0x03, 0x02),
326 /*
327 * MSR_P4_MOB_ESCR0: 0, 1
328 * MSR_P4_MOB_ESCR1: 2, 3
329 */
330
331 P4_OPCODE(P4_EVENT_PAGE_WALK_TYPE) = P4_OPCODE_PACK(0x01, 0x04),
332 /*
333 * MSR_P4_PMH_ESCR0: 0, 1
334 * MSR_P4_PMH_ESCR1: 2, 3
335 */
336
337 P4_OPCODE(P4_EVENT_BSQ_CACHE_REFERENCE) = P4_OPCODE_PACK(0x0c, 0x07),
338 /*
339 * MSR_P4_BSU_ESCR0: 0, 1
340 * MSR_P4_BSU_ESCR1: 2, 3
341 */
342
343 P4_OPCODE(P4_EVENT_IOQ_ALLOCATION) = P4_OPCODE_PACK(0x03, 0x06),
344 /*
345 * MSR_P4_FSB_ESCR0: 0, 1
346 * MSR_P4_FSB_ESCR1: 2, 3
347 */
348
349 P4_OPCODE(P4_EVENT_IOQ_ACTIVE_ENTRIES) = P4_OPCODE_PACK(0x1a, 0x06),
350 /*
351 * MSR_P4_FSB_ESCR1: 2, 3
352 */
353
354 P4_OPCODE(P4_EVENT_FSB_DATA_ACTIVITY) = P4_OPCODE_PACK(0x17, 0x06),
355 /*
356 * MSR_P4_FSB_ESCR0: 0, 1
357 * MSR_P4_FSB_ESCR1: 2, 3
358 */
359
360 P4_OPCODE(P4_EVENT_BSQ_ALLOCATION) = P4_OPCODE_PACK(0x05, 0x07),
361 /*
362 * MSR_P4_BSU_ESCR0: 0, 1
363 */
364
365 P4_OPCODE(P4_EVENT_BSQ_ACTIVE_ENTRIES) = P4_OPCODE_PACK(0x06, 0x07),
366 /*
367 * NOTE: no ESCR name in docs, it's guessed
368 * MSR_P4_BSU_ESCR1: 2, 3
369 */
370
371 P4_OPCODE(P4_EVENT_SSE_INPUT_ASSIST) = P4_OPCODE_PACK(0x34, 0x01),
372 /*
373 * MSR_P4_FIRM_ESCR0: 8, 9
374 * MSR_P4_FIRM_ESCR1: 10, 11
375 */
376
377 P4_OPCODE(P4_EVENT_PACKED_SP_UOP) = P4_OPCODE_PACK(0x08, 0x01),
378 /*
379 * MSR_P4_FIRM_ESCR0: 8, 9
380 * MSR_P4_FIRM_ESCR1: 10, 11
381 */
382
383 P4_OPCODE(P4_EVENT_PACKED_DP_UOP) = P4_OPCODE_PACK(0x0c, 0x01),
384 /*
385 * MSR_P4_FIRM_ESCR0: 8, 9
386 * MSR_P4_FIRM_ESCR1: 10, 11
387 */
388
389 P4_OPCODE(P4_EVENT_SCALAR_SP_UOP) = P4_OPCODE_PACK(0x0a, 0x01),
390 /*
391 * MSR_P4_FIRM_ESCR0: 8, 9
392 * MSR_P4_FIRM_ESCR1: 10, 11
393 */
394
395 P4_OPCODE(P4_EVENT_SCALAR_DP_UOP) = P4_OPCODE_PACK(0x0e, 0x01),
396 /*
397 * MSR_P4_FIRM_ESCR0: 8, 9
398 * MSR_P4_FIRM_ESCR1: 10, 11
399 */
400
401 P4_OPCODE(P4_EVENT_64BIT_MMX_UOP) = P4_OPCODE_PACK(0x02, 0x01),
402 /*
403 * MSR_P4_FIRM_ESCR0: 8, 9
404 * MSR_P4_FIRM_ESCR1: 10, 11
405 */
406
407 P4_OPCODE(P4_EVENT_128BIT_MMX_UOP) = P4_OPCODE_PACK(0x1a, 0x01),
408 /*
409 * MSR_P4_FIRM_ESCR0: 8, 9
410 * MSR_P4_FIRM_ESCR1: 10, 11
411 */
412
413 P4_OPCODE(P4_EVENT_X87_FP_UOP) = P4_OPCODE_PACK(0x04, 0x01),
414 /*
415 * MSR_P4_FIRM_ESCR0: 8, 9
416 * MSR_P4_FIRM_ESCR1: 10, 11
417 */
418
419 P4_OPCODE(P4_EVENT_TC_MISC) = P4_OPCODE_PACK(0x06, 0x01),
420 /*
421 * MSR_P4_TC_ESCR0: 4, 5
422 * MSR_P4_TC_ESCR1: 6, 7
423 */
424
425 P4_OPCODE(P4_EVENT_GLOBAL_POWER_EVENTS) = P4_OPCODE_PACK(0x13, 0x06),
426 /*
427 * MSR_P4_FSB_ESCR0: 0, 1
428 * MSR_P4_FSB_ESCR1: 2, 3
429 */
430
431 P4_OPCODE(P4_EVENT_TC_MS_XFER) = P4_OPCODE_PACK(0x05, 0x00),
432 /*
433 * MSR_P4_MS_ESCR0: 4, 5
434 * MSR_P4_MS_ESCR1: 6, 7
435 */
436
437 P4_OPCODE(P4_EVENT_UOP_QUEUE_WRITES) = P4_OPCODE_PACK(0x09, 0x00),
438 /*
439 * MSR_P4_MS_ESCR0: 4, 5
440 * MSR_P4_MS_ESCR1: 6, 7
441 */
442
443 P4_OPCODE(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE) = P4_OPCODE_PACK(0x05, 0x02),
444 /*
445 * MSR_P4_TBPU_ESCR0: 4, 5
446 * MSR_P4_TBPU_ESCR1: 6, 7
447 */
448
449 P4_OPCODE(P4_EVENT_RETIRED_BRANCH_TYPE) = P4_OPCODE_PACK(0x04, 0x02),
450 /*
451 * MSR_P4_TBPU_ESCR0: 4, 5
452 * MSR_P4_TBPU_ESCR1: 6, 7
453 */
454
455 P4_OPCODE(P4_EVENT_RESOURCE_STALL) = P4_OPCODE_PACK(0x01, 0x01),
456 /*
457 * MSR_P4_ALF_ESCR0: 12, 13, 16
458 * MSR_P4_ALF_ESCR1: 14, 15, 17
459 */
460
461 P4_OPCODE(P4_EVENT_WC_BUFFER) = P4_OPCODE_PACK(0x05, 0x05),
462 /*
463 * MSR_P4_DAC_ESCR0: 8, 9
464 * MSR_P4_DAC_ESCR1: 10, 11
465 */
466
467 P4_OPCODE(P4_EVENT_B2B_CYCLES) = P4_OPCODE_PACK(0x16, 0x03),
468 /*
469 * MSR_P4_FSB_ESCR0: 0, 1
470 * MSR_P4_FSB_ESCR1: 2, 3
471 */
472
473 P4_OPCODE(P4_EVENT_BNR) = P4_OPCODE_PACK(0x08, 0x03),
474 /*
475 * MSR_P4_FSB_ESCR0: 0, 1
476 * MSR_P4_FSB_ESCR1: 2, 3
477 */
478
479 P4_OPCODE(P4_EVENT_SNOOP) = P4_OPCODE_PACK(0x06, 0x03),
480 /*
481 * MSR_P4_FSB_ESCR0: 0, 1
482 * MSR_P4_FSB_ESCR1: 2, 3
483 */
484
485 P4_OPCODE(P4_EVENT_RESPONSE) = P4_OPCODE_PACK(0x04, 0x03),
486 /*
487 * MSR_P4_FSB_ESCR0: 0, 1
488 * MSR_P4_FSB_ESCR1: 2, 3
489 */
490
491 P4_OPCODE(P4_EVENT_FRONT_END_EVENT) = P4_OPCODE_PACK(0x08, 0x05),
492 /*
493 * MSR_P4_CRU_ESCR2: 12, 13, 16
494 * MSR_P4_CRU_ESCR3: 14, 15, 17
495 */
496
497 P4_OPCODE(P4_EVENT_EXECUTION_EVENT) = P4_OPCODE_PACK(0x0c, 0x05),
498 /*
499 * MSR_P4_CRU_ESCR2: 12, 13, 16
500 * MSR_P4_CRU_ESCR3: 14, 15, 17
501 */
502
503 P4_OPCODE(P4_EVENT_REPLAY_EVENT) = P4_OPCODE_PACK(0x09, 0x05),
504 /*
505 * MSR_P4_CRU_ESCR2: 12, 13, 16
506 * MSR_P4_CRU_ESCR3: 14, 15, 17
507 */
508
509 P4_OPCODE(P4_EVENT_INSTR_RETIRED) = P4_OPCODE_PACK(0x02, 0x04),
510 /*
511 * MSR_P4_CRU_ESCR0: 12, 13, 16
512 * MSR_P4_CRU_ESCR1: 14, 15, 17
513 */
514
515 P4_OPCODE(P4_EVENT_UOPS_RETIRED) = P4_OPCODE_PACK(0x01, 0x04),
516 /*
517 * MSR_P4_CRU_ESCR0: 12, 13, 16
518 * MSR_P4_CRU_ESCR1: 14, 15, 17
519 */
520
521 P4_OPCODE(P4_EVENT_UOP_TYPE) = P4_OPCODE_PACK(0x02, 0x02),
522 /*
523 * MSR_P4_RAT_ESCR0: 12, 13, 16
524 * MSR_P4_RAT_ESCR1: 14, 15, 17
525 */
526
527 P4_OPCODE(P4_EVENT_BRANCH_RETIRED) = P4_OPCODE_PACK(0x06, 0x05),
528 /*
529 * MSR_P4_CRU_ESCR2: 12, 13, 16
530 * MSR_P4_CRU_ESCR3: 14, 15, 17
531 */
532
533 P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED) = P4_OPCODE_PACK(0x03, 0x04),
534 /*
535 * MSR_P4_CRU_ESCR0: 12, 13, 16
536 * MSR_P4_CRU_ESCR1: 14, 15, 17
537 */
538
539 P4_OPCODE(P4_EVENT_X87_ASSIST) = P4_OPCODE_PACK(0x03, 0x05),
540 /*
541 * MSR_P4_CRU_ESCR2: 12, 13, 16
542 * MSR_P4_CRU_ESCR3: 14, 15, 17
543 */
544
545 P4_OPCODE(P4_EVENT_MACHINE_CLEAR) = P4_OPCODE_PACK(0x02, 0x05),
546 /*
547 * MSR_P4_CRU_ESCR2: 12, 13, 16
548 * MSR_P4_CRU_ESCR3: 14, 15, 17
549 */
550
551 P4_OPCODE(P4_EVENT_INSTR_COMPLETED) = P4_OPCODE_PACK(0x07, 0x04),
552 /*
553 * MSR_P4_CRU_ESCR0: 12, 13, 16
554 * MSR_P4_CRU_ESCR1: 14, 15, 17
555 */
556};
557
558/*
559 * a caller should use P4_ESCR_EMASK_NAME helper to
560 * pick the EventMask needed, for example
561 *
562 * P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, DD)
563 */
564enum P4_ESCR_EMASKS {
565 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, DD, 0),
566 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, DB, 1),
567 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, DI, 2),
568 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, BD, 3),
569 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, BB, 4),
570 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, BI, 5),
571 P4_GEN_ESCR_EMASK(P4_EVENT_TC_DELIVER_MODE, ID, 6),
572
573 P4_GEN_ESCR_EMASK(P4_EVENT_BPU_FETCH_REQUEST, TCMISS, 0),
574
575 P4_GEN_ESCR_EMASK(P4_EVENT_ITLB_REFERENCE, HIT, 0),
576 P4_GEN_ESCR_EMASK(P4_EVENT_ITLB_REFERENCE, MISS, 1),
577 P4_GEN_ESCR_EMASK(P4_EVENT_ITLB_REFERENCE, HIT_UK, 2),
578
579 P4_GEN_ESCR_EMASK(P4_EVENT_MEMORY_CANCEL, ST_RB_FULL, 2),
580 P4_GEN_ESCR_EMASK(P4_EVENT_MEMORY_CANCEL, 64K_CONF, 3),
581
582 P4_GEN_ESCR_EMASK(P4_EVENT_MEMORY_COMPLETE, LSC, 0),
583 P4_GEN_ESCR_EMASK(P4_EVENT_MEMORY_COMPLETE, SSC, 1),
584
585 P4_GEN_ESCR_EMASK(P4_EVENT_LOAD_PORT_REPLAY, SPLIT_LD, 1),
586
587 P4_GEN_ESCR_EMASK(P4_EVENT_STORE_PORT_REPLAY, SPLIT_ST, 1),
588
589 P4_GEN_ESCR_EMASK(P4_EVENT_MOB_LOAD_REPLAY, NO_STA, 1),
590 P4_GEN_ESCR_EMASK(P4_EVENT_MOB_LOAD_REPLAY, NO_STD, 3),
591 P4_GEN_ESCR_EMASK(P4_EVENT_MOB_LOAD_REPLAY, PARTIAL_DATA, 4),
592 P4_GEN_ESCR_EMASK(P4_EVENT_MOB_LOAD_REPLAY, UNALGN_ADDR, 5),
593
594 P4_GEN_ESCR_EMASK(P4_EVENT_PAGE_WALK_TYPE, DTMISS, 0),
595 P4_GEN_ESCR_EMASK(P4_EVENT_PAGE_WALK_TYPE, ITMISS, 1),
596
597 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITS, 0),
598 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITE, 1),
599 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITM, 2),
600 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITS, 3),
601 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITE, 4),
602 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITM, 5),
603 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_MISS, 8),
604 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_MISS, 9),
605 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_CACHE_REFERENCE, WR_2ndL_MISS, 10),
606
607 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, DEFAULT, 0),
608 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, ALL_READ, 5),
609 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, ALL_WRITE, 6),
610 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, MEM_UC, 7),
611 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, MEM_WC, 8),
612 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, MEM_WT, 9),
613 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, MEM_WP, 10),
614 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, MEM_WB, 11),
615 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, OWN, 13),
616 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, OTHER, 14),
617 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ALLOCATION, PREFETCH, 15),
618
619 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, DEFAULT, 0),
620 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, ALL_READ, 5),
621 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, ALL_WRITE, 6),
622 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_UC, 7),
623 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WC, 8),
624 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WT, 9),
625 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WP, 10),
626 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WB, 11),
627 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, OWN, 13),
628 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, OTHER, 14),
629 P4_GEN_ESCR_EMASK(P4_EVENT_IOQ_ACTIVE_ENTRIES, PREFETCH, 15),
630
631 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_DRV, 0),
632 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OWN, 1),
633 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OTHER, 2),
634 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_DRV, 3),
635 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_OWN, 4),
636 P4_GEN_ESCR_EMASK(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_OTHER, 5),
637
638 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_TYPE0, 0),
639 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_TYPE1, 1),
640 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_LEN0, 2),
641 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_LEN1, 3),
642 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_IO_TYPE, 5),
643 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_LOCK_TYPE, 6),
644 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_CACHE_TYPE, 7),
645 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_SPLIT_TYPE, 8),
646 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_DEM_TYPE, 9),
647 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, REQ_ORD_TYPE, 10),
648 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE0, 11),
649 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE1, 12),
650 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE2, 13),
651
652 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_TYPE0, 0),
653 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_TYPE1, 1),
654 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LEN0, 2),
655 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LEN1, 3),
656 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_IO_TYPE, 5),
657 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LOCK_TYPE, 6),
658 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_CACHE_TYPE, 7),
659 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_SPLIT_TYPE, 8),
660 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_DEM_TYPE, 9),
661 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_ORD_TYPE, 10),
662 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE0, 11),
663 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE1, 12),
664 P4_GEN_ESCR_EMASK(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE2, 13),
665
666 P4_GEN_ESCR_EMASK(P4_EVENT_SSE_INPUT_ASSIST, ALL, 15),
667
668 P4_GEN_ESCR_EMASK(P4_EVENT_PACKED_SP_UOP, ALL, 15),
669
670 P4_GEN_ESCR_EMASK(P4_EVENT_PACKED_DP_UOP, ALL, 15),
671
672 P4_GEN_ESCR_EMASK(P4_EVENT_SCALAR_SP_UOP, ALL, 15),
673
674 P4_GEN_ESCR_EMASK(P4_EVENT_SCALAR_DP_UOP, ALL, 15),
675
676 P4_GEN_ESCR_EMASK(P4_EVENT_64BIT_MMX_UOP, ALL, 15),
677
678 P4_GEN_ESCR_EMASK(P4_EVENT_128BIT_MMX_UOP, ALL, 15),
679
680 P4_GEN_ESCR_EMASK(P4_EVENT_X87_FP_UOP, ALL, 15),
681
682 P4_GEN_ESCR_EMASK(P4_EVENT_TC_MISC, FLUSH, 4),
683
684 P4_GEN_ESCR_EMASK(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING, 0),
685
686 P4_GEN_ESCR_EMASK(P4_EVENT_TC_MS_XFER, CISC, 0),
687
688 P4_GEN_ESCR_EMASK(P4_EVENT_UOP_QUEUE_WRITES, FROM_TC_BUILD, 0),
689 P4_GEN_ESCR_EMASK(P4_EVENT_UOP_QUEUE_WRITES, FROM_TC_DELIVER, 1),
690 P4_GEN_ESCR_EMASK(P4_EVENT_UOP_QUEUE_WRITES, FROM_ROM, 2),
691
692 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, CONDITIONAL, 1),
693 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, CALL, 2),
694 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, RETURN, 3),
695 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, INDIRECT, 4),
696
697 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_BRANCH_TYPE, CONDITIONAL, 1),
698 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_BRANCH_TYPE, CALL, 2),
699 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_BRANCH_TYPE, RETURN, 3),
700 P4_GEN_ESCR_EMASK(P4_EVENT_RETIRED_BRANCH_TYPE, INDIRECT, 4),
701
702 P4_GEN_ESCR_EMASK(P4_EVENT_RESOURCE_STALL, SBFULL, 5),
703
704 P4_GEN_ESCR_EMASK(P4_EVENT_WC_BUFFER, WCB_EVICTS, 0),
705 P4_GEN_ESCR_EMASK(P4_EVENT_WC_BUFFER, WCB_FULL_EVICTS, 1),
706
707 P4_GEN_ESCR_EMASK(P4_EVENT_FRONT_END_EVENT, NBOGUS, 0),
708 P4_GEN_ESCR_EMASK(P4_EVENT_FRONT_END_EVENT, BOGUS, 1),
709
710 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, NBOGUS0, 0),
711 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, NBOGUS1, 1),
712 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, NBOGUS2, 2),
713 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, NBOGUS3, 3),
714 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, BOGUS0, 4),
715 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, BOGUS1, 5),
716 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, BOGUS2, 6),
717 P4_GEN_ESCR_EMASK(P4_EVENT_EXECUTION_EVENT, BOGUS3, 7),
718
719 P4_GEN_ESCR_EMASK(P4_EVENT_REPLAY_EVENT, NBOGUS, 0),
720 P4_GEN_ESCR_EMASK(P4_EVENT_REPLAY_EVENT, BOGUS, 1),
721
722 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_RETIRED, NBOGUSNTAG, 0),
723 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_RETIRED, NBOGUSTAG, 1),
724 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_RETIRED, BOGUSNTAG, 2),
725 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_RETIRED, BOGUSTAG, 3),
726
727 P4_GEN_ESCR_EMASK(P4_EVENT_UOPS_RETIRED, NBOGUS, 0),
728 P4_GEN_ESCR_EMASK(P4_EVENT_UOPS_RETIRED, BOGUS, 1),
729
730 P4_GEN_ESCR_EMASK(P4_EVENT_UOP_TYPE, TAGLOADS, 1),
731 P4_GEN_ESCR_EMASK(P4_EVENT_UOP_TYPE, TAGSTORES, 2),
732
733 P4_GEN_ESCR_EMASK(P4_EVENT_BRANCH_RETIRED, MMNP, 0),
734 P4_GEN_ESCR_EMASK(P4_EVENT_BRANCH_RETIRED, MMNM, 1),
735 P4_GEN_ESCR_EMASK(P4_EVENT_BRANCH_RETIRED, MMTP, 2),
736 P4_GEN_ESCR_EMASK(P4_EVENT_BRANCH_RETIRED, MMTM, 3),
737
738 P4_GEN_ESCR_EMASK(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS, 0),
739
740 P4_GEN_ESCR_EMASK(P4_EVENT_X87_ASSIST, FPSU, 0),
741 P4_GEN_ESCR_EMASK(P4_EVENT_X87_ASSIST, FPSO, 1),
742 P4_GEN_ESCR_EMASK(P4_EVENT_X87_ASSIST, POAO, 2),
743 P4_GEN_ESCR_EMASK(P4_EVENT_X87_ASSIST, POAU, 3),
744 P4_GEN_ESCR_EMASK(P4_EVENT_X87_ASSIST, PREA, 4),
745
746 P4_GEN_ESCR_EMASK(P4_EVENT_MACHINE_CLEAR, CLEAR, 0),
747 P4_GEN_ESCR_EMASK(P4_EVENT_MACHINE_CLEAR, MOCLEAR, 1),
748 P4_GEN_ESCR_EMASK(P4_EVENT_MACHINE_CLEAR, SMCLEAR, 2),
749
750 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_COMPLETED, NBOGUS, 0),
751 P4_GEN_ESCR_EMASK(P4_EVENT_INSTR_COMPLETED, BOGUS, 1),
752};
753
754/*
755 * P4 PEBS specifics (Replay Event only)
756 *
757 * Format (bits):
758 * 0-6: metric from P4_PEBS_METRIC enum
759 * 7 : reserved
760 * 8 : reserved
761 * 9-11 : reserved
762 *
763 * Note we have UOP and PEBS bits reserved for now
764 * just in case if we will need them once
765 */
766#define P4_PEBS_CONFIG_ENABLE (1 << 7)
767#define P4_PEBS_CONFIG_UOP_TAG (1 << 8)
768#define P4_PEBS_CONFIG_METRIC_MASK 0x3f
769#define P4_PEBS_CONFIG_MASK 0xff
770
771/*
772 * mem: Only counters MSR_IQ_COUNTER4 (16) and
773 * MSR_IQ_COUNTER5 (17) are allowed for PEBS sampling
774 */
775#define P4_PEBS_ENABLE 0x02000000U
776#define P4_PEBS_ENABLE_UOP_TAG 0x01000000U
777
778#define p4_config_unpack_metric(v) (((u64)(v)) & P4_PEBS_CONFIG_METRIC_MASK)
779#define p4_config_unpack_pebs(v) (((u64)(v)) & P4_PEBS_CONFIG_MASK)
780
781#define p4_config_pebs_has(v, mask) (p4_config_unpack_pebs(v) & (mask))
782
783enum P4_PEBS_METRIC {
784 P4_PEBS_METRIC__none,
785
786 P4_PEBS_METRIC__1stl_cache_load_miss_retired,
787 P4_PEBS_METRIC__2ndl_cache_load_miss_retired,
788 P4_PEBS_METRIC__dtlb_load_miss_retired,
789 P4_PEBS_METRIC__dtlb_store_miss_retired,
790 P4_PEBS_METRIC__dtlb_all_miss_retired,
791 P4_PEBS_METRIC__tagged_mispred_branch,
792 P4_PEBS_METRIC__mob_load_replay_retired,
793 P4_PEBS_METRIC__split_load_retired,
794 P4_PEBS_METRIC__split_store_retired,
795
796 P4_PEBS_METRIC__max
797};
798
799#endif /* PERF_EVENT_P4_H */
800
diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index 2984a25ff383..f686f49e8b7b 100644
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -26,6 +26,7 @@ struct mm_struct;
26struct vm_area_struct; 26struct vm_area_struct;
27 27
28extern pgd_t swapper_pg_dir[1024]; 28extern pgd_t swapper_pg_dir[1024];
29extern pgd_t trampoline_pg_dir[1024];
29 30
30static inline void pgtable_cache_init(void) { } 31static inline void pgtable_cache_init(void) { }
31static inline void check_pgt_cache(void) { } 32static inline void check_pgt_cache(void) { }
diff --git a/arch/x86/include/asm/pgtable_32_types.h b/arch/x86/include/asm/pgtable_32_types.h
index 5e67c1532314..ed5903be26fe 100644
--- a/arch/x86/include/asm/pgtable_32_types.h
+++ b/arch/x86/include/asm/pgtable_32_types.h
@@ -26,7 +26,7 @@
26 */ 26 */
27#define VMALLOC_OFFSET (8 * 1024 * 1024) 27#define VMALLOC_OFFSET (8 * 1024 * 1024)
28 28
29#ifndef __ASSEMBLER__ 29#ifndef __ASSEMBLY__
30extern bool __vmalloc_start_set; /* set once high_memory is set */ 30extern bool __vmalloc_start_set; /* set once high_memory is set */
31#endif 31#endif
32 32
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 181be528c612..076052cd62be 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -126,8 +126,8 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
126/* x86-64 always has all page tables mapped. */ 126/* x86-64 always has all page tables mapped. */
127#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) 127#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
128#define pte_offset_map_nested(dir, address) pte_offset_kernel((dir), (address)) 128#define pte_offset_map_nested(dir, address) pte_offset_kernel((dir), (address))
129#define pte_unmap(pte) /* NOP */ 129#define pte_unmap(pte) ((void)(pte))/* NOP */
130#define pte_unmap_nested(pte) /* NOP */ 130#define pte_unmap_nested(pte) ((void)(pte)) /* NOP */
131 131
132#define update_mmu_cache(vma, address, ptep) do { } while (0) 132#define update_mmu_cache(vma, address, ptep) do { } while (0)
133 133
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 48426f991475..ebaa04a8d3af 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -21,7 +21,6 @@ struct mm_struct;
21#include <asm/msr.h> 21#include <asm/msr.h>
22#include <asm/desc_defs.h> 22#include <asm/desc_defs.h>
23#include <asm/nops.h> 23#include <asm/nops.h>
24#include <asm/ds.h>
25 24
26#include <linux/personality.h> 25#include <linux/personality.h>
27#include <linux/cpumask.h> 26#include <linux/cpumask.h>
@@ -29,6 +28,7 @@ struct mm_struct;
29#include <linux/threads.h> 28#include <linux/threads.h>
30#include <linux/math64.h> 29#include <linux/math64.h>
31#include <linux/init.h> 30#include <linux/init.h>
31#include <linux/err.h>
32 32
33#define HBP_NUM 4 33#define HBP_NUM 4
34/* 34/*
@@ -113,7 +113,6 @@ struct cpuinfo_x86 {
113 /* Index into per_cpu list: */ 113 /* Index into per_cpu list: */
114 u16 cpu_index; 114 u16 cpu_index;
115#endif 115#endif
116 unsigned int x86_hyper_vendor;
117} __attribute__((__aligned__(SMP_CACHE_BYTES))); 116} __attribute__((__aligned__(SMP_CACHE_BYTES)));
118 117
119#define X86_VENDOR_INTEL 0 118#define X86_VENDOR_INTEL 0
@@ -127,9 +126,6 @@ struct cpuinfo_x86 {
127 126
128#define X86_VENDOR_UNKNOWN 0xff 127#define X86_VENDOR_UNKNOWN 0xff
129 128
130#define X86_HYPER_VENDOR_NONE 0
131#define X86_HYPER_VENDOR_VMWARE 1
132
133/* 129/*
134 * capabilities of CPUs 130 * capabilities of CPUs
135 */ 131 */
@@ -384,6 +380,10 @@ union thread_xstate {
384 struct xsave_struct xsave; 380 struct xsave_struct xsave;
385}; 381};
386 382
383struct fpu {
384 union thread_xstate *state;
385};
386
387#ifdef CONFIG_X86_64 387#ifdef CONFIG_X86_64
388DECLARE_PER_CPU(struct orig_ist, orig_ist); 388DECLARE_PER_CPU(struct orig_ist, orig_ist);
389 389
@@ -461,7 +461,7 @@ struct thread_struct {
461 unsigned long trap_no; 461 unsigned long trap_no;
462 unsigned long error_code; 462 unsigned long error_code;
463 /* floating point and extended processor state */ 463 /* floating point and extended processor state */
464 union thread_xstate *xstate; 464 struct fpu fpu;
465#ifdef CONFIG_X86_32 465#ifdef CONFIG_X86_32
466 /* Virtual 86 mode info */ 466 /* Virtual 86 mode info */
467 struct vm86_struct __user *vm86_info; 467 struct vm86_struct __user *vm86_info;
@@ -477,10 +477,6 @@ struct thread_struct {
477 unsigned long iopl; 477 unsigned long iopl;
478 /* Max allowed port in the bitmap, in bytes: */ 478 /* Max allowed port in the bitmap, in bytes: */
479 unsigned io_bitmap_max; 479 unsigned io_bitmap_max;
480/* MSR_IA32_DEBUGCTLMSR value to switch in if TIF_DEBUGCTLMSR is set. */
481 unsigned long debugctlmsr;
482 /* Debug Store context; see asm/ds.h */
483 struct ds_context *ds_ctx;
484}; 480};
485 481
486static inline unsigned long native_get_debugreg(int regno) 482static inline unsigned long native_get_debugreg(int regno)
@@ -770,6 +766,7 @@ extern void init_c1e_mask(void);
770extern unsigned long boot_option_idle_override; 766extern unsigned long boot_option_idle_override;
771extern unsigned long idle_halt; 767extern unsigned long idle_halt;
772extern unsigned long idle_nomwait; 768extern unsigned long idle_nomwait;
769extern bool c1e_detected;
773 770
774/* 771/*
775 * on systems with caches, caches must be flashed as the absolute 772 * on systems with caches, caches must be flashed as the absolute
@@ -797,6 +794,8 @@ static inline void wbinvd_halt(void)
797extern void enable_sep_cpu(void); 794extern void enable_sep_cpu(void);
798extern int sysenter_setup(void); 795extern int sysenter_setup(void);
799 796
797extern void early_trap_init(void);
798
800/* Defined in head.S */ 799/* Defined in head.S */
801extern struct desc_ptr early_gdt_descr; 800extern struct desc_ptr early_gdt_descr;
802 801
@@ -807,7 +806,7 @@ extern void cpu_init(void);
807 806
808static inline unsigned long get_debugctlmsr(void) 807static inline unsigned long get_debugctlmsr(void)
809{ 808{
810 unsigned long debugctlmsr = 0; 809 unsigned long debugctlmsr = 0;
811 810
812#ifndef CONFIG_X86_DEBUGCTLMSR 811#ifndef CONFIG_X86_DEBUGCTLMSR
813 if (boot_cpu_data.x86 < 6) 812 if (boot_cpu_data.x86 < 6)
@@ -815,21 +814,6 @@ static inline unsigned long get_debugctlmsr(void)
815#endif 814#endif
816 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); 815 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
817 816
818 return debugctlmsr;
819}
820
821static inline unsigned long get_debugctlmsr_on_cpu(int cpu)
822{
823 u64 debugctlmsr = 0;
824 u32 val1, val2;
825
826#ifndef CONFIG_X86_DEBUGCTLMSR
827 if (boot_cpu_data.x86 < 6)
828 return 0;
829#endif
830 rdmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR, &val1, &val2);
831 debugctlmsr = val1 | ((u64)val2 << 32);
832
833 return debugctlmsr; 817 return debugctlmsr;
834} 818}
835 819
@@ -842,18 +826,6 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr)
842 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); 826 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
843} 827}
844 828
845static inline void update_debugctlmsr_on_cpu(int cpu,
846 unsigned long debugctlmsr)
847{
848#ifndef CONFIG_X86_DEBUGCTLMSR
849 if (boot_cpu_data.x86 < 6)
850 return;
851#endif
852 wrmsr_on_cpu(cpu, MSR_IA32_DEBUGCTLMSR,
853 (u32)((u64)debugctlmsr),
854 (u32)((u64)debugctlmsr >> 32));
855}
856
857/* 829/*
858 * from system description table in BIOS. Mostly for MCA use, but 830 * from system description table in BIOS. Mostly for MCA use, but
859 * others may find it useful: 831 * others may find it useful:
@@ -1058,4 +1030,24 @@ unsigned long calc_aperfmperf_ratio(struct aperfmperf *old,
1058 return ratio; 1030 return ratio;
1059} 1031}
1060 1032
1033/*
1034 * AMD errata checking
1035 */
1036#ifdef CONFIG_CPU_SUP_AMD
1037extern const int amd_erratum_383[];
1038extern const int amd_erratum_400[];
1039extern bool cpu_has_amd_erratum(const int *);
1040
1041#define AMD_LEGACY_ERRATUM(...) { -1, __VA_ARGS__, 0 }
1042#define AMD_OSVW_ERRATUM(osvw_id, ...) { osvw_id, __VA_ARGS__, 0 }
1043#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
1044 ((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
1045#define AMD_MODEL_RANGE_FAMILY(range) (((range) >> 24) & 0xff)
1046#define AMD_MODEL_RANGE_START(range) (((range) >> 12) & 0xfff)
1047#define AMD_MODEL_RANGE_END(range) ((range) & 0xfff)
1048
1049#else
1050#define cpu_has_amd_erratum(x) (false)
1051#endif /* CONFIG_CPU_SUP_AMD */
1052
1061#endif /* _ASM_X86_PROCESSOR_H */ 1053#endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/include/asm/ptrace-abi.h b/arch/x86/include/asm/ptrace-abi.h
index 86723035a515..52b098a6eebb 100644
--- a/arch/x86/include/asm/ptrace-abi.h
+++ b/arch/x86/include/asm/ptrace-abi.h
@@ -82,61 +82,6 @@
82 82
83#ifndef __ASSEMBLY__ 83#ifndef __ASSEMBLY__
84#include <linux/types.h> 84#include <linux/types.h>
85 85#endif
86/* configuration/status structure used in PTRACE_BTS_CONFIG and
87 PTRACE_BTS_STATUS commands.
88*/
89struct ptrace_bts_config {
90 /* requested or actual size of BTS buffer in bytes */
91 __u32 size;
92 /* bitmask of below flags */
93 __u32 flags;
94 /* buffer overflow signal */
95 __u32 signal;
96 /* actual size of bts_struct in bytes */
97 __u32 bts_size;
98};
99#endif /* __ASSEMBLY__ */
100
101#define PTRACE_BTS_O_TRACE 0x1 /* branch trace */
102#define PTRACE_BTS_O_SCHED 0x2 /* scheduling events w/ jiffies */
103#define PTRACE_BTS_O_SIGNAL 0x4 /* send SIG<signal> on buffer overflow
104 instead of wrapping around */
105#define PTRACE_BTS_O_ALLOC 0x8 /* (re)allocate buffer */
106
107#define PTRACE_BTS_CONFIG 40
108/* Configure branch trace recording.
109 ADDR points to a struct ptrace_bts_config.
110 DATA gives the size of that buffer.
111 A new buffer is allocated, if requested in the flags.
112 An overflow signal may only be requested for new buffers.
113 Returns the number of bytes read.
114*/
115#define PTRACE_BTS_STATUS 41
116/* Return the current configuration in a struct ptrace_bts_config
117 pointed to by ADDR; DATA gives the size of that buffer.
118 Returns the number of bytes written.
119*/
120#define PTRACE_BTS_SIZE 42
121/* Return the number of available BTS records for draining.
122 DATA and ADDR are ignored.
123*/
124#define PTRACE_BTS_GET 43
125/* Get a single BTS record.
126 DATA defines the index into the BTS array, where 0 is the newest
127 entry, and higher indices refer to older entries.
128 ADDR is pointing to struct bts_struct (see asm/ds.h).
129*/
130#define PTRACE_BTS_CLEAR 44
131/* Clear the BTS buffer.
132 DATA and ADDR are ignored.
133*/
134#define PTRACE_BTS_DRAIN 45
135/* Read all available BTS records and clear the buffer.
136 ADDR points to an array of struct bts_struct.
137 DATA gives the size of that buffer.
138 BTS records are read from oldest to newest.
139 Returns number of BTS records drained.
140*/
141 86
142#endif /* _ASM_X86_PTRACE_ABI_H */ 87#endif /* _ASM_X86_PTRACE_ABI_H */
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 69a686a7dff0..78cd1ea94500 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -289,12 +289,6 @@ extern int do_get_thread_area(struct task_struct *p, int idx,
289extern int do_set_thread_area(struct task_struct *p, int idx, 289extern int do_set_thread_area(struct task_struct *p, int idx,
290 struct user_desc __user *info, int can_allocate); 290 struct user_desc __user *info, int can_allocate);
291 291
292#ifdef CONFIG_X86_PTRACE_BTS
293extern void ptrace_bts_untrace(struct task_struct *tsk);
294
295#define arch_ptrace_untrace(tsk) ptrace_bts_untrace(tsk)
296#endif /* CONFIG_X86_PTRACE_BTS */
297
298#endif /* __KERNEL__ */ 292#endif /* __KERNEL__ */
299 293
300#endif /* !__ASSEMBLY__ */ 294#endif /* !__ASSEMBLY__ */
diff --git a/arch/x86/include/asm/pvclock-abi.h b/arch/x86/include/asm/pvclock-abi.h
index 6d93508f2626..35f2d1948ada 100644
--- a/arch/x86/include/asm/pvclock-abi.h
+++ b/arch/x86/include/asm/pvclock-abi.h
@@ -29,7 +29,8 @@ struct pvclock_vcpu_time_info {
29 u64 system_time; 29 u64 system_time;
30 u32 tsc_to_system_mul; 30 u32 tsc_to_system_mul;
31 s8 tsc_shift; 31 s8 tsc_shift;
32 u8 pad[3]; 32 u8 flags;
33 u8 pad[2];
33} __attribute__((__packed__)); /* 32 bytes */ 34} __attribute__((__packed__)); /* 32 bytes */
34 35
35struct pvclock_wall_clock { 36struct pvclock_wall_clock {
@@ -38,5 +39,6 @@ struct pvclock_wall_clock {
38 u32 nsec; 39 u32 nsec;
39} __attribute__((__packed__)); 40} __attribute__((__packed__));
40 41
42#define PVCLOCK_TSC_STABLE_BIT (1 << 0)
41#endif /* __ASSEMBLY__ */ 43#endif /* __ASSEMBLY__ */
42#endif /* _ASM_X86_PVCLOCK_ABI_H */ 44#endif /* _ASM_X86_PVCLOCK_ABI_H */
diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h
index 53235fd5f8ce..cd02f324aa6b 100644
--- a/arch/x86/include/asm/pvclock.h
+++ b/arch/x86/include/asm/pvclock.h
@@ -6,6 +6,7 @@
6 6
7/* some helper functions for xen and kvm pv clock sources */ 7/* some helper functions for xen and kvm pv clock sources */
8cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src); 8cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src);
9void pvclock_set_flags(u8 flags);
9unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src); 10unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src);
10void pvclock_read_wallclock(struct pvclock_wall_clock *wall, 11void pvclock_read_wallclock(struct pvclock_wall_clock *wall,
11 struct pvclock_vcpu_time_info *vcpu, 12 struct pvclock_vcpu_time_info *vcpu,
diff --git a/arch/x86/include/asm/rdc321x_defs.h b/arch/x86/include/asm/rdc321x_defs.h
deleted file mode 100644
index c8e9c8bed3d0..000000000000
--- a/arch/x86/include/asm/rdc321x_defs.h
+++ /dev/null
@@ -1,12 +0,0 @@
1#define PFX "rdc321x: "
2
3/* General purpose configuration and data registers */
4#define RDC3210_CFGREG_ADDR 0x0CF8
5#define RDC3210_CFGREG_DATA 0x0CFC
6
7#define RDC321X_GPIO_CTRL_REG1 0x48
8#define RDC321X_GPIO_CTRL_REG2 0x84
9#define RDC321X_GPIO_DATA_REG1 0x4c
10#define RDC321X_GPIO_DATA_REG2 0x88
11
12#define RDC321X_MAX_GPIO 58
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
index 64cf2d24fad1..6c7fc25f2c34 100644
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -84,5 +84,7 @@
84#define REQUIRED_MASK5 0 84#define REQUIRED_MASK5 0
85#define REQUIRED_MASK6 0 85#define REQUIRED_MASK6 0
86#define REQUIRED_MASK7 0 86#define REQUIRED_MASK7 0
87#define REQUIRED_MASK8 0
88#define REQUIRED_MASK9 0
87 89
88#endif /* _ASM_X86_REQUIRED_FEATURES_H */ 90#endif /* _ASM_X86_REQUIRED_FEATURES_H */
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index 606ede126972..d1e41b0f9b60 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -118,7 +118,7 @@ static inline void __down_read(struct rw_semaphore *sem)
118{ 118{
119 asm volatile("# beginning down_read\n\t" 119 asm volatile("# beginning down_read\n\t"
120 LOCK_PREFIX _ASM_INC "(%1)\n\t" 120 LOCK_PREFIX _ASM_INC "(%1)\n\t"
121 /* adds 0x00000001, returns the old value */ 121 /* adds 0x00000001 */
122 " jns 1f\n" 122 " jns 1f\n"
123 " call call_rwsem_down_read_failed\n" 123 " call call_rwsem_down_read_failed\n"
124 "1:\n\t" 124 "1:\n\t"
@@ -156,11 +156,9 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
156static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) 156static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
157{ 157{
158 rwsem_count_t tmp; 158 rwsem_count_t tmp;
159
160 tmp = RWSEM_ACTIVE_WRITE_BIAS;
161 asm volatile("# beginning down_write\n\t" 159 asm volatile("# beginning down_write\n\t"
162 LOCK_PREFIX " xadd %1,(%2)\n\t" 160 LOCK_PREFIX " xadd %1,(%2)\n\t"
163 /* subtract 0x0000ffff, returns the old value */ 161 /* adds 0xffff0001, returns the old value */
164 " test %1,%1\n\t" 162 " test %1,%1\n\t"
165 /* was the count 0 before? */ 163 /* was the count 0 before? */
166 " jz 1f\n" 164 " jz 1f\n"
@@ -168,7 +166,7 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
168 "1:\n" 166 "1:\n"
169 "# ending down_write" 167 "# ending down_write"
170 : "+m" (sem->count), "=d" (tmp) 168 : "+m" (sem->count), "=d" (tmp)
171 : "a" (sem), "1" (tmp) 169 : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS)
172 : "memory", "cc"); 170 : "memory", "cc");
173} 171}
174 172
@@ -195,16 +193,16 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
195 */ 193 */
196static inline void __up_read(struct rw_semaphore *sem) 194static inline void __up_read(struct rw_semaphore *sem)
197{ 195{
198 rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS; 196 rwsem_count_t tmp;
199 asm volatile("# beginning __up_read\n\t" 197 asm volatile("# beginning __up_read\n\t"
200 LOCK_PREFIX " xadd %1,(%2)\n\t" 198 LOCK_PREFIX " xadd %1,(%2)\n\t"
201 /* subtracts 1, returns the old value */ 199 /* subtracts 1, returns the old value */
202 " jns 1f\n\t" 200 " jns 1f\n\t"
203 " call call_rwsem_wake\n" 201 " call call_rwsem_wake\n" /* expects old value in %edx */
204 "1:\n" 202 "1:\n"
205 "# ending __up_read\n" 203 "# ending __up_read\n"
206 : "+m" (sem->count), "=d" (tmp) 204 : "+m" (sem->count), "=d" (tmp)
207 : "a" (sem), "1" (tmp) 205 : "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS)
208 : "memory", "cc"); 206 : "memory", "cc");
209} 207}
210 208
@@ -216,10 +214,9 @@ static inline void __up_write(struct rw_semaphore *sem)
216 rwsem_count_t tmp; 214 rwsem_count_t tmp;
217 asm volatile("# beginning __up_write\n\t" 215 asm volatile("# beginning __up_write\n\t"
218 LOCK_PREFIX " xadd %1,(%2)\n\t" 216 LOCK_PREFIX " xadd %1,(%2)\n\t"
219 /* tries to transition 217 /* subtracts 0xffff0001, returns the old value */
220 0xffff0001 -> 0x00000000 */ 218 " jns 1f\n\t"
221 " jz 1f\n" 219 " call call_rwsem_wake\n" /* expects old value in %edx */
222 " call call_rwsem_wake\n"
223 "1:\n\t" 220 "1:\n\t"
224 "# ending __up_write\n" 221 "# ending __up_write\n"
225 : "+m" (sem->count), "=d" (tmp) 222 : "+m" (sem->count), "=d" (tmp)
diff --git a/arch/x86/include/asm/scatterlist.h b/arch/x86/include/asm/scatterlist.h
index 75af592677ec..4240878b9d76 100644
--- a/arch/x86/include/asm/scatterlist.h
+++ b/arch/x86/include/asm/scatterlist.h
@@ -1,8 +1,8 @@
1#ifndef _ASM_X86_SCATTERLIST_H 1#ifndef _ASM_X86_SCATTERLIST_H
2#define _ASM_X86_SCATTERLIST_H 2#define _ASM_X86_SCATTERLIST_H
3 3
4#define ISA_DMA_THRESHOLD (0x00ffffff)
5
6#include <asm-generic/scatterlist.h> 4#include <asm-generic/scatterlist.h>
7 5
6#define ARCH_HAS_SG_CHAIN
7
8#endif /* _ASM_X86_SCATTERLIST_H */ 8#endif /* _ASM_X86_SCATTERLIST_H */
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index 86b1506f4179..ef292c792d74 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -82,7 +82,7 @@ void *extend_brk(size_t size, size_t align);
82 * executable.) 82 * executable.)
83 */ 83 */
84#define RESERVE_BRK(name,sz) \ 84#define RESERVE_BRK(name,sz) \
85 static void __section(.discard) __used \ 85 static void __section(.discard.text) __used \
86 __brk_reservation_fn_##name##__(void) { \ 86 __brk_reservation_fn_##name##__(void) { \
87 asm volatile ( \ 87 asm volatile ( \
88 ".pushsection .brk_reservation,\"aw\",@nobits;" \ 88 ".pushsection .brk_reservation,\"aw\",@nobits;" \
diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h
index 4dab78edbad9..2b16a2ad23dc 100644
--- a/arch/x86/include/asm/stacktrace.h
+++ b/arch/x86/include/asm/stacktrace.h
@@ -1,6 +1,13 @@
1/*
2 * Copyright (C) 1991, 1992 Linus Torvalds
3 * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
4 */
5
1#ifndef _ASM_X86_STACKTRACE_H 6#ifndef _ASM_X86_STACKTRACE_H
2#define _ASM_X86_STACKTRACE_H 7#define _ASM_X86_STACKTRACE_H
3 8
9#include <linux/uaccess.h>
10
4extern int kstack_depth_to_print; 11extern int kstack_depth_to_print;
5 12
6struct thread_info; 13struct thread_info;
@@ -42,4 +49,46 @@ void dump_trace(struct task_struct *tsk, struct pt_regs *regs,
42 unsigned long *stack, unsigned long bp, 49 unsigned long *stack, unsigned long bp,
43 const struct stacktrace_ops *ops, void *data); 50 const struct stacktrace_ops *ops, void *data);
44 51
52#ifdef CONFIG_X86_32
53#define STACKSLOTS_PER_LINE 8
54#define get_bp(bp) asm("movl %%ebp, %0" : "=r" (bp) :)
55#else
56#define STACKSLOTS_PER_LINE 4
57#define get_bp(bp) asm("movq %%rbp, %0" : "=r" (bp) :)
58#endif
59
60extern void
61show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
62 unsigned long *stack, unsigned long bp, char *log_lvl);
63
64extern void
65show_stack_log_lvl(struct task_struct *task, struct pt_regs *regs,
66 unsigned long *sp, unsigned long bp, char *log_lvl);
67
68extern unsigned int code_bytes;
69
70/* The form of the top of the frame on the stack */
71struct stack_frame {
72 struct stack_frame *next_frame;
73 unsigned long return_address;
74};
75
76struct stack_frame_ia32 {
77 u32 next_frame;
78 u32 return_address;
79};
80
81static inline unsigned long caller_frame_pointer(void)
82{
83 struct stack_frame *frame;
84
85 get_bp(frame);
86
87#ifdef CONFIG_FRAME_POINTER
88 frame = frame->next_frame;
89#endif
90
91 return (unsigned long)frame;
92}
93
45#endif /* _ASM_X86_STACKTRACE_H */ 94#endif /* _ASM_X86_STACKTRACE_H */
diff --git a/arch/x86/include/asm/suspend_32.h b/arch/x86/include/asm/suspend_32.h
index 48dcfa62ea07..fd921c3a6841 100644
--- a/arch/x86/include/asm/suspend_32.h
+++ b/arch/x86/include/asm/suspend_32.h
@@ -15,6 +15,8 @@ static inline int arch_prepare_suspend(void) { return 0; }
15struct saved_context { 15struct saved_context {
16 u16 es, fs, gs, ss; 16 u16 es, fs, gs, ss;
17 unsigned long cr0, cr2, cr3, cr4; 17 unsigned long cr0, cr2, cr3, cr4;
18 u64 misc_enable;
19 bool misc_enable_saved;
18 struct desc_ptr gdt; 20 struct desc_ptr gdt;
19 struct desc_ptr idt; 21 struct desc_ptr idt;
20 u16 ldt; 22 u16 ldt;
diff --git a/arch/x86/include/asm/suspend_64.h b/arch/x86/include/asm/suspend_64.h
index 06284f42b759..8d942afae681 100644
--- a/arch/x86/include/asm/suspend_64.h
+++ b/arch/x86/include/asm/suspend_64.h
@@ -27,6 +27,8 @@ struct saved_context {
27 u16 ds, es, fs, gs, ss; 27 u16 ds, es, fs, gs, ss;
28 unsigned long gs_base, gs_kernel_base, fs_base; 28 unsigned long gs_base, gs_kernel_base, fs_base;
29 unsigned long cr0, cr2, cr3, cr4, cr8; 29 unsigned long cr0, cr2, cr3, cr4, cr8;
30 u64 misc_enable;
31 bool misc_enable_saved;
30 unsigned long efer; 32 unsigned long efer;
31 u16 gdt_pad; 33 u16 gdt_pad;
32 u16 gdt_limit; 34 u16 gdt_limit;
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 38638cd2fa4c..0e831059ac5a 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -81,7 +81,9 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
81 u32 event_inj_err; 81 u32 event_inj_err;
82 u64 nested_cr3; 82 u64 nested_cr3;
83 u64 lbr_ctl; 83 u64 lbr_ctl;
84 u8 reserved_5[832]; 84 u64 reserved_5;
85 u64 next_rip;
86 u8 reserved_6[816];
85}; 87};
86 88
87 89
@@ -115,6 +117,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
115#define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) 117#define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT)
116#define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) 118#define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT)
117 119
120#define SVM_VM_CR_VALID_MASK 0x001fULL
121#define SVM_VM_CR_SVM_LOCK_MASK 0x0008ULL
122#define SVM_VM_CR_SVM_DIS_MASK 0x0010ULL
123
118struct __attribute__ ((__packed__)) vmcb_seg { 124struct __attribute__ ((__packed__)) vmcb_seg {
119 u16 selector; 125 u16 selector;
120 u16 attrib; 126 u16 attrib;
@@ -238,6 +244,7 @@ struct __attribute__ ((__packed__)) vmcb {
238 244
239#define SVM_EXITINFOSHIFT_TS_REASON_IRET 36 245#define SVM_EXITINFOSHIFT_TS_REASON_IRET 36
240#define SVM_EXITINFOSHIFT_TS_REASON_JMP 38 246#define SVM_EXITINFOSHIFT_TS_REASON_JMP 38
247#define SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE 44
241 248
242#define SVM_EXIT_READ_CR0 0x000 249#define SVM_EXIT_READ_CR0 0x000
243#define SVM_EXIT_READ_CR3 0x003 250#define SVM_EXIT_READ_CR3 0x003
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h
index 3ad421784ae7..cb238526a9f1 100644
--- a/arch/x86/include/asm/sys_ia32.h
+++ b/arch/x86/include/asm/sys_ia32.h
@@ -18,13 +18,13 @@
18#include <asm/ia32.h> 18#include <asm/ia32.h>
19 19
20/* ia32/sys_ia32.c */ 20/* ia32/sys_ia32.c */
21asmlinkage long sys32_truncate64(char __user *, unsigned long, unsigned long); 21asmlinkage long sys32_truncate64(const char __user *, unsigned long, unsigned long);
22asmlinkage long sys32_ftruncate64(unsigned int, unsigned long, unsigned long); 22asmlinkage long sys32_ftruncate64(unsigned int, unsigned long, unsigned long);
23 23
24asmlinkage long sys32_stat64(char __user *, struct stat64 __user *); 24asmlinkage long sys32_stat64(const char __user *, struct stat64 __user *);
25asmlinkage long sys32_lstat64(char __user *, struct stat64 __user *); 25asmlinkage long sys32_lstat64(const char __user *, struct stat64 __user *);
26asmlinkage long sys32_fstat64(unsigned int, struct stat64 __user *); 26asmlinkage long sys32_fstat64(unsigned int, struct stat64 __user *);
27asmlinkage long sys32_fstatat(unsigned int, char __user *, 27asmlinkage long sys32_fstatat(unsigned int, const char __user *,
28 struct stat64 __user *, int); 28 struct stat64 __user *, int);
29struct mmap_arg_struct32; 29struct mmap_arg_struct32;
30asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *); 30asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *);
@@ -49,12 +49,12 @@ asmlinkage long sys32_rt_sigpending(compat_sigset_t __user *, compat_size_t);
49asmlinkage long sys32_rt_sigqueueinfo(int, int, compat_siginfo_t __user *); 49asmlinkage long sys32_rt_sigqueueinfo(int, int, compat_siginfo_t __user *);
50 50
51asmlinkage long sys32_pread(unsigned int, char __user *, u32, u32, u32); 51asmlinkage long sys32_pread(unsigned int, char __user *, u32, u32, u32);
52asmlinkage long sys32_pwrite(unsigned int, char __user *, u32, u32, u32); 52asmlinkage long sys32_pwrite(unsigned int, const char __user *, u32, u32, u32);
53 53
54asmlinkage long sys32_personality(unsigned long); 54asmlinkage long sys32_personality(unsigned long);
55asmlinkage long sys32_sendfile(int, int, compat_off_t __user *, s32); 55asmlinkage long sys32_sendfile(int, int, compat_off_t __user *, s32);
56 56
57asmlinkage long sys32_execve(char __user *, compat_uptr_t __user *, 57asmlinkage long sys32_execve(const char __user *, compat_uptr_t __user *,
58 compat_uptr_t __user *, struct pt_regs *); 58 compat_uptr_t __user *, struct pt_regs *);
59asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *); 59asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *);
60 60
@@ -80,4 +80,7 @@ asmlinkage long sys32_rt_sigreturn(struct pt_regs *);
80 80
81/* ia32/ipc32.c */ 81/* ia32/ipc32.c */
82asmlinkage long sys32_ipc(u32, int, int, int, compat_uptr_t, u32); 82asmlinkage long sys32_ipc(u32, int, int, int, compat_uptr_t, u32);
83
84asmlinkage long sys32_fanotify_mark(int, unsigned int, u32, u32, int,
85 const char __user *);
83#endif /* _ASM_X86_SYS_IA32_H */ 86#endif /* _ASM_X86_SYS_IA32_H */
diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h
index 5c044b43e9a7..f1d8b441fc77 100644
--- a/arch/x86/include/asm/syscalls.h
+++ b/arch/x86/include/asm/syscalls.h
@@ -23,8 +23,9 @@ long sys_iopl(unsigned int, struct pt_regs *);
23/* kernel/process.c */ 23/* kernel/process.c */
24int sys_fork(struct pt_regs *); 24int sys_fork(struct pt_regs *);
25int sys_vfork(struct pt_regs *); 25int sys_vfork(struct pt_regs *);
26long sys_execve(char __user *, char __user * __user *, 26long sys_execve(const char __user *,
27 char __user * __user *, struct pt_regs *); 27 const char __user *const __user *,
28 const char __user *const __user *, struct pt_regs *);
28long sys_clone(unsigned long, unsigned long, void __user *, 29long sys_clone(unsigned long, unsigned long, void __user *,
29 void __user *, struct pt_regs *); 30 void __user *, struct pt_regs *);
30 31
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
index b8fe48ee2ed9..33ecc3ea8782 100644
--- a/arch/x86/include/asm/system.h
+++ b/arch/x86/include/asm/system.h
@@ -451,10 +451,17 @@ void stop_this_cpu(void *dummy);
451 * 451 *
452 * (Could use an alternative three way for this if there was one.) 452 * (Could use an alternative three way for this if there was one.)
453 */ 453 */
454static inline void rdtsc_barrier(void) 454static __always_inline void rdtsc_barrier(void)
455{ 455{
456 alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC); 456 alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
457 alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC); 457 alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
458} 458}
459 459
460/*
461 * We handle most unaligned accesses in hardware. On the other hand
462 * unaligned DMA can be quite expensive on some Nehalem processors.
463 *
464 * Based on this we disable the IP header alignment in network drivers.
465 */
466#define NET_IP_ALIGN 0
460#endif /* _ASM_X86_SYSTEM_H */ 467#endif /* _ASM_X86_SYSTEM_H */
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index e0d28901e969..f0b6e5dbc5a0 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -87,13 +87,12 @@ struct thread_info {
87#define TIF_NOTSC 16 /* TSC is not accessible in userland */ 87#define TIF_NOTSC 16 /* TSC is not accessible in userland */
88#define TIF_IA32 17 /* 32bit process */ 88#define TIF_IA32 17 /* 32bit process */
89#define TIF_FORK 18 /* ret_from_fork */ 89#define TIF_FORK 18 /* ret_from_fork */
90#define TIF_MEMDIE 20 90#define TIF_MEMDIE 20 /* is terminating due to OOM killer */
91#define TIF_DEBUG 21 /* uses debug registers */ 91#define TIF_DEBUG 21 /* uses debug registers */
92#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ 92#define TIF_IO_BITMAP 22 /* uses I/O bitmap */
93#define TIF_FREEZE 23 /* is freezing for suspend */ 93#define TIF_FREEZE 23 /* is freezing for suspend */
94#define TIF_FORCED_TF 24 /* true if TF in eflags artificially */ 94#define TIF_FORCED_TF 24 /* true if TF in eflags artificially */
95#define TIF_DEBUGCTLMSR 25 /* uses thread_struct.debugctlmsr */ 95#define TIF_BLOCKSTEP 25 /* set when we want DEBUGCTLMSR_BTF */
96#define TIF_DS_AREA_MSR 26 /* uses thread_struct.ds_area_msr */
97#define TIF_LAZY_MMU_UPDATES 27 /* task is updating the mmu lazily */ 96#define TIF_LAZY_MMU_UPDATES 27 /* task is updating the mmu lazily */
98#define TIF_SYSCALL_TRACEPOINT 28 /* syscall tracepoint instrumentation */ 97#define TIF_SYSCALL_TRACEPOINT 28 /* syscall tracepoint instrumentation */
99 98
@@ -115,8 +114,7 @@ struct thread_info {
115#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) 114#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP)
116#define _TIF_FREEZE (1 << TIF_FREEZE) 115#define _TIF_FREEZE (1 << TIF_FREEZE)
117#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) 116#define _TIF_FORCED_TF (1 << TIF_FORCED_TF)
118#define _TIF_DEBUGCTLMSR (1 << TIF_DEBUGCTLMSR) 117#define _TIF_BLOCKSTEP (1 << TIF_BLOCKSTEP)
119#define _TIF_DS_AREA_MSR (1 << TIF_DS_AREA_MSR)
120#define _TIF_LAZY_MMU_UPDATES (1 << TIF_LAZY_MMU_UPDATES) 118#define _TIF_LAZY_MMU_UPDATES (1 << TIF_LAZY_MMU_UPDATES)
121#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) 119#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
122 120
@@ -147,7 +145,7 @@ struct thread_info {
147 145
148/* flags to check in __switch_to() */ 146/* flags to check in __switch_to() */
149#define _TIF_WORK_CTXSW \ 147#define _TIF_WORK_CTXSW \
150 (_TIF_IO_BITMAP|_TIF_DEBUGCTLMSR|_TIF_DS_AREA_MSR|_TIF_NOTSC) 148 (_TIF_IO_BITMAP|_TIF_NOTSC|_TIF_BLOCKSTEP)
151 149
152#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY) 150#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
153#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG) 151#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW|_TIF_DEBUG)
@@ -241,10 +239,9 @@ static inline struct thread_info *current_thread_info(void)
241#define TS_USEDFPU 0x0001 /* FPU was used by this task 239#define TS_USEDFPU 0x0001 /* FPU was used by this task
242 this quantum (SMP) */ 240 this quantum (SMP) */
243#define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/ 241#define TS_COMPAT 0x0002 /* 32bit syscall active (64BIT)*/
244#define TS_POLLING 0x0004 /* true if in idle loop 242#define TS_POLLING 0x0004 /* idle task polling need_resched,
245 and not sleeping */ 243 skip sending interrupt */
246#define TS_RESTORE_SIGMASK 0x0008 /* restore signal mask in do_signal() */ 244#define TS_RESTORE_SIGMASK 0x0008 /* restore signal mask in do_signal() */
247#define TS_XSAVE 0x0010 /* Use xsave/xrstor */
248 245
249#define tsk_is_polling(t) (task_thread_info(t)->status & TS_POLLING) 246#define tsk_is_polling(t) (task_thread_info(t)->status & TS_POLLING)
250 247
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index c5087d796587..21899cc31e52 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -53,33 +53,29 @@
53extern int cpu_to_node_map[]; 53extern int cpu_to_node_map[];
54 54
55/* Returns the number of the node containing CPU 'cpu' */ 55/* Returns the number of the node containing CPU 'cpu' */
56static inline int cpu_to_node(int cpu) 56static inline int __cpu_to_node(int cpu)
57{ 57{
58 return cpu_to_node_map[cpu]; 58 return cpu_to_node_map[cpu];
59} 59}
60#define early_cpu_to_node(cpu) cpu_to_node(cpu) 60#define early_cpu_to_node __cpu_to_node
61#define cpu_to_node __cpu_to_node
61 62
62#else /* CONFIG_X86_64 */ 63#else /* CONFIG_X86_64 */
63 64
64/* Mappings between logical cpu number and node number */ 65/* Mappings between logical cpu number and node number */
65DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map); 66DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map);
66 67
67/* Returns the number of the current Node. */
68DECLARE_PER_CPU(int, node_number);
69#define numa_node_id() percpu_read(node_number)
70
71#ifdef CONFIG_DEBUG_PER_CPU_MAPS 68#ifdef CONFIG_DEBUG_PER_CPU_MAPS
72extern int cpu_to_node(int cpu); 69/*
70 * override generic percpu implementation of cpu_to_node
71 */
72extern int __cpu_to_node(int cpu);
73#define cpu_to_node __cpu_to_node
74
73extern int early_cpu_to_node(int cpu); 75extern int early_cpu_to_node(int cpu);
74 76
75#else /* !CONFIG_DEBUG_PER_CPU_MAPS */ 77#else /* !CONFIG_DEBUG_PER_CPU_MAPS */
76 78
77/* Returns the number of the node containing CPU 'cpu' */
78static inline int cpu_to_node(int cpu)
79{
80 return per_cpu(x86_cpu_to_node_map, cpu);
81}
82
83/* Same function but used if called before per_cpu areas are setup */ 79/* Same function but used if called before per_cpu areas are setup */
84static inline int early_cpu_to_node(int cpu) 80static inline int early_cpu_to_node(int cpu)
85{ 81{
@@ -170,6 +166,10 @@ static inline int numa_node_id(void)
170{ 166{
171 return 0; 167 return 0;
172} 168}
169/*
170 * indicate override:
171 */
172#define numa_node_id numa_node_id
173 173
174static inline int early_cpu_to_node(int cpu) 174static inline int early_cpu_to_node(int cpu)
175{ 175{
diff --git a/arch/x86/include/asm/trampoline.h b/arch/x86/include/asm/trampoline.h
index cb507bb05d79..4dde797c0578 100644
--- a/arch/x86/include/asm/trampoline.h
+++ b/arch/x86/include/asm/trampoline.h
@@ -13,14 +13,17 @@ extern unsigned char *trampoline_base;
13 13
14extern unsigned long init_rsp; 14extern unsigned long init_rsp;
15extern unsigned long initial_code; 15extern unsigned long initial_code;
16extern unsigned long initial_page_table;
16extern unsigned long initial_gs; 17extern unsigned long initial_gs;
17 18
18#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE) 19#define TRAMPOLINE_SIZE roundup(trampoline_end - trampoline_data, PAGE_SIZE)
19 20
20extern unsigned long setup_trampoline(void); 21extern unsigned long setup_trampoline(void);
22extern void __init setup_trampoline_page_table(void);
21extern void __init reserve_trampoline_memory(void); 23extern void __init reserve_trampoline_memory(void);
22#else 24#else
23static inline void reserve_trampoline_memory(void) {}; 25static inline void setup_trampoline_page_table(void) {}
26static inline void reserve_trampoline_memory(void) {}
24#endif /* CONFIG_X86_TRAMPOLINE */ 27#endif /* CONFIG_X86_TRAMPOLINE */
25 28
26#endif /* __ASSEMBLY__ */ 29#endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 4da91ad69e0d..f66cda56781d 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -79,7 +79,7 @@ static inline int get_si_code(unsigned long condition)
79 79
80extern int panic_on_unrecovered_nmi; 80extern int panic_on_unrecovered_nmi;
81 81
82void math_error(void __user *); 82void math_error(struct pt_regs *, int, int);
83void math_emulate(struct math_emu_info *); 83void math_emulate(struct math_emu_info *);
84#ifndef CONFIG_X86_32 84#ifndef CONFIG_X86_32
85asmlinkage void smp_thermal_interrupt(void); 85asmlinkage void smp_thermal_interrupt(void);
diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index c0427295e8f5..1ca132fc0d03 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -59,5 +59,7 @@ extern void check_tsc_sync_source(int cpu);
59extern void check_tsc_sync_target(void); 59extern void check_tsc_sync_target(void);
60 60
61extern int notsc_setup(char *); 61extern int notsc_setup(char *);
62extern void save_sched_clock_state(void);
63extern void restore_sched_clock_state(void);
62 64
63#endif /* _ASM_X86_TSC_H */ 65#endif /* _ASM_X86_TSC_H */
diff --git a/arch/x86/include/asm/unistd_32.h b/arch/x86/include/asm/unistd_32.h
index 987e523c3480..b7ba19acd3f8 100644
--- a/arch/x86/include/asm/unistd_32.h
+++ b/arch/x86/include/asm/unistd_32.h
@@ -343,14 +343,17 @@
343#define __NR_rt_tgsigqueueinfo 335 343#define __NR_rt_tgsigqueueinfo 335
344#define __NR_perf_event_open 336 344#define __NR_perf_event_open 336
345#define __NR_recvmmsg 337 345#define __NR_recvmmsg 337
346#define __NR_fanotify_init 338
347#define __NR_fanotify_mark 339
348#define __NR_prlimit64 340
346 349
347#define __NR_LITMUS 338 350#define __NR_LITMUS 341
348 351
349#include "litmus/unistd_32.h" 352#include "litmus/unistd_32.h"
350 353
351#ifdef __KERNEL__ 354#ifdef __KERNEL__
352 355
353#define NR_syscalls 338 + NR_litmus_syscalls 356#define NR_syscalls 341 + NR_litmus_syscalls
354 357
355#define __ARCH_WANT_IPC_PARSE_VERSION 358#define __ARCH_WANT_IPC_PARSE_VERSION
356#define __ARCH_WANT_OLD_READDIR 359#define __ARCH_WANT_OLD_READDIR
diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h
index b21c3b269aac..332bf3c9c84c 100644
--- a/arch/x86/include/asm/unistd_64.h
+++ b/arch/x86/include/asm/unistd_64.h
@@ -663,8 +663,14 @@ __SYSCALL(__NR_rt_tgsigqueueinfo, sys_rt_tgsigqueueinfo)
663__SYSCALL(__NR_perf_event_open, sys_perf_event_open) 663__SYSCALL(__NR_perf_event_open, sys_perf_event_open)
664#define __NR_recvmmsg 299 664#define __NR_recvmmsg 299
665__SYSCALL(__NR_recvmmsg, sys_recvmmsg) 665__SYSCALL(__NR_recvmmsg, sys_recvmmsg)
666 666#define __NR_fanotify_init 300
667#define __NR_LITMUS 299 667__SYSCALL(__NR_fanotify_init, sys_fanotify_init)
668#define __NR_fanotify_mark 301
669__SYSCALL(__NR_fanotify_mark, sys_fanotify_mark)
670#define __NR_prlimit64 302
671__SYSCALL(__NR_prlimit64, sys_prlimit64)
672
673#define __NR_LITMUS 303
668 674
669#include "litmus/unistd_64.h" 675#include "litmus/unistd_64.h"
670 676
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index b414d2b401f6..42d412fd8b02 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -27,13 +27,15 @@
27 * set 2 is at BASE + 2*512, set 3 at BASE + 3*512, and so on. 27 * set 2 is at BASE + 2*512, set 3 at BASE + 3*512, and so on.
28 * 28 *
29 * We will use 31 sets, one for sending BAU messages from each of the 32 29 * We will use 31 sets, one for sending BAU messages from each of the 32
30 * cpu's on the node. 30 * cpu's on the uvhub.
31 * 31 *
32 * TLB shootdown will use the first of the 8 descriptors of each set. 32 * TLB shootdown will use the first of the 8 descriptors of each set.
33 * Each of the descriptors is 64 bytes in size (8*64 = 512 bytes in a set). 33 * Each of the descriptors is 64 bytes in size (8*64 = 512 bytes in a set).
34 */ 34 */
35 35
36#define UV_ITEMS_PER_DESCRIPTOR 8 36#define UV_ITEMS_PER_DESCRIPTOR 8
37/* the 'throttle' to prevent the hardware stay-busy bug */
38#define MAX_BAU_CONCURRENT 3
37#define UV_CPUS_PER_ACT_STATUS 32 39#define UV_CPUS_PER_ACT_STATUS 32
38#define UV_ACT_STATUS_MASK 0x3 40#define UV_ACT_STATUS_MASK 0x3
39#define UV_ACT_STATUS_SIZE 2 41#define UV_ACT_STATUS_SIZE 2
@@ -44,7 +46,26 @@
44#define UV_DESC_BASE_PNODE_SHIFT 49 46#define UV_DESC_BASE_PNODE_SHIFT 49
45#define UV_PAYLOADQ_PNODE_SHIFT 49 47#define UV_PAYLOADQ_PNODE_SHIFT 49
46#define UV_PTC_BASENAME "sgi_uv/ptc_statistics" 48#define UV_PTC_BASENAME "sgi_uv/ptc_statistics"
49#define UV_BAU_BASENAME "sgi_uv/bau_tunables"
50#define UV_BAU_TUNABLES_DIR "sgi_uv"
51#define UV_BAU_TUNABLES_FILE "bau_tunables"
52#define WHITESPACE " \t\n"
47#define uv_physnodeaddr(x) ((__pa((unsigned long)(x)) & uv_mmask)) 53#define uv_physnodeaddr(x) ((__pa((unsigned long)(x)) & uv_mmask))
54#define UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT 15
55#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT 16
56#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD 0x0000000009UL
57/* [19:16] SOFT_ACK timeout period 19: 1 is urgency 7 17:16 1 is multiplier */
58#define BAU_MISC_CONTROL_MULT_MASK 3
59
60#define UVH_AGING_PRESCALE_SEL 0x000000b000UL
61/* [30:28] URGENCY_7 an index into a table of times */
62#define BAU_URGENCY_7_SHIFT 28
63#define BAU_URGENCY_7_MASK 7
64
65#define UVH_TRANSACTION_TIMEOUT 0x000000b200UL
66/* [45:40] BAU - BAU transaction timeout select - a multiplier */
67#define BAU_TRANS_SHIFT 40
68#define BAU_TRANS_MASK 0x3f
48 69
49/* 70/*
50 * bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1 71 * bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1
@@ -55,15 +76,26 @@
55#define DESC_STATUS_SOURCE_TIMEOUT 3 76#define DESC_STATUS_SOURCE_TIMEOUT 3
56 77
57/* 78/*
58 * source side thresholds at which message retries print a warning 79 * delay for 'plugged' timeout retries, in microseconds
59 */ 80 */
60#define SOURCE_TIMEOUT_LIMIT 20 81#define PLUGGED_DELAY 10
61#define DESTINATION_TIMEOUT_LIMIT 20 82
83/*
84 * threshholds at which to use IPI to free resources
85 */
86/* after this # consecutive 'plugged' timeouts, use IPI to release resources */
87#define PLUGSB4RESET 100
88/* after this many consecutive timeouts, use IPI to release resources */
89#define TIMEOUTSB4RESET 1
90/* at this number uses of IPI to release resources, giveup the request */
91#define IPI_RESET_LIMIT 1
92/* after this # consecutive successes, bump up the throttle if it was lowered */
93#define COMPLETE_THRESHOLD 5
62 94
63/* 95/*
64 * number of entries in the destination side payload queue 96 * number of entries in the destination side payload queue
65 */ 97 */
66#define DEST_Q_SIZE 17 98#define DEST_Q_SIZE 20
67/* 99/*
68 * number of destination side software ack resources 100 * number of destination side software ack resources
69 */ 101 */
@@ -72,9 +104,17 @@
72/* 104/*
73 * completion statuses for sending a TLB flush message 105 * completion statuses for sending a TLB flush message
74 */ 106 */
75#define FLUSH_RETRY 1 107#define FLUSH_RETRY_PLUGGED 1
76#define FLUSH_GIVEUP 2 108#define FLUSH_RETRY_TIMEOUT 2
77#define FLUSH_COMPLETE 3 109#define FLUSH_GIVEUP 3
110#define FLUSH_COMPLETE 4
111
112/*
113 * tuning the action when the numalink network is extremely delayed
114 */
115#define CONGESTED_RESPONSE_US 1000 /* 'long' response time, in microseconds */
116#define CONGESTED_REPS 10 /* long delays averaged over this many broadcasts */
117#define CONGESTED_PERIOD 30 /* time for the bau to be disabled, in seconds */
78 118
79/* 119/*
80 * Distribution: 32 bytes (256 bits) (bytes 0-0x1f of descriptor) 120 * Distribution: 32 bytes (256 bits) (bytes 0-0x1f of descriptor)
@@ -86,14 +126,14 @@
86 * 'base_dest_nodeid' field of the header corresponds to the 126 * 'base_dest_nodeid' field of the header corresponds to the
87 * destination nodeID associated with that specified bit. 127 * destination nodeID associated with that specified bit.
88 */ 128 */
89struct bau_target_nodemask { 129struct bau_target_uvhubmask {
90 unsigned long bits[BITS_TO_LONGS(256)]; 130 unsigned long bits[BITS_TO_LONGS(UV_DISTRIBUTION_SIZE)];
91}; 131};
92 132
93/* 133/*
94 * mask of cpu's on a node 134 * mask of cpu's on a uvhub
95 * (during initialization we need to check that unsigned long has 135 * (during initialization we need to check that unsigned long has
96 * enough bits for max. cpu's per node) 136 * enough bits for max. cpu's per uvhub)
97 */ 137 */
98struct bau_local_cpumask { 138struct bau_local_cpumask {
99 unsigned long bits; 139 unsigned long bits;
@@ -135,8 +175,8 @@ struct bau_msg_payload {
135struct bau_msg_header { 175struct bau_msg_header {
136 unsigned int dest_subnodeid:6; /* must be 0x10, for the LB */ 176 unsigned int dest_subnodeid:6; /* must be 0x10, for the LB */
137 /* bits 5:0 */ 177 /* bits 5:0 */
138 unsigned int base_dest_nodeid:15; /* nasid>>1 (pnode) of */ 178 unsigned int base_dest_nodeid:15; /* nasid (pnode<<1) of */
139 /* bits 20:6 */ /* first bit in node_map */ 179 /* bits 20:6 */ /* first bit in uvhub map */
140 unsigned int command:8; /* message type */ 180 unsigned int command:8; /* message type */
141 /* bits 28:21 */ 181 /* bits 28:21 */
142 /* 0x38: SN3net EndPoint Message */ 182 /* 0x38: SN3net EndPoint Message */
@@ -146,26 +186,38 @@ struct bau_msg_header {
146 unsigned int rsvd_2:9; /* must be zero */ 186 unsigned int rsvd_2:9; /* must be zero */
147 /* bits 40:32 */ 187 /* bits 40:32 */
148 /* Suppl_A is 56-41 */ 188 /* Suppl_A is 56-41 */
149 unsigned int payload_2a:8;/* becomes byte 16 of msg */ 189 unsigned int sequence:16;/* message sequence number */
150 /* bits 48:41 */ /* not currently using */ 190 /* bits 56:41 */ /* becomes bytes 16-17 of msg */
151 unsigned int payload_2b:8;/* becomes byte 17 of msg */
152 /* bits 56:49 */ /* not currently using */
153 /* Address field (96:57) is never used as an 191 /* Address field (96:57) is never used as an
154 address (these are address bits 42:3) */ 192 address (these are address bits 42:3) */
193
155 unsigned int rsvd_3:1; /* must be zero */ 194 unsigned int rsvd_3:1; /* must be zero */
156 /* bit 57 */ 195 /* bit 57 */
157 /* address bits 27:4 are payload */ 196 /* address bits 27:4 are payload */
158 /* these 24 bits become bytes 12-14 of msg */ 197 /* these next 24 (58-81) bits become bytes 12-14 of msg */
198
199 /* bits 65:58 land in byte 12 */
159 unsigned int replied_to:1;/* sent as 0 by the source to byte 12 */ 200 unsigned int replied_to:1;/* sent as 0 by the source to byte 12 */
160 /* bit 58 */ 201 /* bit 58 */
161 202 unsigned int msg_type:3; /* software type of the message*/
162 unsigned int payload_1a:5;/* not currently used */ 203 /* bits 61:59 */
163 /* bits 63:59 */ 204 unsigned int canceled:1; /* message canceled, resource to be freed*/
164 unsigned int payload_1b:8;/* not currently used */ 205 /* bit 62 */
165 /* bits 71:64 */ 206 unsigned int payload_1a:1;/* not currently used */
166 unsigned int payload_1c:8;/* not currently used */ 207 /* bit 63 */
167 /* bits 79:72 */ 208 unsigned int payload_1b:2;/* not currently used */
168 unsigned int payload_1d:2;/* not currently used */ 209 /* bits 65:64 */
210
211 /* bits 73:66 land in byte 13 */
212 unsigned int payload_1ca:6;/* not currently used */
213 /* bits 71:66 */
214 unsigned int payload_1c:2;/* not currently used */
215 /* bits 73:72 */
216
217 /* bits 81:74 land in byte 14 */
218 unsigned int payload_1d:6;/* not currently used */
219 /* bits 79:74 */
220 unsigned int payload_1e:2;/* not currently used */
169 /* bits 81:80 */ 221 /* bits 81:80 */
170 222
171 unsigned int rsvd_4:7; /* must be zero */ 223 unsigned int rsvd_4:7; /* must be zero */
@@ -178,7 +230,7 @@ struct bau_msg_header {
178 /* bits 95:90 */ 230 /* bits 95:90 */
179 unsigned int rsvd_6:5; /* must be zero */ 231 unsigned int rsvd_6:5; /* must be zero */
180 /* bits 100:96 */ 232 /* bits 100:96 */
181 unsigned int int_both:1;/* if 1, interrupt both sockets on the blade */ 233 unsigned int int_both:1;/* if 1, interrupt both sockets on the uvhub */
182 /* bit 101*/ 234 /* bit 101*/
183 unsigned int fairness:3;/* usually zero */ 235 unsigned int fairness:3;/* usually zero */
184 /* bits 104:102 */ 236 /* bits 104:102 */
@@ -191,13 +243,18 @@ struct bau_msg_header {
191 /* bits 127:107 */ 243 /* bits 127:107 */
192}; 244};
193 245
246/* see msg_type: */
247#define MSG_NOOP 0
248#define MSG_REGULAR 1
249#define MSG_RETRY 2
250
194/* 251/*
195 * The activation descriptor: 252 * The activation descriptor:
196 * The format of the message to send, plus all accompanying control 253 * The format of the message to send, plus all accompanying control
197 * Should be 64 bytes 254 * Should be 64 bytes
198 */ 255 */
199struct bau_desc { 256struct bau_desc {
200 struct bau_target_nodemask distribution; 257 struct bau_target_uvhubmask distribution;
201 /* 258 /*
202 * message template, consisting of header and payload: 259 * message template, consisting of header and payload:
203 */ 260 */
@@ -237,19 +294,25 @@ struct bau_payload_queue_entry {
237 unsigned short acknowledge_count; /* filled in by destination */ 294 unsigned short acknowledge_count; /* filled in by destination */
238 /* 16 bits, bytes 10-11 */ 295 /* 16 bits, bytes 10-11 */
239 296
240 unsigned short replied_to:1; /* sent as 0 by the source */ 297 /* these next 3 bytes come from bits 58-81 of the message header */
241 /* 1 bit */ 298 unsigned short replied_to:1; /* sent as 0 by the source */
242 unsigned short unused1:7; /* not currently using */ 299 unsigned short msg_type:3; /* software message type */
243 /* 7 bits: byte 12) */ 300 unsigned short canceled:1; /* sent as 0 by the source */
301 unsigned short unused1:3; /* not currently using */
302 /* byte 12 */
244 303
245 unsigned char unused2[2]; /* not currently using */ 304 unsigned char unused2a; /* not currently using */
246 /* bytes 13-14 */ 305 /* byte 13 */
306 unsigned char unused2; /* not currently using */
307 /* byte 14 */
247 308
248 unsigned char sw_ack_vector; /* filled in by the hardware */ 309 unsigned char sw_ack_vector; /* filled in by the hardware */
249 /* byte 15 (bits 127:120) */ 310 /* byte 15 (bits 127:120) */
250 311
251 unsigned char unused4[3]; /* not currently using bytes 17-19 */ 312 unsigned short sequence; /* message sequence number */
252 /* bytes 17-19 */ 313 /* bytes 16-17 */
314 unsigned char unused4[2]; /* not currently using bytes 18-19 */
315 /* bytes 18-19 */
253 316
254 int number_of_cpus; /* filled in at destination */ 317 int number_of_cpus; /* filled in at destination */
255 /* 32 bits, bytes 20-23 (aligned) */ 318 /* 32 bits, bytes 20-23 (aligned) */
@@ -258,64 +321,126 @@ struct bau_payload_queue_entry {
258 /* bytes 24-31 */ 321 /* bytes 24-31 */
259}; 322};
260 323
261/* 324struct msg_desc {
262 * one for every slot in the destination payload queue 325 struct bau_payload_queue_entry *msg;
263 */ 326 int msg_slot;
264struct bau_msg_status { 327 int sw_ack_slot;
265 struct bau_local_cpumask seen_by; /* map of cpu's */ 328 struct bau_payload_queue_entry *va_queue_first;
329 struct bau_payload_queue_entry *va_queue_last;
330};
331
332struct reset_args {
333 int sender;
266}; 334};
267 335
268/* 336/*
269 * one for every slot in the destination software ack resources 337 * This structure is allocated per_cpu for UV TLB shootdown statistics.
270 */ 338 */
271struct bau_sw_ack_status { 339struct ptc_stats {
272 struct bau_payload_queue_entry *msg; /* associated message */ 340 /* sender statistics */
273 int watcher; /* cpu monitoring, or -1 */ 341 unsigned long s_giveup; /* number of fall backs to IPI-style flushes */
342 unsigned long s_requestor; /* number of shootdown requests */
343 unsigned long s_stimeout; /* source side timeouts */
344 unsigned long s_dtimeout; /* destination side timeouts */
345 unsigned long s_time; /* time spent in sending side */
346 unsigned long s_retriesok; /* successful retries */
347 unsigned long s_ntargcpu; /* total number of cpu's targeted */
348 unsigned long s_ntargself; /* times the sending cpu was targeted */
349 unsigned long s_ntarglocals; /* targets of cpus on the local blade */
350 unsigned long s_ntargremotes; /* targets of cpus on remote blades */
351 unsigned long s_ntarglocaluvhub; /* targets of the local hub */
352 unsigned long s_ntargremoteuvhub; /* remotes hubs targeted */
353 unsigned long s_ntarguvhub; /* total number of uvhubs targeted */
354 unsigned long s_ntarguvhub16; /* number of times target hubs >= 16*/
355 unsigned long s_ntarguvhub8; /* number of times target hubs >= 8 */
356 unsigned long s_ntarguvhub4; /* number of times target hubs >= 4 */
357 unsigned long s_ntarguvhub2; /* number of times target hubs >= 2 */
358 unsigned long s_ntarguvhub1; /* number of times target hubs == 1 */
359 unsigned long s_resets_plug; /* ipi-style resets from plug state */
360 unsigned long s_resets_timeout; /* ipi-style resets from timeouts */
361 unsigned long s_busy; /* status stayed busy past s/w timer */
362 unsigned long s_throttles; /* waits in throttle */
363 unsigned long s_retry_messages; /* retry broadcasts */
364 unsigned long s_bau_reenabled; /* for bau enable/disable */
365 unsigned long s_bau_disabled; /* for bau enable/disable */
366 /* destination statistics */
367 unsigned long d_alltlb; /* times all tlb's on this cpu were flushed */
368 unsigned long d_onetlb; /* times just one tlb on this cpu was flushed */
369 unsigned long d_multmsg; /* interrupts with multiple messages */
370 unsigned long d_nomsg; /* interrupts with no message */
371 unsigned long d_time; /* time spent on destination side */
372 unsigned long d_requestee; /* number of messages processed */
373 unsigned long d_retries; /* number of retry messages processed */
374 unsigned long d_canceled; /* number of messages canceled by retries */
375 unsigned long d_nocanceled; /* retries that found nothing to cancel */
376 unsigned long d_resets; /* number of ipi-style requests processed */
377 unsigned long d_rcanceled; /* number of messages canceled by resets */
274}; 378};
275 379
276/* 380/*
277 * one on every node and per-cpu; to locate the software tables 381 * one per-cpu; to locate the software tables
278 */ 382 */
279struct bau_control { 383struct bau_control {
280 struct bau_desc *descriptor_base; 384 struct bau_desc *descriptor_base;
281 struct bau_payload_queue_entry *bau_msg_head;
282 struct bau_payload_queue_entry *va_queue_first; 385 struct bau_payload_queue_entry *va_queue_first;
283 struct bau_payload_queue_entry *va_queue_last; 386 struct bau_payload_queue_entry *va_queue_last;
284 struct bau_msg_status *msg_statuses; 387 struct bau_payload_queue_entry *bau_msg_head;
285 int *watching; /* pointer to array */ 388 struct bau_control *uvhub_master;
286}; 389 struct bau_control *socket_master;
287 390 struct ptc_stats *statp;
288/* 391 unsigned long timeout_interval;
289 * This structure is allocated per_cpu for UV TLB shootdown statistics. 392 unsigned long set_bau_on_time;
290 */ 393 atomic_t active_descriptor_count;
291struct ptc_stats { 394 int plugged_tries;
292 unsigned long ptc_i; /* number of IPI-style flushes */ 395 int timeout_tries;
293 unsigned long requestor; /* number of nodes this cpu sent to */ 396 int ipi_attempts;
294 unsigned long requestee; /* times cpu was remotely requested */ 397 int conseccompletes;
295 unsigned long alltlb; /* times all tlb's on this cpu were flushed */ 398 int baudisabled;
296 unsigned long onetlb; /* times just one tlb on this cpu was flushed */ 399 int set_bau_off;
297 unsigned long s_retry; /* retries on source side timeouts */ 400 short cpu;
298 unsigned long d_retry; /* retries on destination side timeouts */ 401 short uvhub_cpu;
299 unsigned long sflush; /* cycles spent in uv_flush_tlb_others */ 402 short uvhub;
300 unsigned long dflush; /* cycles spent on destination side */ 403 short cpus_in_socket;
301 unsigned long retriesok; /* successes on retries */ 404 short cpus_in_uvhub;
302 unsigned long nomsg; /* interrupts with no message */ 405 unsigned short message_number;
303 unsigned long multmsg; /* interrupts with multiple messages */ 406 unsigned short uvhub_quiesce;
304 unsigned long ntargeted;/* nodes targeted */ 407 short socket_acknowledge_count[DEST_Q_SIZE];
408 cycles_t send_message;
409 spinlock_t uvhub_lock;
410 spinlock_t queue_lock;
411 /* tunables */
412 int max_bau_concurrent;
413 int max_bau_concurrent_constant;
414 int plugged_delay;
415 int plugsb4reset;
416 int timeoutsb4reset;
417 int ipi_reset_limit;
418 int complete_threshold;
419 int congested_response_us;
420 int congested_reps;
421 int congested_period;
422 cycles_t period_time;
423 long period_requests;
305}; 424};
306 425
307static inline int bau_node_isset(int node, struct bau_target_nodemask *dstp) 426static inline int bau_uvhub_isset(int uvhub, struct bau_target_uvhubmask *dstp)
308{ 427{
309 return constant_test_bit(node, &dstp->bits[0]); 428 return constant_test_bit(uvhub, &dstp->bits[0]);
310} 429}
311static inline void bau_node_set(int node, struct bau_target_nodemask *dstp) 430static inline void bau_uvhub_set(int uvhub, struct bau_target_uvhubmask *dstp)
312{ 431{
313 __set_bit(node, &dstp->bits[0]); 432 __set_bit(uvhub, &dstp->bits[0]);
314} 433}
315static inline void bau_nodes_clear(struct bau_target_nodemask *dstp, int nbits) 434static inline void bau_uvhubs_clear(struct bau_target_uvhubmask *dstp,
435 int nbits)
316{ 436{
317 bitmap_zero(&dstp->bits[0], nbits); 437 bitmap_zero(&dstp->bits[0], nbits);
318} 438}
439static inline int bau_uvhub_weight(struct bau_target_uvhubmask *dstp)
440{
441 return bitmap_weight((unsigned long *)&dstp->bits[0],
442 UV_DISTRIBUTION_SIZE);
443}
319 444
320static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits) 445static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
321{ 446{
@@ -328,4 +453,35 @@ static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits)
328extern void uv_bau_message_intr1(void); 453extern void uv_bau_message_intr1(void);
329extern void uv_bau_timeout_intr1(void); 454extern void uv_bau_timeout_intr1(void);
330 455
456struct atomic_short {
457 short counter;
458};
459
460/**
461 * atomic_read_short - read a short atomic variable
462 * @v: pointer of type atomic_short
463 *
464 * Atomically reads the value of @v.
465 */
466static inline int atomic_read_short(const struct atomic_short *v)
467{
468 return v->counter;
469}
470
471/**
472 * atomic_add_short_return - add and return a short int
473 * @i: short value to add
474 * @v: pointer of type atomic_short
475 *
476 * Atomically adds @i to @v and returns @i + @v
477 */
478static inline int atomic_add_short_return(short i, struct atomic_short *v)
479{
480 short __i = i;
481 asm volatile(LOCK_PREFIX "xaddw %0, %1"
482 : "+r" (i), "+m" (v->counter)
483 : : "memory");
484 return i + __i;
485}
486
331#endif /* _ASM_X86_UV_UV_BAU_H */ 487#endif /* _ASM_X86_UV_UV_BAU_H */
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index 14cc74ba5d23..bf6b88ef8eeb 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -307,7 +307,7 @@ static inline unsigned long uv_read_global_mmr32(int pnode, unsigned long offset
307 * Access Global MMR space using the MMR space located at the top of physical 307 * Access Global MMR space using the MMR space located at the top of physical
308 * memory. 308 * memory.
309 */ 309 */
310static inline unsigned long *uv_global_mmr64_address(int pnode, unsigned long offset) 310static inline volatile void __iomem *uv_global_mmr64_address(int pnode, unsigned long offset)
311{ 311{
312 return __va(UV_GLOBAL_MMR64_BASE | 312 return __va(UV_GLOBAL_MMR64_BASE |
313 UV_GLOBAL_MMR64_PNODE_BITS(pnode) | offset); 313 UV_GLOBAL_MMR64_PNODE_BITS(pnode) | offset);
diff --git a/arch/x86/include/asm/uv/uv_mmrs.h b/arch/x86/include/asm/uv/uv_mmrs.h
index 2cae46c7c8a2..b2f2d2e05cec 100644
--- a/arch/x86/include/asm/uv/uv_mmrs.h
+++ b/arch/x86/include/asm/uv/uv_mmrs.h
@@ -1,4 +1,3 @@
1
2/* 1/*
3 * This file is subject to the terms and conditions of the GNU General Public 2 * This file is subject to the terms and conditions of the GNU General Public
4 * License. See the file "COPYING" in the main directory of this archive 3 * License. See the file "COPYING" in the main directory of this archive
@@ -15,13 +14,25 @@
15#define UV_MMR_ENABLE (1UL << 63) 14#define UV_MMR_ENABLE (1UL << 63)
16 15
17/* ========================================================================= */ 16/* ========================================================================= */
17/* UVH_BAU_DATA_BROADCAST */
18/* ========================================================================= */
19#define UVH_BAU_DATA_BROADCAST 0x61688UL
20#define UVH_BAU_DATA_BROADCAST_32 0x0440
21
22#define UVH_BAU_DATA_BROADCAST_ENABLE_SHFT 0
23#define UVH_BAU_DATA_BROADCAST_ENABLE_MASK 0x0000000000000001UL
24
25union uvh_bau_data_broadcast_u {
26 unsigned long v;
27 struct uvh_bau_data_broadcast_s {
28 unsigned long enable : 1; /* RW */
29 unsigned long rsvd_1_63: 63; /* */
30 } s;
31};
32
33/* ========================================================================= */
18/* UVH_BAU_DATA_CONFIG */ 34/* UVH_BAU_DATA_CONFIG */
19/* ========================================================================= */ 35/* ========================================================================= */
20#define UVH_LB_BAU_MISC_CONTROL 0x320170UL
21#define UV_ENABLE_INTD_SOFT_ACK_MODE_SHIFT 15
22#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHIFT 16
23#define UV_INTD_SOFT_ACK_TIMEOUT_PERIOD 0x000000000bUL
24/* 1011 timebase 7 (168millisec) * 3 ticks -> 500ms */
25#define UVH_BAU_DATA_CONFIG 0x61680UL 36#define UVH_BAU_DATA_CONFIG 0x61680UL
26#define UVH_BAU_DATA_CONFIG_32 0x0438 37#define UVH_BAU_DATA_CONFIG_32 0x0438
27 38
@@ -604,6 +615,68 @@ union uvh_lb_bau_intd_software_acknowledge_u {
604#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS_32 0x0a70 615#define UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS_32 0x0a70
605 616
606/* ========================================================================= */ 617/* ========================================================================= */
618/* UVH_LB_BAU_MISC_CONTROL */
619/* ========================================================================= */
620#define UVH_LB_BAU_MISC_CONTROL 0x320170UL
621#define UVH_LB_BAU_MISC_CONTROL_32 0x00a10
622
623#define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_SHFT 0
624#define UVH_LB_BAU_MISC_CONTROL_REJECTION_DELAY_MASK 0x00000000000000ffUL
625#define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_SHFT 8
626#define UVH_LB_BAU_MISC_CONTROL_APIC_MODE_MASK 0x0000000000000100UL
627#define UVH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_SHFT 9
628#define UVH_LB_BAU_MISC_CONTROL_FORCE_BROADCAST_MASK 0x0000000000000200UL
629#define UVH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_SHFT 10
630#define UVH_LB_BAU_MISC_CONTROL_FORCE_LOCK_NOP_MASK 0x0000000000000400UL
631#define UVH_LB_BAU_MISC_CONTROL_CSI_AGENT_PRESENCE_VECTOR_SHFT 11
632#define UVH_LB_BAU_MISC_CONTROL_CSI_AGENT_PRESENCE_VECTOR_MASK 0x0000000000003800UL
633#define UVH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_SHFT 14
634#define UVH_LB_BAU_MISC_CONTROL_DESCRIPTOR_FETCH_MODE_MASK 0x0000000000004000UL
635#define UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_SHFT 15
636#define UVH_LB_BAU_MISC_CONTROL_ENABLE_INTD_SOFT_ACK_MODE_MASK 0x0000000000008000UL
637#define UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_SHFT 16
638#define UVH_LB_BAU_MISC_CONTROL_INTD_SOFT_ACK_TIMEOUT_PERIOD_MASK 0x00000000000f0000UL
639#define UVH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_SHFT 20
640#define UVH_LB_BAU_MISC_CONTROL_ENABLE_DUAL_MAPPING_MODE_MASK 0x0000000000100000UL
641#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_SHFT 21
642#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_DECODE_ENABLE_MASK 0x0000000000200000UL
643#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_SHFT 22
644#define UVH_LB_BAU_MISC_CONTROL_VGA_IO_PORT_16_BIT_DECODE_MASK 0x0000000000400000UL
645#define UVH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_SHFT 23
646#define UVH_LB_BAU_MISC_CONTROL_SUPPRESS_DEST_REGISTRATION_MASK 0x0000000000800000UL
647#define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_SHFT 24
648#define UVH_LB_BAU_MISC_CONTROL_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000007000000UL
649#define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_SHFT 27
650#define UVH_LB_BAU_MISC_CONTROL_USE_INCOMING_PRIORITY_MASK 0x0000000008000000UL
651#define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_SHFT 28
652#define UVH_LB_BAU_MISC_CONTROL_ENABLE_PROGRAMMED_INITIAL_PRIORITY_MASK 0x0000000010000000UL
653#define UVH_LB_BAU_MISC_CONTROL_FUN_SHFT 48
654#define UVH_LB_BAU_MISC_CONTROL_FUN_MASK 0xffff000000000000UL
655
656union uvh_lb_bau_misc_control_u {
657 unsigned long v;
658 struct uvh_lb_bau_misc_control_s {
659 unsigned long rejection_delay : 8; /* RW */
660 unsigned long apic_mode : 1; /* RW */
661 unsigned long force_broadcast : 1; /* RW */
662 unsigned long force_lock_nop : 1; /* RW */
663 unsigned long csi_agent_presence_vector : 3; /* RW */
664 unsigned long descriptor_fetch_mode : 1; /* RW */
665 unsigned long enable_intd_soft_ack_mode : 1; /* RW */
666 unsigned long intd_soft_ack_timeout_period : 4; /* RW */
667 unsigned long enable_dual_mapping_mode : 1; /* RW */
668 unsigned long vga_io_port_decode_enable : 1; /* RW */
669 unsigned long vga_io_port_16_bit_decode : 1; /* RW */
670 unsigned long suppress_dest_registration : 1; /* RW */
671 unsigned long programmed_initial_priority : 3; /* RW */
672 unsigned long use_incoming_priority : 1; /* RW */
673 unsigned long enable_programmed_initial_priority : 1; /* RW */
674 unsigned long rsvd_29_47 : 19; /* */
675 unsigned long fun : 16; /* RW */
676 } s;
677};
678
679/* ========================================================================= */
607/* UVH_LB_BAU_SB_ACTIVATION_CONTROL */ 680/* UVH_LB_BAU_SB_ACTIVATION_CONTROL */
608/* ========================================================================= */ 681/* ========================================================================= */
609#define UVH_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL 682#define UVH_LB_BAU_SB_ACTIVATION_CONTROL 0x320020UL
@@ -681,334 +754,6 @@ union uvh_lb_bau_sb_descriptor_base_u {
681}; 754};
682 755
683/* ========================================================================= */ 756/* ========================================================================= */
684/* UVH_LB_MCAST_AOERR0_RPT_ENABLE */
685/* ========================================================================= */
686#define UVH_LB_MCAST_AOERR0_RPT_ENABLE 0x50b20UL
687
688#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_OBESE_MSG_SHFT 0
689#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_OBESE_MSG_MASK 0x0000000000000001UL
690#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_DATA_SB_ERR_SHFT 1
691#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_DATA_SB_ERR_MASK 0x0000000000000002UL
692#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_NACK_BUFF_PARITY_SHFT 2
693#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_NACK_BUFF_PARITY_MASK 0x0000000000000004UL
694#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_TIMEOUT_SHFT 3
695#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_TIMEOUT_MASK 0x0000000000000008UL
696#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_INACTIVE_REPLY_SHFT 4
697#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_INACTIVE_REPLY_MASK 0x0000000000000010UL
698#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_UPGRADE_ERROR_SHFT 5
699#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_UPGRADE_ERROR_MASK 0x0000000000000020UL
700#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_REG_COUNT_UNDERFLOW_SHFT 6
701#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_REG_COUNT_UNDERFLOW_MASK 0x0000000000000040UL
702#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_REP_OBESE_MSG_SHFT 7
703#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MCAST_REP_OBESE_MSG_MASK 0x0000000000000080UL
704#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_RUNT_MSG_SHFT 8
705#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_RUNT_MSG_MASK 0x0000000000000100UL
706#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_OBESE_MSG_SHFT 9
707#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_OBESE_MSG_MASK 0x0000000000000200UL
708#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_DATA_SB_ERR_SHFT 10
709#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REQ_DATA_SB_ERR_MASK 0x0000000000000400UL
710#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_RUNT_MSG_SHFT 11
711#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_RUNT_MSG_MASK 0x0000000000000800UL
712#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_OBESE_MSG_SHFT 12
713#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_OBESE_MSG_MASK 0x0000000000001000UL
714#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_DATA_SB_ERR_SHFT 13
715#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_DATA_SB_ERR_MASK 0x0000000000002000UL
716#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_COMMAND_ERR_SHFT 14
717#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_REP_COMMAND_ERR_MASK 0x0000000000004000UL
718#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_PEND_TIMEOUT_SHFT 15
719#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_UCACHE_PEND_TIMEOUT_MASK 0x0000000000008000UL
720#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_RUNT_MSG_SHFT 16
721#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_RUNT_MSG_MASK 0x0000000000010000UL
722#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_OBESE_MSG_SHFT 17
723#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_OBESE_MSG_MASK 0x0000000000020000UL
724#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_DATA_SB_ERR_SHFT 18
725#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REQ_DATA_SB_ERR_MASK 0x0000000000040000UL
726#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_RUNT_MSG_SHFT 19
727#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_RUNT_MSG_MASK 0x0000000000080000UL
728#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_OBESE_MSG_SHFT 20
729#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_OBESE_MSG_MASK 0x0000000000100000UL
730#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_DATA_SB_ERR_SHFT 21
731#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_REP_DATA_SB_ERR_MASK 0x0000000000200000UL
732#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_AMO_TIMEOUT_SHFT 22
733#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_AMO_TIMEOUT_MASK 0x0000000000400000UL
734#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_PUT_TIMEOUT_SHFT 23
735#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_PUT_TIMEOUT_MASK 0x0000000000800000UL
736#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_SPURIOUS_EVENT_SHFT 24
737#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_MACC_SPURIOUS_EVENT_MASK 0x0000000001000000UL
738#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_IOH_DESTINATION_TABLE_PARITY_SHFT 25
739#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_IOH_DESTINATION_TABLE_PARITY_MASK 0x0000000002000000UL
740#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_GET_HAD_ERROR_REPLY_SHFT 26
741#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_GET_HAD_ERROR_REPLY_MASK 0x0000000004000000UL
742#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_GET_TIMEOUT_SHFT 27
743#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_GET_TIMEOUT_MASK 0x0000000008000000UL
744#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_LOCK_MANAGER_HAD_ERROR_REPLY_SHFT 28
745#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_LOCK_MANAGER_HAD_ERROR_REPLY_MASK 0x0000000010000000UL
746#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_PUT_HAD_ERROR_REPLY_SHFT 29
747#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_PUT_HAD_ERROR_REPLY_MASK 0x0000000020000000UL
748#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_PUT_TIMEOUT_SHFT 30
749#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_PUT_TIMEOUT_MASK 0x0000000040000000UL
750#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_SB_ACTIVATION_OVERRUN_SHFT 31
751#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_SB_ACTIVATION_OVERRUN_MASK 0x0000000080000000UL
752#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_COMPLETED_GB_ACTIVATION_HAD_ERROR_REPLY_SHFT 32
753#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_COMPLETED_GB_ACTIVATION_HAD_ERROR_REPLY_MASK 0x0000000100000000UL
754#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_COMPLETED_GB_ACTIVATION_TIMEOUT_SHFT 33
755#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_COMPLETED_GB_ACTIVATION_TIMEOUT_MASK 0x0000000200000000UL
756#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_DESCRIPTOR_BUFFER_0_PARITY_SHFT 34
757#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_DESCRIPTOR_BUFFER_0_PARITY_MASK 0x0000000400000000UL
758#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_DESCRIPTOR_BUFFER_1_PARITY_SHFT 35
759#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_DESCRIPTOR_BUFFER_1_PARITY_MASK 0x0000000800000000UL
760#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_SOCKET_DESTINATION_TABLE_PARITY_SHFT 36
761#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_SOCKET_DESTINATION_TABLE_PARITY_MASK 0x0000001000000000UL
762#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_BAU_REPLY_PAYLOAD_CORRUPTION_SHFT 37
763#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_BAU_REPLY_PAYLOAD_CORRUPTION_MASK 0x0000002000000000UL
764#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_IO_PORT_DESTINATION_TABLE_PARITY_SHFT 38
765#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_IO_PORT_DESTINATION_TABLE_PARITY_MASK 0x0000004000000000UL
766#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INTD_SOFT_ACK_TIMEOUT_SHFT 39
767#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INTD_SOFT_ACK_TIMEOUT_MASK 0x0000008000000000UL
768#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_REP_OBESE_MSG_SHFT 40
769#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_REP_OBESE_MSG_MASK 0x0000010000000000UL
770#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_REP_COMMAND_ERR_SHFT 41
771#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_REP_COMMAND_ERR_MASK 0x0000020000000000UL
772#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_TIMEOUT_SHFT 42
773#define UVH_LB_MCAST_AOERR0_RPT_ENABLE_INT_TIMEOUT_MASK 0x0000040000000000UL
774
775union uvh_lb_mcast_aoerr0_rpt_enable_u {
776 unsigned long v;
777 struct uvh_lb_mcast_aoerr0_rpt_enable_s {
778 unsigned long mcast_obese_msg : 1; /* RW */
779 unsigned long mcast_data_sb_err : 1; /* RW */
780 unsigned long mcast_nack_buff_parity : 1; /* RW */
781 unsigned long mcast_timeout : 1; /* RW */
782 unsigned long mcast_inactive_reply : 1; /* RW */
783 unsigned long mcast_upgrade_error : 1; /* RW */
784 unsigned long mcast_reg_count_underflow : 1; /* RW */
785 unsigned long mcast_rep_obese_msg : 1; /* RW */
786 unsigned long ucache_req_runt_msg : 1; /* RW */
787 unsigned long ucache_req_obese_msg : 1; /* RW */
788 unsigned long ucache_req_data_sb_err : 1; /* RW */
789 unsigned long ucache_rep_runt_msg : 1; /* RW */
790 unsigned long ucache_rep_obese_msg : 1; /* RW */
791 unsigned long ucache_rep_data_sb_err : 1; /* RW */
792 unsigned long ucache_rep_command_err : 1; /* RW */
793 unsigned long ucache_pend_timeout : 1; /* RW */
794 unsigned long macc_req_runt_msg : 1; /* RW */
795 unsigned long macc_req_obese_msg : 1; /* RW */
796 unsigned long macc_req_data_sb_err : 1; /* RW */
797 unsigned long macc_rep_runt_msg : 1; /* RW */
798 unsigned long macc_rep_obese_msg : 1; /* RW */
799 unsigned long macc_rep_data_sb_err : 1; /* RW */
800 unsigned long macc_amo_timeout : 1; /* RW */
801 unsigned long macc_put_timeout : 1; /* RW */
802 unsigned long macc_spurious_event : 1; /* RW */
803 unsigned long ioh_destination_table_parity : 1; /* RW */
804 unsigned long get_had_error_reply : 1; /* RW */
805 unsigned long get_timeout : 1; /* RW */
806 unsigned long lock_manager_had_error_reply : 1; /* RW */
807 unsigned long put_had_error_reply : 1; /* RW */
808 unsigned long put_timeout : 1; /* RW */
809 unsigned long sb_activation_overrun : 1; /* RW */
810 unsigned long completed_gb_activation_had_error_reply : 1; /* RW */
811 unsigned long completed_gb_activation_timeout : 1; /* RW */
812 unsigned long descriptor_buffer_0_parity : 1; /* RW */
813 unsigned long descriptor_buffer_1_parity : 1; /* RW */
814 unsigned long socket_destination_table_parity : 1; /* RW */
815 unsigned long bau_reply_payload_corruption : 1; /* RW */
816 unsigned long io_port_destination_table_parity : 1; /* RW */
817 unsigned long intd_soft_ack_timeout : 1; /* RW */
818 unsigned long int_rep_obese_msg : 1; /* RW */
819 unsigned long int_rep_command_err : 1; /* RW */
820 unsigned long int_timeout : 1; /* RW */
821 unsigned long rsvd_43_63 : 21; /* */
822 } s;
823};
824
825/* ========================================================================= */
826/* UVH_LOCAL_INT0_CONFIG */
827/* ========================================================================= */
828#define UVH_LOCAL_INT0_CONFIG 0x61000UL
829
830#define UVH_LOCAL_INT0_CONFIG_VECTOR_SHFT 0
831#define UVH_LOCAL_INT0_CONFIG_VECTOR_MASK 0x00000000000000ffUL
832#define UVH_LOCAL_INT0_CONFIG_DM_SHFT 8
833#define UVH_LOCAL_INT0_CONFIG_DM_MASK 0x0000000000000700UL
834#define UVH_LOCAL_INT0_CONFIG_DESTMODE_SHFT 11
835#define UVH_LOCAL_INT0_CONFIG_DESTMODE_MASK 0x0000000000000800UL
836#define UVH_LOCAL_INT0_CONFIG_STATUS_SHFT 12
837#define UVH_LOCAL_INT0_CONFIG_STATUS_MASK 0x0000000000001000UL
838#define UVH_LOCAL_INT0_CONFIG_P_SHFT 13
839#define UVH_LOCAL_INT0_CONFIG_P_MASK 0x0000000000002000UL
840#define UVH_LOCAL_INT0_CONFIG_T_SHFT 15
841#define UVH_LOCAL_INT0_CONFIG_T_MASK 0x0000000000008000UL
842#define UVH_LOCAL_INT0_CONFIG_M_SHFT 16
843#define UVH_LOCAL_INT0_CONFIG_M_MASK 0x0000000000010000UL
844#define UVH_LOCAL_INT0_CONFIG_APIC_ID_SHFT 32
845#define UVH_LOCAL_INT0_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
846
847union uvh_local_int0_config_u {
848 unsigned long v;
849 struct uvh_local_int0_config_s {
850 unsigned long vector_ : 8; /* RW */
851 unsigned long dm : 3; /* RW */
852 unsigned long destmode : 1; /* RW */
853 unsigned long status : 1; /* RO */
854 unsigned long p : 1; /* RO */
855 unsigned long rsvd_14 : 1; /* */
856 unsigned long t : 1; /* RO */
857 unsigned long m : 1; /* RW */
858 unsigned long rsvd_17_31: 15; /* */
859 unsigned long apic_id : 32; /* RW */
860 } s;
861};
862
863/* ========================================================================= */
864/* UVH_LOCAL_INT0_ENABLE */
865/* ========================================================================= */
866#define UVH_LOCAL_INT0_ENABLE 0x65000UL
867
868#define UVH_LOCAL_INT0_ENABLE_LB_HCERR_SHFT 0
869#define UVH_LOCAL_INT0_ENABLE_LB_HCERR_MASK 0x0000000000000001UL
870#define UVH_LOCAL_INT0_ENABLE_GR0_HCERR_SHFT 1
871#define UVH_LOCAL_INT0_ENABLE_GR0_HCERR_MASK 0x0000000000000002UL
872#define UVH_LOCAL_INT0_ENABLE_GR1_HCERR_SHFT 2
873#define UVH_LOCAL_INT0_ENABLE_GR1_HCERR_MASK 0x0000000000000004UL
874#define UVH_LOCAL_INT0_ENABLE_LH_HCERR_SHFT 3
875#define UVH_LOCAL_INT0_ENABLE_LH_HCERR_MASK 0x0000000000000008UL
876#define UVH_LOCAL_INT0_ENABLE_RH_HCERR_SHFT 4
877#define UVH_LOCAL_INT0_ENABLE_RH_HCERR_MASK 0x0000000000000010UL
878#define UVH_LOCAL_INT0_ENABLE_XN_HCERR_SHFT 5
879#define UVH_LOCAL_INT0_ENABLE_XN_HCERR_MASK 0x0000000000000020UL
880#define UVH_LOCAL_INT0_ENABLE_SI_HCERR_SHFT 6
881#define UVH_LOCAL_INT0_ENABLE_SI_HCERR_MASK 0x0000000000000040UL
882#define UVH_LOCAL_INT0_ENABLE_LB_AOERR0_SHFT 7
883#define UVH_LOCAL_INT0_ENABLE_LB_AOERR0_MASK 0x0000000000000080UL
884#define UVH_LOCAL_INT0_ENABLE_GR0_AOERR0_SHFT 8
885#define UVH_LOCAL_INT0_ENABLE_GR0_AOERR0_MASK 0x0000000000000100UL
886#define UVH_LOCAL_INT0_ENABLE_GR1_AOERR0_SHFT 9
887#define UVH_LOCAL_INT0_ENABLE_GR1_AOERR0_MASK 0x0000000000000200UL
888#define UVH_LOCAL_INT0_ENABLE_LH_AOERR0_SHFT 10
889#define UVH_LOCAL_INT0_ENABLE_LH_AOERR0_MASK 0x0000000000000400UL
890#define UVH_LOCAL_INT0_ENABLE_RH_AOERR0_SHFT 11
891#define UVH_LOCAL_INT0_ENABLE_RH_AOERR0_MASK 0x0000000000000800UL
892#define UVH_LOCAL_INT0_ENABLE_XN_AOERR0_SHFT 12
893#define UVH_LOCAL_INT0_ENABLE_XN_AOERR0_MASK 0x0000000000001000UL
894#define UVH_LOCAL_INT0_ENABLE_SI_AOERR0_SHFT 13
895#define UVH_LOCAL_INT0_ENABLE_SI_AOERR0_MASK 0x0000000000002000UL
896#define UVH_LOCAL_INT0_ENABLE_LB_AOERR1_SHFT 14
897#define UVH_LOCAL_INT0_ENABLE_LB_AOERR1_MASK 0x0000000000004000UL
898#define UVH_LOCAL_INT0_ENABLE_GR0_AOERR1_SHFT 15
899#define UVH_LOCAL_INT0_ENABLE_GR0_AOERR1_MASK 0x0000000000008000UL
900#define UVH_LOCAL_INT0_ENABLE_GR1_AOERR1_SHFT 16
901#define UVH_LOCAL_INT0_ENABLE_GR1_AOERR1_MASK 0x0000000000010000UL
902#define UVH_LOCAL_INT0_ENABLE_LH_AOERR1_SHFT 17
903#define UVH_LOCAL_INT0_ENABLE_LH_AOERR1_MASK 0x0000000000020000UL
904#define UVH_LOCAL_INT0_ENABLE_RH_AOERR1_SHFT 18
905#define UVH_LOCAL_INT0_ENABLE_RH_AOERR1_MASK 0x0000000000040000UL
906#define UVH_LOCAL_INT0_ENABLE_XN_AOERR1_SHFT 19
907#define UVH_LOCAL_INT0_ENABLE_XN_AOERR1_MASK 0x0000000000080000UL
908#define UVH_LOCAL_INT0_ENABLE_SI_AOERR1_SHFT 20
909#define UVH_LOCAL_INT0_ENABLE_SI_AOERR1_MASK 0x0000000000100000UL
910#define UVH_LOCAL_INT0_ENABLE_RH_VPI_INT_SHFT 21
911#define UVH_LOCAL_INT0_ENABLE_RH_VPI_INT_MASK 0x0000000000200000UL
912#define UVH_LOCAL_INT0_ENABLE_SYSTEM_SHUTDOWN_INT_SHFT 22
913#define UVH_LOCAL_INT0_ENABLE_SYSTEM_SHUTDOWN_INT_MASK 0x0000000000400000UL
914#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_0_SHFT 23
915#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_0_MASK 0x0000000000800000UL
916#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_1_SHFT 24
917#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_1_MASK 0x0000000001000000UL
918#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_2_SHFT 25
919#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_2_MASK 0x0000000002000000UL
920#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_3_SHFT 26
921#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_3_MASK 0x0000000004000000UL
922#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_4_SHFT 27
923#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_4_MASK 0x0000000008000000UL
924#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_5_SHFT 28
925#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_5_MASK 0x0000000010000000UL
926#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_6_SHFT 29
927#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_6_MASK 0x0000000020000000UL
928#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_7_SHFT 30
929#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_7_MASK 0x0000000040000000UL
930#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_8_SHFT 31
931#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_8_MASK 0x0000000080000000UL
932#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_9_SHFT 32
933#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_9_MASK 0x0000000100000000UL
934#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_10_SHFT 33
935#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_10_MASK 0x0000000200000000UL
936#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_11_SHFT 34
937#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_11_MASK 0x0000000400000000UL
938#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_12_SHFT 35
939#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_12_MASK 0x0000000800000000UL
940#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_13_SHFT 36
941#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_13_MASK 0x0000001000000000UL
942#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_14_SHFT 37
943#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_14_MASK 0x0000002000000000UL
944#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_15_SHFT 38
945#define UVH_LOCAL_INT0_ENABLE_LB_IRQ_INT_15_MASK 0x0000004000000000UL
946#define UVH_LOCAL_INT0_ENABLE_L1_NMI_INT_SHFT 39
947#define UVH_LOCAL_INT0_ENABLE_L1_NMI_INT_MASK 0x0000008000000000UL
948#define UVH_LOCAL_INT0_ENABLE_STOP_CLOCK_SHFT 40
949#define UVH_LOCAL_INT0_ENABLE_STOP_CLOCK_MASK 0x0000010000000000UL
950#define UVH_LOCAL_INT0_ENABLE_ASIC_TO_L1_SHFT 41
951#define UVH_LOCAL_INT0_ENABLE_ASIC_TO_L1_MASK 0x0000020000000000UL
952#define UVH_LOCAL_INT0_ENABLE_L1_TO_ASIC_SHFT 42
953#define UVH_LOCAL_INT0_ENABLE_L1_TO_ASIC_MASK 0x0000040000000000UL
954#define UVH_LOCAL_INT0_ENABLE_LTC_INT_SHFT 43
955#define UVH_LOCAL_INT0_ENABLE_LTC_INT_MASK 0x0000080000000000UL
956#define UVH_LOCAL_INT0_ENABLE_LA_SEQ_TRIGGER_SHFT 44
957#define UVH_LOCAL_INT0_ENABLE_LA_SEQ_TRIGGER_MASK 0x0000100000000000UL
958
959union uvh_local_int0_enable_u {
960 unsigned long v;
961 struct uvh_local_int0_enable_s {
962 unsigned long lb_hcerr : 1; /* RW */
963 unsigned long gr0_hcerr : 1; /* RW */
964 unsigned long gr1_hcerr : 1; /* RW */
965 unsigned long lh_hcerr : 1; /* RW */
966 unsigned long rh_hcerr : 1; /* RW */
967 unsigned long xn_hcerr : 1; /* RW */
968 unsigned long si_hcerr : 1; /* RW */
969 unsigned long lb_aoerr0 : 1; /* RW */
970 unsigned long gr0_aoerr0 : 1; /* RW */
971 unsigned long gr1_aoerr0 : 1; /* RW */
972 unsigned long lh_aoerr0 : 1; /* RW */
973 unsigned long rh_aoerr0 : 1; /* RW */
974 unsigned long xn_aoerr0 : 1; /* RW */
975 unsigned long si_aoerr0 : 1; /* RW */
976 unsigned long lb_aoerr1 : 1; /* RW */
977 unsigned long gr0_aoerr1 : 1; /* RW */
978 unsigned long gr1_aoerr1 : 1; /* RW */
979 unsigned long lh_aoerr1 : 1; /* RW */
980 unsigned long rh_aoerr1 : 1; /* RW */
981 unsigned long xn_aoerr1 : 1; /* RW */
982 unsigned long si_aoerr1 : 1; /* RW */
983 unsigned long rh_vpi_int : 1; /* RW */
984 unsigned long system_shutdown_int : 1; /* RW */
985 unsigned long lb_irq_int_0 : 1; /* RW */
986 unsigned long lb_irq_int_1 : 1; /* RW */
987 unsigned long lb_irq_int_2 : 1; /* RW */
988 unsigned long lb_irq_int_3 : 1; /* RW */
989 unsigned long lb_irq_int_4 : 1; /* RW */
990 unsigned long lb_irq_int_5 : 1; /* RW */
991 unsigned long lb_irq_int_6 : 1; /* RW */
992 unsigned long lb_irq_int_7 : 1; /* RW */
993 unsigned long lb_irq_int_8 : 1; /* RW */
994 unsigned long lb_irq_int_9 : 1; /* RW */
995 unsigned long lb_irq_int_10 : 1; /* RW */
996 unsigned long lb_irq_int_11 : 1; /* RW */
997 unsigned long lb_irq_int_12 : 1; /* RW */
998 unsigned long lb_irq_int_13 : 1; /* RW */
999 unsigned long lb_irq_int_14 : 1; /* RW */
1000 unsigned long lb_irq_int_15 : 1; /* RW */
1001 unsigned long l1_nmi_int : 1; /* RW */
1002 unsigned long stop_clock : 1; /* RW */
1003 unsigned long asic_to_l1 : 1; /* RW */
1004 unsigned long l1_to_asic : 1; /* RW */
1005 unsigned long ltc_int : 1; /* RW */
1006 unsigned long la_seq_trigger : 1; /* RW */
1007 unsigned long rsvd_45_63 : 19; /* */
1008 } s;
1009};
1010
1011/* ========================================================================= */
1012/* UVH_NODE_ID */ 757/* UVH_NODE_ID */
1013/* ========================================================================= */ 758/* ========================================================================= */
1014#define UVH_NODE_ID 0x0UL 759#define UVH_NODE_ID 0x0UL
@@ -1112,26 +857,6 @@ union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
1112}; 857};
1113 858
1114/* ========================================================================= */ 859/* ========================================================================= */
1115/* UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR */
1116/* ========================================================================= */
1117#define UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR 0x1600020UL
1118
1119#define UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR_BASE_SHFT 26
1120#define UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR_BASE_MASK 0x00003ffffc000000UL
1121#define UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR_ENABLE_SHFT 63
1122#define UVH_RH_GAM_CFG_OVERLAY_CONFIG_MMR_ENABLE_MASK 0x8000000000000000UL
1123
1124union uvh_rh_gam_cfg_overlay_config_mmr_u {
1125 unsigned long v;
1126 struct uvh_rh_gam_cfg_overlay_config_mmr_s {
1127 unsigned long rsvd_0_25: 26; /* */
1128 unsigned long base : 20; /* RW */
1129 unsigned long rsvd_46_62: 17; /* */
1130 unsigned long enable : 1; /* RW */
1131 } s;
1132};
1133
1134/* ========================================================================= */
1135/* UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR */ 860/* UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR */
1136/* ========================================================================= */ 861/* ========================================================================= */
1137#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL 862#define UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR 0x1600010UL
@@ -1263,101 +988,6 @@ union uvh_rtc1_int_config_u {
1263}; 988};
1264 989
1265/* ========================================================================= */ 990/* ========================================================================= */
1266/* UVH_RTC2_INT_CONFIG */
1267/* ========================================================================= */
1268#define UVH_RTC2_INT_CONFIG 0x61600UL
1269
1270#define UVH_RTC2_INT_CONFIG_VECTOR_SHFT 0
1271#define UVH_RTC2_INT_CONFIG_VECTOR_MASK 0x00000000000000ffUL
1272#define UVH_RTC2_INT_CONFIG_DM_SHFT 8
1273#define UVH_RTC2_INT_CONFIG_DM_MASK 0x0000000000000700UL
1274#define UVH_RTC2_INT_CONFIG_DESTMODE_SHFT 11
1275#define UVH_RTC2_INT_CONFIG_DESTMODE_MASK 0x0000000000000800UL
1276#define UVH_RTC2_INT_CONFIG_STATUS_SHFT 12
1277#define UVH_RTC2_INT_CONFIG_STATUS_MASK 0x0000000000001000UL
1278#define UVH_RTC2_INT_CONFIG_P_SHFT 13
1279#define UVH_RTC2_INT_CONFIG_P_MASK 0x0000000000002000UL
1280#define UVH_RTC2_INT_CONFIG_T_SHFT 15
1281#define UVH_RTC2_INT_CONFIG_T_MASK 0x0000000000008000UL
1282#define UVH_RTC2_INT_CONFIG_M_SHFT 16
1283#define UVH_RTC2_INT_CONFIG_M_MASK 0x0000000000010000UL
1284#define UVH_RTC2_INT_CONFIG_APIC_ID_SHFT 32
1285#define UVH_RTC2_INT_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
1286
1287union uvh_rtc2_int_config_u {
1288 unsigned long v;
1289 struct uvh_rtc2_int_config_s {
1290 unsigned long vector_ : 8; /* RW */
1291 unsigned long dm : 3; /* RW */
1292 unsigned long destmode : 1; /* RW */
1293 unsigned long status : 1; /* RO */
1294 unsigned long p : 1; /* RO */
1295 unsigned long rsvd_14 : 1; /* */
1296 unsigned long t : 1; /* RO */
1297 unsigned long m : 1; /* RW */
1298 unsigned long rsvd_17_31: 15; /* */
1299 unsigned long apic_id : 32; /* RW */
1300 } s;
1301};
1302
1303/* ========================================================================= */
1304/* UVH_RTC3_INT_CONFIG */
1305/* ========================================================================= */
1306#define UVH_RTC3_INT_CONFIG 0x61640UL
1307
1308#define UVH_RTC3_INT_CONFIG_VECTOR_SHFT 0
1309#define UVH_RTC3_INT_CONFIG_VECTOR_MASK 0x00000000000000ffUL
1310#define UVH_RTC3_INT_CONFIG_DM_SHFT 8
1311#define UVH_RTC3_INT_CONFIG_DM_MASK 0x0000000000000700UL
1312#define UVH_RTC3_INT_CONFIG_DESTMODE_SHFT 11
1313#define UVH_RTC3_INT_CONFIG_DESTMODE_MASK 0x0000000000000800UL
1314#define UVH_RTC3_INT_CONFIG_STATUS_SHFT 12
1315#define UVH_RTC3_INT_CONFIG_STATUS_MASK 0x0000000000001000UL
1316#define UVH_RTC3_INT_CONFIG_P_SHFT 13
1317#define UVH_RTC3_INT_CONFIG_P_MASK 0x0000000000002000UL
1318#define UVH_RTC3_INT_CONFIG_T_SHFT 15
1319#define UVH_RTC3_INT_CONFIG_T_MASK 0x0000000000008000UL
1320#define UVH_RTC3_INT_CONFIG_M_SHFT 16
1321#define UVH_RTC3_INT_CONFIG_M_MASK 0x0000000000010000UL
1322#define UVH_RTC3_INT_CONFIG_APIC_ID_SHFT 32
1323#define UVH_RTC3_INT_CONFIG_APIC_ID_MASK 0xffffffff00000000UL
1324
1325union uvh_rtc3_int_config_u {
1326 unsigned long v;
1327 struct uvh_rtc3_int_config_s {
1328 unsigned long vector_ : 8; /* RW */
1329 unsigned long dm : 3; /* RW */
1330 unsigned long destmode : 1; /* RW */
1331 unsigned long status : 1; /* RO */
1332 unsigned long p : 1; /* RO */
1333 unsigned long rsvd_14 : 1; /* */
1334 unsigned long t : 1; /* RO */
1335 unsigned long m : 1; /* RW */
1336 unsigned long rsvd_17_31: 15; /* */
1337 unsigned long apic_id : 32; /* RW */
1338 } s;
1339};
1340
1341/* ========================================================================= */
1342/* UVH_RTC_INC_RATIO */
1343/* ========================================================================= */
1344#define UVH_RTC_INC_RATIO 0x350000UL
1345
1346#define UVH_RTC_INC_RATIO_FRACTION_SHFT 0
1347#define UVH_RTC_INC_RATIO_FRACTION_MASK 0x00000000000fffffUL
1348#define UVH_RTC_INC_RATIO_RATIO_SHFT 20
1349#define UVH_RTC_INC_RATIO_RATIO_MASK 0x0000000000700000UL
1350
1351union uvh_rtc_inc_ratio_u {
1352 unsigned long v;
1353 struct uvh_rtc_inc_ratio_s {
1354 unsigned long fraction : 20; /* RW */
1355 unsigned long ratio : 3; /* RW */
1356 unsigned long rsvd_23_63: 41; /* */
1357 } s;
1358};
1359
1360/* ========================================================================= */
1361/* UVH_SI_ADDR_MAP_CONFIG */ 991/* UVH_SI_ADDR_MAP_CONFIG */
1362/* ========================================================================= */ 992/* ========================================================================= */
1363#define UVH_SI_ADDR_MAP_CONFIG 0xc80000UL 993#define UVH_SI_ADDR_MAP_CONFIG 0xc80000UL
diff --git a/arch/x86/include/asm/vmware.h b/arch/x86/include/asm/vmware.h
deleted file mode 100644
index e49ed6d2fd4e..000000000000
--- a/arch/x86/include/asm/vmware.h
+++ /dev/null
@@ -1,27 +0,0 @@
1/*
2 * Copyright (C) 2008, VMware, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
12 * NON INFRINGEMENT. See the GNU General Public License for more
13 * details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 */
20#ifndef ASM_X86__VMWARE_H
21#define ASM_X86__VMWARE_H
22
23extern void vmware_platform_setup(void);
24extern int vmware_platform(void);
25extern void vmware_set_feature_bits(struct cpuinfo_x86 *c);
26
27#endif
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index fb9a080740ec..9f0cbd987d50 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -25,6 +25,8 @@
25 * 25 *
26 */ 26 */
27 27
28#include <linux/types.h>
29
28/* 30/*
29 * Definitions of Primary Processor-Based VM-Execution Controls. 31 * Definitions of Primary Processor-Based VM-Execution Controls.
30 */ 32 */
@@ -120,6 +122,8 @@ enum vmcs_field {
120 GUEST_IA32_DEBUGCTL_HIGH = 0x00002803, 122 GUEST_IA32_DEBUGCTL_HIGH = 0x00002803,
121 GUEST_IA32_PAT = 0x00002804, 123 GUEST_IA32_PAT = 0x00002804,
122 GUEST_IA32_PAT_HIGH = 0x00002805, 124 GUEST_IA32_PAT_HIGH = 0x00002805,
125 GUEST_IA32_EFER = 0x00002806,
126 GUEST_IA32_EFER_HIGH = 0x00002807,
123 GUEST_PDPTR0 = 0x0000280a, 127 GUEST_PDPTR0 = 0x0000280a,
124 GUEST_PDPTR0_HIGH = 0x0000280b, 128 GUEST_PDPTR0_HIGH = 0x0000280b,
125 GUEST_PDPTR1 = 0x0000280c, 129 GUEST_PDPTR1 = 0x0000280c,
@@ -130,6 +134,8 @@ enum vmcs_field {
130 GUEST_PDPTR3_HIGH = 0x00002811, 134 GUEST_PDPTR3_HIGH = 0x00002811,
131 HOST_IA32_PAT = 0x00002c00, 135 HOST_IA32_PAT = 0x00002c00,
132 HOST_IA32_PAT_HIGH = 0x00002c01, 136 HOST_IA32_PAT_HIGH = 0x00002c01,
137 HOST_IA32_EFER = 0x00002c02,
138 HOST_IA32_EFER_HIGH = 0x00002c03,
133 PIN_BASED_VM_EXEC_CONTROL = 0x00004000, 139 PIN_BASED_VM_EXEC_CONTROL = 0x00004000,
134 CPU_BASED_VM_EXEC_CONTROL = 0x00004002, 140 CPU_BASED_VM_EXEC_CONTROL = 0x00004002,
135 EXCEPTION_BITMAP = 0x00004004, 141 EXCEPTION_BITMAP = 0x00004004,
@@ -251,6 +257,7 @@ enum vmcs_field {
251#define EXIT_REASON_IO_INSTRUCTION 30 257#define EXIT_REASON_IO_INSTRUCTION 30
252#define EXIT_REASON_MSR_READ 31 258#define EXIT_REASON_MSR_READ 31
253#define EXIT_REASON_MSR_WRITE 32 259#define EXIT_REASON_MSR_WRITE 32
260#define EXIT_REASON_INVALID_STATE 33
254#define EXIT_REASON_MWAIT_INSTRUCTION 36 261#define EXIT_REASON_MWAIT_INSTRUCTION 36
255#define EXIT_REASON_MONITOR_INSTRUCTION 39 262#define EXIT_REASON_MONITOR_INSTRUCTION 39
256#define EXIT_REASON_PAUSE_INSTRUCTION 40 263#define EXIT_REASON_PAUSE_INSTRUCTION 40
@@ -260,6 +267,7 @@ enum vmcs_field {
260#define EXIT_REASON_EPT_VIOLATION 48 267#define EXIT_REASON_EPT_VIOLATION 48
261#define EXIT_REASON_EPT_MISCONFIG 49 268#define EXIT_REASON_EPT_MISCONFIG 49
262#define EXIT_REASON_WBINVD 54 269#define EXIT_REASON_WBINVD 54
270#define EXIT_REASON_XSETBV 55
263 271
264/* 272/*
265 * Interruption-information format 273 * Interruption-information format
@@ -369,6 +377,9 @@ enum vmcs_field {
369#define VMX_EPT_EXTENT_CONTEXT_BIT (1ull << 25) 377#define VMX_EPT_EXTENT_CONTEXT_BIT (1ull << 25)
370#define VMX_EPT_EXTENT_GLOBAL_BIT (1ull << 26) 378#define VMX_EPT_EXTENT_GLOBAL_BIT (1ull << 26)
371 379
380#define VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT (1ull << 9) /* (41 - 32) */
381#define VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT (1ull << 10) /* (42 - 32) */
382
372#define VMX_EPT_DEFAULT_GAW 3 383#define VMX_EPT_DEFAULT_GAW 3
373#define VMX_EPT_MAX_GAW 0x4 384#define VMX_EPT_MAX_GAW 0x4
374#define VMX_EPT_MT_EPTE_SHIFT 3 385#define VMX_EPT_MT_EPTE_SHIFT 3
@@ -394,6 +405,10 @@ enum vmcs_field {
394#define ASM_VMX_INVEPT ".byte 0x66, 0x0f, 0x38, 0x80, 0x08" 405#define ASM_VMX_INVEPT ".byte 0x66, 0x0f, 0x38, 0x80, 0x08"
395#define ASM_VMX_INVVPID ".byte 0x66, 0x0f, 0x38, 0x81, 0x08" 406#define ASM_VMX_INVVPID ".byte 0x66, 0x0f, 0x38, 0x81, 0x08"
396 407
397 408struct vmx_msr_entry {
409 u32 index;
410 u32 reserved;
411 u64 value;
412} __aligned(16);
398 413
399#endif 414#endif
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 519b54327d75..baa579c8e038 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -142,6 +142,7 @@ struct x86_cpuinit_ops {
142 * @set_wallclock: set time back to HW clock 142 * @set_wallclock: set time back to HW clock
143 * @is_untracked_pat_range exclude from PAT logic 143 * @is_untracked_pat_range exclude from PAT logic
144 * @nmi_init enable NMI on cpus 144 * @nmi_init enable NMI on cpus
145 * @i8042_detect pre-detect if i8042 controller exists
145 */ 146 */
146struct x86_platform_ops { 147struct x86_platform_ops {
147 unsigned long (*calibrate_tsc)(void); 148 unsigned long (*calibrate_tsc)(void);
@@ -150,6 +151,7 @@ struct x86_platform_ops {
150 void (*iommu_shutdown)(void); 151 void (*iommu_shutdown)(void);
151 bool (*is_untracked_pat_range)(u64 start, u64 end); 152 bool (*is_untracked_pat_range)(u64 start, u64 end);
152 void (*nmi_init)(void); 153 void (*nmi_init)(void);
154 int (*i8042_detect)(void);
153}; 155};
154 156
155extern struct x86_init_ops x86_init; 157extern struct x86_init_ops x86_init;
diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index 9c371e4a9fa6..7fda040a76cd 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -417,6 +417,12 @@ HYPERVISOR_nmi_op(unsigned long op, unsigned long arg)
417 return _hypercall2(int, nmi_op, op, arg); 417 return _hypercall2(int, nmi_op, op, arg);
418} 418}
419 419
420static inline unsigned long __must_check
421HYPERVISOR_hvm_op(int op, void *arg)
422{
423 return _hypercall2(unsigned long, hvm_op, op, arg);
424}
425
420static inline void 426static inline void
421MULTI_fpu_taskswitch(struct multicall_entry *mcl, int set) 427MULTI_fpu_taskswitch(struct multicall_entry *mcl, int set)
422{ 428{
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 018a0a400799..bf5f7d32bd08 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -112,13 +112,9 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine)
112 */ 112 */
113static inline unsigned long mfn_to_local_pfn(unsigned long mfn) 113static inline unsigned long mfn_to_local_pfn(unsigned long mfn)
114{ 114{
115 extern unsigned long max_mapnr;
116 unsigned long pfn = mfn_to_pfn(mfn); 115 unsigned long pfn = mfn_to_pfn(mfn);
117 if ((pfn < max_mapnr) 116 if (get_phys_to_machine(pfn) != mfn)
118 && !xen_feature(XENFEAT_auto_translated_physmap) 117 return -1; /* force !pfn_valid() */
119 && (get_phys_to_machine(pfn) != mfn))
120 return max_mapnr; /* force !pfn_valid() */
121 /* XXX fixme; not true with sparsemem */
122 return pfn; 118 return pfn;
123} 119}
124 120
diff --git a/arch/x86/include/asm/xen/swiotlb-xen.h b/arch/x86/include/asm/xen/swiotlb-xen.h
new file mode 100644
index 000000000000..1be1ab7d6a41
--- /dev/null
+++ b/arch/x86/include/asm/xen/swiotlb-xen.h
@@ -0,0 +1,14 @@
1#ifndef _ASM_X86_SWIOTLB_XEN_H
2#define _ASM_X86_SWIOTLB_XEN_H
3
4#ifdef CONFIG_SWIOTLB_XEN
5extern int xen_swiotlb;
6extern int __init pci_xen_swiotlb_detect(void);
7extern void __init pci_xen_swiotlb_init(void);
8#else
9#define xen_swiotlb (0)
10static inline int __init pci_xen_swiotlb_detect(void) { return 0; }
11static inline void __init pci_xen_swiotlb_init(void) { }
12#endif
13
14#endif /* _ASM_X86_SWIOTLB_XEN_H */
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index ddc04ccad03b..c6ce2452f10c 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -3,7 +3,8 @@
3 3
4#include <linux/types.h> 4#include <linux/types.h>
5#include <asm/processor.h> 5#include <asm/processor.h>
6#include <asm/i387.h> 6
7#define XSTATE_CPUID 0x0000000d
7 8
8#define XSTATE_FP 0x1 9#define XSTATE_FP 0x1
9#define XSTATE_SSE 0x2 10#define XSTATE_SSE 0x2
@@ -13,6 +14,12 @@
13 14
14#define FXSAVE_SIZE 512 15#define FXSAVE_SIZE 512
15 16
17#define XSAVE_HDR_SIZE 64
18#define XSAVE_HDR_OFFSET FXSAVE_SIZE
19
20#define XSAVE_YMM_SIZE 256
21#define XSAVE_YMM_OFFSET (XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET)
22
16/* 23/*
17 * These are the features that the OS can handle currently. 24 * These are the features that the OS can handle currently.
18 */ 25 */
@@ -26,10 +33,8 @@
26 33
27extern unsigned int xstate_size; 34extern unsigned int xstate_size;
28extern u64 pcntxt_mask; 35extern u64 pcntxt_mask;
29extern struct xsave_struct *init_xstate_buf;
30extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; 36extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
31 37
32extern void xsave_cntxt_init(void);
33extern void xsave_init(void); 38extern void xsave_init(void);
34extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); 39extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
35extern int init_fpu(struct task_struct *child); 40extern int init_fpu(struct task_struct *child);
@@ -37,8 +42,9 @@ extern int check_for_xstate(struct i387_fxsave_struct __user *buf,
37 void __user *fpstate, 42 void __user *fpstate,
38 struct _fpx_sw_bytes *sw); 43 struct _fpx_sw_bytes *sw);
39 44
40static inline int xrstor_checking(struct xsave_struct *fx) 45static inline int fpu_xrstor_checking(struct fpu *fpu)
41{ 46{
47 struct xsave_struct *fx = &fpu->state->xsave;
42 int err; 48 int err;
43 49
44 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" 50 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
@@ -58,6 +64,16 @@ static inline int xrstor_checking(struct xsave_struct *fx)
58static inline int xsave_user(struct xsave_struct __user *buf) 64static inline int xsave_user(struct xsave_struct __user *buf)
59{ 65{
60 int err; 66 int err;
67
68 /*
69 * Clear the xsave header first, so that reserved fields are
70 * initialized to zero.
71 */
72 err = __clear_user(&buf->xsave_hdr,
73 sizeof(struct xsave_hdr_struct));
74 if (unlikely(err))
75 return -EFAULT;
76
61 __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" 77 __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x27\n"
62 "2:\n" 78 "2:\n"
63 ".section .fixup,\"ax\"\n" 79 ".section .fixup,\"ax\"\n"
@@ -110,12 +126,25 @@ static inline void xrstor_state(struct xsave_struct *fx, u64 mask)
110 : "memory"); 126 : "memory");
111} 127}
112 128
113static inline void xsave(struct task_struct *tsk) 129static inline void xsave_state(struct xsave_struct *fx, u64 mask)
130{
131 u32 lmask = mask;
132 u32 hmask = mask >> 32;
133
134 asm volatile(".byte " REX_PREFIX "0x0f,0xae,0x27\n\t"
135 : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask)
136 : "memory");
137}
138
139static inline void fpu_xsave(struct fpu *fpu)
114{ 140{
115 /* This, however, we can work around by forcing the compiler to select 141 /* This, however, we can work around by forcing the compiler to select
116 an addressing mode that doesn't require extended registers. */ 142 an addressing mode that doesn't require extended registers. */
117 __asm__ __volatile__(".byte " REX_PREFIX "0x0f,0xae,0x27" 143 alternative_input(
118 : : "D" (&(tsk->thread.xstate->xsave)), 144 ".byte " REX_PREFIX "0x0f,0xae,0x27",
119 "a" (-1), "d"(-1) : "memory"); 145 ".byte " REX_PREFIX "0x0f,0xae,0x37",
146 X86_FEATURE_XSAVEOPT,
147 [fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) :
148 "memory");
120} 149}
121#endif 150#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-15 04:52:57 -0500