aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/amd_nb.c2
-rw-r--r--arch/x86/kernel/aperture_64.c20
-rw-r--r--arch/x86/kernel/cpu/amd.c5
-rw-r--r--arch/x86/kernel/cpu/centaur.c272
-rw-r--r--arch/x86/kernel/cpu/common.c7
-rw-r--r--arch/x86/kernel/cpu/intel.c10
-rw-r--r--arch/x86/kernel/cpu/microcode/amd_early.c43
-rw-r--r--arch/x86/kernel/cpu/mtrr/generic.c4
-rw-r--r--arch/x86/kernel/cpu/perf_event.c11
-rw-r--r--arch/x86/kernel/cpu/perf_event.h1
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c11
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.c13
-rw-r--r--arch/x86/kernel/cpu/perf_event_p6.c48
-rw-r--r--arch/x86/kernel/ftrace.c83
-rw-r--r--arch/x86/kernel/head_32.S7
-rw-r--r--arch/x86/kernel/head_64.S6
-rw-r--r--arch/x86/kernel/i387.c15
-rw-r--r--arch/x86/kernel/irq.c9
-rw-r--r--arch/x86/kernel/machine_kexec_64.c2
-rw-r--r--arch/x86/kernel/pci-dma.c4
-rw-r--r--arch/x86/kernel/quirks.c39
-rw-r--r--arch/x86/kernel/setup.c10
-rw-r--r--arch/x86/kernel/tsc.c11
-rw-r--r--arch/x86/kernel/tsc_msr.c30
24 files changed, 242 insertions, 421 deletions
diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c
index 59554dca96ec..dec8de4e1663 100644
--- a/arch/x86/kernel/amd_nb.c
+++ b/arch/x86/kernel/amd_nb.c
@@ -179,7 +179,7 @@ int amd_get_subcaches(int cpu)
179 return (mask >> (4 * cuid)) & 0xf; 179 return (mask >> (4 * cuid)) & 0xf;
180} 180}
181 181
182int amd_set_subcaches(int cpu, int mask) 182int amd_set_subcaches(int cpu, unsigned long mask)
183{ 183{
184 static unsigned int reset, ban; 184 static unsigned int reset, ban;
185 struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu)); 185 struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu));
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index fd972a3e4cbb..9fa8aa051f54 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -18,7 +18,6 @@
18#include <linux/pci_ids.h> 18#include <linux/pci_ids.h>
19#include <linux/pci.h> 19#include <linux/pci.h>
20#include <linux/bitops.h> 20#include <linux/bitops.h>
21#include <linux/ioport.h>
22#include <linux/suspend.h> 21#include <linux/suspend.h>
23#include <asm/e820.h> 22#include <asm/e820.h>
24#include <asm/io.h> 23#include <asm/io.h>
@@ -54,18 +53,6 @@ int fallback_aper_force __initdata;
54 53
55int fix_aperture __initdata = 1; 54int fix_aperture __initdata = 1;
56 55
57static struct resource gart_resource = {
58 .name = "GART",
59 .flags = IORESOURCE_MEM,
60};
61
62static void __init insert_aperture_resource(u32 aper_base, u32 aper_size)
63{
64 gart_resource.start = aper_base;
65 gart_resource.end = aper_base + aper_size - 1;
66 insert_resource(&iomem_resource, &gart_resource);
67}
68
69/* This code runs before the PCI subsystem is initialized, so just 56/* This code runs before the PCI subsystem is initialized, so just
70 access the northbridge directly. */ 57 access the northbridge directly. */
71 58
@@ -96,7 +83,6 @@ static u32 __init allocate_aperture(void)
96 memblock_reserve(addr, aper_size); 83 memblock_reserve(addr, aper_size);
97 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n", 84 printk(KERN_INFO "Mapping aperture over %d KB of RAM @ %lx\n",
98 aper_size >> 10, addr); 85 aper_size >> 10, addr);
99 insert_aperture_resource((u32)addr, aper_size);
100 register_nosave_region(addr >> PAGE_SHIFT, 86 register_nosave_region(addr >> PAGE_SHIFT,
101 (addr+aper_size) >> PAGE_SHIFT); 87 (addr+aper_size) >> PAGE_SHIFT);
102 88
@@ -444,12 +430,8 @@ int __init gart_iommu_hole_init(void)
444 430
445out: 431out:
446 if (!fix && !fallback_aper_force) { 432 if (!fix && !fallback_aper_force) {
447 if (last_aper_base) { 433 if (last_aper_base)
448 unsigned long n = (32 * 1024 * 1024) << last_aper_order;
449
450 insert_aperture_resource((u32)last_aper_base, n);
451 return 1; 434 return 1;
452 }
453 return 0; 435 return 0;
454 } 436 }
455 437
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index d3153e281d72..c67ffa686064 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -767,10 +767,7 @@ static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
767 767
768static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) 768static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
769{ 769{
770 tlb_flushall_shift = 5; 770 tlb_flushall_shift = 6;
771
772 if (c->x86 <= 0x11)
773 tlb_flushall_shift = 4;
774} 771}
775 772
776static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) 773static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index 8779edab684e..d8fba5c15fbd 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -8,236 +8,6 @@
8 8
9#include "cpu.h" 9#include "cpu.h"
10 10
11#ifdef CONFIG_X86_OOSTORE
12
13static u32 power2(u32 x)
14{
15 u32 s = 1;
16
17 while (s <= x)
18 s <<= 1;
19
20 return s >>= 1;
21}
22
23
24/*
25 * Set up an actual MCR
26 */
27static void centaur_mcr_insert(int reg, u32 base, u32 size, int key)
28{
29 u32 lo, hi;
30
31 hi = base & ~0xFFF;
32 lo = ~(size-1); /* Size is a power of 2 so this makes a mask */
33 lo &= ~0xFFF; /* Remove the ctrl value bits */
34 lo |= key; /* Attribute we wish to set */
35 wrmsr(reg+MSR_IDT_MCR0, lo, hi);
36 mtrr_centaur_report_mcr(reg, lo, hi); /* Tell the mtrr driver */
37}
38
39/*
40 * Figure what we can cover with MCR's
41 *
42 * Shortcut: We know you can't put 4Gig of RAM on a winchip
43 */
44static u32 ramtop(void)
45{
46 u32 clip = 0xFFFFFFFFUL;
47 u32 top = 0;
48 int i;
49
50 for (i = 0; i < e820.nr_map; i++) {
51 unsigned long start, end;
52
53 if (e820.map[i].addr > 0xFFFFFFFFUL)
54 continue;
55 /*
56 * Don't MCR over reserved space. Ignore the ISA hole
57 * we frob around that catastrophe already
58 */
59 if (e820.map[i].type == E820_RESERVED) {
60 if (e820.map[i].addr >= 0x100000UL &&
61 e820.map[i].addr < clip)
62 clip = e820.map[i].addr;
63 continue;
64 }
65 start = e820.map[i].addr;
66 end = e820.map[i].addr + e820.map[i].size;
67 if (start >= end)
68 continue;
69 if (end > top)
70 top = end;
71 }
72 /*
73 * Everything below 'top' should be RAM except for the ISA hole.
74 * Because of the limited MCR's we want to map NV/ACPI into our
75 * MCR range for gunk in RAM
76 *
77 * Clip might cause us to MCR insufficient RAM but that is an
78 * acceptable failure mode and should only bite obscure boxes with
79 * a VESA hole at 15Mb
80 *
81 * The second case Clip sometimes kicks in is when the EBDA is marked
82 * as reserved. Again we fail safe with reasonable results
83 */
84 if (top > clip)
85 top = clip;
86
87 return top;
88}
89
90/*
91 * Compute a set of MCR's to give maximum coverage
92 */
93static int centaur_mcr_compute(int nr, int key)
94{
95 u32 mem = ramtop();
96 u32 root = power2(mem);
97 u32 base = root;
98 u32 top = root;
99 u32 floor = 0;
100 int ct = 0;
101
102 while (ct < nr) {
103 u32 fspace = 0;
104 u32 high;
105 u32 low;
106
107 /*
108 * Find the largest block we will fill going upwards
109 */
110 high = power2(mem-top);
111
112 /*
113 * Find the largest block we will fill going downwards
114 */
115 low = base/2;
116
117 /*
118 * Don't fill below 1Mb going downwards as there
119 * is an ISA hole in the way.
120 */
121 if (base <= 1024*1024)
122 low = 0;
123
124 /*
125 * See how much space we could cover by filling below
126 * the ISA hole
127 */
128
129 if (floor == 0)
130 fspace = 512*1024;
131 else if (floor == 512*1024)
132 fspace = 128*1024;
133
134 /* And forget ROM space */
135
136 /*
137 * Now install the largest coverage we get
138 */
139 if (fspace > high && fspace > low) {
140 centaur_mcr_insert(ct, floor, fspace, key);
141 floor += fspace;
142 } else if (high > low) {
143 centaur_mcr_insert(ct, top, high, key);
144 top += high;
145 } else if (low > 0) {
146 base -= low;
147 centaur_mcr_insert(ct, base, low, key);
148 } else
149 break;
150 ct++;
151 }
152 /*
153 * We loaded ct values. We now need to set the mask. The caller
154 * must do this bit.
155 */
156 return ct;
157}
158
159static void centaur_create_optimal_mcr(void)
160{
161 int used;
162 int i;
163
164 /*
165 * Allocate up to 6 mcrs to mark as much of ram as possible
166 * as write combining and weak write ordered.
167 *
168 * To experiment with: Linux never uses stack operations for
169 * mmio spaces so we could globally enable stack operation wc
170 *
171 * Load the registers with type 31 - full write combining, all
172 * writes weakly ordered.
173 */
174 used = centaur_mcr_compute(6, 31);
175
176 /*
177 * Wipe unused MCRs
178 */
179 for (i = used; i < 8; i++)
180 wrmsr(MSR_IDT_MCR0+i, 0, 0);
181}
182
183static void winchip2_create_optimal_mcr(void)
184{
185 u32 lo, hi;
186 int used;
187 int i;
188
189 /*
190 * Allocate up to 6 mcrs to mark as much of ram as possible
191 * as write combining, weak store ordered.
192 *
193 * Load the registers with type 25
194 * 8 - weak write ordering
195 * 16 - weak read ordering
196 * 1 - write combining
197 */
198 used = centaur_mcr_compute(6, 25);
199
200 /*
201 * Mark the registers we are using.
202 */
203 rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
204 for (i = 0; i < used; i++)
205 lo |= 1<<(9+i);
206 wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
207
208 /*
209 * Wipe unused MCRs
210 */
211
212 for (i = used; i < 8; i++)
213 wrmsr(MSR_IDT_MCR0+i, 0, 0);
214}
215
216/*
217 * Handle the MCR key on the Winchip 2.
218 */
219static void winchip2_unprotect_mcr(void)
220{
221 u32 lo, hi;
222 u32 key;
223
224 rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
225 lo &= ~0x1C0; /* blank bits 8-6 */
226 key = (lo>>17) & 7;
227 lo |= key<<6; /* replace with unlock key */
228 wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
229}
230
231static void winchip2_protect_mcr(void)
232{
233 u32 lo, hi;
234
235 rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
236 lo &= ~0x1C0; /* blank bits 8-6 */
237 wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
238}
239#endif /* CONFIG_X86_OOSTORE */
240
241#define ACE_PRESENT (1 << 6) 11#define ACE_PRESENT (1 << 6)
242#define ACE_ENABLED (1 << 7) 12#define ACE_ENABLED (1 << 7)
243#define ACE_FCR (1 << 28) /* MSR_VIA_FCR */ 13#define ACE_FCR (1 << 28) /* MSR_VIA_FCR */
@@ -362,20 +132,6 @@ static void init_centaur(struct cpuinfo_x86 *c)
362 fcr_clr = DPDC; 132 fcr_clr = DPDC;
363 printk(KERN_NOTICE "Disabling bugged TSC.\n"); 133 printk(KERN_NOTICE "Disabling bugged TSC.\n");
364 clear_cpu_cap(c, X86_FEATURE_TSC); 134 clear_cpu_cap(c, X86_FEATURE_TSC);
365#ifdef CONFIG_X86_OOSTORE
366 centaur_create_optimal_mcr();
367 /*
368 * Enable:
369 * write combining on non-stack, non-string
370 * write combining on string, all types
371 * weak write ordering
372 *
373 * The C6 original lacks weak read order
374 *
375 * Note 0x120 is write only on Winchip 1
376 */
377 wrmsr(MSR_IDT_MCR_CTRL, 0x01F0001F, 0);
378#endif
379 break; 135 break;
380 case 8: 136 case 8:
381 switch (c->x86_mask) { 137 switch (c->x86_mask) {
@@ -392,40 +148,12 @@ static void init_centaur(struct cpuinfo_x86 *c)
392 fcr_set = ECX8|DSMC|DTLOCK|EMMX|EBRPRED|ERETSTK| 148 fcr_set = ECX8|DSMC|DTLOCK|EMMX|EBRPRED|ERETSTK|
393 E2MMX|EAMD3D; 149 E2MMX|EAMD3D;
394 fcr_clr = DPDC; 150 fcr_clr = DPDC;
395#ifdef CONFIG_X86_OOSTORE
396 winchip2_unprotect_mcr();
397 winchip2_create_optimal_mcr();
398 rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
399 /*
400 * Enable:
401 * write combining on non-stack, non-string
402 * write combining on string, all types
403 * weak write ordering
404 */
405 lo |= 31;
406 wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
407 winchip2_protect_mcr();
408#endif
409 break; 151 break;
410 case 9: 152 case 9:
411 name = "3"; 153 name = "3";
412 fcr_set = ECX8|DSMC|DTLOCK|EMMX|EBRPRED|ERETSTK| 154 fcr_set = ECX8|DSMC|DTLOCK|EMMX|EBRPRED|ERETSTK|
413 E2MMX|EAMD3D; 155 E2MMX|EAMD3D;
414 fcr_clr = DPDC; 156 fcr_clr = DPDC;
415#ifdef CONFIG_X86_OOSTORE
416 winchip2_unprotect_mcr();
417 winchip2_create_optimal_mcr();
418 rdmsr(MSR_IDT_MCR_CTRL, lo, hi);
419 /*
420 * Enable:
421 * write combining on non-stack, non-string
422 * write combining on string, all types
423 * weak write ordering
424 */
425 lo |= 31;
426 wrmsr(MSR_IDT_MCR_CTRL, lo, hi);
427 winchip2_protect_mcr();
428#endif
429 break; 157 break;
430 default: 158 default:
431 name = "??"; 159 name = "??";
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 24b6fd10625a..8e28bf2fc3ef 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -284,8 +284,13 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
284 raw_local_save_flags(eflags); 284 raw_local_save_flags(eflags);
285 BUG_ON(eflags & X86_EFLAGS_AC); 285 BUG_ON(eflags & X86_EFLAGS_AC);
286 286
287 if (cpu_has(c, X86_FEATURE_SMAP)) 287 if (cpu_has(c, X86_FEATURE_SMAP)) {
288#ifdef CONFIG_X86_SMAP
288 set_in_cr4(X86_CR4_SMAP); 289 set_in_cr4(X86_CR4_SMAP);
290#else
291 clear_in_cr4(X86_CR4_SMAP);
292#endif
293 }
289} 294}
290 295
291/* 296/*
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3db61c644e44..5cd9bfabd645 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -640,21 +640,17 @@ static void intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
640 case 0x61d: /* six-core 45 nm xeon "Dunnington" */ 640 case 0x61d: /* six-core 45 nm xeon "Dunnington" */
641 tlb_flushall_shift = -1; 641 tlb_flushall_shift = -1;
642 break; 642 break;
643 case 0x63a: /* Ivybridge */
644 tlb_flushall_shift = 2;
645 break;
643 case 0x61a: /* 45 nm nehalem, "Bloomfield" */ 646 case 0x61a: /* 45 nm nehalem, "Bloomfield" */
644 case 0x61e: /* 45 nm nehalem, "Lynnfield" */ 647 case 0x61e: /* 45 nm nehalem, "Lynnfield" */
645 case 0x625: /* 32 nm nehalem, "Clarkdale" */ 648 case 0x625: /* 32 nm nehalem, "Clarkdale" */
646 case 0x62c: /* 32 nm nehalem, "Gulftown" */ 649 case 0x62c: /* 32 nm nehalem, "Gulftown" */
647 case 0x62e: /* 45 nm nehalem-ex, "Beckton" */ 650 case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
648 case 0x62f: /* 32 nm Xeon E7 */ 651 case 0x62f: /* 32 nm Xeon E7 */
649 tlb_flushall_shift = 6;
650 break;
651 case 0x62a: /* SandyBridge */ 652 case 0x62a: /* SandyBridge */
652 case 0x62d: /* SandyBridge, "Romely-EP" */ 653 case 0x62d: /* SandyBridge, "Romely-EP" */
653 tlb_flushall_shift = 5;
654 break;
655 case 0x63a: /* Ivybridge */
656 tlb_flushall_shift = 1;
657 break;
658 default: 654 default:
659 tlb_flushall_shift = 6; 655 tlb_flushall_shift = 6;
660 } 656 }
diff --git a/arch/x86/kernel/cpu/microcode/amd_early.c b/arch/x86/kernel/cpu/microcode/amd_early.c
index 8384c0fa206f..617a9e284245 100644
--- a/arch/x86/kernel/cpu/microcode/amd_early.c
+++ b/arch/x86/kernel/cpu/microcode/amd_early.c
@@ -285,6 +285,15 @@ static void __init collect_cpu_sig_on_bsp(void *arg)
285 285
286 uci->cpu_sig.sig = cpuid_eax(0x00000001); 286 uci->cpu_sig.sig = cpuid_eax(0x00000001);
287} 287}
288
289static void __init get_bsp_sig(void)
290{
291 unsigned int bsp = boot_cpu_data.cpu_index;
292 struct ucode_cpu_info *uci = ucode_cpu_info + bsp;
293
294 if (!uci->cpu_sig.sig)
295 smp_call_function_single(bsp, collect_cpu_sig_on_bsp, NULL, 1);
296}
288#else 297#else
289void load_ucode_amd_ap(void) 298void load_ucode_amd_ap(void)
290{ 299{
@@ -337,31 +346,37 @@ void load_ucode_amd_ap(void)
337 346
338int __init save_microcode_in_initrd_amd(void) 347int __init save_microcode_in_initrd_amd(void)
339{ 348{
349 unsigned long cont;
340 enum ucode_state ret; 350 enum ucode_state ret;
341 u32 eax; 351 u32 eax;
342 352
343#ifdef CONFIG_X86_32 353 if (!container)
344 unsigned int bsp = boot_cpu_data.cpu_index; 354 return -EINVAL;
345 struct ucode_cpu_info *uci = ucode_cpu_info + bsp;
346
347 if (!uci->cpu_sig.sig)
348 smp_call_function_single(bsp, collect_cpu_sig_on_bsp, NULL, 1);
349 355
356#ifdef CONFIG_X86_32
357 get_bsp_sig();
358 cont = (unsigned long)container;
359#else
350 /* 360 /*
351 * Take into account the fact that the ramdisk might get relocated 361 * We need the physical address of the container for both bitness since
352 * and therefore we need to recompute the container's position in 362 * boot_params.hdr.ramdisk_image is a physical address.
353 * virtual memory space.
354 */ 363 */
355 container = (u8 *)(__va((u32)relocated_ramdisk) + 364 cont = __pa(container);
356 ((u32)container - boot_params.hdr.ramdisk_image));
357#endif 365#endif
366
367 /*
368 * Take into account the fact that the ramdisk might get relocated and
369 * therefore we need to recompute the container's position in virtual
370 * memory space.
371 */
372 if (relocated_ramdisk)
373 container = (u8 *)(__va(relocated_ramdisk) +
374 (cont - boot_params.hdr.ramdisk_image));
375
358 if (ucode_new_rev) 376 if (ucode_new_rev)
359 pr_info("microcode: updated early to new patch_level=0x%08x\n", 377 pr_info("microcode: updated early to new patch_level=0x%08x\n",
360 ucode_new_rev); 378 ucode_new_rev);
361 379
362 if (!container)
363 return -EINVAL;
364
365 eax = cpuid_eax(0x00000001); 380 eax = cpuid_eax(0x00000001);
366 eax = ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff); 381 eax = ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff);
367 382
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index ce2d0a2c3e4f..0e25a1bc5ab5 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -683,7 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
683 } 683 }
684 684
685 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ 685 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
686 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 686 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
687 __flush_tlb(); 687 __flush_tlb();
688 688
689 /* Save MTRR state */ 689 /* Save MTRR state */
@@ -697,7 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
697static void post_set(void) __releases(set_atomicity_lock) 697static void post_set(void) __releases(set_atomicity_lock)
698{ 698{
699 /* Flush TLBs (no need to flush caches - they are disabled) */ 699 /* Flush TLBs (no need to flush caches - they are disabled) */
700 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 700 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
701 __flush_tlb(); 701 __flush_tlb();
702 702
703 /* Intel (P6) standard MTRRs */ 703 /* Intel (P6) standard MTRRs */
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index b88645191fe5..79f9f848bee4 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1192,6 +1192,9 @@ static void x86_pmu_del(struct perf_event *event, int flags)
1192 for (i = 0; i < cpuc->n_events; i++) { 1192 for (i = 0; i < cpuc->n_events; i++) {
1193 if (event == cpuc->event_list[i]) { 1193 if (event == cpuc->event_list[i]) {
1194 1194
1195 if (i >= cpuc->n_events - cpuc->n_added)
1196 --cpuc->n_added;
1197
1195 if (x86_pmu.put_event_constraints) 1198 if (x86_pmu.put_event_constraints)
1196 x86_pmu.put_event_constraints(cpuc, event); 1199 x86_pmu.put_event_constraints(cpuc, event);
1197 1200
@@ -1521,6 +1524,8 @@ static int __init init_hw_perf_events(void)
1521 1524
1522 pr_cont("%s PMU driver.\n", x86_pmu.name); 1525 pr_cont("%s PMU driver.\n", x86_pmu.name);
1523 1526
1527 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1528
1524 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1529 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1525 quirk->func(); 1530 quirk->func();
1526 1531
@@ -1534,7 +1539,6 @@ static int __init init_hw_perf_events(void)
1534 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1539 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1535 0, x86_pmu.num_counters, 0, 0); 1540 0, x86_pmu.num_counters, 0, 0);
1536 1541
1537 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1538 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1542 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1539 1543
1540 if (x86_pmu.event_attrs) 1544 if (x86_pmu.event_attrs)
@@ -1820,9 +1824,12 @@ static ssize_t set_attr_rdpmc(struct device *cdev,
1820 if (ret) 1824 if (ret)
1821 return ret; 1825 return ret;
1822 1826
1827 if (x86_pmu.attr_rdpmc_broken)
1828 return -ENOTSUPP;
1829
1823 if (!!val != !!x86_pmu.attr_rdpmc) { 1830 if (!!val != !!x86_pmu.attr_rdpmc) {
1824 x86_pmu.attr_rdpmc = !!val; 1831 x86_pmu.attr_rdpmc = !!val;
1825 smp_call_function(change_rdpmc, (void *)val, 1); 1832 on_each_cpu(change_rdpmc, (void *)val, 1);
1826 } 1833 }
1827 1834
1828 return count; 1835 return count;
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index c1a861829d81..4972c244d0bc 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -409,6 +409,7 @@ struct x86_pmu {
409 /* 409 /*
410 * sysfs attrs 410 * sysfs attrs
411 */ 411 */
412 int attr_rdpmc_broken;
412 int attr_rdpmc; 413 int attr_rdpmc;
413 struct attribute **format_attrs; 414 struct attribute **format_attrs;
414 struct attribute **event_attrs; 415 struct attribute **event_attrs;
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 0fa4f242f050..aa333d966886 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1361,10 +1361,8 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
1361 intel_pmu_disable_all(); 1361 intel_pmu_disable_all();
1362 handled = intel_pmu_drain_bts_buffer(); 1362 handled = intel_pmu_drain_bts_buffer();
1363 status = intel_pmu_get_status(); 1363 status = intel_pmu_get_status();
1364 if (!status) { 1364 if (!status)
1365 intel_pmu_enable_all(0); 1365 goto done;
1366 return handled;
1367 }
1368 1366
1369 loops = 0; 1367 loops = 0;
1370again: 1368again:
@@ -2310,10 +2308,7 @@ __init int intel_pmu_init(void)
2310 if (version > 1) 2308 if (version > 1)
2311 x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3); 2309 x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
2312 2310
2313 /* 2311 if (boot_cpu_has(X86_FEATURE_PDCM)) {
2314 * v2 and above have a perf capabilities MSR
2315 */
2316 if (version > 1) {
2317 u64 capabilities; 2312 u64 capabilities;
2318 2313
2319 rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities); 2314 rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 29c248799ced..047f540cf3f7 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -501,8 +501,11 @@ static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
501 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN, 501 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
502 SNBEP_CBO_PMON_CTL_TID_EN, 0x1), 502 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
503 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4), 503 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
504 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0x6),
504 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4), 505 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
506 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0x6),
505 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4), 507 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
508 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0x6),
506 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6), 509 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
507 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8), 510 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
508 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8), 511 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
@@ -1178,10 +1181,15 @@ static struct extra_reg ivt_uncore_cbox_extra_regs[] = {
1178 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN, 1181 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1179 SNBEP_CBO_PMON_CTL_TID_EN, 0x1), 1182 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1180 SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2), 1183 SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1184 SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
1185 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1186 SNBEP_CBO_EVENT_EXTRA_REG(0x5134, 0xffff, 0xc),
1181 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4), 1187 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1188 SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0xc),
1182 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4), 1189 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1190 SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0xc),
1183 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4), 1191 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1184 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc), 1192 SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0xc),
1185 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10), 1193 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1186 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10), 1194 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1187 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10), 1195 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
@@ -3326,6 +3334,8 @@ static int __init uncore_type_init(struct intel_uncore_type *type)
3326 if (!pmus) 3334 if (!pmus)
3327 return -ENOMEM; 3335 return -ENOMEM;
3328 3336
3337 type->pmus = pmus;
3338
3329 type->unconstrainted = (struct event_constraint) 3339 type->unconstrainted = (struct event_constraint)
3330 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1, 3340 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
3331 0, type->num_counters, 0, 0); 3341 0, type->num_counters, 0, 0);
@@ -3361,7 +3371,6 @@ static int __init uncore_type_init(struct intel_uncore_type *type)
3361 } 3371 }
3362 3372
3363 type->pmu_group = &uncore_pmu_attr_group; 3373 type->pmu_group = &uncore_pmu_attr_group;
3364 type->pmus = pmus;
3365 return 0; 3374 return 0;
3366fail: 3375fail:
3367 uncore_type_exit(type); 3376 uncore_type_exit(type);
diff --git a/arch/x86/kernel/cpu/perf_event_p6.c b/arch/x86/kernel/cpu/perf_event_p6.c
index b1e2fe115323..7c1a0c07b607 100644
--- a/arch/x86/kernel/cpu/perf_event_p6.c
+++ b/arch/x86/kernel/cpu/perf_event_p6.c
@@ -231,31 +231,49 @@ static __initconst const struct x86_pmu p6_pmu = {
231 231
232}; 232};
233 233
234static __init void p6_pmu_rdpmc_quirk(void)
235{
236 if (boot_cpu_data.x86_mask < 9) {
237 /*
238 * PPro erratum 26; fixed in stepping 9 and above.
239 */
240 pr_warn("Userspace RDPMC support disabled due to a CPU erratum\n");
241 x86_pmu.attr_rdpmc_broken = 1;
242 x86_pmu.attr_rdpmc = 0;
243 }
244}
245
234__init int p6_pmu_init(void) 246__init int p6_pmu_init(void)
235{ 247{
248 x86_pmu = p6_pmu;
249
236 switch (boot_cpu_data.x86_model) { 250 switch (boot_cpu_data.x86_model) {
237 case 1: 251 case 1: /* Pentium Pro */
238 case 3: /* Pentium Pro */ 252 x86_add_quirk(p6_pmu_rdpmc_quirk);
239 case 5: 253 break;
240 case 6: /* Pentium II */ 254
241 case 7: 255 case 3: /* Pentium II - Klamath */
242 case 8: 256 case 5: /* Pentium II - Deschutes */
243 case 11: /* Pentium III */ 257 case 6: /* Pentium II - Mendocino */
244 case 9:
245 case 13:
246 /* Pentium M */
247 break; 258 break;
259
260 case 7: /* Pentium III - Katmai */
261 case 8: /* Pentium III - Coppermine */
262 case 10: /* Pentium III Xeon */
263 case 11: /* Pentium III - Tualatin */
264 break;
265
266 case 9: /* Pentium M - Banias */
267 case 13: /* Pentium M - Dothan */
268 break;
269
248 default: 270 default:
249 pr_cont("unsupported p6 CPU model %d ", 271 pr_cont("unsupported p6 CPU model %d ", boot_cpu_data.x86_model);
250 boot_cpu_data.x86_model);
251 return -ENODEV; 272 return -ENODEV;
252 } 273 }
253 274
254 x86_pmu = p6_pmu;
255
256 memcpy(hw_cache_event_ids, p6_hw_cache_event_ids, 275 memcpy(hw_cache_event_ids, p6_hw_cache_event_ids,
257 sizeof(hw_cache_event_ids)); 276 sizeof(hw_cache_event_ids));
258 277
259
260 return 0; 278 return 0;
261} 279}
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index d4bdd253fea7..e6253195a301 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -77,8 +77,7 @@ within(unsigned long addr, unsigned long start, unsigned long end)
77 return addr >= start && addr < end; 77 return addr >= start && addr < end;
78} 78}
79 79
80static int 80static unsigned long text_ip_addr(unsigned long ip)
81do_ftrace_mod_code(unsigned long ip, const void *new_code)
82{ 81{
83 /* 82 /*
84 * On x86_64, kernel text mappings are mapped read-only with 83 * On x86_64, kernel text mappings are mapped read-only with
@@ -91,7 +90,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code)
91 if (within(ip, (unsigned long)_text, (unsigned long)_etext)) 90 if (within(ip, (unsigned long)_text, (unsigned long)_etext))
92 ip = (unsigned long)__va(__pa_symbol(ip)); 91 ip = (unsigned long)__va(__pa_symbol(ip));
93 92
94 return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE); 93 return ip;
95} 94}
96 95
97static const unsigned char *ftrace_nop_replace(void) 96static const unsigned char *ftrace_nop_replace(void)
@@ -123,8 +122,10 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
123 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) 122 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
124 return -EINVAL; 123 return -EINVAL;
125 124
125 ip = text_ip_addr(ip);
126
126 /* replace the text with the new text */ 127 /* replace the text with the new text */
127 if (do_ftrace_mod_code(ip, new_code)) 128 if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
128 return -EPERM; 129 return -EPERM;
129 130
130 sync_core(); 131 sync_core();
@@ -221,37 +222,51 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
221 return -EINVAL; 222 return -EINVAL;
222} 223}
223 224
224int ftrace_update_ftrace_func(ftrace_func_t func) 225static unsigned long ftrace_update_func;
226
227static int update_ftrace_func(unsigned long ip, void *new)
225{ 228{
226 unsigned long ip = (unsigned long)(&ftrace_call); 229 unsigned char old[MCOUNT_INSN_SIZE];
227 unsigned char old[MCOUNT_INSN_SIZE], *new;
228 int ret; 230 int ret;
229 231
230 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE); 232 memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
231 new = ftrace_call_replace(ip, (unsigned long)func); 233
234 ftrace_update_func = ip;
235 /* Make sure the breakpoints see the ftrace_update_func update */
236 smp_wmb();
232 237
233 /* See comment above by declaration of modifying_ftrace_code */ 238 /* See comment above by declaration of modifying_ftrace_code */
234 atomic_inc(&modifying_ftrace_code); 239 atomic_inc(&modifying_ftrace_code);
235 240
236 ret = ftrace_modify_code(ip, old, new); 241 ret = ftrace_modify_code(ip, old, new);
237 242
243 atomic_dec(&modifying_ftrace_code);
244
245 return ret;
246}
247
248int ftrace_update_ftrace_func(ftrace_func_t func)
249{
250 unsigned long ip = (unsigned long)(&ftrace_call);
251 unsigned char *new;
252 int ret;
253
254 new = ftrace_call_replace(ip, (unsigned long)func);
255 ret = update_ftrace_func(ip, new);
256
238 /* Also update the regs callback function */ 257 /* Also update the regs callback function */
239 if (!ret) { 258 if (!ret) {
240 ip = (unsigned long)(&ftrace_regs_call); 259 ip = (unsigned long)(&ftrace_regs_call);
241 memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
242 new = ftrace_call_replace(ip, (unsigned long)func); 260 new = ftrace_call_replace(ip, (unsigned long)func);
243 ret = ftrace_modify_code(ip, old, new); 261 ret = update_ftrace_func(ip, new);
244 } 262 }
245 263
246 atomic_dec(&modifying_ftrace_code);
247
248 return ret; 264 return ret;
249} 265}
250 266
251static int is_ftrace_caller(unsigned long ip) 267static int is_ftrace_caller(unsigned long ip)
252{ 268{
253 if (ip == (unsigned long)(&ftrace_call) || 269 if (ip == ftrace_update_func)
254 ip == (unsigned long)(&ftrace_regs_call))
255 return 1; 270 return 1;
256 271
257 return 0; 272 return 0;
@@ -677,45 +692,41 @@ int __init ftrace_dyn_arch_init(void *data)
677#ifdef CONFIG_DYNAMIC_FTRACE 692#ifdef CONFIG_DYNAMIC_FTRACE
678extern void ftrace_graph_call(void); 693extern void ftrace_graph_call(void);
679 694
680static int ftrace_mod_jmp(unsigned long ip, 695static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
681 int old_offset, int new_offset)
682{ 696{
683 unsigned char code[MCOUNT_INSN_SIZE]; 697 static union ftrace_code_union calc;
684 698
685 if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE)) 699 /* Jmp not a call (ignore the .e8) */
686 return -EFAULT; 700 calc.e8 = 0xe9;
701 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
687 702
688 if (code[0] != 0xe9 || old_offset != *(int *)(&code[1])) 703 /*
689 return -EINVAL; 704 * ftrace external locks synchronize the access to the static variable.
705 */
706 return calc.code;
707}
690 708
691 *(int *)(&code[1]) = new_offset; 709static int ftrace_mod_jmp(unsigned long ip, void *func)
710{
711 unsigned char *new;
692 712
693 if (do_ftrace_mod_code(ip, &code)) 713 new = ftrace_jmp_replace(ip, (unsigned long)func);
694 return -EPERM;
695 714
696 return 0; 715 return update_ftrace_func(ip, new);
697} 716}
698 717
699int ftrace_enable_ftrace_graph_caller(void) 718int ftrace_enable_ftrace_graph_caller(void)
700{ 719{
701 unsigned long ip = (unsigned long)(&ftrace_graph_call); 720 unsigned long ip = (unsigned long)(&ftrace_graph_call);
702 int old_offset, new_offset;
703 721
704 old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE); 722 return ftrace_mod_jmp(ip, &ftrace_graph_caller);
705 new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
706
707 return ftrace_mod_jmp(ip, old_offset, new_offset);
708} 723}
709 724
710int ftrace_disable_ftrace_graph_caller(void) 725int ftrace_disable_ftrace_graph_caller(void)
711{ 726{
712 unsigned long ip = (unsigned long)(&ftrace_graph_call); 727 unsigned long ip = (unsigned long)(&ftrace_graph_call);
713 int old_offset, new_offset;
714
715 old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
716 new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
717 728
718 return ftrace_mod_jmp(ip, old_offset, new_offset); 729 return ftrace_mod_jmp(ip, &ftrace_stub);
719} 730}
720 731
721#endif /* !CONFIG_DYNAMIC_FTRACE */ 732#endif /* !CONFIG_DYNAMIC_FTRACE */
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 81ba27679f18..f36bd42d6f0c 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -544,6 +544,10 @@ ENDPROC(early_idt_handlers)
544 /* This is global to keep gas from relaxing the jumps */ 544 /* This is global to keep gas from relaxing the jumps */
545ENTRY(early_idt_handler) 545ENTRY(early_idt_handler)
546 cld 546 cld
547
548 cmpl $2,(%esp) # X86_TRAP_NMI
549 je is_nmi # Ignore NMI
550
547 cmpl $2,%ss:early_recursion_flag 551 cmpl $2,%ss:early_recursion_flag
548 je hlt_loop 552 je hlt_loop
549 incl %ss:early_recursion_flag 553 incl %ss:early_recursion_flag
@@ -594,8 +598,9 @@ ex_entry:
594 pop %edx 598 pop %edx
595 pop %ecx 599 pop %ecx
596 pop %eax 600 pop %eax
597 addl $8,%esp /* drop vector number and error code */
598 decl %ss:early_recursion_flag 601 decl %ss:early_recursion_flag
602is_nmi:
603 addl $8,%esp /* drop vector number and error code */
599 iret 604 iret
600ENDPROC(early_idt_handler) 605ENDPROC(early_idt_handler)
601 606
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index e1aabdb314c8..a468c0a65c42 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -343,6 +343,9 @@ early_idt_handlers:
343ENTRY(early_idt_handler) 343ENTRY(early_idt_handler)
344 cld 344 cld
345 345
346 cmpl $2,(%rsp) # X86_TRAP_NMI
347 je is_nmi # Ignore NMI
348
346 cmpl $2,early_recursion_flag(%rip) 349 cmpl $2,early_recursion_flag(%rip)
347 jz 1f 350 jz 1f
348 incl early_recursion_flag(%rip) 351 incl early_recursion_flag(%rip)
@@ -405,8 +408,9 @@ ENTRY(early_idt_handler)
405 popq %rdx 408 popq %rdx
406 popq %rcx 409 popq %rcx
407 popq %rax 410 popq %rax
408 addq $16,%rsp # drop vector number and error code
409 decl early_recursion_flag(%rip) 411 decl early_recursion_flag(%rip)
412is_nmi:
413 addq $16,%rsp # drop vector number and error code
410 INTERRUPT_RETURN 414 INTERRUPT_RETURN
411ENDPROC(early_idt_handler) 415ENDPROC(early_idt_handler)
412 416
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index e8368c6dd2a2..d5dd80814419 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -86,10 +86,19 @@ EXPORT_SYMBOL(__kernel_fpu_begin);
86 86
87void __kernel_fpu_end(void) 87void __kernel_fpu_end(void)
88{ 88{
89 if (use_eager_fpu()) 89 if (use_eager_fpu()) {
90 math_state_restore(); 90 /*
91 else 91 * For eager fpu, most the time, tsk_used_math() is true.
92 * Restore the user math as we are done with the kernel usage.
93 * At few instances during thread exit, signal handling etc,
94 * tsk_used_math() is false. Those few places will take proper
95 * actions, so we don't need to restore the math here.
96 */
97 if (likely(tsk_used_math(current)))
98 math_state_restore();
99 } else {
92 stts(); 100 stts();
101 }
93} 102}
94EXPORT_SYMBOL(__kernel_fpu_end); 103EXPORT_SYMBOL(__kernel_fpu_end);
95 104
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index dbb60878b744..d99f31d9a750 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -266,6 +266,14 @@ __visible void smp_trace_x86_platform_ipi(struct pt_regs *regs)
266EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); 266EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
267 267
268#ifdef CONFIG_HOTPLUG_CPU 268#ifdef CONFIG_HOTPLUG_CPU
269
270/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
271 * below, which is protected by stop_machine(). Putting them on the stack
272 * results in a stack frame overflow. Dynamically allocating could result in a
273 * failure so declare these two cpumasks as global.
274 */
275static struct cpumask affinity_new, online_new;
276
269/* 277/*
270 * This cpu is going to be removed and its vectors migrated to the remaining 278 * This cpu is going to be removed and its vectors migrated to the remaining
271 * online cpus. Check to see if there are enough vectors in the remaining cpus. 279 * online cpus. Check to see if there are enough vectors in the remaining cpus.
@@ -277,7 +285,6 @@ int check_irq_vectors_for_cpu_disable(void)
277 unsigned int this_cpu, vector, this_count, count; 285 unsigned int this_cpu, vector, this_count, count;
278 struct irq_desc *desc; 286 struct irq_desc *desc;
279 struct irq_data *data; 287 struct irq_data *data;
280 struct cpumask affinity_new, online_new;
281 288
282 this_cpu = smp_processor_id(); 289 this_cpu = smp_processor_id();
283 cpumask_copy(&online_new, cpu_online_mask); 290 cpumask_copy(&online_new, cpu_online_mask);
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 4eabc160696f..679cef0791cd 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -279,5 +279,7 @@ void arch_crash_save_vmcoreinfo(void)
279 VMCOREINFO_SYMBOL(node_data); 279 VMCOREINFO_SYMBOL(node_data);
280 VMCOREINFO_LENGTH(node_data, MAX_NUMNODES); 280 VMCOREINFO_LENGTH(node_data, MAX_NUMNODES);
281#endif 281#endif
282 vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
283 (unsigned long)&_text - __START_KERNEL);
282} 284}
283 285
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 872079a67e4d..f7d0672481fd 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -100,8 +100,10 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size,
100 flag |= __GFP_ZERO; 100 flag |= __GFP_ZERO;
101again: 101again:
102 page = NULL; 102 page = NULL;
103 if (!(flag & GFP_ATOMIC)) 103 /* CMA can be used only in the context which permits sleeping */
104 if (flag & __GFP_WAIT)
104 page = dma_alloc_from_contiguous(dev, count, get_order(size)); 105 page = dma_alloc_from_contiguous(dev, count, get_order(size));
106 /* fallback */
105 if (!page) 107 if (!page)
106 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 108 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
107 if (!page) 109 if (!page)
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index 04ee1e2e4c02..ff898bbf579d 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -529,7 +529,7 @@ static void quirk_amd_nb_node(struct pci_dev *dev)
529 return; 529 return;
530 530
531 pci_read_config_dword(nb_ht, 0x60, &val); 531 pci_read_config_dword(nb_ht, 0x60, &val);
532 node = val & 7; 532 node = pcibus_to_node(dev->bus) | (val & 7);
533 /* 533 /*
534 * Some hardware may return an invalid node ID, 534 * Some hardware may return an invalid node ID,
535 * so check it first: 535 * so check it first:
@@ -571,3 +571,40 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F5,
571 quirk_amd_nb_node); 571 quirk_amd_nb_node);
572 572
573#endif 573#endif
574
575#ifdef CONFIG_PCI
576/*
577 * Processor does not ensure DRAM scrub read/write sequence
578 * is atomic wrt accesses to CC6 save state area. Therefore
579 * if a concurrent scrub read/write access is to same address
580 * the entry may appear as if it is not written. This quirk
581 * applies to Fam16h models 00h-0Fh
582 *
583 * See "Revision Guide" for AMD F16h models 00h-0fh,
584 * document 51810 rev. 3.04, Nov 2013
585 */
586static void amd_disable_seq_and_redirect_scrub(struct pci_dev *dev)
587{
588 u32 val;
589
590 /*
591 * Suggested workaround:
592 * set D18F3x58[4:0] = 00h and set D18F3x5C[0] = 0b
593 */
594 pci_read_config_dword(dev, 0x58, &val);
595 if (val & 0x1F) {
596 val &= ~(0x1F);
597 pci_write_config_dword(dev, 0x58, val);
598 }
599
600 pci_read_config_dword(dev, 0x5C, &val);
601 if (val & BIT(0)) {
602 val &= ~BIT(0);
603 pci_write_config_dword(dev, 0x5c, val);
604 }
605}
606
607DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3,
608 amd_disable_seq_and_redirect_scrub);
609
610#endif
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 06853e670354..ce72964b2f46 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1239,14 +1239,8 @@ void __init setup_arch(char **cmdline_p)
1239 register_refined_jiffies(CLOCK_TICK_RATE); 1239 register_refined_jiffies(CLOCK_TICK_RATE);
1240 1240
1241#ifdef CONFIG_EFI 1241#ifdef CONFIG_EFI
1242 /* Once setup is done above, unmap the EFI memory map on 1242 if (efi_enabled(EFI_BOOT))
1243 * mismatched firmware/kernel archtectures since there is no 1243 efi_apply_memmap_quirks();
1244 * support for runtime services.
1245 */
1246 if (efi_enabled(EFI_BOOT) && !efi_is_native()) {
1247 pr_info("efi: Setup done, disabling due to 32/64-bit mismatch\n");
1248 efi_unmap_memmap();
1249 }
1250#endif 1244#endif
1251} 1245}
1252 1246
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 19e5adb49a27..cfbe99f88830 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -209,7 +209,7 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)
209 * dance when its actually needed. 209 * dance when its actually needed.
210 */ 210 */
211 211
212 preempt_disable(); 212 preempt_disable_notrace();
213 data = this_cpu_read(cyc2ns.head); 213 data = this_cpu_read(cyc2ns.head);
214 tail = this_cpu_read(cyc2ns.tail); 214 tail = this_cpu_read(cyc2ns.tail);
215 215
@@ -229,7 +229,7 @@ static inline unsigned long long cycles_2_ns(unsigned long long cyc)
229 if (!--data->__count) 229 if (!--data->__count)
230 this_cpu_write(cyc2ns.tail, data); 230 this_cpu_write(cyc2ns.tail, data);
231 } 231 }
232 preempt_enable(); 232 preempt_enable_notrace();
233 233
234 return ns; 234 return ns;
235} 235}
@@ -653,13 +653,10 @@ unsigned long native_calibrate_tsc(void)
653 653
654 /* Calibrate TSC using MSR for Intel Atom SoCs */ 654 /* Calibrate TSC using MSR for Intel Atom SoCs */
655 local_irq_save(flags); 655 local_irq_save(flags);
656 i = try_msr_calibrate_tsc(&fast_calibrate); 656 fast_calibrate = try_msr_calibrate_tsc();
657 local_irq_restore(flags); 657 local_irq_restore(flags);
658 if (i >= 0) { 658 if (fast_calibrate)
659 if (i == 0)
660 pr_warn("Fast TSC calibration using MSR failed\n");
661 return fast_calibrate; 659 return fast_calibrate;
662 }
663 660
664 local_irq_save(flags); 661 local_irq_save(flags);
665 fast_calibrate = quick_pit_calibrate(); 662 fast_calibrate = quick_pit_calibrate();
diff --git a/arch/x86/kernel/tsc_msr.c b/arch/x86/kernel/tsc_msr.c
index 8b5434f4389f..92ae6acac8a7 100644
--- a/arch/x86/kernel/tsc_msr.c
+++ b/arch/x86/kernel/tsc_msr.c
@@ -53,7 +53,7 @@ static struct freq_desc freq_desc_tables[] = {
53 /* TNG */ 53 /* TNG */
54 { 6, 0x4a, 1, { 0, FREQ_100, FREQ_133, 0, 0, 0, 0, 0 } }, 54 { 6, 0x4a, 1, { 0, FREQ_100, FREQ_133, 0, 0, 0, 0, 0 } },
55 /* VLV2 */ 55 /* VLV2 */
56 { 6, 0x37, 1, { 0, FREQ_100, FREQ_133, FREQ_166, 0, 0, 0, 0 } }, 56 { 6, 0x37, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_166, 0, 0, 0, 0 } },
57 /* ANN */ 57 /* ANN */
58 { 6, 0x5a, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_100, 0, 0, 0, 0 } }, 58 { 6, 0x5a, 1, { FREQ_83, FREQ_100, FREQ_133, FREQ_100, 0, 0, 0, 0 } },
59}; 59};
@@ -77,21 +77,18 @@ static int match_cpu(u8 family, u8 model)
77 77
78/* 78/*
79 * Do MSR calibration only for known/supported CPUs. 79 * Do MSR calibration only for known/supported CPUs.
80 * Return values: 80 *
81 * -1: CPU is unknown/unsupported for MSR based calibration 81 * Returns the calibration value or 0 if MSR calibration failed.
82 * 0: CPU is known/supported, but calibration failed
83 * 1: CPU is known/supported, and calibration succeeded
84 */ 82 */
85int try_msr_calibrate_tsc(unsigned long *fast_calibrate) 83unsigned long try_msr_calibrate_tsc(void)
86{ 84{
87 int cpu_index;
88 u32 lo, hi, ratio, freq_id, freq; 85 u32 lo, hi, ratio, freq_id, freq;
86 unsigned long res;
87 int cpu_index;
89 88
90 cpu_index = match_cpu(boot_cpu_data.x86, boot_cpu_data.x86_model); 89 cpu_index = match_cpu(boot_cpu_data.x86, boot_cpu_data.x86_model);
91 if (cpu_index < 0) 90 if (cpu_index < 0)
92 return -1; 91 return 0;
93
94 *fast_calibrate = 0;
95 92
96 if (freq_desc_tables[cpu_index].msr_plat) { 93 if (freq_desc_tables[cpu_index].msr_plat) {
97 rdmsr(MSR_PLATFORM_INFO, lo, hi); 94 rdmsr(MSR_PLATFORM_INFO, lo, hi);
@@ -103,7 +100,7 @@ int try_msr_calibrate_tsc(unsigned long *fast_calibrate)
103 pr_info("Maximum core-clock to bus-clock ratio: 0x%x\n", ratio); 100 pr_info("Maximum core-clock to bus-clock ratio: 0x%x\n", ratio);
104 101
105 if (!ratio) 102 if (!ratio)
106 return 0; 103 goto fail;
107 104
108 /* Get FSB FREQ ID */ 105 /* Get FSB FREQ ID */
109 rdmsr(MSR_FSB_FREQ, lo, hi); 106 rdmsr(MSR_FSB_FREQ, lo, hi);
@@ -112,16 +109,19 @@ int try_msr_calibrate_tsc(unsigned long *fast_calibrate)
112 pr_info("Resolved frequency ID: %u, frequency: %u KHz\n", 109 pr_info("Resolved frequency ID: %u, frequency: %u KHz\n",
113 freq_id, freq); 110 freq_id, freq);
114 if (!freq) 111 if (!freq)
115 return 0; 112 goto fail;
116 113
117 /* TSC frequency = maximum resolved freq * maximum resolved bus ratio */ 114 /* TSC frequency = maximum resolved freq * maximum resolved bus ratio */
118 *fast_calibrate = freq * ratio; 115 res = freq * ratio;
119 pr_info("TSC runs at %lu KHz\n", *fast_calibrate); 116 pr_info("TSC runs at %lu KHz\n", res);
120 117
121#ifdef CONFIG_X86_LOCAL_APIC 118#ifdef CONFIG_X86_LOCAL_APIC
122 lapic_timer_frequency = (freq * 1000) / HZ; 119 lapic_timer_frequency = (freq * 1000) / HZ;
123 pr_info("lapic_timer_frequency = %d\n", lapic_timer_frequency); 120 pr_info("lapic_timer_frequency = %d\n", lapic_timer_frequency);
124#endif 121#endif
122 return res;
125 123
126 return 1; 124fail:
125 pr_warn("Fast TSC calibration using MSR failed\n");
126 return 0;
127} 127}
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-22 01:30:24 -0400