6 files changed, 82 insertions, 6 deletions

diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index d3895dbf4ddb..81f04cee5f74 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -18,6 +18,10 @@ typedef struct {
 #ifdef CONFIG_SMP
         unsigned int irq_resched_count;
         unsigned int irq_call_count;
+        /*
+         * irq_tlb_count is double-counted in irq_call_count, so it must be
+         * subtracted from irq_call_count when displaying irq_call_count
+         */
         unsigned int irq_tlb_count;
 #endif
 #ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 9d92e19039f0..f7e98a2c0d12 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -737,6 +737,72 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
 }
 #endif
 
+static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
+{
+        if (!cpu_has_invlpg)
+                return;
+
+        tlb_flushall_shift = 5;
+
+        if (c->x86 <= 0x11)
+                tlb_flushall_shift = 4;
+}
+
+static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
+{
+        u32 ebx, eax, ecx, edx;
+        u16 mask = 0xfff;
+
+        if (c->x86 < 0xf)
+                return;
+
+        if (c->extended_cpuid_level < 0x80000006)
+                return;
+
+        cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
+
+        tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
+        tlb_lli_4k[ENTRIES] = ebx & mask;
+
+        /*
+         * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
+         * characteristics from the CPUID function 0x80000005 instead.
+         */
+        if (c->x86 == 0xf) {
+                cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
+                mask = 0xff;
+        }
+
+        /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
+        if (!((eax >> 16) & mask)) {
+                u32 a, b, c, d;
+
+                cpuid(0x80000005, &a, &b, &c, &d);
+                tlb_lld_2m[ENTRIES] = (a >> 16) & 0xff;
+        } else {
+                tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
+        }
+
+        /* a 4M entry uses two 2M entries */
+        tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
+
+        /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
+        if (!(eax & mask)) {
+                /* Erratum 658 */
+                if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
+                        tlb_lli_2m[ENTRIES] = 1024;
+                } else {
+                        cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
+                        tlb_lli_2m[ENTRIES] = eax & 0xff;
+                }
+        } else
+                tlb_lli_2m[ENTRIES] = eax & mask;
+
+        tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
+
+        cpu_set_tlb_flushall_shift(c);
+}
+
 static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
         .c_vendor       = "AMD",
         .c_ident        = { "AuthenticAMD" },
@@ -756,6 +822,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
         .c_size_cache   = amd_size_cache,
 #endif
         .c_early_init   = early_init_amd,
+        .c_detect_tlb   = cpu_detect_tlb_amd,
         .c_bsp_init     = bsp_init_amd,
         .c_init         = init_amd,
         .c_x86_vendor   = X86_VENDOR_AMD,
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 761cb3547041..532691b6c8fe 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -476,7 +476,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
 
         printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
                 "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n"          \
-                "tlb_flushall_shift is 0x%x\n",
+                "tlb_flushall_shift: %d\n",
                 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
                 tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
                 tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
@@ -942,8 +942,7 @@ void __init identify_boot_cpu(void)
 #else
         vgetcpu_set_mode();
 #endif
-        if (boot_cpu_data.cpuid_level >= 2)
-                cpu_detect_tlb(&boot_cpu_data);
+        cpu_detect_tlb(&boot_cpu_data);
 }
 
 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 0a4ce2980a5a..198e019a531a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -648,6 +648,10 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
         int i, j, n;
         unsigned int regs[4];
         unsigned char *desc = (unsigned char *)regs;
+
+        if (c->cpuid_level < 2)
+                return;
+
         /* Number of times to iterate */
         n = cpuid_eax(2) & 0xFF;
 
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index d44f7829968e..e4595f105910 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -92,7 +92,8 @@ int arch_show_interrupts(struct seq_file *p, int prec)
         seq_printf(p, "  Rescheduling interrupts\n");
         seq_printf(p, "%*s: ", prec, "CAL");
         for_each_online_cpu(j)
-                seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
+                seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
+                                        irq_stats(j)->irq_tlb_count);
         seq_printf(p, "  Function call interrupts\n");
         seq_printf(p, "%*s: ", prec, "TLB");
         for_each_online_cpu(j)
@@ -147,7 +148,6 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
 #ifdef CONFIG_SMP
         sum += irq_stats(cpu)->irq_resched_count;
         sum += irq_stats(cpu)->irq_call_count;
-        sum += irq_stats(cpu)->irq_tlb_count;
 #endif
 #ifdef CONFIG_X86_THERMAL_VECTOR
         sum += irq_stats(cpu)->irq_thermal_count;
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 613cd83e8c0c..0777f042e400 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -98,6 +98,8 @@ static void flush_tlb_func(void *info)
 {
         struct flush_tlb_info *f = info;
 
+        inc_irq_stat(irq_tlb_count);
+
         if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
                 return;
 
@@ -320,7 +322,7 @@ static ssize_t tlbflush_write_file(struct file *file,
         if (kstrtos8(buf, 0, &shift))
                 return -EINVAL;
 
-        if (shift > 64)
+        if (shift < -1 || shift >= BITS_PER_LONG)
                 return -EINVAL;
 
         tlb_flushall_shift = shift;