Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/mm changes from Peter Anvin:
 "The big change here is the patchset by Alex Shi to use INVLPG to flush
  only the affected pages when we only need to flush a small page range.

  It also removes the special INVALIDATE_TLB_VECTOR interrupts (32
  vectors!) and replace it with an ordinary IPI function call."

Fix up trivial conflicts in arch/x86/include/asm/apic.h (added code next
to changed line)

* 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/tlb: Fix build warning and crash when building for !SMP
  x86/tlb: do flush_tlb_kernel_range by 'invlpg'
  x86/tlb: replace INVALIDATE_TLB_VECTOR by CALL_FUNCTION_VECTOR
  x86/tlb: enable tlb flush range support for x86
  mm/mmu_gather: enable tlb flush range in generic mmu_gather
  x86/tlb: add tlb_flushall_shift knob into debugfs
  x86/tlb: add tlb_flushall_shift for specific CPU
  x86/tlb: fall back to flush all when meet a THP large page
  x86/flush_tlb: try flush_tlb_single one by one in flush_tlb_range
  x86/tlb_info: get last level TLB entry number of CPU
  x86: Add read_mostly declaration/definition to variables from smp.h
  x86: Define early read-mostly per-cpu macros

8 files changed, 224 insertions, 99 deletions

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 98e24131ff3a..24deb3082328 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -75,8 +75,8 @@ physid_mask_t phys_cpu_present_map;
 /*
  * Map cpu index to physical APIC ID
  */
-DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
-DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
 EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
 
@@ -88,7 +88,7 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
  * used for the mapping.  This is where the behaviors of x86_64 and 32
  * actually diverge.  Let's keep it ugly for now.
  */
-DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
 
 /*
  * Knob to control our willingness to enable the local APIC.
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 5bbc082c47ad..46d8786d655e 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -452,6 +452,35 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
         c->x86_cache_size = l2size;
 }
 
+u16 __read_mostly tlb_lli_4k[NR_INFO];
+u16 __read_mostly tlb_lli_2m[NR_INFO];
+u16 __read_mostly tlb_lli_4m[NR_INFO];
+u16 __read_mostly tlb_lld_4k[NR_INFO];
+u16 __read_mostly tlb_lld_2m[NR_INFO];
+u16 __read_mostly tlb_lld_4m[NR_INFO];
+
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ *   flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+s8  __read_mostly tlb_flushall_shift = -1;
+
+void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+        if (this_cpu->c_detect_tlb)
+                this_cpu->c_detect_tlb(c);
+
+        printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+                "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n"          \
+                "tlb_flushall_shift is 0x%x\n",
+                tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
+                tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
+                tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+                tlb_flushall_shift);
+}
+
 void __cpuinit detect_ht(struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_X86_HT
@@ -911,6 +940,8 @@ void __init identify_boot_cpu(void)
 #else
         vgetcpu_set_mode();
 #endif
+        if (boot_cpu_data.cpuid_level >= 2)
+                cpu_detect_tlb(&boot_cpu_data);
 }
 
 void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 8bacc7826fb3..4041c24ae7db 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -20,10 +20,19 @@ struct cpu_dev {
         void            (*c_bsp_init)(struct cpuinfo_x86 *);
         void            (*c_init)(struct cpuinfo_x86 *);
         void            (*c_identify)(struct cpuinfo_x86 *);
+        void            (*c_detect_tlb)(struct cpuinfo_x86 *);
         unsigned int    (*c_size_cache)(struct cpuinfo_x86 *, unsigned int);
         int             c_x86_vendor;
 };
 
+struct _tlb_table {
+        unsigned char descriptor;
+        char tlb_type;
+        unsigned int entries;
+        /* unsigned int ways; */
+        char info[128];
+};
+
 #define cpu_dev_register(cpu_devX) \
         static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \
         __attribute__((__section__(".x86_cpu_dev.init"))) = \
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3e6ff6cbf42a..0a4ce2980a5a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -491,6 +491,181 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i
 }
 #endif
 
+#define TLB_INST_4K     0x01
+#define TLB_INST_4M     0x02
+#define TLB_INST_2M_4M  0x03
+
+#define TLB_INST_ALL    0x05
+#define TLB_INST_1G     0x06
+
+#define TLB_DATA_4K     0x11
+#define TLB_DATA_4M     0x12
+#define TLB_DATA_2M_4M  0x13
+#define TLB_DATA_4K_4M  0x14
+
+#define TLB_DATA_1G     0x16
+
+#define TLB_DATA0_4K    0x21
+#define TLB_DATA0_4M    0x22
+#define TLB_DATA0_2M_4M 0x23
+
+#define STLB_4K         0x41
+
+static const struct _tlb_table intel_tlb_table[] __cpuinitconst = {
+        { 0x01, TLB_INST_4K,            32,     " TLB_INST 4 KByte pages, 4-way set associative" },
+        { 0x02, TLB_INST_4M,            2,      " TLB_INST 4 MByte pages, full associative" },
+        { 0x03, TLB_DATA_4K,            64,     " TLB_DATA 4 KByte pages, 4-way set associative" },
+        { 0x04, TLB_DATA_4M,            8,      " TLB_DATA 4 MByte pages, 4-way set associative" },
+        { 0x05, TLB_DATA_4M,            32,     " TLB_DATA 4 MByte pages, 4-way set associative" },
+        { 0x0b, TLB_INST_4M,            4,      " TLB_INST 4 MByte pages, 4-way set associative" },
+        { 0x4f, TLB_INST_4K,            32,     " TLB_INST 4 KByte pages */" },
+        { 0x50, TLB_INST_ALL,           64,     " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+        { 0x51, TLB_INST_ALL,           128,    " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+        { 0x52, TLB_INST_ALL,           256,    " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+        { 0x55, TLB_INST_2M_4M,         7,      " TLB_INST 2-MByte or 4-MByte pages, fully associative" },
+        { 0x56, TLB_DATA0_4M,           16,     " TLB_DATA0 4 MByte pages, 4-way set associative" },
+        { 0x57, TLB_DATA0_4K,           16,     " TLB_DATA0 4 KByte pages, 4-way associative" },
+        { 0x59, TLB_DATA0_4K,           16,     " TLB_DATA0 4 KByte pages, fully associative" },
+        { 0x5a, TLB_DATA0_2M_4M,        32,     " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" },
+        { 0x5b, TLB_DATA_4K_4M,         64,     " TLB_DATA 4 KByte and 4 MByte pages" },
+        { 0x5c, TLB_DATA_4K_4M,         128,    " TLB_DATA 4 KByte and 4 MByte pages" },
+        { 0x5d, TLB_DATA_4K_4M,         256,    " TLB_DATA 4 KByte and 4 MByte pages" },
+        { 0xb0, TLB_INST_4K,            128,    " TLB_INST 4 KByte pages, 4-way set associative" },
+        { 0xb1, TLB_INST_2M_4M,         4,      " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" },
+        { 0xb2, TLB_INST_4K,            64,     " TLB_INST 4KByte pages, 4-way set associative" },
+        { 0xb3, TLB_DATA_4K,            128,    " TLB_DATA 4 KByte pages, 4-way set associative" },
+        { 0xb4, TLB_DATA_4K,            256,    " TLB_DATA 4 KByte pages, 4-way associative" },
+        { 0xba, TLB_DATA_4K,            64,     " TLB_DATA 4 KByte pages, 4-way associative" },
+        { 0xc0, TLB_DATA_4K_4M,         8,      " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
+        { 0xca, STLB_4K,                512,    " STLB 4 KByte pages, 4-way associative" },
+        { 0x00, 0, 0 }
+};
+
+static void __cpuinit intel_tlb_lookup(const unsigned char desc)
+{
+        unsigned char k;
+        if (desc == 0)
+                return;
+
+        /* look up this descriptor in the table */
+        for (k = 0; intel_tlb_table[k].descriptor != desc && \
+                        intel_tlb_table[k].descriptor != 0; k++)
+                ;
+
+        if (intel_tlb_table[k].tlb_type == 0)
+                return;
+
+        switch (intel_tlb_table[k].tlb_type) {
+        case STLB_4K:
+                if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_INST_ALL:
+                if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_INST_4K:
+                if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_INST_4M:
+                if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_INST_2M_4M:
+                if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_DATA_4K:
+        case TLB_DATA0_4K:
+                if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_DATA_4M:
+        case TLB_DATA0_4M:
+                if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_DATA_2M_4M:
+        case TLB_DATA0_2M_4M:
+                if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        case TLB_DATA_4K_4M:
+                if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+                if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+                        tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+                break;
+        }
+}
+
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+        if (!cpu_has_invlpg) {
+                tlb_flushall_shift = -1;
+                return;
+        }
+        switch ((c->x86 << 8) + c->x86_model) {
+        case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+        case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+        case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+        case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+                tlb_flushall_shift = -1;
+                break;
+        case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+        case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+        case 0x625: /* 32 nm nehalem, "Clarkdale" */
+        case 0x62c: /* 32 nm nehalem, "Gulftown" */
+        case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+        case 0x62f: /* 32 nm Xeon E7 */
+                tlb_flushall_shift = 6;
+                break;
+        case 0x62a: /* SandyBridge */
+        case 0x62d: /* SandyBridge, "Romely-EP" */
+                tlb_flushall_shift = 5;
+                break;
+        case 0x63a: /* Ivybridge */
+                tlb_flushall_shift = 1;
+                break;
+        default:
+                tlb_flushall_shift = 6;
+        }
+}
+
+static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
+{
+        int i, j, n;
+        unsigned int regs[4];
+        unsigned char *desc = (unsigned char *)regs;
+        /* Number of times to iterate */
+        n = cpuid_eax(2) & 0xFF;
+
+        for (i = 0 ; i < n ; i++) {
+                cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
+
+                /* If bit 31 is set, this is an unknown format */
+                for (j = 0 ; j < 3 ; j++)
+                        if (regs[j] & (1 << 31))
+                                regs[j] = 0;
+
+                /* Byte 0 is level count, not a descriptor */
+                for (j = 1 ; j < 16 ; j++)
+                        intel_tlb_lookup(desc[j]);
+        }
+        intel_tlb_flushall_shift_set(c);
+}
+
 static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
         .c_vendor       = "Intel",
         .c_ident        = { "GenuineIntel" },
@@ -546,6 +721,7 @@ static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
         },
         .c_size_cache   = intel_size_cache,
 #endif
+        .c_detect_tlb   = intel_detect_tlb,
         .c_early_init   = early_init_intel,
         .c_init         = init_intel,
         .c_x86_vendor   = X86_VENDOR_INTEL,
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 111f6bbd8b38..69babd8c834f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1048,24 +1048,6 @@ apicinterrupt LOCAL_TIMER_VECTOR \
 apicinterrupt X86_PLATFORM_IPI_VECTOR \
         x86_platform_ipi smp_x86_platform_ipi
 
-#ifdef CONFIG_SMP
-        ALIGN
-        INTR_FRAME
-.irp idx,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
-        16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
-.if NUM_INVALIDATE_TLB_VECTORS > \idx
-ENTRY(invalidate_interrupt\idx)
-        pushq_cfi $~(INVALIDATE_TLB_VECTOR_START+\idx)
-        jmp .Lcommon_invalidate_interrupt0
-        CFI_ADJUST_CFA_OFFSET -8
-END(invalidate_interrupt\idx)
-.endif
-.endr
-        CFI_ENDPROC
-apicinterrupt INVALIDATE_TLB_VECTOR_START, \
-        invalidate_interrupt0, smp_invalidate_interrupt
-#endif
-
 apicinterrupt THRESHOLD_APIC_VECTOR \
         threshold_interrupt smp_threshold_interrupt
 apicinterrupt THERMAL_APIC_VECTOR \
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 252981afd6c4..6e03b0d69138 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -171,79 +171,6 @@ static void __init smp_intr_init(void)
          */
         alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
 
-        /* IPIs for invalidation */
-#define ALLOC_INVTLB_VEC(NR) \
-        alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+NR, \
-                invalidate_interrupt##NR)
-
-        switch (NUM_INVALIDATE_TLB_VECTORS) {
-        default:
-                ALLOC_INVTLB_VEC(31);
-        case 31:
-                ALLOC_INVTLB_VEC(30);
-        case 30:
-                ALLOC_INVTLB_VEC(29);
-        case 29:
-                ALLOC_INVTLB_VEC(28);
-        case 28:
-                ALLOC_INVTLB_VEC(27);
-        case 27:
-                ALLOC_INVTLB_VEC(26);
-        case 26:
-                ALLOC_INVTLB_VEC(25);
-        case 25:
-                ALLOC_INVTLB_VEC(24);
-        case 24:
-                ALLOC_INVTLB_VEC(23);
-        case 23:
-                ALLOC_INVTLB_VEC(22);
-        case 22:
-                ALLOC_INVTLB_VEC(21);
-        case 21:
-                ALLOC_INVTLB_VEC(20);
-        case 20:
-                ALLOC_INVTLB_VEC(19);
-        case 19:
-                ALLOC_INVTLB_VEC(18);
-        case 18:
-                ALLOC_INVTLB_VEC(17);
-        case 17:
-                ALLOC_INVTLB_VEC(16);
-        case 16:
-                ALLOC_INVTLB_VEC(15);
-        case 15:
-                ALLOC_INVTLB_VEC(14);
-        case 14:
-                ALLOC_INVTLB_VEC(13);
-        case 13:
-                ALLOC_INVTLB_VEC(12);
-        case 12:
-                ALLOC_INVTLB_VEC(11);
-        case 11:
-                ALLOC_INVTLB_VEC(10);
-        case 10:
-                ALLOC_INVTLB_VEC(9);
-        case 9:
-                ALLOC_INVTLB_VEC(8);
-        case 8:
-                ALLOC_INVTLB_VEC(7);
-        case 7:
-                ALLOC_INVTLB_VEC(6);
-        case 6:
-                ALLOC_INVTLB_VEC(5);
-        case 5:
-                ALLOC_INVTLB_VEC(4);
-        case 4:
-                ALLOC_INVTLB_VEC(3);
-        case 3:
-                ALLOC_INVTLB_VEC(2);
-        case 2:
-                ALLOC_INVTLB_VEC(1);
-        case 1:
-                ALLOC_INVTLB_VEC(0);
-                break;
-        }
-
         /* IPI for generic function call */
         alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
 
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 5a98aa272184..5cdff0357746 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -21,7 +21,7 @@
 #include <asm/cpu.h>
 #include <asm/stackprotector.h>
 
-DEFINE_PER_CPU(int, cpu_number);
+DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
 EXPORT_PER_CPU_SYMBOL(cpu_number);
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c1a310fb8309..7c5a8c314c02 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -106,17 +106,17 @@ int smp_num_siblings = 1;
 EXPORT_SYMBOL(smp_num_siblings);
 
 /* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
+DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
 
 /* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
 EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
 
 /* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
 EXPORT_PER_CPU_SYMBOL(cpu_core_map);
 
-DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
 
 /* Per CPU bogomips and other parameters */
 DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);