diff options
| -rw-r--r-- | arch/x86/Kconfig | 7 | ||||
| -rw-r--r-- | arch/x86/include/asm/amd_nb.h | 2 | ||||
| -rw-r--r-- | arch/x86/include/asm/tlbflush.h | 6 | ||||
| -rw-r--r-- | arch/x86/kernel/amd_nb.c | 2 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/amd.c | 5 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/intel.c | 10 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/microcode/amd_early.c | 43 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/mtrr/generic.c | 4 | ||||
| -rw-r--r-- | arch/x86/kernel/irq.c | 9 | ||||
| -rw-r--r-- | arch/x86/kernel/quirks.c | 37 | ||||
| -rw-r--r-- | arch/x86/mm/numa_32.c | 2 | ||||
| -rw-r--r-- | arch/x86/mm/srat.c | 16 | ||||
| -rw-r--r-- | arch/x86/mm/tlb.c | 52 | ||||
| -rw-r--r-- | arch/x86/platform/efi/efi-bgrt.c | 10 | ||||
| -rw-r--r-- | include/linux/vm_event_item.h | 4 | ||||
| -rw-r--r-- | include/linux/vmstat.h | 8 | ||||
| -rw-r--r-- | lib/Makefile | 1 | ||||
| -rw-r--r-- | mm/vmstat.c | 4 |
18 files changed, 138 insertions, 84 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 940e50ebfafa..0af5250d914f 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
| @@ -444,6 +444,7 @@ config X86_INTEL_MID | |||
| 444 | bool "Intel MID platform support" | 444 | bool "Intel MID platform support" |
| 445 | depends on X86_32 | 445 | depends on X86_32 |
| 446 | depends on X86_EXTENDED_PLATFORM | 446 | depends on X86_EXTENDED_PLATFORM |
| 447 | depends on X86_PLATFORM_DEVICES | ||
| 447 | depends on PCI | 448 | depends on PCI |
| 448 | depends on PCI_GOANY | 449 | depends on PCI_GOANY |
| 449 | depends on X86_IO_APIC | 450 | depends on X86_IO_APIC |
| @@ -1051,9 +1052,9 @@ config MICROCODE_INTEL | |||
| 1051 | This options enables microcode patch loading support for Intel | 1052 | This options enables microcode patch loading support for Intel |
| 1052 | processors. | 1053 | processors. |
| 1053 | 1054 | ||
| 1054 | For latest news and information on obtaining all the required | 1055 | For the current Intel microcode data package go to |
| 1055 | Intel ingredients for this driver, check: | 1056 | <https://downloadcenter.intel.com> and search for |
| 1056 | <http://www.urbanmyth.org/microcode/>. | 1057 | 'Linux Processor Microcode Data File'. |
| 1057 | 1058 | ||
| 1058 | config MICROCODE_AMD | 1059 | config MICROCODE_AMD |
| 1059 | bool "AMD microcode loading support" | 1060 | bool "AMD microcode loading support" |
diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index a54ee1d054d9..aaac3b2fb746 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h | |||
| @@ -19,7 +19,7 @@ extern int amd_cache_northbridges(void); | |||
| 19 | extern void amd_flush_garts(void); | 19 | extern void amd_flush_garts(void); |
| 20 | extern int amd_numa_init(void); | 20 | extern int amd_numa_init(void); |
| 21 | extern int amd_get_subcaches(int); | 21 | extern int amd_get_subcaches(int); |
| 22 | extern int amd_set_subcaches(int, int); | 22 | extern int amd_set_subcaches(int, unsigned long); |
| 23 | 23 | ||
| 24 | struct amd_l3_cache { | 24 | struct amd_l3_cache { |
| 25 | unsigned indices; | 25 | unsigned indices; |
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index e6d90babc245..04905bfc508b 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h | |||
| @@ -62,7 +62,7 @@ static inline void __flush_tlb_all(void) | |||
| 62 | 62 | ||
| 63 | static inline void __flush_tlb_one(unsigned long addr) | 63 | static inline void __flush_tlb_one(unsigned long addr) |
| 64 | { | 64 | { |
| 65 | count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); | 65 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); |
| 66 | __flush_tlb_single(addr); | 66 | __flush_tlb_single(addr); |
| 67 | } | 67 | } |
| 68 | 68 | ||
| @@ -93,13 +93,13 @@ static inline void __flush_tlb_one(unsigned long addr) | |||
| 93 | */ | 93 | */ |
| 94 | static inline void __flush_tlb_up(void) | 94 | static inline void __flush_tlb_up(void) |
| 95 | { | 95 | { |
| 96 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 96 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 97 | __flush_tlb(); | 97 | __flush_tlb(); |
| 98 | } | 98 | } |
| 99 | 99 | ||
| 100 | static inline void flush_tlb_all(void) | 100 | static inline void flush_tlb_all(void) |
| 101 | { | 101 | { |
| 102 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 102 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 103 | __flush_tlb_all(); | 103 | __flush_tlb_all(); |
| 104 | } | 104 | } |
| 105 | 105 | ||
diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 59554dca96ec..dec8de4e1663 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c | |||
| @@ -179,7 +179,7 @@ int amd_get_subcaches(int cpu) | |||
| 179 | return (mask >> (4 * cuid)) & 0xf; | 179 | return (mask >> (4 * cuid)) & 0xf; |
| 180 | } | 180 | } |
| 181 | 181 | ||
| 182 | int amd_set_subcaches(int cpu, int mask) | 182 | int amd_set_subcaches(int cpu, unsigned long mask) |
| 183 | { | 183 | { |
| 184 | static unsigned int reset, ban; | 184 | static unsigned int reset, ban; |
| 185 | struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu)); | 185 | struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu)); |
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index d3153e281d72..c67ffa686064 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c | |||
| @@ -767,10 +767,7 @@ static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size) | |||
| 767 | 767 | ||
| 768 | static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) | 768 | static void cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c) |
| 769 | { | 769 | { |
| 770 | tlb_flushall_shift = 5; | 770 | tlb_flushall_shift = 6; |
| 771 | |||
| 772 | if (c->x86 <= 0x11) | ||
| 773 | tlb_flushall_shift = 4; | ||
| 774 | } | 771 | } |
| 775 | 772 | ||
| 776 | static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) | 773 | static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c) |
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 3db61c644e44..5cd9bfabd645 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c | |||
| @@ -640,21 +640,17 @@ static void intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) | |||
| 640 | case 0x61d: /* six-core 45 nm xeon "Dunnington" */ | 640 | case 0x61d: /* six-core 45 nm xeon "Dunnington" */ |
| 641 | tlb_flushall_shift = -1; | 641 | tlb_flushall_shift = -1; |
| 642 | break; | 642 | break; |
| 643 | case 0x63a: /* Ivybridge */ | ||
| 644 | tlb_flushall_shift = 2; | ||
| 645 | break; | ||
| 643 | case 0x61a: /* 45 nm nehalem, "Bloomfield" */ | 646 | case 0x61a: /* 45 nm nehalem, "Bloomfield" */ |
| 644 | case 0x61e: /* 45 nm nehalem, "Lynnfield" */ | 647 | case 0x61e: /* 45 nm nehalem, "Lynnfield" */ |
| 645 | case 0x625: /* 32 nm nehalem, "Clarkdale" */ | 648 | case 0x625: /* 32 nm nehalem, "Clarkdale" */ |
| 646 | case 0x62c: /* 32 nm nehalem, "Gulftown" */ | 649 | case 0x62c: /* 32 nm nehalem, "Gulftown" */ |
| 647 | case 0x62e: /* 45 nm nehalem-ex, "Beckton" */ | 650 | case 0x62e: /* 45 nm nehalem-ex, "Beckton" */ |
| 648 | case 0x62f: /* 32 nm Xeon E7 */ | 651 | case 0x62f: /* 32 nm Xeon E7 */ |
| 649 | tlb_flushall_shift = 6; | ||
| 650 | break; | ||
| 651 | case 0x62a: /* SandyBridge */ | 652 | case 0x62a: /* SandyBridge */ |
| 652 | case 0x62d: /* SandyBridge, "Romely-EP" */ | 653 | case 0x62d: /* SandyBridge, "Romely-EP" */ |
| 653 | tlb_flushall_shift = 5; | ||
| 654 | break; | ||
| 655 | case 0x63a: /* Ivybridge */ | ||
| 656 | tlb_flushall_shift = 1; | ||
| 657 | break; | ||
| 658 | default: | 654 | default: |
| 659 | tlb_flushall_shift = 6; | 655 | tlb_flushall_shift = 6; |
| 660 | } | 656 | } |
diff --git a/arch/x86/kernel/cpu/microcode/amd_early.c b/arch/x86/kernel/cpu/microcode/amd_early.c index 8384c0fa206f..617a9e284245 100644 --- a/arch/x86/kernel/cpu/microcode/amd_early.c +++ b/arch/x86/kernel/cpu/microcode/amd_early.c | |||
| @@ -285,6 +285,15 @@ static void __init collect_cpu_sig_on_bsp(void *arg) | |||
| 285 | 285 | ||
| 286 | uci->cpu_sig.sig = cpuid_eax(0x00000001); | 286 | uci->cpu_sig.sig = cpuid_eax(0x00000001); |
| 287 | } | 287 | } |
| 288 | |||
| 289 | static void __init get_bsp_sig(void) | ||
| 290 | { | ||
| 291 | unsigned int bsp = boot_cpu_data.cpu_index; | ||
| 292 | struct ucode_cpu_info *uci = ucode_cpu_info + bsp; | ||
| 293 | |||
| 294 | if (!uci->cpu_sig.sig) | ||
| 295 | smp_call_function_single(bsp, collect_cpu_sig_on_bsp, NULL, 1); | ||
| 296 | } | ||
| 288 | #else | 297 | #else |
| 289 | void load_ucode_amd_ap(void) | 298 | void load_ucode_amd_ap(void) |
| 290 | { | 299 | { |
| @@ -337,31 +346,37 @@ void load_ucode_amd_ap(void) | |||
| 337 | 346 | ||
| 338 | int __init save_microcode_in_initrd_amd(void) | 347 | int __init save_microcode_in_initrd_amd(void) |
| 339 | { | 348 | { |
| 349 | unsigned long cont; | ||
| 340 | enum ucode_state ret; | 350 | enum ucode_state ret; |
| 341 | u32 eax; | 351 | u32 eax; |
| 342 | 352 | ||
| 343 | #ifdef CONFIG_X86_32 | 353 | if (!container) |
| 344 | unsigned int bsp = boot_cpu_data.cpu_index; | 354 | return -EINVAL; |
| 345 | struct ucode_cpu_info *uci = ucode_cpu_info + bsp; | ||
| 346 | |||
| 347 | if (!uci->cpu_sig.sig) | ||
| 348 | smp_call_function_single(bsp, collect_cpu_sig_on_bsp, NULL, 1); | ||
| 349 | 355 | ||
| 356 | #ifdef CONFIG_X86_32 | ||
| 357 | get_bsp_sig(); | ||
| 358 | cont = (unsigned long)container; | ||
| 359 | #else | ||
| 350 | /* | 360 | /* |
| 351 | * Take into account the fact that the ramdisk might get relocated | 361 | * We need the physical address of the container for both bitness since |
| 352 | * and therefore we need to recompute the container's position in | 362 | * boot_params.hdr.ramdisk_image is a physical address. |
| 353 | * virtual memory space. | ||
| 354 | */ | 363 | */ |
| 355 | container = (u8 *)(__va((u32)relocated_ramdisk) + | 364 | cont = __pa(container); |
| 356 | ((u32)container - boot_params.hdr.ramdisk_image)); | ||
| 357 | #endif | 365 | #endif |
| 366 | |||
| 367 | /* | ||
| 368 | * Take into account the fact that the ramdisk might get relocated and | ||
| 369 | * therefore we need to recompute the container's position in virtual | ||
| 370 | * memory space. | ||
| 371 | */ | ||
| 372 | if (relocated_ramdisk) | ||
| 373 | container = (u8 *)(__va(relocated_ramdisk) + | ||
| 374 | (cont - boot_params.hdr.ramdisk_image)); | ||
| 375 | |||
| 358 | if (ucode_new_rev) | 376 | if (ucode_new_rev) |
| 359 | pr_info("microcode: updated early to new patch_level=0x%08x\n", | 377 | pr_info("microcode: updated early to new patch_level=0x%08x\n", |
| 360 | ucode_new_rev); | 378 | ucode_new_rev); |
| 361 | 379 | ||
| 362 | if (!container) | ||
| 363 | return -EINVAL; | ||
| 364 | |||
| 365 | eax = cpuid_eax(0x00000001); | 380 | eax = cpuid_eax(0x00000001); |
| 366 | eax = ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff); | 381 | eax = ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff); |
| 367 | 382 | ||
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index ce2d0a2c3e4f..0e25a1bc5ab5 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c | |||
| @@ -683,7 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) | |||
| 683 | } | 683 | } |
| 684 | 684 | ||
| 685 | /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ | 685 | /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ |
| 686 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 686 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 687 | __flush_tlb(); | 687 | __flush_tlb(); |
| 688 | 688 | ||
| 689 | /* Save MTRR state */ | 689 | /* Save MTRR state */ |
| @@ -697,7 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) | |||
| 697 | static void post_set(void) __releases(set_atomicity_lock) | 697 | static void post_set(void) __releases(set_atomicity_lock) |
| 698 | { | 698 | { |
| 699 | /* Flush TLBs (no need to flush caches - they are disabled) */ | 699 | /* Flush TLBs (no need to flush caches - they are disabled) */ |
| 700 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 700 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 701 | __flush_tlb(); | 701 | __flush_tlb(); |
| 702 | 702 | ||
| 703 | /* Intel (P6) standard MTRRs */ | 703 | /* Intel (P6) standard MTRRs */ |
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index dbb60878b744..d99f31d9a750 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c | |||
| @@ -266,6 +266,14 @@ __visible void smp_trace_x86_platform_ipi(struct pt_regs *regs) | |||
| 266 | EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); | 266 | EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq); |
| 267 | 267 | ||
| 268 | #ifdef CONFIG_HOTPLUG_CPU | 268 | #ifdef CONFIG_HOTPLUG_CPU |
| 269 | |||
| 270 | /* These two declarations are only used in check_irq_vectors_for_cpu_disable() | ||
| 271 | * below, which is protected by stop_machine(). Putting them on the stack | ||
| 272 | * results in a stack frame overflow. Dynamically allocating could result in a | ||
| 273 | * failure so declare these two cpumasks as global. | ||
| 274 | */ | ||
| 275 | static struct cpumask affinity_new, online_new; | ||
| 276 | |||
| 269 | /* | 277 | /* |
| 270 | * This cpu is going to be removed and its vectors migrated to the remaining | 278 | * This cpu is going to be removed and its vectors migrated to the remaining |
| 271 | * online cpus. Check to see if there are enough vectors in the remaining cpus. | 279 | * online cpus. Check to see if there are enough vectors in the remaining cpus. |
| @@ -277,7 +285,6 @@ int check_irq_vectors_for_cpu_disable(void) | |||
| 277 | unsigned int this_cpu, vector, this_count, count; | 285 | unsigned int this_cpu, vector, this_count, count; |
| 278 | struct irq_desc *desc; | 286 | struct irq_desc *desc; |
| 279 | struct irq_data *data; | 287 | struct irq_data *data; |
| 280 | struct cpumask affinity_new, online_new; | ||
| 281 | 288 | ||
| 282 | this_cpu = smp_processor_id(); | 289 | this_cpu = smp_processor_id(); |
| 283 | cpumask_copy(&online_new, cpu_online_mask); | 290 | cpumask_copy(&online_new, cpu_online_mask); |
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c index 04ee1e2e4c02..7c6acd4b8995 100644 --- a/arch/x86/kernel/quirks.c +++ b/arch/x86/kernel/quirks.c | |||
| @@ -571,3 +571,40 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_15H_NB_F5, | |||
| 571 | quirk_amd_nb_node); | 571 | quirk_amd_nb_node); |
| 572 | 572 | ||
| 573 | #endif | 573 | #endif |
| 574 | |||
| 575 | #ifdef CONFIG_PCI | ||
| 576 | /* | ||
| 577 | * Processor does not ensure DRAM scrub read/write sequence | ||
| 578 | * is atomic wrt accesses to CC6 save state area. Therefore | ||
| 579 | * if a concurrent scrub read/write access is to same address | ||
| 580 | * the entry may appear as if it is not written. This quirk | ||
| 581 | * applies to Fam16h models 00h-0Fh | ||
| 582 | * | ||
| 583 | * See "Revision Guide" for AMD F16h models 00h-0fh, | ||
| 584 | * document 51810 rev. 3.04, Nov 2013 | ||
| 585 | */ | ||
| 586 | static void amd_disable_seq_and_redirect_scrub(struct pci_dev *dev) | ||
| 587 | { | ||
| 588 | u32 val; | ||
| 589 | |||
| 590 | /* | ||
| 591 | * Suggested workaround: | ||
| 592 | * set D18F3x58[4:0] = 00h and set D18F3x5C[0] = 0b | ||
| 593 | */ | ||
| 594 | pci_read_config_dword(dev, 0x58, &val); | ||
| 595 | if (val & 0x1F) { | ||
| 596 | val &= ~(0x1F); | ||
| 597 | pci_write_config_dword(dev, 0x58, val); | ||
| 598 | } | ||
| 599 | |||
| 600 | pci_read_config_dword(dev, 0x5C, &val); | ||
| 601 | if (val & BIT(0)) { | ||
| 602 | val &= ~BIT(0); | ||
| 603 | pci_write_config_dword(dev, 0x5c, val); | ||
| 604 | } | ||
| 605 | } | ||
| 606 | |||
| 607 | DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3, | ||
| 608 | amd_disable_seq_and_redirect_scrub); | ||
| 609 | |||
| 610 | #endif | ||
diff --git a/arch/x86/mm/numa_32.c b/arch/x86/mm/numa_32.c index 0342d27ca798..47b6436e41c2 100644 --- a/arch/x86/mm/numa_32.c +++ b/arch/x86/mm/numa_32.c | |||
| @@ -52,6 +52,8 @@ void memory_present(int nid, unsigned long start, unsigned long end) | |||
| 52 | nid, start, end); | 52 | nid, start, end); |
| 53 | printk(KERN_DEBUG " Setting physnode_map array to node %d for pfns:\n", nid); | 53 | printk(KERN_DEBUG " Setting physnode_map array to node %d for pfns:\n", nid); |
| 54 | printk(KERN_DEBUG " "); | 54 | printk(KERN_DEBUG " "); |
| 55 | start = round_down(start, PAGES_PER_SECTION); | ||
| 56 | end = round_up(end, PAGES_PER_SECTION); | ||
| 55 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { | 57 | for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { |
| 56 | physnode_map[pfn / PAGES_PER_SECTION] = nid; | 58 | physnode_map[pfn / PAGES_PER_SECTION] = nid; |
| 57 | printk(KERN_CONT "%lx ", pfn); | 59 | printk(KERN_CONT "%lx ", pfn); |
diff --git a/arch/x86/mm/srat.c b/arch/x86/mm/srat.c index 1a25187e151e..1953e9c9391a 100644 --- a/arch/x86/mm/srat.c +++ b/arch/x86/mm/srat.c | |||
| @@ -42,15 +42,25 @@ static __init inline int srat_disabled(void) | |||
| 42 | return acpi_numa < 0; | 42 | return acpi_numa < 0; |
| 43 | } | 43 | } |
| 44 | 44 | ||
| 45 | /* Callback for SLIT parsing */ | 45 | /* |
| 46 | * Callback for SLIT parsing. pxm_to_node() returns NUMA_NO_NODE for | ||
| 47 | * I/O localities since SRAT does not list them. I/O localities are | ||
| 48 | * not supported at this point. | ||
| 49 | */ | ||
| 46 | void __init acpi_numa_slit_init(struct acpi_table_slit *slit) | 50 | void __init acpi_numa_slit_init(struct acpi_table_slit *slit) |
| 47 | { | 51 | { |
| 48 | int i, j; | 52 | int i, j; |
| 49 | 53 | ||
| 50 | for (i = 0; i < slit->locality_count; i++) | 54 | for (i = 0; i < slit->locality_count; i++) { |
| 51 | for (j = 0; j < slit->locality_count; j++) | 55 | if (pxm_to_node(i) == NUMA_NO_NODE) |
| 56 | continue; | ||
| 57 | for (j = 0; j < slit->locality_count; j++) { | ||
| 58 | if (pxm_to_node(j) == NUMA_NO_NODE) | ||
| 59 | continue; | ||
| 52 | numa_set_distance(pxm_to_node(i), pxm_to_node(j), | 60 | numa_set_distance(pxm_to_node(i), pxm_to_node(j), |
| 53 | slit->entry[slit->locality_count * i + j]); | 61 | slit->entry[slit->locality_count * i + j]); |
| 62 | } | ||
| 63 | } | ||
| 54 | } | 64 | } |
| 55 | 65 | ||
| 56 | /* Callback for Proximity Domain -> x2APIC mapping */ | 66 | /* Callback for Proximity Domain -> x2APIC mapping */ |
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index ae699b3bbac8..dd8dda167a24 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c | |||
| @@ -103,7 +103,7 @@ static void flush_tlb_func(void *info) | |||
| 103 | if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) | 103 | if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) |
| 104 | return; | 104 | return; |
| 105 | 105 | ||
| 106 | count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); | 106 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); |
| 107 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { | 107 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { |
| 108 | if (f->flush_end == TLB_FLUSH_ALL) | 108 | if (f->flush_end == TLB_FLUSH_ALL) |
| 109 | local_flush_tlb(); | 109 | local_flush_tlb(); |
| @@ -131,7 +131,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask, | |||
| 131 | info.flush_start = start; | 131 | info.flush_start = start; |
| 132 | info.flush_end = end; | 132 | info.flush_end = end; |
| 133 | 133 | ||
| 134 | count_vm_event(NR_TLB_REMOTE_FLUSH); | 134 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); |
| 135 | if (is_uv_system()) { | 135 | if (is_uv_system()) { |
| 136 | unsigned int cpu; | 136 | unsigned int cpu; |
| 137 | 137 | ||
| @@ -151,44 +151,19 @@ void flush_tlb_current_task(void) | |||
| 151 | 151 | ||
| 152 | preempt_disable(); | 152 | preempt_disable(); |
| 153 | 153 | ||
| 154 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 154 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 155 | local_flush_tlb(); | 155 | local_flush_tlb(); |
| 156 | if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) | 156 | if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) |
| 157 | flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); | 157 | flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); |
| 158 | preempt_enable(); | 158 | preempt_enable(); |
| 159 | } | 159 | } |
| 160 | 160 | ||
| 161 | /* | ||
| 162 | * It can find out the THP large page, or | ||
| 163 | * HUGETLB page in tlb_flush when THP disabled | ||
| 164 | */ | ||
| 165 | static inline unsigned long has_large_page(struct mm_struct *mm, | ||
| 166 | unsigned long start, unsigned long end) | ||
| 167 | { | ||
| 168 | pgd_t *pgd; | ||
| 169 | pud_t *pud; | ||
| 170 | pmd_t *pmd; | ||
| 171 | unsigned long addr = ALIGN(start, HPAGE_SIZE); | ||
| 172 | for (; addr < end; addr += HPAGE_SIZE) { | ||
| 173 | pgd = pgd_offset(mm, addr); | ||
| 174 | if (likely(!pgd_none(*pgd))) { | ||
| 175 | pud = pud_offset(pgd, addr); | ||
| 176 | if (likely(!pud_none(*pud))) { | ||
| 177 | pmd = pmd_offset(pud, addr); | ||
| 178 | if (likely(!pmd_none(*pmd))) | ||
| 179 | if (pmd_large(*pmd)) | ||
| 180 | return addr; | ||
| 181 | } | ||
| 182 | } | ||
| 183 | } | ||
| 184 | return 0; | ||
| 185 | } | ||
| 186 | |||
| 187 | void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, | 161 | void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, |
| 188 | unsigned long end, unsigned long vmflag) | 162 | unsigned long end, unsigned long vmflag) |
| 189 | { | 163 | { |
| 190 | unsigned long addr; | 164 | unsigned long addr; |
| 191 | unsigned act_entries, tlb_entries = 0; | 165 | unsigned act_entries, tlb_entries = 0; |
| 166 | unsigned long nr_base_pages; | ||
| 192 | 167 | ||
| 193 | preempt_disable(); | 168 | preempt_disable(); |
| 194 | if (current->active_mm != mm) | 169 | if (current->active_mm != mm) |
| @@ -210,21 +185,20 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, | |||
| 210 | tlb_entries = tlb_lli_4k[ENTRIES]; | 185 | tlb_entries = tlb_lli_4k[ENTRIES]; |
| 211 | else | 186 | else |
| 212 | tlb_entries = tlb_lld_4k[ENTRIES]; | 187 | tlb_entries = tlb_lld_4k[ENTRIES]; |
| 188 | |||
| 213 | /* Assume all of TLB entries was occupied by this task */ | 189 | /* Assume all of TLB entries was occupied by this task */ |
| 214 | act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm; | 190 | act_entries = tlb_entries >> tlb_flushall_shift; |
| 191 | act_entries = mm->total_vm > act_entries ? act_entries : mm->total_vm; | ||
| 192 | nr_base_pages = (end - start) >> PAGE_SHIFT; | ||
| 215 | 193 | ||
| 216 | /* tlb_flushall_shift is on balance point, details in commit log */ | 194 | /* tlb_flushall_shift is on balance point, details in commit log */ |
| 217 | if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) { | 195 | if (nr_base_pages > act_entries) { |
| 218 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 196 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
| 219 | local_flush_tlb(); | 197 | local_flush_tlb(); |
| 220 | } else { | 198 | } else { |
| 221 | if (has_large_page(mm, start, end)) { | ||
| 222 | local_flush_tlb(); | ||
| 223 | goto flush_all; | ||
| 224 | } | ||
| 225 | /* flush range by one by one 'invlpg' */ | 199 | /* flush range by one by one 'invlpg' */ |
| 226 | for (addr = start; addr < end; addr += PAGE_SIZE) { | 200 | for (addr = start; addr < end; addr += PAGE_SIZE) { |
| 227 | count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); | 201 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); |
| 228 | __flush_tlb_single(addr); | 202 | __flush_tlb_single(addr); |
| 229 | } | 203 | } |
| 230 | 204 | ||
| @@ -262,7 +236,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start) | |||
| 262 | 236 | ||
| 263 | static void do_flush_tlb_all(void *info) | 237 | static void do_flush_tlb_all(void *info) |
| 264 | { | 238 | { |
| 265 | count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); | 239 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); |
| 266 | __flush_tlb_all(); | 240 | __flush_tlb_all(); |
| 267 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY) | 241 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY) |
| 268 | leave_mm(smp_processor_id()); | 242 | leave_mm(smp_processor_id()); |
| @@ -270,7 +244,7 @@ static void do_flush_tlb_all(void *info) | |||
| 270 | 244 | ||
| 271 | void flush_tlb_all(void) | 245 | void flush_tlb_all(void) |
| 272 | { | 246 | { |
| 273 | count_vm_event(NR_TLB_REMOTE_FLUSH); | 247 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); |
| 274 | on_each_cpu(do_flush_tlb_all, NULL, 1); | 248 | on_each_cpu(do_flush_tlb_all, NULL, 1); |
| 275 | } | 249 | } |
| 276 | 250 | ||
diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c index 7145ec63c520..4df9591eadad 100644 --- a/arch/x86/platform/efi/efi-bgrt.c +++ b/arch/x86/platform/efi/efi-bgrt.c | |||
| @@ -49,7 +49,8 @@ void __init efi_bgrt_init(void) | |||
| 49 | 49 | ||
| 50 | image = efi_lookup_mapped_addr(bgrt_tab->image_address); | 50 | image = efi_lookup_mapped_addr(bgrt_tab->image_address); |
| 51 | if (!image) { | 51 | if (!image) { |
| 52 | image = ioremap(bgrt_tab->image_address, sizeof(bmp_header)); | 52 | image = early_memremap(bgrt_tab->image_address, |
| 53 | sizeof(bmp_header)); | ||
| 53 | ioremapped = true; | 54 | ioremapped = true; |
| 54 | if (!image) | 55 | if (!image) |
| 55 | return; | 56 | return; |
| @@ -57,7 +58,7 @@ void __init efi_bgrt_init(void) | |||
| 57 | 58 | ||
| 58 | memcpy_fromio(&bmp_header, image, sizeof(bmp_header)); | 59 | memcpy_fromio(&bmp_header, image, sizeof(bmp_header)); |
| 59 | if (ioremapped) | 60 | if (ioremapped) |
| 60 | iounmap(image); | 61 | early_iounmap(image, sizeof(bmp_header)); |
| 61 | bgrt_image_size = bmp_header.size; | 62 | bgrt_image_size = bmp_header.size; |
| 62 | 63 | ||
| 63 | bgrt_image = kmalloc(bgrt_image_size, GFP_KERNEL); | 64 | bgrt_image = kmalloc(bgrt_image_size, GFP_KERNEL); |
| @@ -65,7 +66,8 @@ void __init efi_bgrt_init(void) | |||
| 65 | return; | 66 | return; |
| 66 | 67 | ||
| 67 | if (ioremapped) { | 68 | if (ioremapped) { |
| 68 | image = ioremap(bgrt_tab->image_address, bmp_header.size); | 69 | image = early_memremap(bgrt_tab->image_address, |
| 70 | bmp_header.size); | ||
| 69 | if (!image) { | 71 | if (!image) { |
| 70 | kfree(bgrt_image); | 72 | kfree(bgrt_image); |
| 71 | bgrt_image = NULL; | 73 | bgrt_image = NULL; |
| @@ -75,5 +77,5 @@ void __init efi_bgrt_init(void) | |||
| 75 | 77 | ||
| 76 | memcpy_fromio(bgrt_image, image, bgrt_image_size); | 78 | memcpy_fromio(bgrt_image, image, bgrt_image_size); |
| 77 | if (ioremapped) | 79 | if (ioremapped) |
| 78 | iounmap(image); | 80 | early_iounmap(image, bmp_header.size); |
| 79 | } | 81 | } |
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index c557c6d096de..3a712e2e7d76 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h | |||
| @@ -71,12 +71,14 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, | |||
| 71 | THP_ZERO_PAGE_ALLOC, | 71 | THP_ZERO_PAGE_ALLOC, |
| 72 | THP_ZERO_PAGE_ALLOC_FAILED, | 72 | THP_ZERO_PAGE_ALLOC_FAILED, |
| 73 | #endif | 73 | #endif |
| 74 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
| 74 | #ifdef CONFIG_SMP | 75 | #ifdef CONFIG_SMP |
| 75 | NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ | 76 | NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ |
| 76 | NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ | 77 | NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ |
| 77 | #endif | 78 | #endif /* CONFIG_SMP */ |
| 78 | NR_TLB_LOCAL_FLUSH_ALL, | 79 | NR_TLB_LOCAL_FLUSH_ALL, |
| 79 | NR_TLB_LOCAL_FLUSH_ONE, | 80 | NR_TLB_LOCAL_FLUSH_ONE, |
| 81 | #endif /* CONFIG_DEBUG_TLBFLUSH */ | ||
| 80 | NR_VM_EVENT_ITEMS | 82 | NR_VM_EVENT_ITEMS |
| 81 | }; | 83 | }; |
| 82 | 84 | ||
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index a67b38415768..67ce70c8279b 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h | |||
| @@ -83,6 +83,14 @@ static inline void vm_events_fold_cpu(int cpu) | |||
| 83 | #define count_vm_numa_events(x, y) do { (void)(y); } while (0) | 83 | #define count_vm_numa_events(x, y) do { (void)(y); } while (0) |
| 84 | #endif /* CONFIG_NUMA_BALANCING */ | 84 | #endif /* CONFIG_NUMA_BALANCING */ |
| 85 | 85 | ||
| 86 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
| 87 | #define count_vm_tlb_event(x) count_vm_event(x) | ||
| 88 | #define count_vm_tlb_events(x, y) count_vm_events(x, y) | ||
| 89 | #else | ||
| 90 | #define count_vm_tlb_event(x) do {} while (0) | ||
| 91 | #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) | ||
| 92 | #endif | ||
| 93 | |||
| 86 | #define __count_zone_vm_events(item, zone, delta) \ | 94 | #define __count_zone_vm_events(item, zone, delta) \ |
| 87 | __count_vm_events(item##_NORMAL - ZONE_NORMAL + \ | 95 | __count_vm_events(item##_NORMAL - ZONE_NORMAL + \ |
| 88 | zone_idx(zone), delta) | 96 | zone_idx(zone), delta) |
diff --git a/lib/Makefile b/lib/Makefile index 126b34f2eb16..48140e3ba73f 100644 --- a/lib/Makefile +++ b/lib/Makefile | |||
| @@ -45,6 +45,7 @@ obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o | |||
| 45 | obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o | 45 | obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o |
| 46 | obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o | 46 | obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o |
| 47 | 47 | ||
| 48 | GCOV_PROFILE_hweight.o := n | ||
| 48 | CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) | 49 | CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) |
| 49 | obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o | 50 | obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o |
| 50 | 51 | ||
diff --git a/mm/vmstat.c b/mm/vmstat.c index 72496140ac08..def5dd2fbe61 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c | |||
| @@ -851,12 +851,14 @@ const char * const vmstat_text[] = { | |||
| 851 | "thp_zero_page_alloc", | 851 | "thp_zero_page_alloc", |
| 852 | "thp_zero_page_alloc_failed", | 852 | "thp_zero_page_alloc_failed", |
| 853 | #endif | 853 | #endif |
| 854 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
| 854 | #ifdef CONFIG_SMP | 855 | #ifdef CONFIG_SMP |
| 855 | "nr_tlb_remote_flush", | 856 | "nr_tlb_remote_flush", |
| 856 | "nr_tlb_remote_flush_received", | 857 | "nr_tlb_remote_flush_received", |
| 857 | #endif | 858 | #endif /* CONFIG_SMP */ |
| 858 | "nr_tlb_local_flush_all", | 859 | "nr_tlb_local_flush_all", |
| 859 | "nr_tlb_local_flush_one", | 860 | "nr_tlb_local_flush_one", |
| 861 | #endif /* CONFIG_DEBUG_TLBFLUSH */ | ||
| 860 | 862 | ||
| 861 | #endif /* CONFIG_VM_EVENTS_COUNTERS */ | 863 | #endif /* CONFIG_VM_EVENTS_COUNTERS */ |
| 862 | }; | 864 | }; |
