diff options
-rw-r--r-- | arch/x86/include/asm/tlbflush.h | 6 | ||||
-rw-r--r-- | arch/x86/kernel/cpu/mtrr/generic.c | 4 | ||||
-rw-r--r-- | arch/x86/mm/tlb.c | 14 | ||||
-rw-r--r-- | include/linux/vm_event_item.h | 4 | ||||
-rw-r--r-- | include/linux/vmstat.h | 8 | ||||
-rw-r--r-- | mm/vmstat.c | 4 |
6 files changed, 26 insertions, 14 deletions
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index e6d90babc245..04905bfc508b 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h | |||
@@ -62,7 +62,7 @@ static inline void __flush_tlb_all(void) | |||
62 | 62 | ||
63 | static inline void __flush_tlb_one(unsigned long addr) | 63 | static inline void __flush_tlb_one(unsigned long addr) |
64 | { | 64 | { |
65 | count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); | 65 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); |
66 | __flush_tlb_single(addr); | 66 | __flush_tlb_single(addr); |
67 | } | 67 | } |
68 | 68 | ||
@@ -93,13 +93,13 @@ static inline void __flush_tlb_one(unsigned long addr) | |||
93 | */ | 93 | */ |
94 | static inline void __flush_tlb_up(void) | 94 | static inline void __flush_tlb_up(void) |
95 | { | 95 | { |
96 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 96 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
97 | __flush_tlb(); | 97 | __flush_tlb(); |
98 | } | 98 | } |
99 | 99 | ||
100 | static inline void flush_tlb_all(void) | 100 | static inline void flush_tlb_all(void) |
101 | { | 101 | { |
102 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 102 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
103 | __flush_tlb_all(); | 103 | __flush_tlb_all(); |
104 | } | 104 | } |
105 | 105 | ||
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c index ce2d0a2c3e4f..0e25a1bc5ab5 100644 --- a/arch/x86/kernel/cpu/mtrr/generic.c +++ b/arch/x86/kernel/cpu/mtrr/generic.c | |||
@@ -683,7 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) | |||
683 | } | 683 | } |
684 | 684 | ||
685 | /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ | 685 | /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ |
686 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 686 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
687 | __flush_tlb(); | 687 | __flush_tlb(); |
688 | 688 | ||
689 | /* Save MTRR state */ | 689 | /* Save MTRR state */ |
@@ -697,7 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock) | |||
697 | static void post_set(void) __releases(set_atomicity_lock) | 697 | static void post_set(void) __releases(set_atomicity_lock) |
698 | { | 698 | { |
699 | /* Flush TLBs (no need to flush caches - they are disabled) */ | 699 | /* Flush TLBs (no need to flush caches - they are disabled) */ |
700 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 700 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
701 | __flush_tlb(); | 701 | __flush_tlb(); |
702 | 702 | ||
703 | /* Intel (P6) standard MTRRs */ | 703 | /* Intel (P6) standard MTRRs */ |
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index ae699b3bbac8..05446c1cccfe 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c | |||
@@ -103,7 +103,7 @@ static void flush_tlb_func(void *info) | |||
103 | if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) | 103 | if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) |
104 | return; | 104 | return; |
105 | 105 | ||
106 | count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); | 106 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); |
107 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { | 107 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { |
108 | if (f->flush_end == TLB_FLUSH_ALL) | 108 | if (f->flush_end == TLB_FLUSH_ALL) |
109 | local_flush_tlb(); | 109 | local_flush_tlb(); |
@@ -131,7 +131,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask, | |||
131 | info.flush_start = start; | 131 | info.flush_start = start; |
132 | info.flush_end = end; | 132 | info.flush_end = end; |
133 | 133 | ||
134 | count_vm_event(NR_TLB_REMOTE_FLUSH); | 134 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); |
135 | if (is_uv_system()) { | 135 | if (is_uv_system()) { |
136 | unsigned int cpu; | 136 | unsigned int cpu; |
137 | 137 | ||
@@ -151,7 +151,7 @@ void flush_tlb_current_task(void) | |||
151 | 151 | ||
152 | preempt_disable(); | 152 | preempt_disable(); |
153 | 153 | ||
154 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 154 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
155 | local_flush_tlb(); | 155 | local_flush_tlb(); |
156 | if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) | 156 | if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) |
157 | flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); | 157 | flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); |
@@ -215,7 +215,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, | |||
215 | 215 | ||
216 | /* tlb_flushall_shift is on balance point, details in commit log */ | 216 | /* tlb_flushall_shift is on balance point, details in commit log */ |
217 | if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) { | 217 | if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) { |
218 | count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); | 218 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); |
219 | local_flush_tlb(); | 219 | local_flush_tlb(); |
220 | } else { | 220 | } else { |
221 | if (has_large_page(mm, start, end)) { | 221 | if (has_large_page(mm, start, end)) { |
@@ -224,7 +224,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, | |||
224 | } | 224 | } |
225 | /* flush range by one by one 'invlpg' */ | 225 | /* flush range by one by one 'invlpg' */ |
226 | for (addr = start; addr < end; addr += PAGE_SIZE) { | 226 | for (addr = start; addr < end; addr += PAGE_SIZE) { |
227 | count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); | 227 | count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); |
228 | __flush_tlb_single(addr); | 228 | __flush_tlb_single(addr); |
229 | } | 229 | } |
230 | 230 | ||
@@ -262,7 +262,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start) | |||
262 | 262 | ||
263 | static void do_flush_tlb_all(void *info) | 263 | static void do_flush_tlb_all(void *info) |
264 | { | 264 | { |
265 | count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); | 265 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); |
266 | __flush_tlb_all(); | 266 | __flush_tlb_all(); |
267 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY) | 267 | if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY) |
268 | leave_mm(smp_processor_id()); | 268 | leave_mm(smp_processor_id()); |
@@ -270,7 +270,7 @@ static void do_flush_tlb_all(void *info) | |||
270 | 270 | ||
271 | void flush_tlb_all(void) | 271 | void flush_tlb_all(void) |
272 | { | 272 | { |
273 | count_vm_event(NR_TLB_REMOTE_FLUSH); | 273 | count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); |
274 | on_each_cpu(do_flush_tlb_all, NULL, 1); | 274 | on_each_cpu(do_flush_tlb_all, NULL, 1); |
275 | } | 275 | } |
276 | 276 | ||
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index c557c6d096de..3a712e2e7d76 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h | |||
@@ -71,12 +71,14 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, | |||
71 | THP_ZERO_PAGE_ALLOC, | 71 | THP_ZERO_PAGE_ALLOC, |
72 | THP_ZERO_PAGE_ALLOC_FAILED, | 72 | THP_ZERO_PAGE_ALLOC_FAILED, |
73 | #endif | 73 | #endif |
74 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
74 | #ifdef CONFIG_SMP | 75 | #ifdef CONFIG_SMP |
75 | NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ | 76 | NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ |
76 | NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ | 77 | NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ |
77 | #endif | 78 | #endif /* CONFIG_SMP */ |
78 | NR_TLB_LOCAL_FLUSH_ALL, | 79 | NR_TLB_LOCAL_FLUSH_ALL, |
79 | NR_TLB_LOCAL_FLUSH_ONE, | 80 | NR_TLB_LOCAL_FLUSH_ONE, |
81 | #endif /* CONFIG_DEBUG_TLBFLUSH */ | ||
80 | NR_VM_EVENT_ITEMS | 82 | NR_VM_EVENT_ITEMS |
81 | }; | 83 | }; |
82 | 84 | ||
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index e4b948080d20..80ebba9c2e87 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h | |||
@@ -83,6 +83,14 @@ static inline void vm_events_fold_cpu(int cpu) | |||
83 | #define count_vm_numa_events(x, y) do { (void)(y); } while (0) | 83 | #define count_vm_numa_events(x, y) do { (void)(y); } while (0) |
84 | #endif /* CONFIG_NUMA_BALANCING */ | 84 | #endif /* CONFIG_NUMA_BALANCING */ |
85 | 85 | ||
86 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
87 | #define count_vm_tlb_event(x) count_vm_event(x) | ||
88 | #define count_vm_tlb_events(x, y) count_vm_events(x, y) | ||
89 | #else | ||
90 | #define count_vm_tlb_event(x) do {} while (0) | ||
91 | #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) | ||
92 | #endif | ||
93 | |||
86 | #define __count_zone_vm_events(item, zone, delta) \ | 94 | #define __count_zone_vm_events(item, zone, delta) \ |
87 | __count_vm_events(item##_NORMAL - ZONE_NORMAL + \ | 95 | __count_vm_events(item##_NORMAL - ZONE_NORMAL + \ |
88 | zone_idx(zone), delta) | 96 | zone_idx(zone), delta) |
diff --git a/mm/vmstat.c b/mm/vmstat.c index 72496140ac08..def5dd2fbe61 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c | |||
@@ -851,12 +851,14 @@ const char * const vmstat_text[] = { | |||
851 | "thp_zero_page_alloc", | 851 | "thp_zero_page_alloc", |
852 | "thp_zero_page_alloc_failed", | 852 | "thp_zero_page_alloc_failed", |
853 | #endif | 853 | #endif |
854 | #ifdef CONFIG_DEBUG_TLBFLUSH | ||
854 | #ifdef CONFIG_SMP | 855 | #ifdef CONFIG_SMP |
855 | "nr_tlb_remote_flush", | 856 | "nr_tlb_remote_flush", |
856 | "nr_tlb_remote_flush_received", | 857 | "nr_tlb_remote_flush_received", |
857 | #endif | 858 | #endif /* CONFIG_SMP */ |
858 | "nr_tlb_local_flush_all", | 859 | "nr_tlb_local_flush_all", |
859 | "nr_tlb_local_flush_one", | 860 | "nr_tlb_local_flush_one", |
861 | #endif /* CONFIG_DEBUG_TLBFLUSH */ | ||
860 | 862 | ||
861 | #endif /* CONFIG_VM_EVENTS_COUNTERS */ | 863 | #endif /* CONFIG_VM_EVENTS_COUNTERS */ |
862 | }; | 864 | }; |