aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--arch/x86/include/asm/tlbflush.h6
-rw-r--r--arch/x86/kernel/cpu/mtrr/generic.c4
-rw-r--r--arch/x86/mm/tlb.c14
-rw-r--r--include/linux/vm_event_item.h4
-rw-r--r--include/linux/vmstat.h8
-rw-r--r--mm/vmstat.c4
6 files changed, 26 insertions, 14 deletions
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index e6d90babc245..04905bfc508b 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -62,7 +62,7 @@ static inline void __flush_tlb_all(void)
62 62
63static inline void __flush_tlb_one(unsigned long addr) 63static inline void __flush_tlb_one(unsigned long addr)
64{ 64{
65 count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); 65 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
66 __flush_tlb_single(addr); 66 __flush_tlb_single(addr);
67} 67}
68 68
@@ -93,13 +93,13 @@ static inline void __flush_tlb_one(unsigned long addr)
93 */ 93 */
94static inline void __flush_tlb_up(void) 94static inline void __flush_tlb_up(void)
95{ 95{
96 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 96 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
97 __flush_tlb(); 97 __flush_tlb();
98} 98}
99 99
100static inline void flush_tlb_all(void) 100static inline void flush_tlb_all(void)
101{ 101{
102 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 102 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
103 __flush_tlb_all(); 103 __flush_tlb_all();
104} 104}
105 105
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index ce2d0a2c3e4f..0e25a1bc5ab5 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -683,7 +683,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
683 } 683 }
684 684
685 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ 685 /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
686 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 686 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
687 __flush_tlb(); 687 __flush_tlb();
688 688
689 /* Save MTRR state */ 689 /* Save MTRR state */
@@ -697,7 +697,7 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
697static void post_set(void) __releases(set_atomicity_lock) 697static void post_set(void) __releases(set_atomicity_lock)
698{ 698{
699 /* Flush TLBs (no need to flush caches - they are disabled) */ 699 /* Flush TLBs (no need to flush caches - they are disabled) */
700 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 700 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
701 __flush_tlb(); 701 __flush_tlb();
702 702
703 /* Intel (P6) standard MTRRs */ 703 /* Intel (P6) standard MTRRs */
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index ae699b3bbac8..05446c1cccfe 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -103,7 +103,7 @@ static void flush_tlb_func(void *info)
103 if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) 103 if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
104 return; 104 return;
105 105
106 count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); 106 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
107 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { 107 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
108 if (f->flush_end == TLB_FLUSH_ALL) 108 if (f->flush_end == TLB_FLUSH_ALL)
109 local_flush_tlb(); 109 local_flush_tlb();
@@ -131,7 +131,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
131 info.flush_start = start; 131 info.flush_start = start;
132 info.flush_end = end; 132 info.flush_end = end;
133 133
134 count_vm_event(NR_TLB_REMOTE_FLUSH); 134 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
135 if (is_uv_system()) { 135 if (is_uv_system()) {
136 unsigned int cpu; 136 unsigned int cpu;
137 137
@@ -151,7 +151,7 @@ void flush_tlb_current_task(void)
151 151
152 preempt_disable(); 152 preempt_disable();
153 153
154 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 154 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
155 local_flush_tlb(); 155 local_flush_tlb();
156 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) 156 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
157 flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); 157 flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
@@ -215,7 +215,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
215 215
216 /* tlb_flushall_shift is on balance point, details in commit log */ 216 /* tlb_flushall_shift is on balance point, details in commit log */
217 if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) { 217 if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) {
218 count_vm_event(NR_TLB_LOCAL_FLUSH_ALL); 218 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
219 local_flush_tlb(); 219 local_flush_tlb();
220 } else { 220 } else {
221 if (has_large_page(mm, start, end)) { 221 if (has_large_page(mm, start, end)) {
@@ -224,7 +224,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
224 } 224 }
225 /* flush range by one by one 'invlpg' */ 225 /* flush range by one by one 'invlpg' */
226 for (addr = start; addr < end; addr += PAGE_SIZE) { 226 for (addr = start; addr < end; addr += PAGE_SIZE) {
227 count_vm_event(NR_TLB_LOCAL_FLUSH_ONE); 227 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
228 __flush_tlb_single(addr); 228 __flush_tlb_single(addr);
229 } 229 }
230 230
@@ -262,7 +262,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
262 262
263static void do_flush_tlb_all(void *info) 263static void do_flush_tlb_all(void *info)
264{ 264{
265 count_vm_event(NR_TLB_REMOTE_FLUSH_RECEIVED); 265 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
266 __flush_tlb_all(); 266 __flush_tlb_all();
267 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY) 267 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
268 leave_mm(smp_processor_id()); 268 leave_mm(smp_processor_id());
@@ -270,7 +270,7 @@ static void do_flush_tlb_all(void *info)
270 270
271void flush_tlb_all(void) 271void flush_tlb_all(void)
272{ 272{
273 count_vm_event(NR_TLB_REMOTE_FLUSH); 273 count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
274 on_each_cpu(do_flush_tlb_all, NULL, 1); 274 on_each_cpu(do_flush_tlb_all, NULL, 1);
275} 275}
276 276
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index c557c6d096de..3a712e2e7d76 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -71,12 +71,14 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
71 THP_ZERO_PAGE_ALLOC, 71 THP_ZERO_PAGE_ALLOC,
72 THP_ZERO_PAGE_ALLOC_FAILED, 72 THP_ZERO_PAGE_ALLOC_FAILED,
73#endif 73#endif
74#ifdef CONFIG_DEBUG_TLBFLUSH
74#ifdef CONFIG_SMP 75#ifdef CONFIG_SMP
75 NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */ 76 NR_TLB_REMOTE_FLUSH, /* cpu tried to flush others' tlbs */
76 NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */ 77 NR_TLB_REMOTE_FLUSH_RECEIVED,/* cpu received ipi for flush */
77#endif 78#endif /* CONFIG_SMP */
78 NR_TLB_LOCAL_FLUSH_ALL, 79 NR_TLB_LOCAL_FLUSH_ALL,
79 NR_TLB_LOCAL_FLUSH_ONE, 80 NR_TLB_LOCAL_FLUSH_ONE,
81#endif /* CONFIG_DEBUG_TLBFLUSH */
80 NR_VM_EVENT_ITEMS 82 NR_VM_EVENT_ITEMS
81}; 83};
82 84
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index e4b948080d20..80ebba9c2e87 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -83,6 +83,14 @@ static inline void vm_events_fold_cpu(int cpu)
83#define count_vm_numa_events(x, y) do { (void)(y); } while (0) 83#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
84#endif /* CONFIG_NUMA_BALANCING */ 84#endif /* CONFIG_NUMA_BALANCING */
85 85
86#ifdef CONFIG_DEBUG_TLBFLUSH
87#define count_vm_tlb_event(x) count_vm_event(x)
88#define count_vm_tlb_events(x, y) count_vm_events(x, y)
89#else
90#define count_vm_tlb_event(x) do {} while (0)
91#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
92#endif
93
86#define __count_zone_vm_events(item, zone, delta) \ 94#define __count_zone_vm_events(item, zone, delta) \
87 __count_vm_events(item##_NORMAL - ZONE_NORMAL + \ 95 __count_vm_events(item##_NORMAL - ZONE_NORMAL + \
88 zone_idx(zone), delta) 96 zone_idx(zone), delta)
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 72496140ac08..def5dd2fbe61 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -851,12 +851,14 @@ const char * const vmstat_text[] = {
851 "thp_zero_page_alloc", 851 "thp_zero_page_alloc",
852 "thp_zero_page_alloc_failed", 852 "thp_zero_page_alloc_failed",
853#endif 853#endif
854#ifdef CONFIG_DEBUG_TLBFLUSH
854#ifdef CONFIG_SMP 855#ifdef CONFIG_SMP
855 "nr_tlb_remote_flush", 856 "nr_tlb_remote_flush",
856 "nr_tlb_remote_flush_received", 857 "nr_tlb_remote_flush_received",
857#endif 858#endif /* CONFIG_SMP */
858 "nr_tlb_local_flush_all", 859 "nr_tlb_local_flush_all",
859 "nr_tlb_local_flush_one", 860 "nr_tlb_local_flush_one",
861#endif /* CONFIG_DEBUG_TLBFLUSH */
860 862
861#endif /* CONFIG_VM_EVENTS_COUNTERS */ 863#endif /* CONFIG_VM_EVENTS_COUNTERS */
862}; 864};