diff options
author | Hugh Dickins <hugh@veritas.com> | 2005-10-29 21:16:03 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@g5.osdl.org> | 2005-10-30 00:40:37 -0400 |
commit | fc2acab31be8e869b2d5f6de12f557f6f054f19c (patch) | |
tree | 60cf419f5e88c3c46d39675a14649ea1e5849f03 | |
parent | 4d6ddfa9242bc3d27fb0f7248f6fdee0299c731f (diff) |
[PATCH] mm: tlb_finish_mmu forget rss
zap_pte_range has been counting the pages it frees in tlb->freed, then
tlb_finish_mmu has used that to update the mm's rss. That got stranger when I
added anon_rss, yet updated it by a different route; and stranger when rss and
anon_rss became mm_counters with special access macros. And it would no
longer be viable if we're relying on page_table_lock to stabilize the
mm_counter, but calling tlb_finish_mmu outside that lock.
Remove the mmu_gather's freed field, let tlb_finish_mmu stick to its own
business, just decrement the rss mm_counter in zap_pte_range (yes, there was
some point to batching the update, and a subsequent patch restores that). And
forget the anal paranoia of first reading the counter to avoid going negative
- if rss does go negative, just fix that bug.
Remove the mmu_gather's flushes and avoided_flushes from arm and arm26: no use
was being made of them. But arm26 alone was actually using the freed, in the
way some others use need_flush: give it a need_flush. arm26 seems to prefer
spaces to tabs here: respect that.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r-- | arch/sparc64/mm/tlb.c | 3 | ||||
-rw-r--r-- | include/asm-arm/tlb.h | 15 | ||||
-rw-r--r-- | include/asm-arm26/tlb.h | 35 | ||||
-rw-r--r-- | include/asm-generic/tlb.h | 9 | ||||
-rw-r--r-- | include/asm-ia64/tlb.h | 9 | ||||
-rw-r--r-- | include/asm-sparc64/tlb.h | 14 | ||||
-rw-r--r-- | mm/memory.c | 2 |
7 files changed, 18 insertions, 69 deletions
diff --git a/arch/sparc64/mm/tlb.c b/arch/sparc64/mm/tlb.c index 6a43f7cd090e..8b104be4662b 100644 --- a/arch/sparc64/mm/tlb.c +++ b/arch/sparc64/mm/tlb.c | |||
@@ -18,8 +18,7 @@ | |||
18 | 18 | ||
19 | /* Heavily inspired by the ppc64 code. */ | 19 | /* Heavily inspired by the ppc64 code. */ |
20 | 20 | ||
21 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers) = | 21 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers) = { 0, }; |
22 | { NULL, 0, 0, 0, 0, 0, { 0 }, { NULL }, }; | ||
23 | 22 | ||
24 | void flush_tlb_pending(void) | 23 | void flush_tlb_pending(void) |
25 | { | 24 | { |
diff --git a/include/asm-arm/tlb.h b/include/asm-arm/tlb.h index a35ab0f2e25e..f49bfb78c221 100644 --- a/include/asm-arm/tlb.h +++ b/include/asm-arm/tlb.h | |||
@@ -27,11 +27,7 @@ | |||
27 | */ | 27 | */ |
28 | struct mmu_gather { | 28 | struct mmu_gather { |
29 | struct mm_struct *mm; | 29 | struct mm_struct *mm; |
30 | unsigned int freed; | ||
31 | unsigned int fullmm; | 30 | unsigned int fullmm; |
32 | |||
33 | unsigned int flushes; | ||
34 | unsigned int avoided_flushes; | ||
35 | }; | 31 | }; |
36 | 32 | ||
37 | DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); | 33 | DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); |
@@ -42,7 +38,6 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) | |||
42 | struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); | 38 | struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); |
43 | 39 | ||
44 | tlb->mm = mm; | 40 | tlb->mm = mm; |
45 | tlb->freed = 0; | ||
46 | tlb->fullmm = full_mm_flush; | 41 | tlb->fullmm = full_mm_flush; |
47 | 42 | ||
48 | return tlb; | 43 | return tlb; |
@@ -51,16 +46,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) | |||
51 | static inline void | 46 | static inline void |
52 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | 47 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) |
53 | { | 48 | { |
54 | struct mm_struct *mm = tlb->mm; | ||
55 | unsigned long freed = tlb->freed; | ||
56 | int rss = get_mm_counter(mm, rss); | ||
57 | |||
58 | if (rss < freed) | ||
59 | freed = rss; | ||
60 | add_mm_counter(mm, rss, -freed); | ||
61 | |||
62 | if (tlb->fullmm) | 49 | if (tlb->fullmm) |
63 | flush_tlb_mm(mm); | 50 | flush_tlb_mm(tlb->mm); |
64 | 51 | ||
65 | /* keep the page table cache within bounds */ | 52 | /* keep the page table cache within bounds */ |
66 | check_pgt_cache(); | 53 | check_pgt_cache(); |
diff --git a/include/asm-arm26/tlb.h b/include/asm-arm26/tlb.h index c7d54ca0a239..08ddd85b8d35 100644 --- a/include/asm-arm26/tlb.h +++ b/include/asm-arm26/tlb.h | |||
@@ -10,11 +10,8 @@ | |||
10 | */ | 10 | */ |
11 | struct mmu_gather { | 11 | struct mmu_gather { |
12 | struct mm_struct *mm; | 12 | struct mm_struct *mm; |
13 | unsigned int freed; | 13 | unsigned int need_flush; |
14 | unsigned int fullmm; | 14 | unsigned int fullmm; |
15 | |||
16 | unsigned int flushes; | ||
17 | unsigned int avoided_flushes; | ||
18 | }; | 15 | }; |
19 | 16 | ||
20 | DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); | 17 | DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); |
@@ -25,8 +22,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) | |||
25 | struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); | 22 | struct mmu_gather *tlb = &get_cpu_var(mmu_gathers); |
26 | 23 | ||
27 | tlb->mm = mm; | 24 | tlb->mm = mm; |
28 | tlb->freed = 0; | 25 | tlb->need_flush = 0; |
29 | tlb->fullmm = full_mm_flush; | 26 | tlb->fullmm = full_mm_flush; |
30 | 27 | ||
31 | return tlb; | 28 | return tlb; |
32 | } | 29 | } |
@@ -34,20 +31,8 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) | |||
34 | static inline void | 31 | static inline void |
35 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | 32 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) |
36 | { | 33 | { |
37 | struct mm_struct *mm = tlb->mm; | 34 | if (tlb->need_flush) |
38 | unsigned long freed = tlb->freed; | 35 | flush_tlb_mm(tlb->mm); |
39 | int rss = get_mm_counter(mm, rss); | ||
40 | |||
41 | if (rss < freed) | ||
42 | freed = rss; | ||
43 | add_mm_counter(mm, rss, -freed); | ||
44 | |||
45 | if (freed) { | ||
46 | flush_tlb_mm(mm); | ||
47 | tlb->flushes++; | ||
48 | } else { | ||
49 | tlb->avoided_flushes++; | ||
50 | } | ||
51 | 36 | ||
52 | /* keep the page table cache within bounds */ | 37 | /* keep the page table cache within bounds */ |
53 | check_pgt_cache(); | 38 | check_pgt_cache(); |
@@ -65,7 +50,13 @@ tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | |||
65 | } while (0) | 50 | } while (0) |
66 | #define tlb_end_vma(tlb,vma) do { } while (0) | 51 | #define tlb_end_vma(tlb,vma) do { } while (0) |
67 | 52 | ||
68 | #define tlb_remove_page(tlb,page) free_page_and_swap_cache(page) | 53 | static inline void |
54 | tlb_remove_page(struct mmu_gather *tlb, struct page *page) | ||
55 | { | ||
56 | tlb->need_flush = 1; | ||
57 | free_page_and_swap_cache(page); | ||
58 | } | ||
59 | |||
69 | #define pte_free_tlb(tlb,ptep) pte_free(ptep) | 60 | #define pte_free_tlb(tlb,ptep) pte_free(ptep) |
70 | #define pmd_free_tlb(tlb,pmdp) pmd_free(pmdp) | 61 | #define pmd_free_tlb(tlb,pmdp) pmd_free(pmdp) |
71 | 62 | ||
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 5d352a70f004..cdd4145243cd 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h | |||
@@ -42,7 +42,6 @@ struct mmu_gather { | |||
42 | unsigned int nr; /* set to ~0U means fast mode */ | 42 | unsigned int nr; /* set to ~0U means fast mode */ |
43 | unsigned int need_flush;/* Really unmapped some ptes? */ | 43 | unsigned int need_flush;/* Really unmapped some ptes? */ |
44 | unsigned int fullmm; /* non-zero means full mm flush */ | 44 | unsigned int fullmm; /* non-zero means full mm flush */ |
45 | unsigned long freed; | ||
46 | struct page * pages[FREE_PTE_NR]; | 45 | struct page * pages[FREE_PTE_NR]; |
47 | }; | 46 | }; |
48 | 47 | ||
@@ -63,7 +62,6 @@ tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) | |||
63 | tlb->nr = num_online_cpus() > 1 ? 0U : ~0U; | 62 | tlb->nr = num_online_cpus() > 1 ? 0U : ~0U; |
64 | 63 | ||
65 | tlb->fullmm = full_mm_flush; | 64 | tlb->fullmm = full_mm_flush; |
66 | tlb->freed = 0; | ||
67 | 65 | ||
68 | return tlb; | 66 | return tlb; |
69 | } | 67 | } |
@@ -88,13 +86,6 @@ tlb_flush_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | |||
88 | static inline void | 86 | static inline void |
89 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) | 87 | tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end) |
90 | { | 88 | { |
91 | int freed = tlb->freed; | ||
92 | struct mm_struct *mm = tlb->mm; | ||
93 | int rss = get_mm_counter(mm, rss); | ||
94 | |||
95 | if (rss < freed) | ||
96 | freed = rss; | ||
97 | add_mm_counter(mm, rss, -freed); | ||
98 | tlb_flush_mmu(tlb, start, end); | 89 | tlb_flush_mmu(tlb, start, end); |
99 | 90 | ||
100 | /* keep the page table cache within bounds */ | 91 | /* keep the page table cache within bounds */ |
diff --git a/include/asm-ia64/tlb.h b/include/asm-ia64/tlb.h index 0bbd79f6a793..834370b9dea1 100644 --- a/include/asm-ia64/tlb.h +++ b/include/asm-ia64/tlb.h | |||
@@ -60,7 +60,6 @@ struct mmu_gather { | |||
60 | unsigned int nr; /* == ~0U => fast mode */ | 60 | unsigned int nr; /* == ~0U => fast mode */ |
61 | unsigned char fullmm; /* non-zero means full mm flush */ | 61 | unsigned char fullmm; /* non-zero means full mm flush */ |
62 | unsigned char need_flush; /* really unmapped some PTEs? */ | 62 | unsigned char need_flush; /* really unmapped some PTEs? */ |
63 | unsigned long freed; /* number of pages freed */ | ||
64 | unsigned long start_addr; | 63 | unsigned long start_addr; |
65 | unsigned long end_addr; | 64 | unsigned long end_addr; |
66 | struct page *pages[FREE_PTE_NR]; | 65 | struct page *pages[FREE_PTE_NR]; |
@@ -147,7 +146,6 @@ tlb_gather_mmu (struct mm_struct *mm, unsigned int full_mm_flush) | |||
147 | */ | 146 | */ |
148 | tlb->nr = (num_online_cpus() == 1) ? ~0U : 0; | 147 | tlb->nr = (num_online_cpus() == 1) ? ~0U : 0; |
149 | tlb->fullmm = full_mm_flush; | 148 | tlb->fullmm = full_mm_flush; |
150 | tlb->freed = 0; | ||
151 | tlb->start_addr = ~0UL; | 149 | tlb->start_addr = ~0UL; |
152 | return tlb; | 150 | return tlb; |
153 | } | 151 | } |
@@ -159,13 +157,6 @@ tlb_gather_mmu (struct mm_struct *mm, unsigned int full_mm_flush) | |||
159 | static inline void | 157 | static inline void |
160 | tlb_finish_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end) | 158 | tlb_finish_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end) |
161 | { | 159 | { |
162 | unsigned long freed = tlb->freed; | ||
163 | struct mm_struct *mm = tlb->mm; | ||
164 | unsigned long rss = get_mm_counter(mm, rss); | ||
165 | |||
166 | if (rss < freed) | ||
167 | freed = rss; | ||
168 | add_mm_counter(mm, rss, -freed); | ||
169 | /* | 160 | /* |
170 | * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and | 161 | * Note: tlb->nr may be 0 at this point, so we can't rely on tlb->start_addr and |
171 | * tlb->end_addr. | 162 | * tlb->end_addr. |
diff --git a/include/asm-sparc64/tlb.h b/include/asm-sparc64/tlb.h index 5d194eae870c..66138d959df5 100644 --- a/include/asm-sparc64/tlb.h +++ b/include/asm-sparc64/tlb.h | |||
@@ -27,7 +27,6 @@ struct mmu_gather { | |||
27 | unsigned int need_flush; | 27 | unsigned int need_flush; |
28 | unsigned int fullmm; | 28 | unsigned int fullmm; |
29 | unsigned int tlb_nr; | 29 | unsigned int tlb_nr; |
30 | unsigned long freed; | ||
31 | unsigned long vaddrs[TLB_BATCH_NR]; | 30 | unsigned long vaddrs[TLB_BATCH_NR]; |
32 | struct page *pages[FREE_PTE_NR]; | 31 | struct page *pages[FREE_PTE_NR]; |
33 | }; | 32 | }; |
@@ -51,7 +50,6 @@ static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, unsigned i | |||
51 | mp->mm = mm; | 50 | mp->mm = mm; |
52 | mp->pages_nr = num_online_cpus() > 1 ? 0U : ~0U; | 51 | mp->pages_nr = num_online_cpus() > 1 ? 0U : ~0U; |
53 | mp->fullmm = full_mm_flush; | 52 | mp->fullmm = full_mm_flush; |
54 | mp->freed = 0; | ||
55 | 53 | ||
56 | return mp; | 54 | return mp; |
57 | } | 55 | } |
@@ -78,19 +76,11 @@ extern void smp_flush_tlb_mm(struct mm_struct *mm); | |||
78 | 76 | ||
79 | static inline void tlb_finish_mmu(struct mmu_gather *mp, unsigned long start, unsigned long end) | 77 | static inline void tlb_finish_mmu(struct mmu_gather *mp, unsigned long start, unsigned long end) |
80 | { | 78 | { |
81 | unsigned long freed = mp->freed; | ||
82 | struct mm_struct *mm = mp->mm; | ||
83 | unsigned long rss = get_mm_counter(mm, rss); | ||
84 | |||
85 | if (rss < freed) | ||
86 | freed = rss; | ||
87 | add_mm_counter(mm, rss, -freed); | ||
88 | |||
89 | tlb_flush_mmu(mp); | 79 | tlb_flush_mmu(mp); |
90 | 80 | ||
91 | if (mp->fullmm) { | 81 | if (mp->fullmm) { |
92 | if (CTX_VALID(mm->context)) | 82 | if (CTX_VALID(mp->mm->context)) |
93 | do_flush_tlb_mm(mm); | 83 | do_flush_tlb_mm(mp->mm); |
94 | mp->fullmm = 0; | 84 | mp->fullmm = 0; |
95 | } else | 85 | } else |
96 | flush_tlb_pending(); | 86 | flush_tlb_pending(); |
diff --git a/mm/memory.c b/mm/memory.c index 585bb4e0b97f..51eb38574830 100644 --- a/mm/memory.c +++ b/mm/memory.c | |||
@@ -582,7 +582,7 @@ static void zap_pte_range(struct mmu_gather *tlb, pmd_t *pmd, | |||
582 | if (pte_young(ptent)) | 582 | if (pte_young(ptent)) |
583 | mark_page_accessed(page); | 583 | mark_page_accessed(page); |
584 | } | 584 | } |
585 | tlb->freed++; | 585 | dec_mm_counter(tlb->mm, rss); |
586 | page_remove_rmap(page); | 586 | page_remove_rmap(page); |
587 | tlb_remove_page(tlb, page); | 587 | tlb_remove_page(tlb, page); |
588 | continue; | 588 | continue; |