aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNick Piggin <npiggin@suse.de>2006-03-22 03:07:58 -0500
committerLinus Torvalds <torvalds@g5.osdl.org>2006-03-22 10:53:56 -0500
commit46453a6e194a8c55fe6cf3dc8e1c4f24e2abc013 (patch)
treee407665e6c8cdf33814007750c49fdbea0a21e0e
parent2492ecc1a16b8ccf679d2999dca4f1b48aef07ee (diff)
[PATCH] mm: never ClearPageLRU released pages
If vmscan finds a zero refcount page on the lru list, never ClearPageLRU it. This means the release code need not hold ->lru_lock to stabilise PageLRU, so that lock may be skipped entirely when releasing !PageLRU pages (because we know PageLRU won't have been temporarily cleared by vmscan, which was previously guaranteed by holding the lock to synchronise against vmscan). Signed-off-by: Nick Piggin <npiggin@suse.de> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r--mm/swap.c52
-rw-r--r--mm/vmscan.c18
2 files changed, 38 insertions, 32 deletions
diff --git a/mm/swap.c b/mm/swap.c
index b524ea90bddb..3045a0f4c451 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -209,19 +209,18 @@ int lru_add_drain_all(void)
209 */ 209 */
210void fastcall __page_cache_release(struct page *page) 210void fastcall __page_cache_release(struct page *page)
211{ 211{
212 unsigned long flags; 212 if (PageLRU(page)) {
213 struct zone *zone = page_zone(page); 213 unsigned long flags;
214 struct zone *zone = page_zone(page);
214 215
215 spin_lock_irqsave(&zone->lru_lock, flags); 216 spin_lock_irqsave(&zone->lru_lock, flags);
216 if (TestClearPageLRU(page)) 217 if (!TestClearPageLRU(page))
218 BUG();
217 del_page_from_lru(zone, page); 219 del_page_from_lru(zone, page);
218 if (page_count(page) != 0) 220 spin_unlock_irqrestore(&zone->lru_lock, flags);
219 page = NULL; 221 }
220 spin_unlock_irqrestore(&zone->lru_lock, flags); 222 free_hot_page(page);
221 if (page)
222 free_hot_page(page);
223} 223}
224
225EXPORT_SYMBOL(__page_cache_release); 224EXPORT_SYMBOL(__page_cache_release);
226 225
227/* 226/*
@@ -245,7 +244,6 @@ void release_pages(struct page **pages, int nr, int cold)
245 pagevec_init(&pages_to_free, cold); 244 pagevec_init(&pages_to_free, cold);
246 for (i = 0; i < nr; i++) { 245 for (i = 0; i < nr; i++) {
247 struct page *page = pages[i]; 246 struct page *page = pages[i];
248 struct zone *pagezone;
249 247
250 if (unlikely(PageCompound(page))) { 248 if (unlikely(PageCompound(page))) {
251 if (zone) { 249 if (zone) {
@@ -259,23 +257,27 @@ void release_pages(struct page **pages, int nr, int cold)
259 if (!put_page_testzero(page)) 257 if (!put_page_testzero(page))
260 continue; 258 continue;
261 259
262 pagezone = page_zone(page); 260 if (PageLRU(page)) {
263 if (pagezone != zone) { 261 struct zone *pagezone = page_zone(page);
264 if (zone) 262 if (pagezone != zone) {
265 spin_unlock_irq(&zone->lru_lock); 263 if (zone)
266 zone = pagezone; 264 spin_unlock_irq(&zone->lru_lock);
267 spin_lock_irq(&zone->lru_lock); 265 zone = pagezone;
268 } 266 spin_lock_irq(&zone->lru_lock);
269 if (TestClearPageLRU(page)) 267 }
268 if (!TestClearPageLRU(page))
269 BUG();
270 del_page_from_lru(zone, page); 270 del_page_from_lru(zone, page);
271 if (page_count(page) == 0) { 271 }
272 if (!pagevec_add(&pages_to_free, page)) { 272
273 if (!pagevec_add(&pages_to_free, page)) {
274 if (zone) {
273 spin_unlock_irq(&zone->lru_lock); 275 spin_unlock_irq(&zone->lru_lock);
274 __pagevec_free(&pages_to_free); 276 zone = NULL;
275 pagevec_reinit(&pages_to_free);
276 zone = NULL; /* No lock is held */
277 } 277 }
278 } 278 __pagevec_free(&pages_to_free);
279 pagevec_reinit(&pages_to_free);
280 }
279 } 281 }
280 if (zone) 282 if (zone)
281 spin_unlock_irq(&zone->lru_lock); 283 spin_unlock_irq(&zone->lru_lock);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4fe7e3aa02e2..acb7611cd525 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1085,21 +1085,25 @@ static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
1085 page = lru_to_page(src); 1085 page = lru_to_page(src);
1086 prefetchw_prev_lru_page(page, src, flags); 1086 prefetchw_prev_lru_page(page, src, flags);
1087 1087
1088 if (!TestClearPageLRU(page))
1089 BUG();
1090 list_del(&page->lru); 1088 list_del(&page->lru);
1091 if (get_page_testone(page)) { 1089 if (unlikely(get_page_testone(page))) {
1092 /* 1090 /*
1093 * It is being freed elsewhere 1091 * It is being freed elsewhere
1094 */ 1092 */
1095 __put_page(page); 1093 __put_page(page);
1096 SetPageLRU(page);
1097 list_add(&page->lru, src); 1094 list_add(&page->lru, src);
1098 continue; 1095 continue;
1099 } else {
1100 list_add(&page->lru, dst);
1101 nr_taken++;
1102 } 1096 }
1097
1098 /*
1099 * Be careful not to clear PageLRU until after we're sure
1100 * the page is not being freed elsewhere -- the page release
1101 * code relies on it.
1102 */
1103 if (!TestClearPageLRU(page))
1104 BUG();
1105 list_add(&page->lru, dst);
1106 nr_taken++;
1103 } 1107 }
1104 1108
1105 *scanned = scan; 1109 *scanned = scan;