aboutsummaryrefslogtreecommitdiffstats
path: root/mm/page_alloc.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/page_alloc.c')
-rw-r--r--mm/page_alloc.c64
1 files changed, 9 insertions, 55 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4ba5e37127f..df2022ff0c8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -221,11 +221,6 @@ EXPORT_SYMBOL(nr_online_nodes);
221 221
222int page_group_by_mobility_disabled __read_mostly; 222int page_group_by_mobility_disabled __read_mostly;
223 223
224/*
225 * NOTE:
226 * Don't use set_pageblock_migratetype(page, MIGRATE_ISOLATE) directly.
227 * Instead, use {un}set_pageblock_isolate.
228 */
229void set_pageblock_migratetype(struct page *page, int migratetype) 224void set_pageblock_migratetype(struct page *page, int migratetype)
230{ 225{
231 226
@@ -1389,14 +1384,8 @@ void split_page(struct page *page, unsigned int order)
1389 set_page_refcounted(page + i); 1384 set_page_refcounted(page + i);
1390} 1385}
1391 1386
1392/* 1387static int __isolate_free_page(struct page *page, unsigned int order)
1393 * Similar to the split_page family of functions except that the page
1394 * required at the given order and being isolated now to prevent races
1395 * with parallel allocators
1396 */
1397int capture_free_page(struct page *page, int alloc_order, int migratetype)
1398{ 1388{
1399 unsigned int order;
1400 unsigned long watermark; 1389 unsigned long watermark;
1401 struct zone *zone; 1390 struct zone *zone;
1402 int mt; 1391 int mt;
@@ -1404,7 +1393,6 @@ int capture_free_page(struct page *page, int alloc_order, int migratetype)
1404 BUG_ON(!PageBuddy(page)); 1393 BUG_ON(!PageBuddy(page));
1405 1394
1406 zone = page_zone(page); 1395 zone = page_zone(page);
1407 order = page_order(page);
1408 mt = get_pageblock_migratetype(page); 1396 mt = get_pageblock_migratetype(page);
1409 1397
1410 if (mt != MIGRATE_ISOLATE) { 1398 if (mt != MIGRATE_ISOLATE) {
@@ -1413,7 +1401,7 @@ int capture_free_page(struct page *page, int alloc_order, int migratetype)
1413 if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) 1401 if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
1414 return 0; 1402 return 0;
1415 1403
1416 __mod_zone_freepage_state(zone, -(1UL << alloc_order), mt); 1404 __mod_zone_freepage_state(zone, -(1UL << order), mt);
1417 } 1405 }
1418 1406
1419 /* Remove page from free list */ 1407 /* Remove page from free list */
@@ -1421,11 +1409,7 @@ int capture_free_page(struct page *page, int alloc_order, int migratetype)
1421 zone->free_area[order].nr_free--; 1409 zone->free_area[order].nr_free--;
1422 rmv_page_order(page); 1410 rmv_page_order(page);
1423 1411
1424 if (alloc_order != order) 1412 /* Set the pageblock if the isolated page is at least a pageblock */
1425 expand(zone, page, alloc_order, order,
1426 &zone->free_area[order], migratetype);
1427
1428 /* Set the pageblock if the captured page is at least a pageblock */
1429 if (order >= pageblock_order - 1) { 1413 if (order >= pageblock_order - 1) {
1430 struct page *endpage = page + (1 << order) - 1; 1414 struct page *endpage = page + (1 << order) - 1;
1431 for (; page < endpage; page += pageblock_nr_pages) { 1415 for (; page < endpage; page += pageblock_nr_pages) {
@@ -1436,7 +1420,7 @@ int capture_free_page(struct page *page, int alloc_order, int migratetype)
1436 } 1420 }
1437 } 1421 }
1438 1422
1439 return 1UL << alloc_order; 1423 return 1UL << order;
1440} 1424}
1441 1425
1442/* 1426/*
@@ -1454,10 +1438,9 @@ int split_free_page(struct page *page)
1454 unsigned int order; 1438 unsigned int order;
1455 int nr_pages; 1439 int nr_pages;
1456 1440
1457 BUG_ON(!PageBuddy(page));
1458 order = page_order(page); 1441 order = page_order(page);
1459 1442
1460 nr_pages = capture_free_page(page, order, 0); 1443 nr_pages = __isolate_free_page(page, order);
1461 if (!nr_pages) 1444 if (!nr_pages)
1462 return 0; 1445 return 0;
1463 1446
@@ -1655,20 +1638,6 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
1655 return true; 1638 return true;
1656} 1639}
1657 1640
1658#ifdef CONFIG_MEMORY_ISOLATION
1659static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
1660{
1661 if (unlikely(zone->nr_pageblock_isolate))
1662 return zone->nr_pageblock_isolate * pageblock_nr_pages;
1663 return 0;
1664}
1665#else
1666static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
1667{
1668 return 0;
1669}
1670#endif
1671
1672bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, 1641bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
1673 int classzone_idx, int alloc_flags) 1642 int classzone_idx, int alloc_flags)
1674{ 1643{
@@ -1684,14 +1653,6 @@ bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
1684 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) 1653 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
1685 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); 1654 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
1686 1655
1687 /*
1688 * If the zone has MIGRATE_ISOLATE type free pages, we should consider
1689 * it. nr_zone_isolate_freepages is never accurate so kswapd might not
1690 * sleep although it could do so. But this is more desirable for memory
1691 * hotplug than sleeping which can cause a livelock in the direct
1692 * reclaim path.
1693 */
1694 free_pages -= nr_zone_isolate_freepages(z);
1695 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, 1656 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
1696 free_pages); 1657 free_pages);
1697} 1658}
@@ -2163,8 +2124,6 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2163 bool *contended_compaction, bool *deferred_compaction, 2124 bool *contended_compaction, bool *deferred_compaction,
2164 unsigned long *did_some_progress) 2125 unsigned long *did_some_progress)
2165{ 2126{
2166 struct page *page = NULL;
2167
2168 if (!order) 2127 if (!order)
2169 return NULL; 2128 return NULL;
2170 2129
@@ -2176,16 +2135,12 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2176 current->flags |= PF_MEMALLOC; 2135 current->flags |= PF_MEMALLOC;
2177 *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask, 2136 *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
2178 nodemask, sync_migration, 2137 nodemask, sync_migration,
2179 contended_compaction, &page); 2138 contended_compaction);
2180 current->flags &= ~PF_MEMALLOC; 2139 current->flags &= ~PF_MEMALLOC;
2181 2140
2182 /* If compaction captured a page, prep and use it */
2183 if (page) {
2184 prep_new_page(page, order, gfp_mask);
2185 goto got_page;
2186 }
2187
2188 if (*did_some_progress != COMPACT_SKIPPED) { 2141 if (*did_some_progress != COMPACT_SKIPPED) {
2142 struct page *page;
2143
2189 /* Page migration frees to the PCP lists but we want merging */ 2144 /* Page migration frees to the PCP lists but we want merging */
2190 drain_pages(get_cpu()); 2145 drain_pages(get_cpu());
2191 put_cpu(); 2146 put_cpu();
@@ -2195,7 +2150,6 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2195 alloc_flags & ~ALLOC_NO_WATERMARKS, 2150 alloc_flags & ~ALLOC_NO_WATERMARKS,
2196 preferred_zone, migratetype); 2151 preferred_zone, migratetype);
2197 if (page) { 2152 if (page) {
2198got_page:
2199 preferred_zone->compact_blockskip_flush = false; 2153 preferred_zone->compact_blockskip_flush = false;
2200 preferred_zone->compact_considered = 0; 2154 preferred_zone->compact_considered = 0;
2201 preferred_zone->compact_defer_shift = 0; 2155 preferred_zone->compact_defer_shift = 0;
@@ -5631,7 +5585,7 @@ static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
5631 pfn &= (PAGES_PER_SECTION-1); 5585 pfn &= (PAGES_PER_SECTION-1);
5632 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 5586 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
5633#else 5587#else
5634 pfn = pfn - zone->zone_start_pfn; 5588 pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
5635 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; 5589 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
5636#endif /* CONFIG_SPARSEMEM */ 5590#endif /* CONFIG_SPARSEMEM */
5637} 5591}