aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/compaction.c30
-rw-r--r--mm/internal.h1
2 files changed, 16 insertions, 15 deletions
diff --git a/mm/compaction.c b/mm/compaction.c
index 8f64d3533990..c5c627aae996 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -1115,6 +1115,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
1115 struct compact_control *cc) 1115 struct compact_control *cc)
1116{ 1116{
1117 unsigned long low_pfn, end_pfn; 1117 unsigned long low_pfn, end_pfn;
1118 unsigned long isolate_start_pfn;
1118 struct page *page; 1119 struct page *page;
1119 const isolate_mode_t isolate_mode = 1120 const isolate_mode_t isolate_mode =
1120 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | 1121 (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) |
@@ -1163,6 +1164,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
1163 continue; 1164 continue;
1164 1165
1165 /* Perform the isolation */ 1166 /* Perform the isolation */
1167 isolate_start_pfn = low_pfn;
1166 low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn, 1168 low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn,
1167 isolate_mode); 1169 isolate_mode);
1168 1170
@@ -1172,6 +1174,15 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
1172 } 1174 }
1173 1175
1174 /* 1176 /*
1177 * Record where we could have freed pages by migration and not
1178 * yet flushed them to buddy allocator.
1179 * - this is the lowest page that could have been isolated and
1180 * then freed by migration.
1181 */
1182 if (cc->nr_migratepages && !cc->last_migrated_pfn)
1183 cc->last_migrated_pfn = isolate_start_pfn;
1184
1185 /*
1175 * Either we isolated something and proceed with migration. Or 1186 * Either we isolated something and proceed with migration. Or
1176 * we failed and compact_zone should decide if we should 1187 * we failed and compact_zone should decide if we should
1177 * continue or not. 1188 * continue or not.
@@ -1342,7 +1353,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
1342 unsigned long end_pfn = zone_end_pfn(zone); 1353 unsigned long end_pfn = zone_end_pfn(zone);
1343 const int migratetype = gfpflags_to_migratetype(cc->gfp_mask); 1354 const int migratetype = gfpflags_to_migratetype(cc->gfp_mask);
1344 const bool sync = cc->mode != MIGRATE_ASYNC; 1355 const bool sync = cc->mode != MIGRATE_ASYNC;
1345 unsigned long last_migrated_pfn = 0;
1346 1356
1347 ret = compaction_suitable(zone, cc->order, cc->alloc_flags, 1357 ret = compaction_suitable(zone, cc->order, cc->alloc_flags,
1348 cc->classzone_idx); 1358 cc->classzone_idx);
@@ -1380,6 +1390,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
1380 zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; 1390 zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn;
1381 zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; 1391 zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn;
1382 } 1392 }
1393 cc->last_migrated_pfn = 0;
1383 1394
1384 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, 1395 trace_mm_compaction_begin(start_pfn, cc->migrate_pfn,
1385 cc->free_pfn, end_pfn, sync); 1396 cc->free_pfn, end_pfn, sync);
@@ -1389,7 +1400,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
1389 while ((ret = compact_finished(zone, cc, migratetype)) == 1400 while ((ret = compact_finished(zone, cc, migratetype)) ==
1390 COMPACT_CONTINUE) { 1401 COMPACT_CONTINUE) {
1391 int err; 1402 int err;
1392 unsigned long isolate_start_pfn = cc->migrate_pfn;
1393 1403
1394 switch (isolate_migratepages(zone, cc)) { 1404 switch (isolate_migratepages(zone, cc)) {
1395 case ISOLATE_ABORT: 1405 case ISOLATE_ABORT:
@@ -1429,16 +1439,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc)
1429 } 1439 }
1430 } 1440 }
1431 1441
1432 /*
1433 * Record where we could have freed pages by migration and not
1434 * yet flushed them to buddy allocator. We use the pfn that
1435 * isolate_migratepages() started from in this loop iteration
1436 * - this is the lowest page that could have been isolated and
1437 * then freed by migration.
1438 */
1439 if (!last_migrated_pfn)
1440 last_migrated_pfn = isolate_start_pfn;
1441
1442check_drain: 1442check_drain:
1443 /* 1443 /*
1444 * Has the migration scanner moved away from the previous 1444 * Has the migration scanner moved away from the previous
@@ -1447,18 +1447,18 @@ check_drain:
1447 * compact_finished() can detect immediately if allocation 1447 * compact_finished() can detect immediately if allocation
1448 * would succeed. 1448 * would succeed.
1449 */ 1449 */
1450 if (cc->order > 0 && last_migrated_pfn) { 1450 if (cc->order > 0 && cc->last_migrated_pfn) {
1451 int cpu; 1451 int cpu;
1452 unsigned long current_block_start = 1452 unsigned long current_block_start =
1453 cc->migrate_pfn & ~((1UL << cc->order) - 1); 1453 cc->migrate_pfn & ~((1UL << cc->order) - 1);
1454 1454
1455 if (last_migrated_pfn < current_block_start) { 1455 if (cc->last_migrated_pfn < current_block_start) {
1456 cpu = get_cpu(); 1456 cpu = get_cpu();
1457 lru_add_drain_cpu(cpu); 1457 lru_add_drain_cpu(cpu);
1458 drain_local_pages(zone); 1458 drain_local_pages(zone);
1459 put_cpu(); 1459 put_cpu();
1460 /* No more flushing until we migrate again */ 1460 /* No more flushing until we migrate again */
1461 last_migrated_pfn = 0; 1461 cc->last_migrated_pfn = 0;
1462 } 1462 }
1463 } 1463 }
1464 1464
diff --git a/mm/internal.h b/mm/internal.h
index 1195dd2d6a2b..bc0fa9a69e46 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -182,6 +182,7 @@ struct compact_control {
182 unsigned long nr_migratepages; /* Number of pages to migrate */ 182 unsigned long nr_migratepages; /* Number of pages to migrate */
183 unsigned long free_pfn; /* isolate_freepages search base */ 183 unsigned long free_pfn; /* isolate_freepages search base */
184 unsigned long migrate_pfn; /* isolate_migratepages search base */ 184 unsigned long migrate_pfn; /* isolate_migratepages search base */
185 unsigned long last_migrated_pfn;/* Not yet flushed page being freed */
185 enum migrate_mode mode; /* Async or sync migration mode */ 186 enum migrate_mode mode; /* Async or sync migration mode */
186 bool ignore_skip_hint; /* Scan blocks even if marked skip */ 187 bool ignore_skip_hint; /* Scan blocks even if marked skip */
187 int order; /* order a direct compactor needs */ 188 int order; /* order a direct compactor needs */