diff options
Diffstat (limited to 'mm')
-rw-r--r-- | mm/compaction.c | 30 | ||||
-rw-r--r-- | mm/internal.h | 1 |
2 files changed, 16 insertions, 15 deletions
diff --git a/mm/compaction.c b/mm/compaction.c index 8f64d3533990..c5c627aae996 100644 --- a/mm/compaction.c +++ b/mm/compaction.c | |||
@@ -1115,6 +1115,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, | |||
1115 | struct compact_control *cc) | 1115 | struct compact_control *cc) |
1116 | { | 1116 | { |
1117 | unsigned long low_pfn, end_pfn; | 1117 | unsigned long low_pfn, end_pfn; |
1118 | unsigned long isolate_start_pfn; | ||
1118 | struct page *page; | 1119 | struct page *page; |
1119 | const isolate_mode_t isolate_mode = | 1120 | const isolate_mode_t isolate_mode = |
1120 | (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | | 1121 | (sysctl_compact_unevictable_allowed ? ISOLATE_UNEVICTABLE : 0) | |
@@ -1163,6 +1164,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, | |||
1163 | continue; | 1164 | continue; |
1164 | 1165 | ||
1165 | /* Perform the isolation */ | 1166 | /* Perform the isolation */ |
1167 | isolate_start_pfn = low_pfn; | ||
1166 | low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn, | 1168 | low_pfn = isolate_migratepages_block(cc, low_pfn, end_pfn, |
1167 | isolate_mode); | 1169 | isolate_mode); |
1168 | 1170 | ||
@@ -1172,6 +1174,15 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, | |||
1172 | } | 1174 | } |
1173 | 1175 | ||
1174 | /* | 1176 | /* |
1177 | * Record where we could have freed pages by migration and not | ||
1178 | * yet flushed them to buddy allocator. | ||
1179 | * - this is the lowest page that could have been isolated and | ||
1180 | * then freed by migration. | ||
1181 | */ | ||
1182 | if (cc->nr_migratepages && !cc->last_migrated_pfn) | ||
1183 | cc->last_migrated_pfn = isolate_start_pfn; | ||
1184 | |||
1185 | /* | ||
1175 | * Either we isolated something and proceed with migration. Or | 1186 | * Either we isolated something and proceed with migration. Or |
1176 | * we failed and compact_zone should decide if we should | 1187 | * we failed and compact_zone should decide if we should |
1177 | * continue or not. | 1188 | * continue or not. |
@@ -1342,7 +1353,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) | |||
1342 | unsigned long end_pfn = zone_end_pfn(zone); | 1353 | unsigned long end_pfn = zone_end_pfn(zone); |
1343 | const int migratetype = gfpflags_to_migratetype(cc->gfp_mask); | 1354 | const int migratetype = gfpflags_to_migratetype(cc->gfp_mask); |
1344 | const bool sync = cc->mode != MIGRATE_ASYNC; | 1355 | const bool sync = cc->mode != MIGRATE_ASYNC; |
1345 | unsigned long last_migrated_pfn = 0; | ||
1346 | 1356 | ||
1347 | ret = compaction_suitable(zone, cc->order, cc->alloc_flags, | 1357 | ret = compaction_suitable(zone, cc->order, cc->alloc_flags, |
1348 | cc->classzone_idx); | 1358 | cc->classzone_idx); |
@@ -1380,6 +1390,7 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) | |||
1380 | zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; | 1390 | zone->compact_cached_migrate_pfn[0] = cc->migrate_pfn; |
1381 | zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; | 1391 | zone->compact_cached_migrate_pfn[1] = cc->migrate_pfn; |
1382 | } | 1392 | } |
1393 | cc->last_migrated_pfn = 0; | ||
1383 | 1394 | ||
1384 | trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, | 1395 | trace_mm_compaction_begin(start_pfn, cc->migrate_pfn, |
1385 | cc->free_pfn, end_pfn, sync); | 1396 | cc->free_pfn, end_pfn, sync); |
@@ -1389,7 +1400,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) | |||
1389 | while ((ret = compact_finished(zone, cc, migratetype)) == | 1400 | while ((ret = compact_finished(zone, cc, migratetype)) == |
1390 | COMPACT_CONTINUE) { | 1401 | COMPACT_CONTINUE) { |
1391 | int err; | 1402 | int err; |
1392 | unsigned long isolate_start_pfn = cc->migrate_pfn; | ||
1393 | 1403 | ||
1394 | switch (isolate_migratepages(zone, cc)) { | 1404 | switch (isolate_migratepages(zone, cc)) { |
1395 | case ISOLATE_ABORT: | 1405 | case ISOLATE_ABORT: |
@@ -1429,16 +1439,6 @@ static int compact_zone(struct zone *zone, struct compact_control *cc) | |||
1429 | } | 1439 | } |
1430 | } | 1440 | } |
1431 | 1441 | ||
1432 | /* | ||
1433 | * Record where we could have freed pages by migration and not | ||
1434 | * yet flushed them to buddy allocator. We use the pfn that | ||
1435 | * isolate_migratepages() started from in this loop iteration | ||
1436 | * - this is the lowest page that could have been isolated and | ||
1437 | * then freed by migration. | ||
1438 | */ | ||
1439 | if (!last_migrated_pfn) | ||
1440 | last_migrated_pfn = isolate_start_pfn; | ||
1441 | |||
1442 | check_drain: | 1442 | check_drain: |
1443 | /* | 1443 | /* |
1444 | * Has the migration scanner moved away from the previous | 1444 | * Has the migration scanner moved away from the previous |
@@ -1447,18 +1447,18 @@ check_drain: | |||
1447 | * compact_finished() can detect immediately if allocation | 1447 | * compact_finished() can detect immediately if allocation |
1448 | * would succeed. | 1448 | * would succeed. |
1449 | */ | 1449 | */ |
1450 | if (cc->order > 0 && last_migrated_pfn) { | 1450 | if (cc->order > 0 && cc->last_migrated_pfn) { |
1451 | int cpu; | 1451 | int cpu; |
1452 | unsigned long current_block_start = | 1452 | unsigned long current_block_start = |
1453 | cc->migrate_pfn & ~((1UL << cc->order) - 1); | 1453 | cc->migrate_pfn & ~((1UL << cc->order) - 1); |
1454 | 1454 | ||
1455 | if (last_migrated_pfn < current_block_start) { | 1455 | if (cc->last_migrated_pfn < current_block_start) { |
1456 | cpu = get_cpu(); | 1456 | cpu = get_cpu(); |
1457 | lru_add_drain_cpu(cpu); | 1457 | lru_add_drain_cpu(cpu); |
1458 | drain_local_pages(zone); | 1458 | drain_local_pages(zone); |
1459 | put_cpu(); | 1459 | put_cpu(); |
1460 | /* No more flushing until we migrate again */ | 1460 | /* No more flushing until we migrate again */ |
1461 | last_migrated_pfn = 0; | 1461 | cc->last_migrated_pfn = 0; |
1462 | } | 1462 | } |
1463 | } | 1463 | } |
1464 | 1464 | ||
diff --git a/mm/internal.h b/mm/internal.h index 1195dd2d6a2b..bc0fa9a69e46 100644 --- a/mm/internal.h +++ b/mm/internal.h | |||
@@ -182,6 +182,7 @@ struct compact_control { | |||
182 | unsigned long nr_migratepages; /* Number of pages to migrate */ | 182 | unsigned long nr_migratepages; /* Number of pages to migrate */ |
183 | unsigned long free_pfn; /* isolate_freepages search base */ | 183 | unsigned long free_pfn; /* isolate_freepages search base */ |
184 | unsigned long migrate_pfn; /* isolate_migratepages search base */ | 184 | unsigned long migrate_pfn; /* isolate_migratepages search base */ |
185 | unsigned long last_migrated_pfn;/* Not yet flushed page being freed */ | ||
185 | enum migrate_mode mode; /* Async or sync migration mode */ | 186 | enum migrate_mode mode; /* Async or sync migration mode */ |
186 | bool ignore_skip_hint; /* Scan blocks even if marked skip */ | 187 | bool ignore_skip_hint; /* Scan blocks even if marked skip */ |
187 | int order; /* order a direct compactor needs */ | 188 | int order; /* order a direct compactor needs */ |