diff options
Diffstat (limited to 'mm/vmscan.c')
-rw-r--r-- | mm/vmscan.c | 91 |
1 files changed, 47 insertions, 44 deletions
diff --git a/mm/vmscan.c b/mm/vmscan.c index 2624edcfb420..83f4d0e85601 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -1760,28 +1760,6 @@ static bool in_reclaim_compaction(struct scan_control *sc) | |||
1760 | return false; | 1760 | return false; |
1761 | } | 1761 | } |
1762 | 1762 | ||
1763 | #ifdef CONFIG_COMPACTION | ||
1764 | /* | ||
1765 | * If compaction is deferred for sc->order then scale the number of pages | ||
1766 | * reclaimed based on the number of consecutive allocation failures | ||
1767 | */ | ||
1768 | static unsigned long scale_for_compaction(unsigned long pages_for_compaction, | ||
1769 | struct lruvec *lruvec, struct scan_control *sc) | ||
1770 | { | ||
1771 | struct zone *zone = lruvec_zone(lruvec); | ||
1772 | |||
1773 | if (zone->compact_order_failed <= sc->order) | ||
1774 | pages_for_compaction <<= zone->compact_defer_shift; | ||
1775 | return pages_for_compaction; | ||
1776 | } | ||
1777 | #else | ||
1778 | static unsigned long scale_for_compaction(unsigned long pages_for_compaction, | ||
1779 | struct lruvec *lruvec, struct scan_control *sc) | ||
1780 | { | ||
1781 | return pages_for_compaction; | ||
1782 | } | ||
1783 | #endif | ||
1784 | |||
1785 | /* | 1763 | /* |
1786 | * Reclaim/compaction is used for high-order allocation requests. It reclaims | 1764 | * Reclaim/compaction is used for high-order allocation requests. It reclaims |
1787 | * order-0 pages before compacting the zone. should_continue_reclaim() returns | 1765 | * order-0 pages before compacting the zone. should_continue_reclaim() returns |
@@ -1829,9 +1807,6 @@ static inline bool should_continue_reclaim(struct lruvec *lruvec, | |||
1829 | * inactive lists are large enough, continue reclaiming | 1807 | * inactive lists are large enough, continue reclaiming |
1830 | */ | 1808 | */ |
1831 | pages_for_compaction = (2UL << sc->order); | 1809 | pages_for_compaction = (2UL << sc->order); |
1832 | |||
1833 | pages_for_compaction = scale_for_compaction(pages_for_compaction, | ||
1834 | lruvec, sc); | ||
1835 | inactive_lru_pages = get_lru_size(lruvec, LRU_INACTIVE_FILE); | 1810 | inactive_lru_pages = get_lru_size(lruvec, LRU_INACTIVE_FILE); |
1836 | if (nr_swap_pages > 0) | 1811 | if (nr_swap_pages > 0) |
1837 | inactive_lru_pages += get_lru_size(lruvec, LRU_INACTIVE_ANON); | 1812 | inactive_lru_pages += get_lru_size(lruvec, LRU_INACTIVE_ANON); |
@@ -2232,9 +2207,12 @@ static bool pfmemalloc_watermark_ok(pg_data_t *pgdat) | |||
2232 | * Throttle direct reclaimers if backing storage is backed by the network | 2207 | * Throttle direct reclaimers if backing storage is backed by the network |
2233 | * and the PFMEMALLOC reserve for the preferred node is getting dangerously | 2208 | * and the PFMEMALLOC reserve for the preferred node is getting dangerously |
2234 | * depleted. kswapd will continue to make progress and wake the processes | 2209 | * depleted. kswapd will continue to make progress and wake the processes |
2235 | * when the low watermark is reached | 2210 | * when the low watermark is reached. |
2211 | * | ||
2212 | * Returns true if a fatal signal was delivered during throttling. If this | ||
2213 | * happens, the page allocator should not consider triggering the OOM killer. | ||
2236 | */ | 2214 | */ |
2237 | static void throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, | 2215 | static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, |
2238 | nodemask_t *nodemask) | 2216 | nodemask_t *nodemask) |
2239 | { | 2217 | { |
2240 | struct zone *zone; | 2218 | struct zone *zone; |
@@ -2249,13 +2227,20 @@ static void throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, | |||
2249 | * processes to block on log_wait_commit(). | 2227 | * processes to block on log_wait_commit(). |
2250 | */ | 2228 | */ |
2251 | if (current->flags & PF_KTHREAD) | 2229 | if (current->flags & PF_KTHREAD) |
2252 | return; | 2230 | goto out; |
2231 | |||
2232 | /* | ||
2233 | * If a fatal signal is pending, this process should not throttle. | ||
2234 | * It should return quickly so it can exit and free its memory | ||
2235 | */ | ||
2236 | if (fatal_signal_pending(current)) | ||
2237 | goto out; | ||
2253 | 2238 | ||
2254 | /* Check if the pfmemalloc reserves are ok */ | 2239 | /* Check if the pfmemalloc reserves are ok */ |
2255 | first_zones_zonelist(zonelist, high_zoneidx, NULL, &zone); | 2240 | first_zones_zonelist(zonelist, high_zoneidx, NULL, &zone); |
2256 | pgdat = zone->zone_pgdat; | 2241 | pgdat = zone->zone_pgdat; |
2257 | if (pfmemalloc_watermark_ok(pgdat)) | 2242 | if (pfmemalloc_watermark_ok(pgdat)) |
2258 | return; | 2243 | goto out; |
2259 | 2244 | ||
2260 | /* Account for the throttling */ | 2245 | /* Account for the throttling */ |
2261 | count_vm_event(PGSCAN_DIRECT_THROTTLE); | 2246 | count_vm_event(PGSCAN_DIRECT_THROTTLE); |
@@ -2271,12 +2256,20 @@ static void throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, | |||
2271 | if (!(gfp_mask & __GFP_FS)) { | 2256 | if (!(gfp_mask & __GFP_FS)) { |
2272 | wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, | 2257 | wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, |
2273 | pfmemalloc_watermark_ok(pgdat), HZ); | 2258 | pfmemalloc_watermark_ok(pgdat), HZ); |
2274 | return; | 2259 | |
2260 | goto check_pending; | ||
2275 | } | 2261 | } |
2276 | 2262 | ||
2277 | /* Throttle until kswapd wakes the process */ | 2263 | /* Throttle until kswapd wakes the process */ |
2278 | wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, | 2264 | wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, |
2279 | pfmemalloc_watermark_ok(pgdat)); | 2265 | pfmemalloc_watermark_ok(pgdat)); |
2266 | |||
2267 | check_pending: | ||
2268 | if (fatal_signal_pending(current)) | ||
2269 | return true; | ||
2270 | |||
2271 | out: | ||
2272 | return false; | ||
2280 | } | 2273 | } |
2281 | 2274 | ||
2282 | unsigned long try_to_free_pages(struct zonelist *zonelist, int order, | 2275 | unsigned long try_to_free_pages(struct zonelist *zonelist, int order, |
@@ -2298,13 +2291,12 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, | |||
2298 | .gfp_mask = sc.gfp_mask, | 2291 | .gfp_mask = sc.gfp_mask, |
2299 | }; | 2292 | }; |
2300 | 2293 | ||
2301 | throttle_direct_reclaim(gfp_mask, zonelist, nodemask); | ||
2302 | |||
2303 | /* | 2294 | /* |
2304 | * Do not enter reclaim if fatal signal is pending. 1 is returned so | 2295 | * Do not enter reclaim if fatal signal was delivered while throttled. |
2305 | * that the page allocator does not consider triggering OOM | 2296 | * 1 is returned so that the page allocator does not OOM kill at this |
2297 | * point. | ||
2306 | */ | 2298 | */ |
2307 | if (fatal_signal_pending(current)) | 2299 | if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask)) |
2308 | return 1; | 2300 | return 1; |
2309 | 2301 | ||
2310 | trace_mm_vmscan_direct_reclaim_begin(order, | 2302 | trace_mm_vmscan_direct_reclaim_begin(order, |
@@ -2422,6 +2414,19 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc) | |||
2422 | } while (memcg); | 2414 | } while (memcg); |
2423 | } | 2415 | } |
2424 | 2416 | ||
2417 | static bool zone_balanced(struct zone *zone, int order, | ||
2418 | unsigned long balance_gap, int classzone_idx) | ||
2419 | { | ||
2420 | if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) + | ||
2421 | balance_gap, classzone_idx, 0)) | ||
2422 | return false; | ||
2423 | |||
2424 | if (COMPACTION_BUILD && order && !compaction_suitable(zone, order)) | ||
2425 | return false; | ||
2426 | |||
2427 | return true; | ||
2428 | } | ||
2429 | |||
2425 | /* | 2430 | /* |
2426 | * pgdat_balanced is used when checking if a node is balanced for high-order | 2431 | * pgdat_balanced is used when checking if a node is balanced for high-order |
2427 | * allocations. Only zones that meet watermarks and are in a zone allowed | 2432 | * allocations. Only zones that meet watermarks and are in a zone allowed |
@@ -2500,8 +2505,7 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining, | |||
2500 | continue; | 2505 | continue; |
2501 | } | 2506 | } |
2502 | 2507 | ||
2503 | if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone), | 2508 | if (!zone_balanced(zone, order, 0, i)) |
2504 | i, 0)) | ||
2505 | all_zones_ok = false; | 2509 | all_zones_ok = false; |
2506 | else | 2510 | else |
2507 | balanced += zone->present_pages; | 2511 | balanced += zone->present_pages; |
@@ -2610,8 +2614,7 @@ loop_again: | |||
2610 | break; | 2614 | break; |
2611 | } | 2615 | } |
2612 | 2616 | ||
2613 | if (!zone_watermark_ok_safe(zone, order, | 2617 | if (!zone_balanced(zone, order, 0, 0)) { |
2614 | high_wmark_pages(zone), 0, 0)) { | ||
2615 | end_zone = i; | 2618 | end_zone = i; |
2616 | break; | 2619 | break; |
2617 | } else { | 2620 | } else { |
@@ -2687,9 +2690,8 @@ loop_again: | |||
2687 | testorder = 0; | 2690 | testorder = 0; |
2688 | 2691 | ||
2689 | if ((buffer_heads_over_limit && is_highmem_idx(i)) || | 2692 | if ((buffer_heads_over_limit && is_highmem_idx(i)) || |
2690 | !zone_watermark_ok_safe(zone, testorder, | 2693 | !zone_balanced(zone, testorder, |
2691 | high_wmark_pages(zone) + balance_gap, | 2694 | balance_gap, end_zone)) { |
2692 | end_zone, 0)) { | ||
2693 | shrink_zone(zone, &sc); | 2695 | shrink_zone(zone, &sc); |
2694 | 2696 | ||
2695 | reclaim_state->reclaimed_slab = 0; | 2697 | reclaim_state->reclaimed_slab = 0; |
@@ -2716,8 +2718,7 @@ loop_again: | |||
2716 | continue; | 2718 | continue; |
2717 | } | 2719 | } |
2718 | 2720 | ||
2719 | if (!zone_watermark_ok_safe(zone, testorder, | 2721 | if (!zone_balanced(zone, testorder, 0, end_zone)) { |
2720 | high_wmark_pages(zone), end_zone, 0)) { | ||
2721 | all_zones_ok = 0; | 2722 | all_zones_ok = 0; |
2722 | /* | 2723 | /* |
2723 | * We are still under min water mark. This | 2724 | * We are still under min water mark. This |
@@ -3017,6 +3018,8 @@ static int kswapd(void *p) | |||
3017 | &balanced_classzone_idx); | 3018 | &balanced_classzone_idx); |
3018 | } | 3019 | } |
3019 | } | 3020 | } |
3021 | |||
3022 | current->reclaim_state = NULL; | ||
3020 | return 0; | 3023 | return 0; |
3021 | } | 3024 | } |
3022 | 3025 | ||