diff options
-rw-r--r-- | mm/memcontrol.c | 38 |
1 files changed, 23 insertions, 15 deletions
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e9d33dc151a5..e286e1603e4f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c | |||
@@ -1077,9 +1077,11 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, | |||
1077 | if (scan >= nr_to_scan) | 1077 | if (scan >= nr_to_scan) |
1078 | break; | 1078 | break; |
1079 | 1079 | ||
1080 | page = pc->page; | ||
1081 | if (unlikely(!PageCgroupUsed(pc))) | 1080 | if (unlikely(!PageCgroupUsed(pc))) |
1082 | continue; | 1081 | continue; |
1082 | |||
1083 | page = pc->page; | ||
1084 | |||
1083 | if (unlikely(!PageLRU(page))) | 1085 | if (unlikely(!PageLRU(page))) |
1084 | continue; | 1086 | continue; |
1085 | 1087 | ||
@@ -2108,6 +2110,7 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) | |||
2108 | } | 2110 | } |
2109 | 2111 | ||
2110 | static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, | 2112 | static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, |
2113 | struct page *page, | ||
2111 | struct page_cgroup *pc, | 2114 | struct page_cgroup *pc, |
2112 | enum charge_type ctype, | 2115 | enum charge_type ctype, |
2113 | int page_size) | 2116 | int page_size) |
@@ -2154,7 +2157,7 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, | |||
2154 | * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. | 2157 | * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. |
2155 | * if they exceeds softlimit. | 2158 | * if they exceeds softlimit. |
2156 | */ | 2159 | */ |
2157 | memcg_check_events(mem, pc->page); | 2160 | memcg_check_events(mem, page); |
2158 | } | 2161 | } |
2159 | 2162 | ||
2160 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 2163 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
@@ -2201,6 +2204,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) | |||
2201 | 2204 | ||
2202 | /** | 2205 | /** |
2203 | * mem_cgroup_move_account - move account of the page | 2206 | * mem_cgroup_move_account - move account of the page |
2207 | * @page: the page | ||
2204 | * @pc: page_cgroup of the page. | 2208 | * @pc: page_cgroup of the page. |
2205 | * @from: mem_cgroup which the page is moved from. | 2209 | * @from: mem_cgroup which the page is moved from. |
2206 | * @to: mem_cgroup which the page is moved to. @from != @to. | 2210 | * @to: mem_cgroup which the page is moved to. @from != @to. |
@@ -2216,7 +2220,7 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail) | |||
2216 | * true, this function does "uncharge" from old cgroup, but it doesn't if | 2220 | * true, this function does "uncharge" from old cgroup, but it doesn't if |
2217 | * @uncharge is false, so a caller should do "uncharge". | 2221 | * @uncharge is false, so a caller should do "uncharge". |
2218 | */ | 2222 | */ |
2219 | static int mem_cgroup_move_account(struct page_cgroup *pc, | 2223 | static int mem_cgroup_move_account(struct page *page, struct page_cgroup *pc, |
2220 | struct mem_cgroup *from, struct mem_cgroup *to, | 2224 | struct mem_cgroup *from, struct mem_cgroup *to, |
2221 | bool uncharge, int charge_size) | 2225 | bool uncharge, int charge_size) |
2222 | { | 2226 | { |
@@ -2225,7 +2229,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, | |||
2225 | int ret; | 2229 | int ret; |
2226 | 2230 | ||
2227 | VM_BUG_ON(from == to); | 2231 | VM_BUG_ON(from == to); |
2228 | VM_BUG_ON(PageLRU(pc->page)); | 2232 | VM_BUG_ON(PageLRU(page)); |
2229 | /* | 2233 | /* |
2230 | * The page is isolated from LRU. So, collapse function | 2234 | * The page is isolated from LRU. So, collapse function |
2231 | * will not handle this page. But page splitting can happen. | 2235 | * will not handle this page. But page splitting can happen. |
@@ -2233,7 +2237,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, | |||
2233 | * hold it. | 2237 | * hold it. |
2234 | */ | 2238 | */ |
2235 | ret = -EBUSY; | 2239 | ret = -EBUSY; |
2236 | if (charge_size > PAGE_SIZE && !PageTransHuge(pc->page)) | 2240 | if (charge_size > PAGE_SIZE && !PageTransHuge(page)) |
2237 | goto out; | 2241 | goto out; |
2238 | 2242 | ||
2239 | lock_page_cgroup(pc); | 2243 | lock_page_cgroup(pc); |
@@ -2273,8 +2277,8 @@ unlock: | |||
2273 | /* | 2277 | /* |
2274 | * check events | 2278 | * check events |
2275 | */ | 2279 | */ |
2276 | memcg_check_events(to, pc->page); | 2280 | memcg_check_events(to, page); |
2277 | memcg_check_events(from, pc->page); | 2281 | memcg_check_events(from, page); |
2278 | out: | 2282 | out: |
2279 | return ret; | 2283 | return ret; |
2280 | } | 2284 | } |
@@ -2283,11 +2287,11 @@ out: | |||
2283 | * move charges to its parent. | 2287 | * move charges to its parent. |
2284 | */ | 2288 | */ |
2285 | 2289 | ||
2286 | static int mem_cgroup_move_parent(struct page_cgroup *pc, | 2290 | static int mem_cgroup_move_parent(struct page *page, |
2291 | struct page_cgroup *pc, | ||
2287 | struct mem_cgroup *child, | 2292 | struct mem_cgroup *child, |
2288 | gfp_t gfp_mask) | 2293 | gfp_t gfp_mask) |
2289 | { | 2294 | { |
2290 | struct page *page = pc->page; | ||
2291 | struct cgroup *cg = child->css.cgroup; | 2295 | struct cgroup *cg = child->css.cgroup; |
2292 | struct cgroup *pcg = cg->parent; | 2296 | struct cgroup *pcg = cg->parent; |
2293 | struct mem_cgroup *parent; | 2297 | struct mem_cgroup *parent; |
@@ -2317,7 +2321,7 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc, | |||
2317 | if (page_size > PAGE_SIZE) | 2321 | if (page_size > PAGE_SIZE) |
2318 | flags = compound_lock_irqsave(page); | 2322 | flags = compound_lock_irqsave(page); |
2319 | 2323 | ||
2320 | ret = mem_cgroup_move_account(pc, child, parent, true, page_size); | 2324 | ret = mem_cgroup_move_account(page, pc, child, parent, true, page_size); |
2321 | if (ret) | 2325 | if (ret) |
2322 | mem_cgroup_cancel_charge(parent, page_size); | 2326 | mem_cgroup_cancel_charge(parent, page_size); |
2323 | 2327 | ||
@@ -2363,7 +2367,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, | |||
2363 | if (ret || !mem) | 2367 | if (ret || !mem) |
2364 | return ret; | 2368 | return ret; |
2365 | 2369 | ||
2366 | __mem_cgroup_commit_charge(mem, pc, ctype, page_size); | 2370 | __mem_cgroup_commit_charge(mem, page, pc, ctype, page_size); |
2367 | return 0; | 2371 | return 0; |
2368 | } | 2372 | } |
2369 | 2373 | ||
@@ -2501,7 +2505,7 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, | |||
2501 | cgroup_exclude_rmdir(&ptr->css); | 2505 | cgroup_exclude_rmdir(&ptr->css); |
2502 | pc = lookup_page_cgroup(page); | 2506 | pc = lookup_page_cgroup(page); |
2503 | mem_cgroup_lru_del_before_commit_swapcache(page); | 2507 | mem_cgroup_lru_del_before_commit_swapcache(page); |
2504 | __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE); | 2508 | __mem_cgroup_commit_charge(ptr, page, pc, ctype, PAGE_SIZE); |
2505 | mem_cgroup_lru_add_after_commit_swapcache(page); | 2509 | mem_cgroup_lru_add_after_commit_swapcache(page); |
2506 | /* | 2510 | /* |
2507 | * Now swap is on-memory. This means this page may be | 2511 | * Now swap is on-memory. This means this page may be |
@@ -2956,7 +2960,7 @@ int mem_cgroup_prepare_migration(struct page *page, | |||
2956 | ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; | 2960 | ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; |
2957 | else | 2961 | else |
2958 | ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; | 2962 | ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; |
2959 | __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE); | 2963 | __mem_cgroup_commit_charge(mem, page, pc, ctype, PAGE_SIZE); |
2960 | return ret; | 2964 | return ret; |
2961 | } | 2965 | } |
2962 | 2966 | ||
@@ -3323,6 +3327,8 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, | |||
3323 | loop += 256; | 3327 | loop += 256; |
3324 | busy = NULL; | 3328 | busy = NULL; |
3325 | while (loop--) { | 3329 | while (loop--) { |
3330 | struct page *page; | ||
3331 | |||
3326 | ret = 0; | 3332 | ret = 0; |
3327 | spin_lock_irqsave(&zone->lru_lock, flags); | 3333 | spin_lock_irqsave(&zone->lru_lock, flags); |
3328 | if (list_empty(list)) { | 3334 | if (list_empty(list)) { |
@@ -3338,7 +3344,9 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, | |||
3338 | } | 3344 | } |
3339 | spin_unlock_irqrestore(&zone->lru_lock, flags); | 3345 | spin_unlock_irqrestore(&zone->lru_lock, flags); |
3340 | 3346 | ||
3341 | ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL); | 3347 | page = pc->page; |
3348 | |||
3349 | ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL); | ||
3342 | if (ret == -ENOMEM) | 3350 | if (ret == -ENOMEM) |
3343 | break; | 3351 | break; |
3344 | 3352 | ||
@@ -4956,7 +4964,7 @@ retry: | |||
4956 | if (isolate_lru_page(page)) | 4964 | if (isolate_lru_page(page)) |
4957 | goto put; | 4965 | goto put; |
4958 | pc = lookup_page_cgroup(page); | 4966 | pc = lookup_page_cgroup(page); |
4959 | if (!mem_cgroup_move_account(pc, | 4967 | if (!mem_cgroup_move_account(page, pc, |
4960 | mc.from, mc.to, false, PAGE_SIZE)) { | 4968 | mc.from, mc.to, false, PAGE_SIZE)) { |
4961 | mc.precharge--; | 4969 | mc.precharge--; |
4962 | /* we uncharge from mc.from later. */ | 4970 | /* we uncharge from mc.from later. */ |