diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Kconfig | 4 | ||||
| -rw-r--r-- | mm/filemap.c | 2 | ||||
| -rw-r--r-- | mm/filemap_xip.c | 2 | ||||
| -rw-r--r-- | mm/hugetlb.c | 2 | ||||
| -rw-r--r-- | mm/memcontrol.c | 127 | ||||
| -rw-r--r-- | mm/mmap.c | 2 | ||||
| -rw-r--r-- | mm/nommu.c | 2 | ||||
| -rw-r--r-- | mm/percpu.c | 83 | ||||
| -rw-r--r-- | mm/rmap.c | 4 | ||||
| -rw-r--r-- | mm/shmem.c | 4 |
10 files changed, 136 insertions, 96 deletions
diff --git a/mm/Kconfig b/mm/Kconfig index 247760729593..edd300aca173 100644 --- a/mm/Kconfig +++ b/mm/Kconfig | |||
| @@ -244,10 +244,12 @@ config DEFAULT_MMAP_MIN_ADDR | |||
| 244 | This value can be changed after boot using the | 244 | This value can be changed after boot using the |
| 245 | /proc/sys/vm/mmap_min_addr tunable. | 245 | /proc/sys/vm/mmap_min_addr tunable. |
| 246 | 246 | ||
| 247 | config ARCH_SUPPORTS_MEMORY_FAILURE | ||
| 248 | bool | ||
| 247 | 249 | ||
| 248 | config MEMORY_FAILURE | 250 | config MEMORY_FAILURE |
| 249 | depends on MMU | 251 | depends on MMU |
| 250 | depends on X86_MCE | 252 | depends on ARCH_SUPPORTS_MEMORY_FAILURE |
| 251 | bool "Enable recovery from hardware memory errors" | 253 | bool "Enable recovery from hardware memory errors" |
| 252 | help | 254 | help |
| 253 | Enables code to recover from some memory failures on systems | 255 | Enables code to recover from some memory failures on systems |
diff --git a/mm/filemap.c b/mm/filemap.c index 6c84e598b4a9..ef169f37156d 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
| @@ -1611,7 +1611,7 @@ page_not_uptodate: | |||
| 1611 | } | 1611 | } |
| 1612 | EXPORT_SYMBOL(filemap_fault); | 1612 | EXPORT_SYMBOL(filemap_fault); |
| 1613 | 1613 | ||
| 1614 | struct vm_operations_struct generic_file_vm_ops = { | 1614 | const struct vm_operations_struct generic_file_vm_ops = { |
| 1615 | .fault = filemap_fault, | 1615 | .fault = filemap_fault, |
| 1616 | }; | 1616 | }; |
| 1617 | 1617 | ||
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index 427dfe3ce78c..1888b2d71bb8 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c | |||
| @@ -296,7 +296,7 @@ out: | |||
| 296 | } | 296 | } |
| 297 | } | 297 | } |
| 298 | 298 | ||
| 299 | static struct vm_operations_struct xip_file_vm_ops = { | 299 | static const struct vm_operations_struct xip_file_vm_ops = { |
| 300 | .fault = xip_file_fault, | 300 | .fault = xip_file_fault, |
| 301 | }; | 301 | }; |
| 302 | 302 | ||
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 6f048fcc749c..5d7601b02874 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c | |||
| @@ -1721,7 +1721,7 @@ static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | |||
| 1721 | return 0; | 1721 | return 0; |
| 1722 | } | 1722 | } |
| 1723 | 1723 | ||
| 1724 | struct vm_operations_struct hugetlb_vm_ops = { | 1724 | const struct vm_operations_struct hugetlb_vm_ops = { |
| 1725 | .fault = hugetlb_vm_op_fault, | 1725 | .fault = hugetlb_vm_op_fault, |
| 1726 | .open = hugetlb_vm_op_open, | 1726 | .open = hugetlb_vm_op_open, |
| 1727 | .close = hugetlb_vm_op_close, | 1727 | .close = hugetlb_vm_op_close, |
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e2b98a6875c0..f99f5991d6bb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c | |||
| @@ -313,7 +313,8 @@ soft_limit_tree_from_page(struct page *page) | |||
| 313 | static void | 313 | static void |
| 314 | __mem_cgroup_insert_exceeded(struct mem_cgroup *mem, | 314 | __mem_cgroup_insert_exceeded(struct mem_cgroup *mem, |
| 315 | struct mem_cgroup_per_zone *mz, | 315 | struct mem_cgroup_per_zone *mz, |
| 316 | struct mem_cgroup_tree_per_zone *mctz) | 316 | struct mem_cgroup_tree_per_zone *mctz, |
| 317 | unsigned long long new_usage_in_excess) | ||
| 317 | { | 318 | { |
| 318 | struct rb_node **p = &mctz->rb_root.rb_node; | 319 | struct rb_node **p = &mctz->rb_root.rb_node; |
| 319 | struct rb_node *parent = NULL; | 320 | struct rb_node *parent = NULL; |
| @@ -322,7 +323,9 @@ __mem_cgroup_insert_exceeded(struct mem_cgroup *mem, | |||
| 322 | if (mz->on_tree) | 323 | if (mz->on_tree) |
| 323 | return; | 324 | return; |
| 324 | 325 | ||
| 325 | mz->usage_in_excess = res_counter_soft_limit_excess(&mem->res); | 326 | mz->usage_in_excess = new_usage_in_excess; |
| 327 | if (!mz->usage_in_excess) | ||
| 328 | return; | ||
| 326 | while (*p) { | 329 | while (*p) { |
| 327 | parent = *p; | 330 | parent = *p; |
| 328 | mz_node = rb_entry(parent, struct mem_cgroup_per_zone, | 331 | mz_node = rb_entry(parent, struct mem_cgroup_per_zone, |
| @@ -353,16 +356,6 @@ __mem_cgroup_remove_exceeded(struct mem_cgroup *mem, | |||
| 353 | } | 356 | } |
| 354 | 357 | ||
| 355 | static void | 358 | static void |
| 356 | mem_cgroup_insert_exceeded(struct mem_cgroup *mem, | ||
| 357 | struct mem_cgroup_per_zone *mz, | ||
| 358 | struct mem_cgroup_tree_per_zone *mctz) | ||
| 359 | { | ||
| 360 | spin_lock(&mctz->lock); | ||
| 361 | __mem_cgroup_insert_exceeded(mem, mz, mctz); | ||
| 362 | spin_unlock(&mctz->lock); | ||
| 363 | } | ||
| 364 | |||
| 365 | static void | ||
| 366 | mem_cgroup_remove_exceeded(struct mem_cgroup *mem, | 359 | mem_cgroup_remove_exceeded(struct mem_cgroup *mem, |
| 367 | struct mem_cgroup_per_zone *mz, | 360 | struct mem_cgroup_per_zone *mz, |
| 368 | struct mem_cgroup_tree_per_zone *mctz) | 361 | struct mem_cgroup_tree_per_zone *mctz) |
| @@ -392,34 +385,36 @@ static bool mem_cgroup_soft_limit_check(struct mem_cgroup *mem) | |||
| 392 | 385 | ||
| 393 | static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) | 386 | static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) |
| 394 | { | 387 | { |
| 395 | unsigned long long prev_usage_in_excess, new_usage_in_excess; | 388 | unsigned long long excess; |
| 396 | bool updated_tree = false; | ||
| 397 | struct mem_cgroup_per_zone *mz; | 389 | struct mem_cgroup_per_zone *mz; |
| 398 | struct mem_cgroup_tree_per_zone *mctz; | 390 | struct mem_cgroup_tree_per_zone *mctz; |
| 399 | 391 | int nid = page_to_nid(page); | |
| 400 | mz = mem_cgroup_zoneinfo(mem, page_to_nid(page), page_zonenum(page)); | 392 | int zid = page_zonenum(page); |
| 401 | mctz = soft_limit_tree_from_page(page); | 393 | mctz = soft_limit_tree_from_page(page); |
| 402 | 394 | ||
| 403 | /* | 395 | /* |
| 404 | * We do updates in lazy mode, mem's are removed | 396 | * Necessary to update all ancestors when hierarchy is used. |
| 405 | * lazily from the per-zone, per-node rb tree | 397 | * because their event counter is not touched. |
| 406 | */ | 398 | */ |
| 407 | prev_usage_in_excess = mz->usage_in_excess; | 399 | for (; mem; mem = parent_mem_cgroup(mem)) { |
| 408 | 400 | mz = mem_cgroup_zoneinfo(mem, nid, zid); | |
| 409 | new_usage_in_excess = res_counter_soft_limit_excess(&mem->res); | 401 | excess = res_counter_soft_limit_excess(&mem->res); |
| 410 | if (prev_usage_in_excess) { | 402 | /* |
| 411 | mem_cgroup_remove_exceeded(mem, mz, mctz); | 403 | * We have to update the tree if mz is on RB-tree or |
| 412 | updated_tree = true; | 404 | * mem is over its softlimit. |
| 413 | } | 405 | */ |
| 414 | if (!new_usage_in_excess) | 406 | if (excess || mz->on_tree) { |
| 415 | goto done; | 407 | spin_lock(&mctz->lock); |
| 416 | mem_cgroup_insert_exceeded(mem, mz, mctz); | 408 | /* if on-tree, remove it */ |
| 417 | 409 | if (mz->on_tree) | |
| 418 | done: | 410 | __mem_cgroup_remove_exceeded(mem, mz, mctz); |
| 419 | if (updated_tree) { | 411 | /* |
| 420 | spin_lock(&mctz->lock); | 412 | * Insert again. mz->usage_in_excess will be updated. |
| 421 | mz->usage_in_excess = new_usage_in_excess; | 413 | * If excess is 0, no tree ops. |
| 422 | spin_unlock(&mctz->lock); | 414 | */ |
| 415 | __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); | ||
| 416 | spin_unlock(&mctz->lock); | ||
| 417 | } | ||
| 423 | } | 418 | } |
| 424 | } | 419 | } |
| 425 | 420 | ||
| @@ -447,9 +442,10 @@ static struct mem_cgroup_per_zone * | |||
| 447 | __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) | 442 | __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) |
| 448 | { | 443 | { |
| 449 | struct rb_node *rightmost = NULL; | 444 | struct rb_node *rightmost = NULL; |
| 450 | struct mem_cgroup_per_zone *mz = NULL; | 445 | struct mem_cgroup_per_zone *mz; |
| 451 | 446 | ||
| 452 | retry: | 447 | retry: |
| 448 | mz = NULL; | ||
| 453 | rightmost = rb_last(&mctz->rb_root); | 449 | rightmost = rb_last(&mctz->rb_root); |
| 454 | if (!rightmost) | 450 | if (!rightmost) |
| 455 | goto done; /* Nothing to reclaim from */ | 451 | goto done; /* Nothing to reclaim from */ |
| @@ -1270,9 +1266,9 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, | |||
| 1270 | gfp_t gfp_mask, struct mem_cgroup **memcg, | 1266 | gfp_t gfp_mask, struct mem_cgroup **memcg, |
| 1271 | bool oom, struct page *page) | 1267 | bool oom, struct page *page) |
| 1272 | { | 1268 | { |
| 1273 | struct mem_cgroup *mem, *mem_over_limit, *mem_over_soft_limit; | 1269 | struct mem_cgroup *mem, *mem_over_limit; |
| 1274 | int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; | 1270 | int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; |
| 1275 | struct res_counter *fail_res, *soft_fail_res = NULL; | 1271 | struct res_counter *fail_res; |
| 1276 | 1272 | ||
| 1277 | if (unlikely(test_thread_flag(TIF_MEMDIE))) { | 1273 | if (unlikely(test_thread_flag(TIF_MEMDIE))) { |
| 1278 | /* Don't account this! */ | 1274 | /* Don't account this! */ |
| @@ -1304,17 +1300,16 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, | |||
| 1304 | 1300 | ||
| 1305 | if (mem_cgroup_is_root(mem)) | 1301 | if (mem_cgroup_is_root(mem)) |
| 1306 | goto done; | 1302 | goto done; |
| 1307 | ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res, | 1303 | ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res); |
| 1308 | &soft_fail_res); | ||
| 1309 | if (likely(!ret)) { | 1304 | if (likely(!ret)) { |
| 1310 | if (!do_swap_account) | 1305 | if (!do_swap_account) |
| 1311 | break; | 1306 | break; |
| 1312 | ret = res_counter_charge(&mem->memsw, PAGE_SIZE, | 1307 | ret = res_counter_charge(&mem->memsw, PAGE_SIZE, |
| 1313 | &fail_res, NULL); | 1308 | &fail_res); |
| 1314 | if (likely(!ret)) | 1309 | if (likely(!ret)) |
| 1315 | break; | 1310 | break; |
| 1316 | /* mem+swap counter fails */ | 1311 | /* mem+swap counter fails */ |
| 1317 | res_counter_uncharge(&mem->res, PAGE_SIZE, NULL); | 1312 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 1318 | flags |= MEM_CGROUP_RECLAIM_NOSWAP; | 1313 | flags |= MEM_CGROUP_RECLAIM_NOSWAP; |
| 1319 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, | 1314 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, |
| 1320 | memsw); | 1315 | memsw); |
| @@ -1353,16 +1348,11 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, | |||
| 1353 | } | 1348 | } |
| 1354 | } | 1349 | } |
| 1355 | /* | 1350 | /* |
| 1356 | * Insert just the ancestor, we should trickle down to the correct | 1351 | * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. |
| 1357 | * cgroup for reclaim, since the other nodes will be below their | 1352 | * if they exceeds softlimit. |
| 1358 | * soft limit | ||
| 1359 | */ | 1353 | */ |
| 1360 | if (soft_fail_res) { | 1354 | if (mem_cgroup_soft_limit_check(mem)) |
| 1361 | mem_over_soft_limit = | 1355 | mem_cgroup_update_tree(mem, page); |
| 1362 | mem_cgroup_from_res_counter(soft_fail_res, res); | ||
| 1363 | if (mem_cgroup_soft_limit_check(mem_over_soft_limit)) | ||
| 1364 | mem_cgroup_update_tree(mem_over_soft_limit, page); | ||
| 1365 | } | ||
| 1366 | done: | 1356 | done: |
| 1367 | return 0; | 1357 | return 0; |
| 1368 | nomem: | 1358 | nomem: |
| @@ -1437,10 +1427,9 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, | |||
| 1437 | if (unlikely(PageCgroupUsed(pc))) { | 1427 | if (unlikely(PageCgroupUsed(pc))) { |
| 1438 | unlock_page_cgroup(pc); | 1428 | unlock_page_cgroup(pc); |
| 1439 | if (!mem_cgroup_is_root(mem)) { | 1429 | if (!mem_cgroup_is_root(mem)) { |
| 1440 | res_counter_uncharge(&mem->res, PAGE_SIZE, NULL); | 1430 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 1441 | if (do_swap_account) | 1431 | if (do_swap_account) |
| 1442 | res_counter_uncharge(&mem->memsw, PAGE_SIZE, | 1432 | res_counter_uncharge(&mem->memsw, PAGE_SIZE); |
| 1443 | NULL); | ||
| 1444 | } | 1433 | } |
| 1445 | css_put(&mem->css); | 1434 | css_put(&mem->css); |
| 1446 | return; | 1435 | return; |
| @@ -1519,7 +1508,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, | |||
| 1519 | goto out; | 1508 | goto out; |
| 1520 | 1509 | ||
| 1521 | if (!mem_cgroup_is_root(from)) | 1510 | if (!mem_cgroup_is_root(from)) |
| 1522 | res_counter_uncharge(&from->res, PAGE_SIZE, NULL); | 1511 | res_counter_uncharge(&from->res, PAGE_SIZE); |
| 1523 | mem_cgroup_charge_statistics(from, pc, false); | 1512 | mem_cgroup_charge_statistics(from, pc, false); |
| 1524 | 1513 | ||
| 1525 | page = pc->page; | 1514 | page = pc->page; |
| @@ -1539,7 +1528,7 @@ static int mem_cgroup_move_account(struct page_cgroup *pc, | |||
| 1539 | } | 1528 | } |
| 1540 | 1529 | ||
| 1541 | if (do_swap_account && !mem_cgroup_is_root(from)) | 1530 | if (do_swap_account && !mem_cgroup_is_root(from)) |
| 1542 | res_counter_uncharge(&from->memsw, PAGE_SIZE, NULL); | 1531 | res_counter_uncharge(&from->memsw, PAGE_SIZE); |
| 1543 | css_put(&from->css); | 1532 | css_put(&from->css); |
| 1544 | 1533 | ||
| 1545 | css_get(&to->css); | 1534 | css_get(&to->css); |
| @@ -1610,9 +1599,9 @@ uncharge: | |||
| 1610 | css_put(&parent->css); | 1599 | css_put(&parent->css); |
| 1611 | /* uncharge if move fails */ | 1600 | /* uncharge if move fails */ |
| 1612 | if (!mem_cgroup_is_root(parent)) { | 1601 | if (!mem_cgroup_is_root(parent)) { |
| 1613 | res_counter_uncharge(&parent->res, PAGE_SIZE, NULL); | 1602 | res_counter_uncharge(&parent->res, PAGE_SIZE); |
| 1614 | if (do_swap_account) | 1603 | if (do_swap_account) |
| 1615 | res_counter_uncharge(&parent->memsw, PAGE_SIZE, NULL); | 1604 | res_counter_uncharge(&parent->memsw, PAGE_SIZE); |
| 1616 | } | 1605 | } |
| 1617 | return ret; | 1606 | return ret; |
| 1618 | } | 1607 | } |
| @@ -1803,8 +1792,7 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, | |||
| 1803 | * calling css_tryget | 1792 | * calling css_tryget |
| 1804 | */ | 1793 | */ |
| 1805 | if (!mem_cgroup_is_root(memcg)) | 1794 | if (!mem_cgroup_is_root(memcg)) |
| 1806 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE, | 1795 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE); |
| 1807 | NULL); | ||
| 1808 | mem_cgroup_swap_statistics(memcg, false); | 1796 | mem_cgroup_swap_statistics(memcg, false); |
| 1809 | mem_cgroup_put(memcg); | 1797 | mem_cgroup_put(memcg); |
| 1810 | } | 1798 | } |
| @@ -1831,9 +1819,9 @@ void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) | |||
| 1831 | if (!mem) | 1819 | if (!mem) |
| 1832 | return; | 1820 | return; |
| 1833 | if (!mem_cgroup_is_root(mem)) { | 1821 | if (!mem_cgroup_is_root(mem)) { |
| 1834 | res_counter_uncharge(&mem->res, PAGE_SIZE, NULL); | 1822 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 1835 | if (do_swap_account) | 1823 | if (do_swap_account) |
| 1836 | res_counter_uncharge(&mem->memsw, PAGE_SIZE, NULL); | 1824 | res_counter_uncharge(&mem->memsw, PAGE_SIZE); |
| 1837 | } | 1825 | } |
| 1838 | css_put(&mem->css); | 1826 | css_put(&mem->css); |
| 1839 | } | 1827 | } |
| @@ -1848,7 +1836,6 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) | |||
| 1848 | struct page_cgroup *pc; | 1836 | struct page_cgroup *pc; |
| 1849 | struct mem_cgroup *mem = NULL; | 1837 | struct mem_cgroup *mem = NULL; |
| 1850 | struct mem_cgroup_per_zone *mz; | 1838 | struct mem_cgroup_per_zone *mz; |
| 1851 | bool soft_limit_excess = false; | ||
| 1852 | 1839 | ||
| 1853 | if (mem_cgroup_disabled()) | 1840 | if (mem_cgroup_disabled()) |
| 1854 | return NULL; | 1841 | return NULL; |
| @@ -1888,10 +1875,10 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) | |||
| 1888 | } | 1875 | } |
| 1889 | 1876 | ||
| 1890 | if (!mem_cgroup_is_root(mem)) { | 1877 | if (!mem_cgroup_is_root(mem)) { |
| 1891 | res_counter_uncharge(&mem->res, PAGE_SIZE, &soft_limit_excess); | 1878 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 1892 | if (do_swap_account && | 1879 | if (do_swap_account && |
| 1893 | (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)) | 1880 | (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)) |
| 1894 | res_counter_uncharge(&mem->memsw, PAGE_SIZE, NULL); | 1881 | res_counter_uncharge(&mem->memsw, PAGE_SIZE); |
| 1895 | } | 1882 | } |
| 1896 | if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) | 1883 | if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) |
| 1897 | mem_cgroup_swap_statistics(mem, true); | 1884 | mem_cgroup_swap_statistics(mem, true); |
| @@ -1908,7 +1895,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) | |||
| 1908 | mz = page_cgroup_zoneinfo(pc); | 1895 | mz = page_cgroup_zoneinfo(pc); |
| 1909 | unlock_page_cgroup(pc); | 1896 | unlock_page_cgroup(pc); |
| 1910 | 1897 | ||
| 1911 | if (soft_limit_excess && mem_cgroup_soft_limit_check(mem)) | 1898 | if (mem_cgroup_soft_limit_check(mem)) |
| 1912 | mem_cgroup_update_tree(mem, page); | 1899 | mem_cgroup_update_tree(mem, page); |
| 1913 | /* at swapout, this memcg will be accessed to record to swap */ | 1900 | /* at swapout, this memcg will be accessed to record to swap */ |
| 1914 | if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT) | 1901 | if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT) |
| @@ -1986,7 +1973,7 @@ void mem_cgroup_uncharge_swap(swp_entry_t ent) | |||
| 1986 | * This memcg can be obsolete one. We avoid calling css_tryget | 1973 | * This memcg can be obsolete one. We avoid calling css_tryget |
| 1987 | */ | 1974 | */ |
| 1988 | if (!mem_cgroup_is_root(memcg)) | 1975 | if (!mem_cgroup_is_root(memcg)) |
| 1989 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE, NULL); | 1976 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE); |
| 1990 | mem_cgroup_swap_statistics(memcg, false); | 1977 | mem_cgroup_swap_statistics(memcg, false); |
| 1991 | mem_cgroup_put(memcg); | 1978 | mem_cgroup_put(memcg); |
| 1992 | } | 1979 | } |
| @@ -2233,6 +2220,7 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, | |||
| 2233 | unsigned long reclaimed; | 2220 | unsigned long reclaimed; |
| 2234 | int loop = 0; | 2221 | int loop = 0; |
| 2235 | struct mem_cgroup_tree_per_zone *mctz; | 2222 | struct mem_cgroup_tree_per_zone *mctz; |
| 2223 | unsigned long long excess; | ||
| 2236 | 2224 | ||
| 2237 | if (order > 0) | 2225 | if (order > 0) |
| 2238 | return 0; | 2226 | return 0; |
| @@ -2284,9 +2272,8 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, | |||
| 2284 | break; | 2272 | break; |
| 2285 | } while (1); | 2273 | } while (1); |
| 2286 | } | 2274 | } |
| 2287 | mz->usage_in_excess = | ||
| 2288 | res_counter_soft_limit_excess(&mz->mem->res); | ||
| 2289 | __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); | 2275 | __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); |
| 2276 | excess = res_counter_soft_limit_excess(&mz->mem->res); | ||
| 2290 | /* | 2277 | /* |
| 2291 | * One school of thought says that we should not add | 2278 | * One school of thought says that we should not add |
| 2292 | * back the node to the tree if reclaim returns 0. | 2279 | * back the node to the tree if reclaim returns 0. |
| @@ -2295,8 +2282,8 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, | |||
| 2295 | * memory to reclaim from. Consider this as a longer | 2282 | * memory to reclaim from. Consider this as a longer |
| 2296 | * term TODO. | 2283 | * term TODO. |
| 2297 | */ | 2284 | */ |
| 2298 | if (mz->usage_in_excess) | 2285 | /* If excess == 0, no tree ops */ |
| 2299 | __mem_cgroup_insert_exceeded(mz->mem, mz, mctz); | 2286 | __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess); |
| 2300 | spin_unlock(&mctz->lock); | 2287 | spin_unlock(&mctz->lock); |
| 2301 | css_put(&mz->mem->css); | 2288 | css_put(&mz->mem->css); |
| 2302 | loop++; | 2289 | loop++; |
| @@ -2282,7 +2282,7 @@ static void special_mapping_close(struct vm_area_struct *vma) | |||
| 2282 | { | 2282 | { |
| 2283 | } | 2283 | } |
| 2284 | 2284 | ||
| 2285 | static struct vm_operations_struct special_mapping_vmops = { | 2285 | static const struct vm_operations_struct special_mapping_vmops = { |
| 2286 | .close = special_mapping_close, | 2286 | .close = special_mapping_close, |
| 2287 | .fault = special_mapping_fault, | 2287 | .fault = special_mapping_fault, |
| 2288 | }; | 2288 | }; |
diff --git a/mm/nommu.c b/mm/nommu.c index c73aa4753d79..5189b5aed8c0 100644 --- a/mm/nommu.c +++ b/mm/nommu.c | |||
| @@ -79,7 +79,7 @@ static struct kmem_cache *vm_region_jar; | |||
| 79 | struct rb_root nommu_region_tree = RB_ROOT; | 79 | struct rb_root nommu_region_tree = RB_ROOT; |
| 80 | DECLARE_RWSEM(nommu_region_sem); | 80 | DECLARE_RWSEM(nommu_region_sem); |
| 81 | 81 | ||
| 82 | struct vm_operations_struct generic_file_vm_ops = { | 82 | const struct vm_operations_struct generic_file_vm_ops = { |
| 83 | }; | 83 | }; |
| 84 | 84 | ||
| 85 | /* | 85 | /* |
diff --git a/mm/percpu.c b/mm/percpu.c index 43d8cacfdaa5..4a048abad043 100644 --- a/mm/percpu.c +++ b/mm/percpu.c | |||
| @@ -1043,7 +1043,9 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void) | |||
| 1043 | */ | 1043 | */ |
| 1044 | static void *pcpu_alloc(size_t size, size_t align, bool reserved) | 1044 | static void *pcpu_alloc(size_t size, size_t align, bool reserved) |
| 1045 | { | 1045 | { |
| 1046 | static int warn_limit = 10; | ||
| 1046 | struct pcpu_chunk *chunk; | 1047 | struct pcpu_chunk *chunk; |
| 1048 | const char *err; | ||
| 1047 | int slot, off; | 1049 | int slot, off; |
| 1048 | 1050 | ||
| 1049 | if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { | 1051 | if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { |
| @@ -1059,11 +1061,14 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved) | |||
| 1059 | if (reserved && pcpu_reserved_chunk) { | 1061 | if (reserved && pcpu_reserved_chunk) { |
| 1060 | chunk = pcpu_reserved_chunk; | 1062 | chunk = pcpu_reserved_chunk; |
| 1061 | if (size > chunk->contig_hint || | 1063 | if (size > chunk->contig_hint || |
| 1062 | pcpu_extend_area_map(chunk) < 0) | 1064 | pcpu_extend_area_map(chunk) < 0) { |
| 1065 | err = "failed to extend area map of reserved chunk"; | ||
| 1063 | goto fail_unlock; | 1066 | goto fail_unlock; |
| 1067 | } | ||
| 1064 | off = pcpu_alloc_area(chunk, size, align); | 1068 | off = pcpu_alloc_area(chunk, size, align); |
| 1065 | if (off >= 0) | 1069 | if (off >= 0) |
| 1066 | goto area_found; | 1070 | goto area_found; |
| 1071 | err = "alloc from reserved chunk failed"; | ||
| 1067 | goto fail_unlock; | 1072 | goto fail_unlock; |
| 1068 | } | 1073 | } |
| 1069 | 1074 | ||
| @@ -1080,6 +1085,7 @@ restart: | |||
| 1080 | case 1: | 1085 | case 1: |
| 1081 | goto restart; /* pcpu_lock dropped, restart */ | 1086 | goto restart; /* pcpu_lock dropped, restart */ |
| 1082 | default: | 1087 | default: |
| 1088 | err = "failed to extend area map"; | ||
| 1083 | goto fail_unlock; | 1089 | goto fail_unlock; |
| 1084 | } | 1090 | } |
| 1085 | 1091 | ||
| @@ -1093,8 +1099,10 @@ restart: | |||
| 1093 | spin_unlock_irq(&pcpu_lock); | 1099 | spin_unlock_irq(&pcpu_lock); |
| 1094 | 1100 | ||
| 1095 | chunk = alloc_pcpu_chunk(); | 1101 | chunk = alloc_pcpu_chunk(); |
| 1096 | if (!chunk) | 1102 | if (!chunk) { |
| 1103 | err = "failed to allocate new chunk"; | ||
| 1097 | goto fail_unlock_mutex; | 1104 | goto fail_unlock_mutex; |
| 1105 | } | ||
| 1098 | 1106 | ||
| 1099 | spin_lock_irq(&pcpu_lock); | 1107 | spin_lock_irq(&pcpu_lock); |
| 1100 | pcpu_chunk_relocate(chunk, -1); | 1108 | pcpu_chunk_relocate(chunk, -1); |
| @@ -1107,6 +1115,7 @@ area_found: | |||
| 1107 | if (pcpu_populate_chunk(chunk, off, size)) { | 1115 | if (pcpu_populate_chunk(chunk, off, size)) { |
| 1108 | spin_lock_irq(&pcpu_lock); | 1116 | spin_lock_irq(&pcpu_lock); |
| 1109 | pcpu_free_area(chunk, off); | 1117 | pcpu_free_area(chunk, off); |
| 1118 | err = "failed to populate"; | ||
| 1110 | goto fail_unlock; | 1119 | goto fail_unlock; |
| 1111 | } | 1120 | } |
| 1112 | 1121 | ||
| @@ -1119,6 +1128,13 @@ fail_unlock: | |||
| 1119 | spin_unlock_irq(&pcpu_lock); | 1128 | spin_unlock_irq(&pcpu_lock); |
| 1120 | fail_unlock_mutex: | 1129 | fail_unlock_mutex: |
| 1121 | mutex_unlock(&pcpu_alloc_mutex); | 1130 | mutex_unlock(&pcpu_alloc_mutex); |
| 1131 | if (warn_limit) { | ||
| 1132 | pr_warning("PERCPU: allocation failed, size=%zu align=%zu, " | ||
| 1133 | "%s\n", size, align, err); | ||
| 1134 | dump_stack(); | ||
| 1135 | if (!--warn_limit) | ||
| 1136 | pr_info("PERCPU: limit reached, disable warning\n"); | ||
| 1137 | } | ||
| 1122 | return NULL; | 1138 | return NULL; |
| 1123 | } | 1139 | } |
| 1124 | 1140 | ||
| @@ -1347,6 +1363,10 @@ struct pcpu_alloc_info * __init pcpu_build_alloc_info( | |||
| 1347 | struct pcpu_alloc_info *ai; | 1363 | struct pcpu_alloc_info *ai; |
| 1348 | unsigned int *cpu_map; | 1364 | unsigned int *cpu_map; |
| 1349 | 1365 | ||
| 1366 | /* this function may be called multiple times */ | ||
| 1367 | memset(group_map, 0, sizeof(group_map)); | ||
| 1368 | memset(group_cnt, 0, sizeof(group_map)); | ||
| 1369 | |||
| 1350 | /* | 1370 | /* |
| 1351 | * Determine min_unit_size, alloc_size and max_upa such that | 1371 | * Determine min_unit_size, alloc_size and max_upa such that |
| 1352 | * alloc_size is multiple of atom_size and is the smallest | 1372 | * alloc_size is multiple of atom_size and is the smallest |
| @@ -1574,6 +1594,7 @@ static void pcpu_dump_alloc_info(const char *lvl, | |||
| 1574 | int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | 1594 | int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, |
| 1575 | void *base_addr) | 1595 | void *base_addr) |
| 1576 | { | 1596 | { |
| 1597 | static char cpus_buf[4096] __initdata; | ||
| 1577 | static int smap[2], dmap[2]; | 1598 | static int smap[2], dmap[2]; |
| 1578 | size_t dyn_size = ai->dyn_size; | 1599 | size_t dyn_size = ai->dyn_size; |
| 1579 | size_t size_sum = ai->static_size + ai->reserved_size + dyn_size; | 1600 | size_t size_sum = ai->static_size + ai->reserved_size + dyn_size; |
| @@ -1585,17 +1606,26 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |||
| 1585 | int *unit_map; | 1606 | int *unit_map; |
| 1586 | int group, unit, i; | 1607 | int group, unit, i; |
| 1587 | 1608 | ||
| 1609 | cpumask_scnprintf(cpus_buf, sizeof(cpus_buf), cpu_possible_mask); | ||
| 1610 | |||
| 1611 | #define PCPU_SETUP_BUG_ON(cond) do { \ | ||
| 1612 | if (unlikely(cond)) { \ | ||
| 1613 | pr_emerg("PERCPU: failed to initialize, %s", #cond); \ | ||
| 1614 | pr_emerg("PERCPU: cpu_possible_mask=%s\n", cpus_buf); \ | ||
| 1615 | pcpu_dump_alloc_info(KERN_EMERG, ai); \ | ||
| 1616 | BUG(); \ | ||
| 1617 | } \ | ||
| 1618 | } while (0) | ||
| 1619 | |||
| 1588 | /* sanity checks */ | 1620 | /* sanity checks */ |
| 1589 | BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC || | 1621 | BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC || |
| 1590 | ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); | 1622 | ARRAY_SIZE(dmap) >= PCPU_DFL_MAP_ALLOC); |
| 1591 | BUG_ON(ai->nr_groups <= 0); | 1623 | PCPU_SETUP_BUG_ON(ai->nr_groups <= 0); |
| 1592 | BUG_ON(!ai->static_size); | 1624 | PCPU_SETUP_BUG_ON(!ai->static_size); |
| 1593 | BUG_ON(!base_addr); | 1625 | PCPU_SETUP_BUG_ON(!base_addr); |
| 1594 | BUG_ON(ai->unit_size < size_sum); | 1626 | PCPU_SETUP_BUG_ON(ai->unit_size < size_sum); |
| 1595 | BUG_ON(ai->unit_size & ~PAGE_MASK); | 1627 | PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK); |
| 1596 | BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); | 1628 | PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE); |
| 1597 | |||
| 1598 | pcpu_dump_alloc_info(KERN_DEBUG, ai); | ||
| 1599 | 1629 | ||
| 1600 | /* process group information and build config tables accordingly */ | 1630 | /* process group information and build config tables accordingly */ |
| 1601 | group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0])); | 1631 | group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0])); |
| @@ -1604,7 +1634,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |||
| 1604 | unit_off = alloc_bootmem(nr_cpu_ids * sizeof(unit_off[0])); | 1634 | unit_off = alloc_bootmem(nr_cpu_ids * sizeof(unit_off[0])); |
| 1605 | 1635 | ||
| 1606 | for (cpu = 0; cpu < nr_cpu_ids; cpu++) | 1636 | for (cpu = 0; cpu < nr_cpu_ids; cpu++) |
| 1607 | unit_map[cpu] = NR_CPUS; | 1637 | unit_map[cpu] = UINT_MAX; |
| 1608 | pcpu_first_unit_cpu = NR_CPUS; | 1638 | pcpu_first_unit_cpu = NR_CPUS; |
| 1609 | 1639 | ||
| 1610 | for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) { | 1640 | for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) { |
| @@ -1618,8 +1648,9 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |||
| 1618 | if (cpu == NR_CPUS) | 1648 | if (cpu == NR_CPUS) |
| 1619 | continue; | 1649 | continue; |
| 1620 | 1650 | ||
| 1621 | BUG_ON(cpu > nr_cpu_ids || !cpu_possible(cpu)); | 1651 | PCPU_SETUP_BUG_ON(cpu > nr_cpu_ids); |
| 1622 | BUG_ON(unit_map[cpu] != NR_CPUS); | 1652 | PCPU_SETUP_BUG_ON(!cpu_possible(cpu)); |
| 1653 | PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX); | ||
| 1623 | 1654 | ||
| 1624 | unit_map[cpu] = unit + i; | 1655 | unit_map[cpu] = unit + i; |
| 1625 | unit_off[cpu] = gi->base_offset + i * ai->unit_size; | 1656 | unit_off[cpu] = gi->base_offset + i * ai->unit_size; |
| @@ -1632,7 +1663,11 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |||
| 1632 | pcpu_nr_units = unit; | 1663 | pcpu_nr_units = unit; |
| 1633 | 1664 | ||
| 1634 | for_each_possible_cpu(cpu) | 1665 | for_each_possible_cpu(cpu) |
| 1635 | BUG_ON(unit_map[cpu] == NR_CPUS); | 1666 | PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX); |
| 1667 | |||
| 1668 | /* we're done parsing the input, undefine BUG macro and dump config */ | ||
| 1669 | #undef PCPU_SETUP_BUG_ON | ||
| 1670 | pcpu_dump_alloc_info(KERN_INFO, ai); | ||
| 1636 | 1671 | ||
| 1637 | pcpu_nr_groups = ai->nr_groups; | 1672 | pcpu_nr_groups = ai->nr_groups; |
| 1638 | pcpu_group_offsets = group_offsets; | 1673 | pcpu_group_offsets = group_offsets; |
| @@ -1782,7 +1817,7 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size, | |||
| 1782 | void *base = (void *)ULONG_MAX; | 1817 | void *base = (void *)ULONG_MAX; |
| 1783 | void **areas = NULL; | 1818 | void **areas = NULL; |
| 1784 | struct pcpu_alloc_info *ai; | 1819 | struct pcpu_alloc_info *ai; |
| 1785 | size_t size_sum, areas_size; | 1820 | size_t size_sum, areas_size, max_distance; |
| 1786 | int group, i, rc; | 1821 | int group, i, rc; |
| 1787 | 1822 | ||
| 1788 | ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, | 1823 | ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, |
| @@ -1832,8 +1867,24 @@ int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size, | |||
| 1832 | } | 1867 | } |
| 1833 | 1868 | ||
| 1834 | /* base address is now known, determine group base offsets */ | 1869 | /* base address is now known, determine group base offsets */ |
| 1835 | for (group = 0; group < ai->nr_groups; group++) | 1870 | max_distance = 0; |
| 1871 | for (group = 0; group < ai->nr_groups; group++) { | ||
| 1836 | ai->groups[group].base_offset = areas[group] - base; | 1872 | ai->groups[group].base_offset = areas[group] - base; |
| 1873 | max_distance = max(max_distance, ai->groups[group].base_offset); | ||
| 1874 | } | ||
| 1875 | max_distance += ai->unit_size; | ||
| 1876 | |||
| 1877 | /* warn if maximum distance is further than 75% of vmalloc space */ | ||
| 1878 | if (max_distance > (VMALLOC_END - VMALLOC_START) * 3 / 4) { | ||
| 1879 | pr_warning("PERCPU: max_distance=0x%lx too large for vmalloc " | ||
| 1880 | "space 0x%lx\n", | ||
| 1881 | max_distance, VMALLOC_END - VMALLOC_START); | ||
| 1882 | #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK | ||
| 1883 | /* and fail if we have fallback */ | ||
| 1884 | rc = -EINVAL; | ||
| 1885 | goto out_free; | ||
| 1886 | #endif | ||
| 1887 | } | ||
| 1837 | 1888 | ||
| 1838 | pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n", | 1889 | pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n", |
| 1839 | PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size, | 1890 | PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size, |
| @@ -242,8 +242,8 @@ vma_address(struct page *page, struct vm_area_struct *vma) | |||
| 242 | } | 242 | } |
| 243 | 243 | ||
| 244 | /* | 244 | /* |
| 245 | * At what user virtual address is page expected in vma? checking that the | 245 | * At what user virtual address is page expected in vma? |
| 246 | * page matches the vma: currently only used on anon pages, by unuse_vma; | 246 | * checking that the page matches the vma. |
| 247 | */ | 247 | */ |
| 248 | unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) | 248 | unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) |
| 249 | { | 249 | { |
diff --git a/mm/shmem.c b/mm/shmem.c index ccf446a9faa1..356dd99566ec 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
| @@ -218,7 +218,7 @@ static const struct file_operations shmem_file_operations; | |||
| 218 | static const struct inode_operations shmem_inode_operations; | 218 | static const struct inode_operations shmem_inode_operations; |
| 219 | static const struct inode_operations shmem_dir_inode_operations; | 219 | static const struct inode_operations shmem_dir_inode_operations; |
| 220 | static const struct inode_operations shmem_special_inode_operations; | 220 | static const struct inode_operations shmem_special_inode_operations; |
| 221 | static struct vm_operations_struct shmem_vm_ops; | 221 | static const struct vm_operations_struct shmem_vm_ops; |
| 222 | 222 | ||
| 223 | static struct backing_dev_info shmem_backing_dev_info __read_mostly = { | 223 | static struct backing_dev_info shmem_backing_dev_info __read_mostly = { |
| 224 | .ra_pages = 0, /* No readahead */ | 224 | .ra_pages = 0, /* No readahead */ |
| @@ -2498,7 +2498,7 @@ static const struct super_operations shmem_ops = { | |||
| 2498 | .put_super = shmem_put_super, | 2498 | .put_super = shmem_put_super, |
| 2499 | }; | 2499 | }; |
| 2500 | 2500 | ||
| 2501 | static struct vm_operations_struct shmem_vm_ops = { | 2501 | static const struct vm_operations_struct shmem_vm_ops = { |
| 2502 | .fault = shmem_fault, | 2502 | .fault = shmem_fault, |
| 2503 | #ifdef CONFIG_NUMA | 2503 | #ifdef CONFIG_NUMA |
| 2504 | .set_policy = shmem_set_policy, | 2504 | .set_policy = shmem_set_policy, |
