diff options
Diffstat (limited to 'mm/hugetlb.c')
-rw-r--r-- | mm/hugetlb.c | 111 |
1 files changed, 36 insertions, 75 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 85855240933d..bb0b7c128015 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c | |||
@@ -394,71 +394,6 @@ static int vma_has_reserves(struct vm_area_struct *vma) | |||
394 | return 0; | 394 | return 0; |
395 | } | 395 | } |
396 | 396 | ||
397 | static void clear_gigantic_page(struct page *page, | ||
398 | unsigned long addr, unsigned long sz) | ||
399 | { | ||
400 | int i; | ||
401 | struct page *p = page; | ||
402 | |||
403 | might_sleep(); | ||
404 | for (i = 0; i < sz/PAGE_SIZE; i++, p = mem_map_next(p, page, i)) { | ||
405 | cond_resched(); | ||
406 | clear_user_highpage(p, addr + i * PAGE_SIZE); | ||
407 | } | ||
408 | } | ||
409 | static void clear_huge_page(struct page *page, | ||
410 | unsigned long addr, unsigned long sz) | ||
411 | { | ||
412 | int i; | ||
413 | |||
414 | if (unlikely(sz/PAGE_SIZE > MAX_ORDER_NR_PAGES)) { | ||
415 | clear_gigantic_page(page, addr, sz); | ||
416 | return; | ||
417 | } | ||
418 | |||
419 | might_sleep(); | ||
420 | for (i = 0; i < sz/PAGE_SIZE; i++) { | ||
421 | cond_resched(); | ||
422 | clear_user_highpage(page + i, addr + i * PAGE_SIZE); | ||
423 | } | ||
424 | } | ||
425 | |||
426 | static void copy_user_gigantic_page(struct page *dst, struct page *src, | ||
427 | unsigned long addr, struct vm_area_struct *vma) | ||
428 | { | ||
429 | int i; | ||
430 | struct hstate *h = hstate_vma(vma); | ||
431 | struct page *dst_base = dst; | ||
432 | struct page *src_base = src; | ||
433 | |||
434 | for (i = 0; i < pages_per_huge_page(h); ) { | ||
435 | cond_resched(); | ||
436 | copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); | ||
437 | |||
438 | i++; | ||
439 | dst = mem_map_next(dst, dst_base, i); | ||
440 | src = mem_map_next(src, src_base, i); | ||
441 | } | ||
442 | } | ||
443 | |||
444 | static void copy_user_huge_page(struct page *dst, struct page *src, | ||
445 | unsigned long addr, struct vm_area_struct *vma) | ||
446 | { | ||
447 | int i; | ||
448 | struct hstate *h = hstate_vma(vma); | ||
449 | |||
450 | if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES)) { | ||
451 | copy_user_gigantic_page(dst, src, addr, vma); | ||
452 | return; | ||
453 | } | ||
454 | |||
455 | might_sleep(); | ||
456 | for (i = 0; i < pages_per_huge_page(h); i++) { | ||
457 | cond_resched(); | ||
458 | copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE, vma); | ||
459 | } | ||
460 | } | ||
461 | |||
462 | static void copy_gigantic_page(struct page *dst, struct page *src) | 397 | static void copy_gigantic_page(struct page *dst, struct page *src) |
463 | { | 398 | { |
464 | int i; | 399 | int i; |
@@ -1428,6 +1363,7 @@ static ssize_t nr_hugepages_show_common(struct kobject *kobj, | |||
1428 | 1363 | ||
1429 | return sprintf(buf, "%lu\n", nr_huge_pages); | 1364 | return sprintf(buf, "%lu\n", nr_huge_pages); |
1430 | } | 1365 | } |
1366 | |||
1431 | static ssize_t nr_hugepages_store_common(bool obey_mempolicy, | 1367 | static ssize_t nr_hugepages_store_common(bool obey_mempolicy, |
1432 | struct kobject *kobj, struct kobj_attribute *attr, | 1368 | struct kobject *kobj, struct kobj_attribute *attr, |
1433 | const char *buf, size_t len) | 1369 | const char *buf, size_t len) |
@@ -1440,9 +1376,14 @@ static ssize_t nr_hugepages_store_common(bool obey_mempolicy, | |||
1440 | 1376 | ||
1441 | err = strict_strtoul(buf, 10, &count); | 1377 | err = strict_strtoul(buf, 10, &count); |
1442 | if (err) | 1378 | if (err) |
1443 | return 0; | 1379 | goto out; |
1444 | 1380 | ||
1445 | h = kobj_to_hstate(kobj, &nid); | 1381 | h = kobj_to_hstate(kobj, &nid); |
1382 | if (h->order >= MAX_ORDER) { | ||
1383 | err = -EINVAL; | ||
1384 | goto out; | ||
1385 | } | ||
1386 | |||
1446 | if (nid == NUMA_NO_NODE) { | 1387 | if (nid == NUMA_NO_NODE) { |
1447 | /* | 1388 | /* |
1448 | * global hstate attribute | 1389 | * global hstate attribute |
@@ -1468,6 +1409,9 @@ static ssize_t nr_hugepages_store_common(bool obey_mempolicy, | |||
1468 | NODEMASK_FREE(nodes_allowed); | 1409 | NODEMASK_FREE(nodes_allowed); |
1469 | 1410 | ||
1470 | return len; | 1411 | return len; |
1412 | out: | ||
1413 | NODEMASK_FREE(nodes_allowed); | ||
1414 | return err; | ||
1471 | } | 1415 | } |
1472 | 1416 | ||
1473 | static ssize_t nr_hugepages_show(struct kobject *kobj, | 1417 | static ssize_t nr_hugepages_show(struct kobject *kobj, |
@@ -1510,6 +1454,7 @@ static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj, | |||
1510 | struct hstate *h = kobj_to_hstate(kobj, NULL); | 1454 | struct hstate *h = kobj_to_hstate(kobj, NULL); |
1511 | return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); | 1455 | return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages); |
1512 | } | 1456 | } |
1457 | |||
1513 | static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, | 1458 | static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, |
1514 | struct kobj_attribute *attr, const char *buf, size_t count) | 1459 | struct kobj_attribute *attr, const char *buf, size_t count) |
1515 | { | 1460 | { |
@@ -1517,9 +1462,12 @@ static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, | |||
1517 | unsigned long input; | 1462 | unsigned long input; |
1518 | struct hstate *h = kobj_to_hstate(kobj, NULL); | 1463 | struct hstate *h = kobj_to_hstate(kobj, NULL); |
1519 | 1464 | ||
1465 | if (h->order >= MAX_ORDER) | ||
1466 | return -EINVAL; | ||
1467 | |||
1520 | err = strict_strtoul(buf, 10, &input); | 1468 | err = strict_strtoul(buf, 10, &input); |
1521 | if (err) | 1469 | if (err) |
1522 | return 0; | 1470 | return err; |
1523 | 1471 | ||
1524 | spin_lock(&hugetlb_lock); | 1472 | spin_lock(&hugetlb_lock); |
1525 | h->nr_overcommit_huge_pages = input; | 1473 | h->nr_overcommit_huge_pages = input; |
@@ -1922,13 +1870,19 @@ static int hugetlb_sysctl_handler_common(bool obey_mempolicy, | |||
1922 | { | 1870 | { |
1923 | struct hstate *h = &default_hstate; | 1871 | struct hstate *h = &default_hstate; |
1924 | unsigned long tmp; | 1872 | unsigned long tmp; |
1873 | int ret; | ||
1925 | 1874 | ||
1926 | if (!write) | 1875 | if (!write) |
1927 | tmp = h->max_huge_pages; | 1876 | tmp = h->max_huge_pages; |
1928 | 1877 | ||
1878 | if (write && h->order >= MAX_ORDER) | ||
1879 | return -EINVAL; | ||
1880 | |||
1929 | table->data = &tmp; | 1881 | table->data = &tmp; |
1930 | table->maxlen = sizeof(unsigned long); | 1882 | table->maxlen = sizeof(unsigned long); |
1931 | proc_doulongvec_minmax(table, write, buffer, length, ppos); | 1883 | ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); |
1884 | if (ret) | ||
1885 | goto out; | ||
1932 | 1886 | ||
1933 | if (write) { | 1887 | if (write) { |
1934 | NODEMASK_ALLOC(nodemask_t, nodes_allowed, | 1888 | NODEMASK_ALLOC(nodemask_t, nodes_allowed, |
@@ -1943,8 +1897,8 @@ static int hugetlb_sysctl_handler_common(bool obey_mempolicy, | |||
1943 | if (nodes_allowed != &node_states[N_HIGH_MEMORY]) | 1897 | if (nodes_allowed != &node_states[N_HIGH_MEMORY]) |
1944 | NODEMASK_FREE(nodes_allowed); | 1898 | NODEMASK_FREE(nodes_allowed); |
1945 | } | 1899 | } |
1946 | 1900 | out: | |
1947 | return 0; | 1901 | return ret; |
1948 | } | 1902 | } |
1949 | 1903 | ||
1950 | int hugetlb_sysctl_handler(struct ctl_table *table, int write, | 1904 | int hugetlb_sysctl_handler(struct ctl_table *table, int write, |
@@ -1982,21 +1936,27 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write, | |||
1982 | { | 1936 | { |
1983 | struct hstate *h = &default_hstate; | 1937 | struct hstate *h = &default_hstate; |
1984 | unsigned long tmp; | 1938 | unsigned long tmp; |
1939 | int ret; | ||
1985 | 1940 | ||
1986 | if (!write) | 1941 | if (!write) |
1987 | tmp = h->nr_overcommit_huge_pages; | 1942 | tmp = h->nr_overcommit_huge_pages; |
1988 | 1943 | ||
1944 | if (write && h->order >= MAX_ORDER) | ||
1945 | return -EINVAL; | ||
1946 | |||
1989 | table->data = &tmp; | 1947 | table->data = &tmp; |
1990 | table->maxlen = sizeof(unsigned long); | 1948 | table->maxlen = sizeof(unsigned long); |
1991 | proc_doulongvec_minmax(table, write, buffer, length, ppos); | 1949 | ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); |
1950 | if (ret) | ||
1951 | goto out; | ||
1992 | 1952 | ||
1993 | if (write) { | 1953 | if (write) { |
1994 | spin_lock(&hugetlb_lock); | 1954 | spin_lock(&hugetlb_lock); |
1995 | h->nr_overcommit_huge_pages = tmp; | 1955 | h->nr_overcommit_huge_pages = tmp; |
1996 | spin_unlock(&hugetlb_lock); | 1956 | spin_unlock(&hugetlb_lock); |
1997 | } | 1957 | } |
1998 | 1958 | out: | |
1999 | return 0; | 1959 | return ret; |
2000 | } | 1960 | } |
2001 | 1961 | ||
2002 | #endif /* CONFIG_SYSCTL */ | 1962 | #endif /* CONFIG_SYSCTL */ |
@@ -2454,7 +2414,8 @@ retry_avoidcopy: | |||
2454 | return VM_FAULT_OOM; | 2414 | return VM_FAULT_OOM; |
2455 | } | 2415 | } |
2456 | 2416 | ||
2457 | copy_user_huge_page(new_page, old_page, address, vma); | 2417 | copy_user_huge_page(new_page, old_page, address, vma, |
2418 | pages_per_huge_page(h)); | ||
2458 | __SetPageUptodate(new_page); | 2419 | __SetPageUptodate(new_page); |
2459 | 2420 | ||
2460 | /* | 2421 | /* |
@@ -2558,7 +2519,7 @@ retry: | |||
2558 | ret = -PTR_ERR(page); | 2519 | ret = -PTR_ERR(page); |
2559 | goto out; | 2520 | goto out; |
2560 | } | 2521 | } |
2561 | clear_huge_page(page, address, huge_page_size(h)); | 2522 | clear_huge_page(page, address, pages_per_huge_page(h)); |
2562 | __SetPageUptodate(page); | 2523 | __SetPageUptodate(page); |
2563 | 2524 | ||
2564 | if (vma->vm_flags & VM_MAYSHARE) { | 2525 | if (vma->vm_flags & VM_MAYSHARE) { |