aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/bootmem.c6
-rw-r--r--mm/hugetlb.c2
-rw-r--r--mm/memory.c69
-rw-r--r--mm/migrate.c10
-rw-r--r--mm/mmap.c12
-rw-r--r--mm/nommu.c21
-rw-r--r--mm/page_alloc.c42
-rw-r--r--mm/pagewalk.c42
-rw-r--r--mm/slab.c5
-rw-r--r--mm/slob.c5
-rw-r--r--mm/slub.c5
-rw-r--r--mm/swap.c4
-rw-r--r--mm/vmscan.c2
13 files changed, 142 insertions, 83 deletions
diff --git a/mm/bootmem.c b/mm/bootmem.c
index e8fb927392b9..8d9f60e06f62 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -442,15 +442,17 @@ unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
442 return init_bootmem_core(pgdat, freepfn, startpfn, endpfn); 442 return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
443} 443}
444 444
445void __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 445int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
446 unsigned long size, int flags) 446 unsigned long size, int flags)
447{ 447{
448 int ret; 448 int ret;
449 449
450 ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); 450 ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
451 if (ret < 0) 451 if (ret < 0)
452 return; 452 return -ENOMEM;
453 reserve_bootmem_core(pgdat->bdata, physaddr, size, flags); 453 reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
454
455 return 0;
454} 456}
455 457
456void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr, 458void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index bbf953eeb58b..ab171274ef21 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -785,7 +785,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
785 continue; 785 continue;
786 786
787 spin_lock(&dst->page_table_lock); 787 spin_lock(&dst->page_table_lock);
788 spin_lock(&src->page_table_lock); 788 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
789 if (!huge_pte_none(huge_ptep_get(src_pte))) { 789 if (!huge_pte_none(huge_ptep_get(src_pte))) {
790 if (cow) 790 if (cow)
791 huge_ptep_set_wrprotect(src, addr, src_pte); 791 huge_ptep_set_wrprotect(src, addr, src_pte);
diff --git a/mm/memory.c b/mm/memory.c
index fb5608a120ed..d14b251a25a6 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -999,17 +999,15 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
999 goto no_page_table; 999 goto no_page_table;
1000 1000
1001 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 1001 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
1002 if (!ptep)
1003 goto out;
1004 1002
1005 pte = *ptep; 1003 pte = *ptep;
1006 if (!pte_present(pte)) 1004 if (!pte_present(pte))
1007 goto unlock; 1005 goto no_page;
1008 if ((flags & FOLL_WRITE) && !pte_write(pte)) 1006 if ((flags & FOLL_WRITE) && !pte_write(pte))
1009 goto unlock; 1007 goto unlock;
1010 page = vm_normal_page(vma, address, pte); 1008 page = vm_normal_page(vma, address, pte);
1011 if (unlikely(!page)) 1009 if (unlikely(!page))
1012 goto unlock; 1010 goto bad_page;
1013 1011
1014 if (flags & FOLL_GET) 1012 if (flags & FOLL_GET)
1015 get_page(page); 1013 get_page(page);
@@ -1024,6 +1022,15 @@ unlock:
1024out: 1022out:
1025 return page; 1023 return page;
1026 1024
1025bad_page:
1026 pte_unmap_unlock(ptep, ptl);
1027 return ERR_PTR(-EFAULT);
1028
1029no_page:
1030 pte_unmap_unlock(ptep, ptl);
1031 if (!pte_none(pte))
1032 return page;
1033 /* Fall through to ZERO_PAGE handling */
1027no_page_table: 1034no_page_table:
1028 /* 1035 /*
1029 * When core dumping an enormous anonymous area that nobody 1036 * When core dumping an enormous anonymous area that nobody
@@ -1038,6 +1045,26 @@ no_page_table:
1038 return page; 1045 return page;
1039} 1046}
1040 1047
1048/* Can we do the FOLL_ANON optimization? */
1049static inline int use_zero_page(struct vm_area_struct *vma)
1050{
1051 /*
1052 * We don't want to optimize FOLL_ANON for make_pages_present()
1053 * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
1054 * we want to get the page from the page tables to make sure
1055 * that we serialize and update with any other user of that
1056 * mapping.
1057 */
1058 if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
1059 return 0;
1060 /*
1061 * And if we have a fault or a nopfn routine, it's not an
1062 * anonymous region.
1063 */
1064 return !vma->vm_ops ||
1065 (!vma->vm_ops->fault && !vma->vm_ops->nopfn);
1066}
1067
1041int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1068int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1042 unsigned long start, int len, int write, int force, 1069 unsigned long start, int len, int write, int force,
1043 struct page **pages, struct vm_area_struct **vmas) 1070 struct page **pages, struct vm_area_struct **vmas)
@@ -1112,8 +1139,7 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1112 foll_flags = FOLL_TOUCH; 1139 foll_flags = FOLL_TOUCH;
1113 if (pages) 1140 if (pages)
1114 foll_flags |= FOLL_GET; 1141 foll_flags |= FOLL_GET;
1115 if (!write && !(vma->vm_flags & VM_LOCKED) && 1142 if (!write && use_zero_page(vma))
1116 (!vma->vm_ops || !vma->vm_ops->fault))
1117 foll_flags |= FOLL_ANON; 1143 foll_flags |= FOLL_ANON;
1118 1144
1119 do { 1145 do {
@@ -1159,6 +1185,8 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1159 1185
1160 cond_resched(); 1186 cond_resched();
1161 } 1187 }
1188 if (IS_ERR(page))
1189 return i ? i : PTR_ERR(page);
1162 if (pages) { 1190 if (pages) {
1163 pages[i] = page; 1191 pages[i] = page;
1164 1192
@@ -1757,7 +1785,6 @@ gotten:
1757 page_table = pte_offset_map_lock(mm, pmd, address, &ptl); 1785 page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
1758 if (likely(pte_same(*page_table, orig_pte))) { 1786 if (likely(pte_same(*page_table, orig_pte))) {
1759 if (old_page) { 1787 if (old_page) {
1760 page_remove_rmap(old_page, vma);
1761 if (!PageAnon(old_page)) { 1788 if (!PageAnon(old_page)) {
1762 dec_mm_counter(mm, file_rss); 1789 dec_mm_counter(mm, file_rss);
1763 inc_mm_counter(mm, anon_rss); 1790 inc_mm_counter(mm, anon_rss);
@@ -1779,6 +1806,32 @@ gotten:
1779 lru_cache_add_active(new_page); 1806 lru_cache_add_active(new_page);
1780 page_add_new_anon_rmap(new_page, vma, address); 1807 page_add_new_anon_rmap(new_page, vma, address);
1781 1808
1809 if (old_page) {
1810 /*
1811 * Only after switching the pte to the new page may
1812 * we remove the mapcount here. Otherwise another
1813 * process may come and find the rmap count decremented
1814 * before the pte is switched to the new page, and
1815 * "reuse" the old page writing into it while our pte
1816 * here still points into it and can be read by other
1817 * threads.
1818 *
1819 * The critical issue is to order this
1820 * page_remove_rmap with the ptp_clear_flush above.
1821 * Those stores are ordered by (if nothing else,)
1822 * the barrier present in the atomic_add_negative
1823 * in page_remove_rmap.
1824 *
1825 * Then the TLB flush in ptep_clear_flush ensures that
1826 * no process can access the old page before the
1827 * decremented mapcount is visible. And the old page
1828 * cannot be reused until after the decremented
1829 * mapcount is visible. So transitively, TLBs to
1830 * old page will be flushed before it can be reused.
1831 */
1832 page_remove_rmap(old_page, vma);
1833 }
1834
1782 /* Free the old page.. */ 1835 /* Free the old page.. */
1783 new_page = old_page; 1836 new_page = old_page;
1784 ret |= VM_FAULT_WRITE; 1837 ret |= VM_FAULT_WRITE;
@@ -2295,8 +2348,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2295 vmf.flags = flags; 2348 vmf.flags = flags;
2296 vmf.page = NULL; 2349 vmf.page = NULL;
2297 2350
2298 BUG_ON(vma->vm_flags & VM_PFNMAP);
2299
2300 ret = vma->vm_ops->fault(vma, &vmf); 2351 ret = vma->vm_ops->fault(vma, &vmf);
2301 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2352 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2302 return ret; 2353 return ret;
diff --git a/mm/migrate.c b/mm/migrate.c
index 449d77d409f5..112bcaeaa104 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -865,6 +865,11 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
865 goto set_status; 865 goto set_status;
866 866
867 page = follow_page(vma, pp->addr, FOLL_GET); 867 page = follow_page(vma, pp->addr, FOLL_GET);
868
869 err = PTR_ERR(page);
870 if (IS_ERR(page))
871 goto set_status;
872
868 err = -ENOENT; 873 err = -ENOENT;
869 if (!page) 874 if (!page)
870 goto set_status; 875 goto set_status;
@@ -928,6 +933,11 @@ static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
928 goto set_status; 933 goto set_status;
929 934
930 page = follow_page(vma, pm->addr, 0); 935 page = follow_page(vma, pm->addr, 0);
936
937 err = PTR_ERR(page);
938 if (IS_ERR(page))
939 goto set_status;
940
931 err = -ENOENT; 941 err = -ENOENT;
932 /* Use PageReserved to check for zero page */ 942 /* Use PageReserved to check for zero page */
933 if (!page || PageReserved(page)) 943 if (!page || PageReserved(page))
diff --git a/mm/mmap.c b/mm/mmap.c
index fac66337da2a..3354fdd83d4b 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -80,7 +80,7 @@ EXPORT_SYMBOL(vm_get_page_prot);
80int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 80int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
81int sysctl_overcommit_ratio = 50; /* default is 50% */ 81int sysctl_overcommit_ratio = 50; /* default is 50% */
82int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT; 82int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
83atomic_t vm_committed_space = ATOMIC_INIT(0); 83atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
84 84
85/* 85/*
86 * Check that a process has enough memory to allocate a new virtual 86 * Check that a process has enough memory to allocate a new virtual
@@ -177,7 +177,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
177 * cast `allowed' as a signed long because vm_committed_space 177 * cast `allowed' as a signed long because vm_committed_space
178 * sometimes has a negative value 178 * sometimes has a negative value
179 */ 179 */
180 if (atomic_read(&vm_committed_space) < (long)allowed) 180 if (atomic_long_read(&vm_committed_space) < (long)allowed)
181 return 0; 181 return 0;
182error: 182error:
183 vm_unacct_memory(pages); 183 vm_unacct_memory(pages);
@@ -245,10 +245,16 @@ asmlinkage unsigned long sys_brk(unsigned long brk)
245 unsigned long rlim, retval; 245 unsigned long rlim, retval;
246 unsigned long newbrk, oldbrk; 246 unsigned long newbrk, oldbrk;
247 struct mm_struct *mm = current->mm; 247 struct mm_struct *mm = current->mm;
248 unsigned long min_brk;
248 249
249 down_write(&mm->mmap_sem); 250 down_write(&mm->mmap_sem);
250 251
251 if (brk < mm->start_brk) 252#ifdef CONFIG_COMPAT_BRK
253 min_brk = mm->end_code;
254#else
255 min_brk = mm->start_brk;
256#endif
257 if (brk < min_brk)
252 goto out; 258 goto out;
253 259
254 /* 260 /*
diff --git a/mm/nommu.c b/mm/nommu.c
index ef8c62cec697..4462b6a3fcb9 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -39,7 +39,7 @@ struct page *mem_map;
39unsigned long max_mapnr; 39unsigned long max_mapnr;
40unsigned long num_physpages; 40unsigned long num_physpages;
41unsigned long askedalloc, realalloc; 41unsigned long askedalloc, realalloc;
42atomic_t vm_committed_space = ATOMIC_INIT(0); 42atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
43int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 43int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
44int sysctl_overcommit_ratio = 50; /* default is 50% */ 44int sysctl_overcommit_ratio = 50; /* default is 50% */
45int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 45int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
@@ -109,16 +109,23 @@ unsigned int kobjsize(const void *objp)
109 * If the object we have should not have ksize performed on it, 109 * If the object we have should not have ksize performed on it,
110 * return size of 0 110 * return size of 0
111 */ 111 */
112 if (!objp || (unsigned long)objp >= memory_end || !((page = virt_to_page(objp)))) 112 if (!objp || !virt_addr_valid(objp))
113 return 0; 113 return 0;
114 114
115 page = virt_to_head_page(objp);
116
117 /*
118 * If the allocator sets PageSlab, we know the pointer came from
119 * kmalloc().
120 */
115 if (PageSlab(page)) 121 if (PageSlab(page))
116 return ksize(objp); 122 return ksize(objp);
117 123
118 BUG_ON(page->index < 0); 124 /*
119 BUG_ON(page->index >= MAX_ORDER); 125 * The ksize() function is only guaranteed to work for pointers
120 126 * returned by kmalloc(). So handle arbitrary pointers here.
121 return (PAGE_SIZE << page->index); 127 */
128 return PAGE_SIZE << compound_order(page);
122} 129}
123 130
124/* 131/*
@@ -1410,7 +1417,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1410 * cast `allowed' as a signed long because vm_committed_space 1417 * cast `allowed' as a signed long because vm_committed_space
1411 * sometimes has a negative value 1418 * sometimes has a negative value
1412 */ 1419 */
1413 if (atomic_read(&vm_committed_space) < (long)allowed) 1420 if (atomic_long_read(&vm_committed_space) < (long)allowed)
1414 return 0; 1421 return 0;
1415error: 1422error:
1416 vm_unacct_memory(pages); 1423 vm_unacct_memory(pages);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 63835579323a..2f552955a02f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -237,16 +237,7 @@ static void bad_page(struct page *page)
237 printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n" 237 printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n"
238 KERN_EMERG "Backtrace:\n"); 238 KERN_EMERG "Backtrace:\n");
239 dump_stack(); 239 dump_stack();
240 page->flags &= ~(1 << PG_lru | 240 page->flags &= ~PAGE_FLAGS_CLEAR_WHEN_BAD;
241 1 << PG_private |
242 1 << PG_locked |
243 1 << PG_active |
244 1 << PG_dirty |
245 1 << PG_reclaim |
246 1 << PG_slab |
247 1 << PG_swapcache |
248 1 << PG_writeback |
249 1 << PG_buddy );
250 set_page_count(page, 0); 241 set_page_count(page, 0);
251 reset_page_mapcount(page); 242 reset_page_mapcount(page);
252 page->mapping = NULL; 243 page->mapping = NULL;
@@ -463,16 +454,7 @@ static inline int free_pages_check(struct page *page)
463 (page->mapping != NULL) | 454 (page->mapping != NULL) |
464 (page_get_page_cgroup(page) != NULL) | 455 (page_get_page_cgroup(page) != NULL) |
465 (page_count(page) != 0) | 456 (page_count(page) != 0) |
466 (page->flags & ( 457 (page->flags & PAGE_FLAGS_CHECK_AT_FREE)))
467 1 << PG_lru |
468 1 << PG_private |
469 1 << PG_locked |
470 1 << PG_active |
471 1 << PG_slab |
472 1 << PG_swapcache |
473 1 << PG_writeback |
474 1 << PG_reserved |
475 1 << PG_buddy ))))
476 bad_page(page); 458 bad_page(page);
477 if (PageDirty(page)) 459 if (PageDirty(page))
478 __ClearPageDirty(page); 460 __ClearPageDirty(page);
@@ -616,17 +598,7 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
616 (page->mapping != NULL) | 598 (page->mapping != NULL) |
617 (page_get_page_cgroup(page) != NULL) | 599 (page_get_page_cgroup(page) != NULL) |
618 (page_count(page) != 0) | 600 (page_count(page) != 0) |
619 (page->flags & ( 601 (page->flags & PAGE_FLAGS_CHECK_AT_PREP)))
620 1 << PG_lru |
621 1 << PG_private |
622 1 << PG_locked |
623 1 << PG_active |
624 1 << PG_dirty |
625 1 << PG_slab |
626 1 << PG_swapcache |
627 1 << PG_writeback |
628 1 << PG_reserved |
629 1 << PG_buddy ))))
630 bad_page(page); 602 bad_page(page);
631 603
632 /* 604 /*
@@ -1396,6 +1368,9 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
1396 1368
1397 (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask, 1369 (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask,
1398 &preferred_zone); 1370 &preferred_zone);
1371 if (!preferred_zone)
1372 return NULL;
1373
1399 classzone_idx = zone_idx(preferred_zone); 1374 classzone_idx = zone_idx(preferred_zone);
1400 1375
1401zonelist_scan: 1376zonelist_scan:
@@ -2804,7 +2779,7 @@ int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
2804 alloc_size = zone->wait_table_hash_nr_entries 2779 alloc_size = zone->wait_table_hash_nr_entries
2805 * sizeof(wait_queue_head_t); 2780 * sizeof(wait_queue_head_t);
2806 2781
2807 if (system_state == SYSTEM_BOOTING) { 2782 if (!slab_is_available()) {
2808 zone->wait_table = (wait_queue_head_t *) 2783 zone->wait_table = (wait_queue_head_t *)
2809 alloc_bootmem_node(pgdat, alloc_size); 2784 alloc_bootmem_node(pgdat, alloc_size);
2810 } else { 2785 } else {
@@ -3378,7 +3353,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
3378 * is used by this zone for memmap. This affects the watermark 3353 * is used by this zone for memmap. This affects the watermark
3379 * and per-cpu initialisations 3354 * and per-cpu initialisations
3380 */ 3355 */
3381 memmap_pages = (size * sizeof(struct page)) >> PAGE_SHIFT; 3356 memmap_pages =
3357 PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
3382 if (realsize >= memmap_pages) { 3358 if (realsize >= memmap_pages) {
3383 realsize -= memmap_pages; 3359 realsize -= memmap_pages;
3384 printk(KERN_DEBUG 3360 printk(KERN_DEBUG
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 0afd2387e507..d5878bed7841 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -3,14 +3,14 @@
3#include <linux/sched.h> 3#include <linux/sched.h>
4 4
5static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 5static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
6 const struct mm_walk *walk, void *private) 6 struct mm_walk *walk)
7{ 7{
8 pte_t *pte; 8 pte_t *pte;
9 int err = 0; 9 int err = 0;
10 10
11 pte = pte_offset_map(pmd, addr); 11 pte = pte_offset_map(pmd, addr);
12 for (;;) { 12 for (;;) {
13 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, private); 13 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
14 if (err) 14 if (err)
15 break; 15 break;
16 addr += PAGE_SIZE; 16 addr += PAGE_SIZE;
@@ -24,7 +24,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
24} 24}
25 25
26static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 26static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
27 const struct mm_walk *walk, void *private) 27 struct mm_walk *walk)
28{ 28{
29 pmd_t *pmd; 29 pmd_t *pmd;
30 unsigned long next; 30 unsigned long next;
@@ -35,15 +35,15 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
35 next = pmd_addr_end(addr, end); 35 next = pmd_addr_end(addr, end);
36 if (pmd_none_or_clear_bad(pmd)) { 36 if (pmd_none_or_clear_bad(pmd)) {
37 if (walk->pte_hole) 37 if (walk->pte_hole)
38 err = walk->pte_hole(addr, next, private); 38 err = walk->pte_hole(addr, next, walk);
39 if (err) 39 if (err)
40 break; 40 break;
41 continue; 41 continue;
42 } 42 }
43 if (walk->pmd_entry) 43 if (walk->pmd_entry)
44 err = walk->pmd_entry(pmd, addr, next, private); 44 err = walk->pmd_entry(pmd, addr, next, walk);
45 if (!err && walk->pte_entry) 45 if (!err && walk->pte_entry)
46 err = walk_pte_range(pmd, addr, next, walk, private); 46 err = walk_pte_range(pmd, addr, next, walk);
47 if (err) 47 if (err)
48 break; 48 break;
49 } while (pmd++, addr = next, addr != end); 49 } while (pmd++, addr = next, addr != end);
@@ -52,7 +52,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
52} 52}
53 53
54static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end, 54static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
55 const struct mm_walk *walk, void *private) 55 struct mm_walk *walk)
56{ 56{
57 pud_t *pud; 57 pud_t *pud;
58 unsigned long next; 58 unsigned long next;
@@ -63,15 +63,15 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
63 next = pud_addr_end(addr, end); 63 next = pud_addr_end(addr, end);
64 if (pud_none_or_clear_bad(pud)) { 64 if (pud_none_or_clear_bad(pud)) {
65 if (walk->pte_hole) 65 if (walk->pte_hole)
66 err = walk->pte_hole(addr, next, private); 66 err = walk->pte_hole(addr, next, walk);
67 if (err) 67 if (err)
68 break; 68 break;
69 continue; 69 continue;
70 } 70 }
71 if (walk->pud_entry) 71 if (walk->pud_entry)
72 err = walk->pud_entry(pud, addr, next, private); 72 err = walk->pud_entry(pud, addr, next, walk);
73 if (!err && (walk->pmd_entry || walk->pte_entry)) 73 if (!err && (walk->pmd_entry || walk->pte_entry))
74 err = walk_pmd_range(pud, addr, next, walk, private); 74 err = walk_pmd_range(pud, addr, next, walk);
75 if (err) 75 if (err)
76 break; 76 break;
77 } while (pud++, addr = next, addr != end); 77 } while (pud++, addr = next, addr != end);
@@ -85,15 +85,15 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
85 * @addr: starting address 85 * @addr: starting address
86 * @end: ending address 86 * @end: ending address
87 * @walk: set of callbacks to invoke for each level of the tree 87 * @walk: set of callbacks to invoke for each level of the tree
88 * @private: private data passed to the callback function
89 * 88 *
90 * Recursively walk the page table for the memory area in a VMA, 89 * Recursively walk the page table for the memory area in a VMA,
91 * calling supplied callbacks. Callbacks are called in-order (first 90 * calling supplied callbacks. Callbacks are called in-order (first
92 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD, 91 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
93 * etc.). If lower-level callbacks are omitted, walking depth is reduced. 92 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
94 * 93 *
95 * Each callback receives an entry pointer, the start and end of the 94 * Each callback receives an entry pointer and the start and end of the
96 * associated range, and a caller-supplied private data pointer. 95 * associated range, and a copy of the original mm_walk for access to
96 * the ->private or ->mm fields.
97 * 97 *
98 * No locks are taken, but the bottom level iterator will map PTE 98 * No locks are taken, but the bottom level iterator will map PTE
99 * directories from highmem if necessary. 99 * directories from highmem if necessary.
@@ -101,9 +101,8 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
101 * If any callback returns a non-zero value, the walk is aborted and 101 * If any callback returns a non-zero value, the walk is aborted and
102 * the return value is propagated back to the caller. Otherwise 0 is returned. 102 * the return value is propagated back to the caller. Otherwise 0 is returned.
103 */ 103 */
104int walk_page_range(const struct mm_struct *mm, 104int walk_page_range(unsigned long addr, unsigned long end,
105 unsigned long addr, unsigned long end, 105 struct mm_walk *walk)
106 const struct mm_walk *walk, void *private)
107{ 106{
108 pgd_t *pgd; 107 pgd_t *pgd;
109 unsigned long next; 108 unsigned long next;
@@ -112,21 +111,24 @@ int walk_page_range(const struct mm_struct *mm,
112 if (addr >= end) 111 if (addr >= end)
113 return err; 112 return err;
114 113
115 pgd = pgd_offset(mm, addr); 114 if (!walk->mm)
115 return -EINVAL;
116
117 pgd = pgd_offset(walk->mm, addr);
116 do { 118 do {
117 next = pgd_addr_end(addr, end); 119 next = pgd_addr_end(addr, end);
118 if (pgd_none_or_clear_bad(pgd)) { 120 if (pgd_none_or_clear_bad(pgd)) {
119 if (walk->pte_hole) 121 if (walk->pte_hole)
120 err = walk->pte_hole(addr, next, private); 122 err = walk->pte_hole(addr, next, walk);
121 if (err) 123 if (err)
122 break; 124 break;
123 continue; 125 continue;
124 } 126 }
125 if (walk->pgd_entry) 127 if (walk->pgd_entry)
126 err = walk->pgd_entry(pgd, addr, next, private); 128 err = walk->pgd_entry(pgd, addr, next, walk);
127 if (!err && 129 if (!err &&
128 (walk->pud_entry || walk->pmd_entry || walk->pte_entry)) 130 (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
129 err = walk_pud_range(pgd, addr, next, walk, private); 131 err = walk_pud_range(pgd, addr, next, walk);
130 if (err) 132 if (err)
131 break; 133 break;
132 } while (pgd++, addr = next, addr != end); 134 } while (pgd++, addr = next, addr != end);
diff --git a/mm/slab.c b/mm/slab.c
index 06236e4ddc1b..046607f05f3e 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3263,9 +3263,12 @@ retry:
3263 3263
3264 if (cpuset_zone_allowed_hardwall(zone, flags) && 3264 if (cpuset_zone_allowed_hardwall(zone, flags) &&
3265 cache->nodelists[nid] && 3265 cache->nodelists[nid] &&
3266 cache->nodelists[nid]->free_objects) 3266 cache->nodelists[nid]->free_objects) {
3267 obj = ____cache_alloc_node(cache, 3267 obj = ____cache_alloc_node(cache,
3268 flags | GFP_THISNODE, nid); 3268 flags | GFP_THISNODE, nid);
3269 if (obj)
3270 break;
3271 }
3269 } 3272 }
3270 3273
3271 if (!obj) { 3274 if (!obj) {
diff --git a/mm/slob.c b/mm/slob.c
index 6038cbadf796..a3ad6671adf1 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -469,8 +469,9 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
469 return ZERO_SIZE_PTR; 469 return ZERO_SIZE_PTR;
470 470
471 m = slob_alloc(size + align, gfp, align, node); 471 m = slob_alloc(size + align, gfp, align, node);
472 if (m) 472 if (!m)
473 *m = size; 473 return NULL;
474 *m = size;
474 return (void *)m + align; 475 return (void *)m + align;
475 } else { 476 } else {
476 void *ret; 477 void *ret;
diff --git a/mm/slub.c b/mm/slub.c
index a505a828ef41..0987d1cd943c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2726,9 +2726,10 @@ size_t ksize(const void *object)
2726 2726
2727 page = virt_to_head_page(object); 2727 page = virt_to_head_page(object);
2728 2728
2729 if (unlikely(!PageSlab(page))) 2729 if (unlikely(!PageSlab(page))) {
2730 WARN_ON(!PageCompound(page));
2730 return PAGE_SIZE << compound_order(page); 2731 return PAGE_SIZE << compound_order(page);
2731 2732 }
2732 s = page->slab; 2733 s = page->slab;
2733 2734
2734#ifdef CONFIG_SLUB_DEBUG 2735#ifdef CONFIG_SLUB_DEBUG
diff --git a/mm/swap.c b/mm/swap.c
index 91e194445a5e..45c9f25a8a3b 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -503,7 +503,7 @@ void vm_acct_memory(long pages)
503 local = &__get_cpu_var(committed_space); 503 local = &__get_cpu_var(committed_space);
504 *local += pages; 504 *local += pages;
505 if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 505 if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) {
506 atomic_add(*local, &vm_committed_space); 506 atomic_long_add(*local, &vm_committed_space);
507 *local = 0; 507 *local = 0;
508 } 508 }
509 preempt_enable(); 509 preempt_enable();
@@ -520,7 +520,7 @@ static int cpu_swap_callback(struct notifier_block *nfb,
520 520
521 committed = &per_cpu(committed_space, (long)hcpu); 521 committed = &per_cpu(committed_space, (long)hcpu);
522 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 522 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
523 atomic_add(*committed, &vm_committed_space); 523 atomic_long_add(*committed, &vm_committed_space);
524 *committed = 0; 524 *committed = 0;
525 drain_cpu_pagevecs((long)hcpu); 525 drain_cpu_pagevecs((long)hcpu);
526 } 526 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 9a29901ad3b3..967d30ccd92b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1307,7 +1307,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
1307 struct scan_control *sc) 1307 struct scan_control *sc)
1308{ 1308{
1309 int priority; 1309 int priority;
1310 int ret = 0; 1310 unsigned long ret = 0;
1311 unsigned long total_scanned = 0; 1311 unsigned long total_scanned = 0;
1312 unsigned long nr_reclaimed = 0; 1312 unsigned long nr_reclaimed = 0;
1313 struct reclaim_state *reclaim_state = current->reclaim_state; 1313 struct reclaim_state *reclaim_state = current->reclaim_state;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-14 09:38:03 -0500