aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2008-06-16 05:27:45 -0400
committerIngo Molnar <mingo@elte.hu>2008-06-16 05:27:45 -0400
commitd939d2851fd12568e2eabb2916769e8a57ba5c89 (patch)
treef3158a5ddd41541a61126f9e48de1ce89c632f64 /mm
parent9f5314fb4d556d3132c784d0df47352b2830ca53 (diff)
parent066519068ad2fbe98c7f45552b1f592903a9c8c8 (diff)
Merge branch 'linus' into x86/irq
Diffstat (limited to 'mm')
-rw-r--r--mm/backing-dev.c12
-rw-r--r--mm/filemap.c5
-rw-r--r--mm/hugetlb.c2
-rw-r--r--mm/memory.c23
-rw-r--r--mm/memory_hotplug.c86
-rw-r--r--mm/mmap.c12
-rw-r--r--mm/mprotect.c11
-rw-r--r--mm/nommu.c21
-rw-r--r--mm/page_alloc.c45
-rw-r--r--mm/pagewalk.c42
-rw-r--r--mm/pdflush.c4
-rw-r--r--mm/slob.c5
-rw-r--r--mm/slub.c5
-rw-r--r--mm/swap.c4
-rw-r--r--mm/vmscan.c2
-rw-r--r--mm/vmstat.c2
16 files changed, 154 insertions, 127 deletions
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 7c4f9e097095..f2e574dbc300 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -172,30 +172,22 @@ postcore_initcall(bdi_class_init);
172int bdi_register(struct backing_dev_info *bdi, struct device *parent, 172int bdi_register(struct backing_dev_info *bdi, struct device *parent,
173 const char *fmt, ...) 173 const char *fmt, ...)
174{ 174{
175 char *name;
176 va_list args; 175 va_list args;
177 int ret = 0; 176 int ret = 0;
178 struct device *dev; 177 struct device *dev;
179 178
180 va_start(args, fmt); 179 va_start(args, fmt);
181 name = kvasprintf(GFP_KERNEL, fmt, args); 180 dev = device_create_vargs(bdi_class, parent, MKDEV(0, 0), bdi, fmt, args);
182 va_end(args); 181 va_end(args);
183
184 if (!name)
185 return -ENOMEM;
186
187 dev = device_create(bdi_class, parent, MKDEV(0, 0), name);
188 if (IS_ERR(dev)) { 182 if (IS_ERR(dev)) {
189 ret = PTR_ERR(dev); 183 ret = PTR_ERR(dev);
190 goto exit; 184 goto exit;
191 } 185 }
192 186
193 bdi->dev = dev; 187 bdi->dev = dev;
194 dev_set_drvdata(bdi->dev, bdi); 188 bdi_debug_register(bdi, dev_name(dev));
195 bdi_debug_register(bdi, name);
196 189
197exit: 190exit:
198 kfree(name);
199 return ret; 191 return ret;
200} 192}
201EXPORT_SYMBOL(bdi_register); 193EXPORT_SYMBOL(bdi_register);
diff --git a/mm/filemap.c b/mm/filemap.c
index 2dead9adf8b7..1e6a7d34874f 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -1461,6 +1461,11 @@ page_not_uptodate:
1461 */ 1461 */
1462 ClearPageError(page); 1462 ClearPageError(page);
1463 error = mapping->a_ops->readpage(file, page); 1463 error = mapping->a_ops->readpage(file, page);
1464 if (!error) {
1465 wait_on_page_locked(page);
1466 if (!PageUptodate(page))
1467 error = -EIO;
1468 }
1464 page_cache_release(page); 1469 page_cache_release(page);
1465 1470
1466 if (!error || error == AOP_TRUNCATED_PAGE) 1471 if (!error || error == AOP_TRUNCATED_PAGE)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index bbf953eeb58b..ab171274ef21 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -785,7 +785,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
785 continue; 785 continue;
786 786
787 spin_lock(&dst->page_table_lock); 787 spin_lock(&dst->page_table_lock);
788 spin_lock(&src->page_table_lock); 788 spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING);
789 if (!huge_pte_none(huge_ptep_get(src_pte))) { 789 if (!huge_pte_none(huge_ptep_get(src_pte))) {
790 if (cow) 790 if (cow)
791 huge_ptep_set_wrprotect(src, addr, src_pte); 791 huge_ptep_set_wrprotect(src, addr, src_pte);
diff --git a/mm/memory.c b/mm/memory.c
index 48c122d42ed7..19e0ae9beecb 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -311,6 +311,21 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
311 if (!new) 311 if (!new)
312 return -ENOMEM; 312 return -ENOMEM;
313 313
314 /*
315 * Ensure all pte setup (eg. pte page lock and page clearing) are
316 * visible before the pte is made visible to other CPUs by being
317 * put into page tables.
318 *
319 * The other side of the story is the pointer chasing in the page
320 * table walking code (when walking the page table without locking;
321 * ie. most of the time). Fortunately, these data accesses consist
322 * of a chain of data-dependent loads, meaning most CPUs (alpha
323 * being the notable exception) will already guarantee loads are
324 * seen in-order. See the alpha page table accessors for the
325 * smp_read_barrier_depends() barriers in page table walking code.
326 */
327 smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
328
314 spin_lock(&mm->page_table_lock); 329 spin_lock(&mm->page_table_lock);
315 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 330 if (!pmd_present(*pmd)) { /* Has another populated it ? */
316 mm->nr_ptes++; 331 mm->nr_ptes++;
@@ -329,6 +344,8 @@ int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
329 if (!new) 344 if (!new)
330 return -ENOMEM; 345 return -ENOMEM;
331 346
347 smp_wmb(); /* See comment in __pte_alloc */
348
332 spin_lock(&init_mm.page_table_lock); 349 spin_lock(&init_mm.page_table_lock);
333 if (!pmd_present(*pmd)) { /* Has another populated it ? */ 350 if (!pmd_present(*pmd)) { /* Has another populated it ? */
334 pmd_populate_kernel(&init_mm, pmd, new); 351 pmd_populate_kernel(&init_mm, pmd, new);
@@ -2278,8 +2295,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2278 vmf.flags = flags; 2295 vmf.flags = flags;
2279 vmf.page = NULL; 2296 vmf.page = NULL;
2280 2297
2281 BUG_ON(vma->vm_flags & VM_PFNMAP);
2282
2283 ret = vma->vm_ops->fault(vma, &vmf); 2298 ret = vma->vm_ops->fault(vma, &vmf);
2284 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) 2299 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
2285 return ret; 2300 return ret;
@@ -2619,6 +2634,8 @@ int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
2619 if (!new) 2634 if (!new)
2620 return -ENOMEM; 2635 return -ENOMEM;
2621 2636
2637 smp_wmb(); /* See comment in __pte_alloc */
2638
2622 spin_lock(&mm->page_table_lock); 2639 spin_lock(&mm->page_table_lock);
2623 if (pgd_present(*pgd)) /* Another has populated it */ 2640 if (pgd_present(*pgd)) /* Another has populated it */
2624 pud_free(mm, new); 2641 pud_free(mm, new);
@@ -2640,6 +2657,8 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
2640 if (!new) 2657 if (!new)
2641 return -ENOMEM; 2658 return -ENOMEM;
2642 2659
2660 smp_wmb(); /* See comment in __pte_alloc */
2661
2643 spin_lock(&mm->page_table_lock); 2662 spin_lock(&mm->page_table_lock);
2644#ifndef __ARCH_HAS_4LEVEL_HACK 2663#ifndef __ARCH_HAS_4LEVEL_HACK
2645 if (pud_present(*pud)) /* Another has populated it */ 2664 if (pud_present(*pud)) /* Another has populated it */
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index b17dca7249f8..833f854eabe5 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -159,21 +159,58 @@ void register_page_bootmem_info_node(struct pglist_data *pgdat)
159} 159}
160#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 160#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
161 161
162static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
163 unsigned long end_pfn)
164{
165 unsigned long old_zone_end_pfn;
166
167 zone_span_writelock(zone);
168
169 old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
170 if (start_pfn < zone->zone_start_pfn)
171 zone->zone_start_pfn = start_pfn;
172
173 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
174 zone->zone_start_pfn;
175
176 zone_span_writeunlock(zone);
177}
178
179static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
180 unsigned long end_pfn)
181{
182 unsigned long old_pgdat_end_pfn =
183 pgdat->node_start_pfn + pgdat->node_spanned_pages;
184
185 if (start_pfn < pgdat->node_start_pfn)
186 pgdat->node_start_pfn = start_pfn;
187
188 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
189 pgdat->node_start_pfn;
190}
191
162static int __add_zone(struct zone *zone, unsigned long phys_start_pfn) 192static int __add_zone(struct zone *zone, unsigned long phys_start_pfn)
163{ 193{
164 struct pglist_data *pgdat = zone->zone_pgdat; 194 struct pglist_data *pgdat = zone->zone_pgdat;
165 int nr_pages = PAGES_PER_SECTION; 195 int nr_pages = PAGES_PER_SECTION;
166 int nid = pgdat->node_id; 196 int nid = pgdat->node_id;
167 int zone_type; 197 int zone_type;
198 unsigned long flags;
168 199
169 zone_type = zone - pgdat->node_zones; 200 zone_type = zone - pgdat->node_zones;
170 if (!zone->wait_table) { 201 if (!zone->wait_table) {
171 int ret = 0; 202 int ret;
203
172 ret = init_currently_empty_zone(zone, phys_start_pfn, 204 ret = init_currently_empty_zone(zone, phys_start_pfn,
173 nr_pages, MEMMAP_HOTPLUG); 205 nr_pages, MEMMAP_HOTPLUG);
174 if (ret < 0) 206 if (ret)
175 return ret; 207 return ret;
176 } 208 }
209 pgdat_resize_lock(zone->zone_pgdat, &flags);
210 grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
211 grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
212 phys_start_pfn + nr_pages);
213 pgdat_resize_unlock(zone->zone_pgdat, &flags);
177 memmap_init_zone(nr_pages, nid, zone_type, 214 memmap_init_zone(nr_pages, nid, zone_type,
178 phys_start_pfn, MEMMAP_HOTPLUG); 215 phys_start_pfn, MEMMAP_HOTPLUG);
179 return 0; 216 return 0;
@@ -299,36 +336,6 @@ int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
299} 336}
300EXPORT_SYMBOL_GPL(__remove_pages); 337EXPORT_SYMBOL_GPL(__remove_pages);
301 338
302static void grow_zone_span(struct zone *zone,
303 unsigned long start_pfn, unsigned long end_pfn)
304{
305 unsigned long old_zone_end_pfn;
306
307 zone_span_writelock(zone);
308
309 old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
310 if (start_pfn < zone->zone_start_pfn)
311 zone->zone_start_pfn = start_pfn;
312
313 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
314 zone->zone_start_pfn;
315
316 zone_span_writeunlock(zone);
317}
318
319static void grow_pgdat_span(struct pglist_data *pgdat,
320 unsigned long start_pfn, unsigned long end_pfn)
321{
322 unsigned long old_pgdat_end_pfn =
323 pgdat->node_start_pfn + pgdat->node_spanned_pages;
324
325 if (start_pfn < pgdat->node_start_pfn)
326 pgdat->node_start_pfn = start_pfn;
327
328 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
329 pgdat->node_start_pfn;
330}
331
332void online_page(struct page *page) 339void online_page(struct page *page)
333{ 340{
334 totalram_pages++; 341 totalram_pages++;
@@ -367,7 +374,6 @@ static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
367 374
368int online_pages(unsigned long pfn, unsigned long nr_pages) 375int online_pages(unsigned long pfn, unsigned long nr_pages)
369{ 376{
370 unsigned long flags;
371 unsigned long onlined_pages = 0; 377 unsigned long onlined_pages = 0;
372 struct zone *zone; 378 struct zone *zone;
373 int need_zonelists_rebuild = 0; 379 int need_zonelists_rebuild = 0;
@@ -395,11 +401,6 @@ int online_pages(unsigned long pfn, unsigned long nr_pages)
395 * memory_block->state_mutex. 401 * memory_block->state_mutex.
396 */ 402 */
397 zone = page_zone(pfn_to_page(pfn)); 403 zone = page_zone(pfn_to_page(pfn));
398 pgdat_resize_lock(zone->zone_pgdat, &flags);
399 grow_zone_span(zone, pfn, pfn + nr_pages);
400 grow_pgdat_span(zone->zone_pgdat, pfn, pfn + nr_pages);
401 pgdat_resize_unlock(zone->zone_pgdat, &flags);
402
403 /* 404 /*
404 * If this zone is not populated, then it is not in zonelist. 405 * If this zone is not populated, then it is not in zonelist.
405 * This means the page allocator ignores this zone. 406 * This means the page allocator ignores this zone.
@@ -408,8 +409,15 @@ int online_pages(unsigned long pfn, unsigned long nr_pages)
408 if (!populated_zone(zone)) 409 if (!populated_zone(zone))
409 need_zonelists_rebuild = 1; 410 need_zonelists_rebuild = 1;
410 411
411 walk_memory_resource(pfn, nr_pages, &onlined_pages, 412 ret = walk_memory_resource(pfn, nr_pages, &onlined_pages,
412 online_pages_range); 413 online_pages_range);
414 if (ret) {
415 printk(KERN_DEBUG "online_pages %lx at %lx failed\n",
416 nr_pages, pfn);
417 memory_notify(MEM_CANCEL_ONLINE, &arg);
418 return ret;
419 }
420
413 zone->present_pages += onlined_pages; 421 zone->present_pages += onlined_pages;
414 zone->zone_pgdat->node_present_pages += onlined_pages; 422 zone->zone_pgdat->node_present_pages += onlined_pages;
415 423
diff --git a/mm/mmap.c b/mm/mmap.c
index fac66337da2a..3354fdd83d4b 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -80,7 +80,7 @@ EXPORT_SYMBOL(vm_get_page_prot);
80int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 80int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
81int sysctl_overcommit_ratio = 50; /* default is 50% */ 81int sysctl_overcommit_ratio = 50; /* default is 50% */
82int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT; 82int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
83atomic_t vm_committed_space = ATOMIC_INIT(0); 83atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
84 84
85/* 85/*
86 * Check that a process has enough memory to allocate a new virtual 86 * Check that a process has enough memory to allocate a new virtual
@@ -177,7 +177,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
177 * cast `allowed' as a signed long because vm_committed_space 177 * cast `allowed' as a signed long because vm_committed_space
178 * sometimes has a negative value 178 * sometimes has a negative value
179 */ 179 */
180 if (atomic_read(&vm_committed_space) < (long)allowed) 180 if (atomic_long_read(&vm_committed_space) < (long)allowed)
181 return 0; 181 return 0;
182error: 182error:
183 vm_unacct_memory(pages); 183 vm_unacct_memory(pages);
@@ -245,10 +245,16 @@ asmlinkage unsigned long sys_brk(unsigned long brk)
245 unsigned long rlim, retval; 245 unsigned long rlim, retval;
246 unsigned long newbrk, oldbrk; 246 unsigned long newbrk, oldbrk;
247 struct mm_struct *mm = current->mm; 247 struct mm_struct *mm = current->mm;
248 unsigned long min_brk;
248 249
249 down_write(&mm->mmap_sem); 250 down_write(&mm->mmap_sem);
250 251
251 if (brk < mm->start_brk) 252#ifdef CONFIG_COMPAT_BRK
253 min_brk = mm->end_code;
254#else
255 min_brk = mm->start_brk;
256#endif
257 if (brk < min_brk)
252 goto out; 258 goto out;
253 259
254 /* 260 /*
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 4de546899dc1..a5bf31c27375 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -26,6 +26,13 @@
26#include <asm/cacheflush.h> 26#include <asm/cacheflush.h>
27#include <asm/tlbflush.h> 27#include <asm/tlbflush.h>
28 28
29#ifndef pgprot_modify
30static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
31{
32 return newprot;
33}
34#endif
35
29static void change_pte_range(struct mm_struct *mm, pmd_t *pmd, 36static void change_pte_range(struct mm_struct *mm, pmd_t *pmd,
30 unsigned long addr, unsigned long end, pgprot_t newprot, 37 unsigned long addr, unsigned long end, pgprot_t newprot,
31 int dirty_accountable) 38 int dirty_accountable)
@@ -192,7 +199,9 @@ success:
192 * held in write mode. 199 * held in write mode.
193 */ 200 */
194 vma->vm_flags = newflags; 201 vma->vm_flags = newflags;
195 vma->vm_page_prot = vm_get_page_prot(newflags); 202 vma->vm_page_prot = pgprot_modify(vma->vm_page_prot,
203 vm_get_page_prot(newflags));
204
196 if (vma_wants_writenotify(vma)) { 205 if (vma_wants_writenotify(vma)) {
197 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED); 206 vma->vm_page_prot = vm_get_page_prot(newflags & ~VM_SHARED);
198 dirty_accountable = 1; 207 dirty_accountable = 1;
diff --git a/mm/nommu.c b/mm/nommu.c
index ef8c62cec697..4462b6a3fcb9 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -39,7 +39,7 @@ struct page *mem_map;
39unsigned long max_mapnr; 39unsigned long max_mapnr;
40unsigned long num_physpages; 40unsigned long num_physpages;
41unsigned long askedalloc, realalloc; 41unsigned long askedalloc, realalloc;
42atomic_t vm_committed_space = ATOMIC_INIT(0); 42atomic_long_t vm_committed_space = ATOMIC_LONG_INIT(0);
43int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */ 43int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
44int sysctl_overcommit_ratio = 50; /* default is 50% */ 44int sysctl_overcommit_ratio = 50; /* default is 50% */
45int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT; 45int sysctl_max_map_count = DEFAULT_MAX_MAP_COUNT;
@@ -109,16 +109,23 @@ unsigned int kobjsize(const void *objp)
109 * If the object we have should not have ksize performed on it, 109 * If the object we have should not have ksize performed on it,
110 * return size of 0 110 * return size of 0
111 */ 111 */
112 if (!objp || (unsigned long)objp >= memory_end || !((page = virt_to_page(objp)))) 112 if (!objp || !virt_addr_valid(objp))
113 return 0; 113 return 0;
114 114
115 page = virt_to_head_page(objp);
116
117 /*
118 * If the allocator sets PageSlab, we know the pointer came from
119 * kmalloc().
120 */
115 if (PageSlab(page)) 121 if (PageSlab(page))
116 return ksize(objp); 122 return ksize(objp);
117 123
118 BUG_ON(page->index < 0); 124 /*
119 BUG_ON(page->index >= MAX_ORDER); 125 * The ksize() function is only guaranteed to work for pointers
120 126 * returned by kmalloc(). So handle arbitrary pointers here.
121 return (PAGE_SIZE << page->index); 127 */
128 return PAGE_SIZE << compound_order(page);
122} 129}
123 130
124/* 131/*
@@ -1410,7 +1417,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
1410 * cast `allowed' as a signed long because vm_committed_space 1417 * cast `allowed' as a signed long because vm_committed_space
1411 * sometimes has a negative value 1418 * sometimes has a negative value
1412 */ 1419 */
1413 if (atomic_read(&vm_committed_space) < (long)allowed) 1420 if (atomic_long_read(&vm_committed_space) < (long)allowed)
1414 return 0; 1421 return 0;
1415error: 1422error:
1416 vm_unacct_memory(pages); 1423 vm_unacct_memory(pages);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index bdd5c432c426..2f552955a02f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -237,16 +237,7 @@ static void bad_page(struct page *page)
237 printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n" 237 printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n"
238 KERN_EMERG "Backtrace:\n"); 238 KERN_EMERG "Backtrace:\n");
239 dump_stack(); 239 dump_stack();
240 page->flags &= ~(1 << PG_lru | 240 page->flags &= ~PAGE_FLAGS_CLEAR_WHEN_BAD;
241 1 << PG_private |
242 1 << PG_locked |
243 1 << PG_active |
244 1 << PG_dirty |
245 1 << PG_reclaim |
246 1 << PG_slab |
247 1 << PG_swapcache |
248 1 << PG_writeback |
249 1 << PG_buddy );
250 set_page_count(page, 0); 241 set_page_count(page, 0);
251 reset_page_mapcount(page); 242 reset_page_mapcount(page);
252 page->mapping = NULL; 243 page->mapping = NULL;
@@ -463,16 +454,7 @@ static inline int free_pages_check(struct page *page)
463 (page->mapping != NULL) | 454 (page->mapping != NULL) |
464 (page_get_page_cgroup(page) != NULL) | 455 (page_get_page_cgroup(page) != NULL) |
465 (page_count(page) != 0) | 456 (page_count(page) != 0) |
466 (page->flags & ( 457 (page->flags & PAGE_FLAGS_CHECK_AT_FREE)))
467 1 << PG_lru |
468 1 << PG_private |
469 1 << PG_locked |
470 1 << PG_active |
471 1 << PG_slab |
472 1 << PG_swapcache |
473 1 << PG_writeback |
474 1 << PG_reserved |
475 1 << PG_buddy ))))
476 bad_page(page); 458 bad_page(page);
477 if (PageDirty(page)) 459 if (PageDirty(page))
478 __ClearPageDirty(page); 460 __ClearPageDirty(page);
@@ -616,17 +598,7 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
616 (page->mapping != NULL) | 598 (page->mapping != NULL) |
617 (page_get_page_cgroup(page) != NULL) | 599 (page_get_page_cgroup(page) != NULL) |
618 (page_count(page) != 0) | 600 (page_count(page) != 0) |
619 (page->flags & ( 601 (page->flags & PAGE_FLAGS_CHECK_AT_PREP)))
620 1 << PG_lru |
621 1 << PG_private |
622 1 << PG_locked |
623 1 << PG_active |
624 1 << PG_dirty |
625 1 << PG_slab |
626 1 << PG_swapcache |
627 1 << PG_writeback |
628 1 << PG_reserved |
629 1 << PG_buddy ))))
630 bad_page(page); 602 bad_page(page);
631 603
632 /* 604 /*
@@ -1396,6 +1368,9 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
1396 1368
1397 (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask, 1369 (void)first_zones_zonelist(zonelist, high_zoneidx, nodemask,
1398 &preferred_zone); 1370 &preferred_zone);
1371 if (!preferred_zone)
1372 return NULL;
1373
1399 classzone_idx = zone_idx(preferred_zone); 1374 classzone_idx = zone_idx(preferred_zone);
1400 1375
1401zonelist_scan: 1376zonelist_scan:
@@ -2804,7 +2779,7 @@ int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
2804 alloc_size = zone->wait_table_hash_nr_entries 2779 alloc_size = zone->wait_table_hash_nr_entries
2805 * sizeof(wait_queue_head_t); 2780 * sizeof(wait_queue_head_t);
2806 2781
2807 if (system_state == SYSTEM_BOOTING) { 2782 if (!slab_is_available()) {
2808 zone->wait_table = (wait_queue_head_t *) 2783 zone->wait_table = (wait_queue_head_t *)
2809 alloc_bootmem_node(pgdat, alloc_size); 2784 alloc_bootmem_node(pgdat, alloc_size);
2810 } else { 2785 } else {
@@ -2862,8 +2837,6 @@ __meminit int init_currently_empty_zone(struct zone *zone,
2862 2837
2863 zone->zone_start_pfn = zone_start_pfn; 2838 zone->zone_start_pfn = zone_start_pfn;
2864 2839
2865 memmap_init(size, pgdat->node_id, zone_idx(zone), zone_start_pfn);
2866
2867 zone_init_free_lists(zone); 2840 zone_init_free_lists(zone);
2868 2841
2869 return 0; 2842 return 0;
@@ -3380,7 +3353,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
3380 * is used by this zone for memmap. This affects the watermark 3353 * is used by this zone for memmap. This affects the watermark
3381 * and per-cpu initialisations 3354 * and per-cpu initialisations
3382 */ 3355 */
3383 memmap_pages = (size * sizeof(struct page)) >> PAGE_SHIFT; 3356 memmap_pages =
3357 PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
3384 if (realsize >= memmap_pages) { 3358 if (realsize >= memmap_pages) {
3385 realsize -= memmap_pages; 3359 realsize -= memmap_pages;
3386 printk(KERN_DEBUG 3360 printk(KERN_DEBUG
@@ -3433,6 +3407,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
3433 ret = init_currently_empty_zone(zone, zone_start_pfn, 3407 ret = init_currently_empty_zone(zone, zone_start_pfn,
3434 size, MEMMAP_EARLY); 3408 size, MEMMAP_EARLY);
3435 BUG_ON(ret); 3409 BUG_ON(ret);
3410 memmap_init(size, nid, j, zone_start_pfn);
3436 zone_start_pfn += size; 3411 zone_start_pfn += size;
3437 } 3412 }
3438} 3413}
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index 0afd2387e507..d5878bed7841 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -3,14 +3,14 @@
3#include <linux/sched.h> 3#include <linux/sched.h>
4 4
5static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 5static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
6 const struct mm_walk *walk, void *private) 6 struct mm_walk *walk)
7{ 7{
8 pte_t *pte; 8 pte_t *pte;
9 int err = 0; 9 int err = 0;
10 10
11 pte = pte_offset_map(pmd, addr); 11 pte = pte_offset_map(pmd, addr);
12 for (;;) { 12 for (;;) {
13 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, private); 13 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
14 if (err) 14 if (err)
15 break; 15 break;
16 addr += PAGE_SIZE; 16 addr += PAGE_SIZE;
@@ -24,7 +24,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
24} 24}
25 25
26static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, 26static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
27 const struct mm_walk *walk, void *private) 27 struct mm_walk *walk)
28{ 28{
29 pmd_t *pmd; 29 pmd_t *pmd;
30 unsigned long next; 30 unsigned long next;
@@ -35,15 +35,15 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
35 next = pmd_addr_end(addr, end); 35 next = pmd_addr_end(addr, end);
36 if (pmd_none_or_clear_bad(pmd)) { 36 if (pmd_none_or_clear_bad(pmd)) {
37 if (walk->pte_hole) 37 if (walk->pte_hole)
38 err = walk->pte_hole(addr, next, private); 38 err = walk->pte_hole(addr, next, walk);
39 if (err) 39 if (err)
40 break; 40 break;
41 continue; 41 continue;
42 } 42 }
43 if (walk->pmd_entry) 43 if (walk->pmd_entry)
44 err = walk->pmd_entry(pmd, addr, next, private); 44 err = walk->pmd_entry(pmd, addr, next, walk);
45 if (!err && walk->pte_entry) 45 if (!err && walk->pte_entry)
46 err = walk_pte_range(pmd, addr, next, walk, private); 46 err = walk_pte_range(pmd, addr, next, walk);
47 if (err) 47 if (err)
48 break; 48 break;
49 } while (pmd++, addr = next, addr != end); 49 } while (pmd++, addr = next, addr != end);
@@ -52,7 +52,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
52} 52}
53 53
54static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end, 54static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
55 const struct mm_walk *walk, void *private) 55 struct mm_walk *walk)
56{ 56{
57 pud_t *pud; 57 pud_t *pud;
58 unsigned long next; 58 unsigned long next;
@@ -63,15 +63,15 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
63 next = pud_addr_end(addr, end); 63 next = pud_addr_end(addr, end);
64 if (pud_none_or_clear_bad(pud)) { 64 if (pud_none_or_clear_bad(pud)) {
65 if (walk->pte_hole) 65 if (walk->pte_hole)
66 err = walk->pte_hole(addr, next, private); 66 err = walk->pte_hole(addr, next, walk);
67 if (err) 67 if (err)
68 break; 68 break;
69 continue; 69 continue;
70 } 70 }
71 if (walk->pud_entry) 71 if (walk->pud_entry)
72 err = walk->pud_entry(pud, addr, next, private); 72 err = walk->pud_entry(pud, addr, next, walk);
73 if (!err && (walk->pmd_entry || walk->pte_entry)) 73 if (!err && (walk->pmd_entry || walk->pte_entry))
74 err = walk_pmd_range(pud, addr, next, walk, private); 74 err = walk_pmd_range(pud, addr, next, walk);
75 if (err) 75 if (err)
76 break; 76 break;
77 } while (pud++, addr = next, addr != end); 77 } while (pud++, addr = next, addr != end);
@@ -85,15 +85,15 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
85 * @addr: starting address 85 * @addr: starting address
86 * @end: ending address 86 * @end: ending address
87 * @walk: set of callbacks to invoke for each level of the tree 87 * @walk: set of callbacks to invoke for each level of the tree
88 * @private: private data passed to the callback function
89 * 88 *
90 * Recursively walk the page table for the memory area in a VMA, 89 * Recursively walk the page table for the memory area in a VMA,
91 * calling supplied callbacks. Callbacks are called in-order (first 90 * calling supplied callbacks. Callbacks are called in-order (first
92 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD, 91 * PGD, first PUD, first PMD, first PTE, second PTE... second PMD,
93 * etc.). If lower-level callbacks are omitted, walking depth is reduced. 92 * etc.). If lower-level callbacks are omitted, walking depth is reduced.
94 * 93 *
95 * Each callback receives an entry pointer, the start and end of the 94 * Each callback receives an entry pointer and the start and end of the
96 * associated range, and a caller-supplied private data pointer. 95 * associated range, and a copy of the original mm_walk for access to
96 * the ->private or ->mm fields.
97 * 97 *
98 * No locks are taken, but the bottom level iterator will map PTE 98 * No locks are taken, but the bottom level iterator will map PTE
99 * directories from highmem if necessary. 99 * directories from highmem if necessary.
@@ -101,9 +101,8 @@ static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
101 * If any callback returns a non-zero value, the walk is aborted and 101 * If any callback returns a non-zero value, the walk is aborted and
102 * the return value is propagated back to the caller. Otherwise 0 is returned. 102 * the return value is propagated back to the caller. Otherwise 0 is returned.
103 */ 103 */
104int walk_page_range(const struct mm_struct *mm, 104int walk_page_range(unsigned long addr, unsigned long end,
105 unsigned long addr, unsigned long end, 105 struct mm_walk *walk)
106 const struct mm_walk *walk, void *private)
107{ 106{
108 pgd_t *pgd; 107 pgd_t *pgd;
109 unsigned long next; 108 unsigned long next;
@@ -112,21 +111,24 @@ int walk_page_range(const struct mm_struct *mm,
112 if (addr >= end) 111 if (addr >= end)
113 return err; 112 return err;
114 113
115 pgd = pgd_offset(mm, addr); 114 if (!walk->mm)
115 return -EINVAL;
116
117 pgd = pgd_offset(walk->mm, addr);
116 do { 118 do {
117 next = pgd_addr_end(addr, end); 119 next = pgd_addr_end(addr, end);
118 if (pgd_none_or_clear_bad(pgd)) { 120 if (pgd_none_or_clear_bad(pgd)) {
119 if (walk->pte_hole) 121 if (walk->pte_hole)
120 err = walk->pte_hole(addr, next, private); 122 err = walk->pte_hole(addr, next, walk);
121 if (err) 123 if (err)
122 break; 124 break;
123 continue; 125 continue;
124 } 126 }
125 if (walk->pgd_entry) 127 if (walk->pgd_entry)
126 err = walk->pgd_entry(pgd, addr, next, private); 128 err = walk->pgd_entry(pgd, addr, next, walk);
127 if (!err && 129 if (!err &&
128 (walk->pud_entry || walk->pmd_entry || walk->pte_entry)) 130 (walk->pud_entry || walk->pmd_entry || walk->pte_entry))
129 err = walk_pud_range(pgd, addr, next, walk, private); 131 err = walk_pud_range(pgd, addr, next, walk);
130 if (err) 132 if (err)
131 break; 133 break;
132 } while (pgd++, addr = next, addr != end); 134 } while (pgd++, addr = next, addr != end);
diff --git a/mm/pdflush.c b/mm/pdflush.c
index 1c96cfc9e040..9d834aa4b979 100644
--- a/mm/pdflush.c
+++ b/mm/pdflush.c
@@ -207,7 +207,6 @@ int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
207 207
208 spin_lock_irqsave(&pdflush_lock, flags); 208 spin_lock_irqsave(&pdflush_lock, flags);
209 if (list_empty(&pdflush_list)) { 209 if (list_empty(&pdflush_list)) {
210 spin_unlock_irqrestore(&pdflush_lock, flags);
211 ret = -1; 210 ret = -1;
212 } else { 211 } else {
213 struct pdflush_work *pdf; 212 struct pdflush_work *pdf;
@@ -219,8 +218,9 @@ int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
219 pdf->fn = fn; 218 pdf->fn = fn;
220 pdf->arg0 = arg0; 219 pdf->arg0 = arg0;
221 wake_up_process(pdf->who); 220 wake_up_process(pdf->who);
222 spin_unlock_irqrestore(&pdflush_lock, flags);
223 } 221 }
222 spin_unlock_irqrestore(&pdflush_lock, flags);
223
224 return ret; 224 return ret;
225} 225}
226 226
diff --git a/mm/slob.c b/mm/slob.c
index 6038cbadf796..a3ad6671adf1 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -469,8 +469,9 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
469 return ZERO_SIZE_PTR; 469 return ZERO_SIZE_PTR;
470 470
471 m = slob_alloc(size + align, gfp, align, node); 471 m = slob_alloc(size + align, gfp, align, node);
472 if (m) 472 if (!m)
473 *m = size; 473 return NULL;
474 *m = size;
474 return (void *)m + align; 475 return (void *)m + align;
475 } else { 476 } else {
476 void *ret; 477 void *ret;
diff --git a/mm/slub.c b/mm/slub.c
index a505a828ef41..0987d1cd943c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2726,9 +2726,10 @@ size_t ksize(const void *object)
2726 2726
2727 page = virt_to_head_page(object); 2727 page = virt_to_head_page(object);
2728 2728
2729 if (unlikely(!PageSlab(page))) 2729 if (unlikely(!PageSlab(page))) {
2730 WARN_ON(!PageCompound(page));
2730 return PAGE_SIZE << compound_order(page); 2731 return PAGE_SIZE << compound_order(page);
2731 2732 }
2732 s = page->slab; 2733 s = page->slab;
2733 2734
2734#ifdef CONFIG_SLUB_DEBUG 2735#ifdef CONFIG_SLUB_DEBUG
diff --git a/mm/swap.c b/mm/swap.c
index 91e194445a5e..45c9f25a8a3b 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -503,7 +503,7 @@ void vm_acct_memory(long pages)
503 local = &__get_cpu_var(committed_space); 503 local = &__get_cpu_var(committed_space);
504 *local += pages; 504 *local += pages;
505 if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) { 505 if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) {
506 atomic_add(*local, &vm_committed_space); 506 atomic_long_add(*local, &vm_committed_space);
507 *local = 0; 507 *local = 0;
508 } 508 }
509 preempt_enable(); 509 preempt_enable();
@@ -520,7 +520,7 @@ static int cpu_swap_callback(struct notifier_block *nfb,
520 520
521 committed = &per_cpu(committed_space, (long)hcpu); 521 committed = &per_cpu(committed_space, (long)hcpu);
522 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 522 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
523 atomic_add(*committed, &vm_committed_space); 523 atomic_long_add(*committed, &vm_committed_space);
524 *committed = 0; 524 *committed = 0;
525 drain_cpu_pagevecs((long)hcpu); 525 drain_cpu_pagevecs((long)hcpu);
526 } 526 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 9a29901ad3b3..967d30ccd92b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1307,7 +1307,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
1307 struct scan_control *sc) 1307 struct scan_control *sc)
1308{ 1308{
1309 int priority; 1309 int priority;
1310 int ret = 0; 1310 unsigned long ret = 0;
1311 unsigned long total_scanned = 0; 1311 unsigned long total_scanned = 0;
1312 unsigned long nr_reclaimed = 0; 1312 unsigned long nr_reclaimed = 0;
1313 struct reclaim_state *reclaim_state = current->reclaim_state; 1313 struct reclaim_state *reclaim_state = current->reclaim_state;
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 1a32130b958c..db9eabb2c5b3 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -41,7 +41,9 @@ static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask)
41*/ 41*/
42void all_vm_events(unsigned long *ret) 42void all_vm_events(unsigned long *ret)
43{ 43{
44 get_online_cpus();
44 sum_vm_events(ret, &cpu_online_map); 45 sum_vm_events(ret, &cpu_online_map);
46 put_online_cpus();
45} 47}
46EXPORT_SYMBOL_GPL(all_vm_events); 48EXPORT_SYMBOL_GPL(all_vm_events);
47 49