aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2008-11-07 01:43:03 -0500
committerDavid S. Miller <davem@davemloft.net>2008-11-07 01:43:03 -0500
commit9eeda9abd1faf489f3df9a1f557975f4c8650363 (patch)
tree3e0a58e25b776cfbee193195460324dccb1886c7 /mm
parent61c9eaf90081cbe6dc4f389e0056bff76eca19ec (diff)
parent4bab0ea1d42dd1927af9df6fbf0003fc00617c50 (diff)
Merge branch 'master' of master.kernel.org:/pub/scm/linux/kernel/git/davem/net-2.6
Conflicts: drivers/net/wireless/ath5k/base.c net/8021q/vlan_core.c
Diffstat (limited to 'mm')
-rw-r--r--mm/hugetlb.c49
-rw-r--r--mm/internal.h29
-rw-r--r--mm/mempolicy.c18
-rw-r--r--mm/migrate.c2
-rw-r--r--mm/oom_kill.c3
-rw-r--r--mm/page_alloc.c28
-rw-r--r--mm/page_isolation.c5
-rw-r--r--mm/sparse-vmemmap.c2
-rw-r--r--mm/vmalloc.c2
9 files changed, 116 insertions, 22 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 421aee99b84a..d143ab67be44 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -354,11 +354,26 @@ static int vma_has_reserves(struct vm_area_struct *vma)
354 return 0; 354 return 0;
355} 355}
356 356
357static void clear_gigantic_page(struct page *page,
358 unsigned long addr, unsigned long sz)
359{
360 int i;
361 struct page *p = page;
362
363 might_sleep();
364 for (i = 0; i < sz/PAGE_SIZE; i++, p = mem_map_next(p, page, i)) {
365 cond_resched();
366 clear_user_highpage(p, addr + i * PAGE_SIZE);
367 }
368}
357static void clear_huge_page(struct page *page, 369static void clear_huge_page(struct page *page,
358 unsigned long addr, unsigned long sz) 370 unsigned long addr, unsigned long sz)
359{ 371{
360 int i; 372 int i;
361 373
374 if (unlikely(sz > MAX_ORDER_NR_PAGES))
375 return clear_gigantic_page(page, addr, sz);
376
362 might_sleep(); 377 might_sleep();
363 for (i = 0; i < sz/PAGE_SIZE; i++) { 378 for (i = 0; i < sz/PAGE_SIZE; i++) {
364 cond_resched(); 379 cond_resched();
@@ -366,12 +381,32 @@ static void clear_huge_page(struct page *page,
366 } 381 }
367} 382}
368 383
384static void copy_gigantic_page(struct page *dst, struct page *src,
385 unsigned long addr, struct vm_area_struct *vma)
386{
387 int i;
388 struct hstate *h = hstate_vma(vma);
389 struct page *dst_base = dst;
390 struct page *src_base = src;
391 might_sleep();
392 for (i = 0; i < pages_per_huge_page(h); ) {
393 cond_resched();
394 copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma);
395
396 i++;
397 dst = mem_map_next(dst, dst_base, i);
398 src = mem_map_next(src, src_base, i);
399 }
400}
369static void copy_huge_page(struct page *dst, struct page *src, 401static void copy_huge_page(struct page *dst, struct page *src,
370 unsigned long addr, struct vm_area_struct *vma) 402 unsigned long addr, struct vm_area_struct *vma)
371{ 403{
372 int i; 404 int i;
373 struct hstate *h = hstate_vma(vma); 405 struct hstate *h = hstate_vma(vma);
374 406
407 if (unlikely(pages_per_huge_page(h) > MAX_ORDER_NR_PAGES))
408 return copy_gigantic_page(dst, src, addr, vma);
409
375 might_sleep(); 410 might_sleep();
376 for (i = 0; i < pages_per_huge_page(h); i++) { 411 for (i = 0; i < pages_per_huge_page(h); i++) {
377 cond_resched(); 412 cond_resched();
@@ -456,6 +491,8 @@ static void update_and_free_page(struct hstate *h, struct page *page)
456{ 491{
457 int i; 492 int i;
458 493
494 VM_BUG_ON(h->order >= MAX_ORDER);
495
459 h->nr_huge_pages--; 496 h->nr_huge_pages--;
460 h->nr_huge_pages_node[page_to_nid(page)]--; 497 h->nr_huge_pages_node[page_to_nid(page)]--;
461 for (i = 0; i < pages_per_huge_page(h); i++) { 498 for (i = 0; i < pages_per_huge_page(h); i++) {
@@ -970,6 +1007,14 @@ found:
970 return 1; 1007 return 1;
971} 1008}
972 1009
1010static void prep_compound_huge_page(struct page *page, int order)
1011{
1012 if (unlikely(order > (MAX_ORDER - 1)))
1013 prep_compound_gigantic_page(page, order);
1014 else
1015 prep_compound_page(page, order);
1016}
1017
973/* Put bootmem huge pages into the standard lists after mem_map is up */ 1018/* Put bootmem huge pages into the standard lists after mem_map is up */
974static void __init gather_bootmem_prealloc(void) 1019static void __init gather_bootmem_prealloc(void)
975{ 1020{
@@ -980,7 +1025,7 @@ static void __init gather_bootmem_prealloc(void)
980 struct hstate *h = m->hstate; 1025 struct hstate *h = m->hstate;
981 __ClearPageReserved(page); 1026 __ClearPageReserved(page);
982 WARN_ON(page_count(page) != 1); 1027 WARN_ON(page_count(page) != 1);
983 prep_compound_page(page, h->order); 1028 prep_compound_huge_page(page, h->order);
984 prep_new_huge_page(h, page, page_to_nid(page)); 1029 prep_new_huge_page(h, page, page_to_nid(page));
985 } 1030 }
986} 1031}
@@ -2130,7 +2175,7 @@ same_page:
2130 if (zeropage_ok) 2175 if (zeropage_ok)
2131 pages[i] = ZERO_PAGE(0); 2176 pages[i] = ZERO_PAGE(0);
2132 else 2177 else
2133 pages[i] = page + pfn_offset; 2178 pages[i] = mem_map_offset(page, pfn_offset);
2134 get_page(pages[i]); 2179 get_page(pages[i]);
2135 } 2180 }
2136 2181
diff --git a/mm/internal.h b/mm/internal.h
index e4e728bdf324..13333bc2eb68 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -17,6 +17,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
17 unsigned long floor, unsigned long ceiling); 17 unsigned long floor, unsigned long ceiling);
18 18
19extern void prep_compound_page(struct page *page, unsigned long order); 19extern void prep_compound_page(struct page *page, unsigned long order);
20extern void prep_compound_gigantic_page(struct page *page, unsigned long order);
20 21
21static inline void set_page_count(struct page *page, int v) 22static inline void set_page_count(struct page *page, int v)
22{ 23{
@@ -176,6 +177,34 @@ static inline void free_page_mlock(struct page *page) { }
176#endif /* CONFIG_UNEVICTABLE_LRU */ 177#endif /* CONFIG_UNEVICTABLE_LRU */
177 178
178/* 179/*
180 * Return the mem_map entry representing the 'offset' subpage within
181 * the maximally aligned gigantic page 'base'. Handle any discontiguity
182 * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
183 */
184static inline struct page *mem_map_offset(struct page *base, int offset)
185{
186 if (unlikely(offset >= MAX_ORDER_NR_PAGES))
187 return pfn_to_page(page_to_pfn(base) + offset);
188 return base + offset;
189}
190
191/*
192 * Iterator over all subpages withing the maximally aligned gigantic
193 * page 'base'. Handle any discontiguity in the mem_map.
194 */
195static inline struct page *mem_map_next(struct page *iter,
196 struct page *base, int offset)
197{
198 if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
199 unsigned long pfn = page_to_pfn(base) + offset;
200 if (!pfn_valid(pfn))
201 return NULL;
202 return pfn_to_page(pfn);
203 }
204 return iter + 1;
205}
206
207/*
179 * FLATMEM and DISCONTIGMEM configurations use alloc_bootmem_node, 208 * FLATMEM and DISCONTIGMEM configurations use alloc_bootmem_node,
180 * so all functions starting at paging_init should be marked __init 209 * so all functions starting at paging_init should be marked __init
181 * in those cases. SPARSEMEM, however, allows for memory hotplug, 210 * in those cases. SPARSEMEM, however, allows for memory hotplug,
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 36f42573a335..e9493b1c1117 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -489,12 +489,6 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
489 int err; 489 int err;
490 struct vm_area_struct *first, *vma, *prev; 490 struct vm_area_struct *first, *vma, *prev;
491 491
492 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
493
494 err = migrate_prep();
495 if (err)
496 return ERR_PTR(err);
497 }
498 492
499 first = find_vma(mm, start); 493 first = find_vma(mm, start);
500 if (!first) 494 if (!first)
@@ -809,9 +803,13 @@ int do_migrate_pages(struct mm_struct *mm,
809 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags) 803 const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
810{ 804{
811 int busy = 0; 805 int busy = 0;
812 int err = 0; 806 int err;
813 nodemask_t tmp; 807 nodemask_t tmp;
814 808
809 err = migrate_prep();
810 if (err)
811 return err;
812
815 down_read(&mm->mmap_sem); 813 down_read(&mm->mmap_sem);
816 814
817 err = migrate_vmas(mm, from_nodes, to_nodes, flags); 815 err = migrate_vmas(mm, from_nodes, to_nodes, flags);
@@ -974,6 +972,12 @@ static long do_mbind(unsigned long start, unsigned long len,
974 start, start + len, mode, mode_flags, 972 start, start + len, mode, mode_flags,
975 nmask ? nodes_addr(*nmask)[0] : -1); 973 nmask ? nodes_addr(*nmask)[0] : -1);
976 974
975 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
976
977 err = migrate_prep();
978 if (err)
979 return err;
980 }
977 down_write(&mm->mmap_sem); 981 down_write(&mm->mmap_sem);
978 vma = check_range(mm, start, end, nmask, 982 vma = check_range(mm, start, end, nmask,
979 flags | MPOL_MF_INVERT, &pagelist); 983 flags | MPOL_MF_INVERT, &pagelist);
diff --git a/mm/migrate.c b/mm/migrate.c
index 6602941bfab0..385db89f0c33 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -841,12 +841,12 @@ static int do_move_page_to_node_array(struct mm_struct *mm,
841 struct page_to_node *pp; 841 struct page_to_node *pp;
842 LIST_HEAD(pagelist); 842 LIST_HEAD(pagelist);
843 843
844 migrate_prep();
844 down_read(&mm->mmap_sem); 845 down_read(&mm->mmap_sem);
845 846
846 /* 847 /*
847 * Build a list of pages to migrate 848 * Build a list of pages to migrate
848 */ 849 */
849 migrate_prep();
850 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 850 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
851 struct vm_area_struct *vma; 851 struct vm_area_struct *vma;
852 struct page *page; 852 struct page *page;
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 64e5b4bcd964..a0a01902f551 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -38,7 +38,6 @@ static DEFINE_SPINLOCK(zone_scan_mutex);
38 * badness - calculate a numeric value for how bad this task has been 38 * badness - calculate a numeric value for how bad this task has been
39 * @p: task struct of which task we should calculate 39 * @p: task struct of which task we should calculate
40 * @uptime: current uptime in seconds 40 * @uptime: current uptime in seconds
41 * @mem: target memory controller
42 * 41 *
43 * The formula used is relatively simple and documented inline in the 42 * The formula used is relatively simple and documented inline in the
44 * function. The main rationale is that we want to select a good task 43 * function. The main rationale is that we want to select a good task
@@ -295,6 +294,8 @@ static void dump_tasks(const struct mem_cgroup *mem)
295 continue; 294 continue;
296 if (mem && !task_in_mem_cgroup(p, mem)) 295 if (mem && !task_in_mem_cgroup(p, mem))
297 continue; 296 continue;
297 if (!thread_group_leader(p))
298 continue;
298 299
299 task_lock(p); 300 task_lock(p);
300 printk(KERN_INFO "[%5d] %5d %5d %8lu %8lu %3d %3d %s\n", 301 printk(KERN_INFO "[%5d] %5d %5d %8lu %8lu %3d %3d %s\n",
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index d0a240fbb8bf..54069e64e3a8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -263,24 +263,39 @@ void prep_compound_page(struct page *page, unsigned long order)
263{ 263{
264 int i; 264 int i;
265 int nr_pages = 1 << order; 265 int nr_pages = 1 << order;
266
267 set_compound_page_dtor(page, free_compound_page);
268 set_compound_order(page, order);
269 __SetPageHead(page);
270 for (i = 1; i < nr_pages; i++) {
271 struct page *p = page + i;
272
273 __SetPageTail(p);
274 p->first_page = page;
275 }
276}
277
278#ifdef CONFIG_HUGETLBFS
279void prep_compound_gigantic_page(struct page *page, unsigned long order)
280{
281 int i;
282 int nr_pages = 1 << order;
266 struct page *p = page + 1; 283 struct page *p = page + 1;
267 284
268 set_compound_page_dtor(page, free_compound_page); 285 set_compound_page_dtor(page, free_compound_page);
269 set_compound_order(page, order); 286 set_compound_order(page, order);
270 __SetPageHead(page); 287 __SetPageHead(page);
271 for (i = 1; i < nr_pages; i++, p++) { 288 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
272 if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0))
273 p = pfn_to_page(page_to_pfn(page) + i);
274 __SetPageTail(p); 289 __SetPageTail(p);
275 p->first_page = page; 290 p->first_page = page;
276 } 291 }
277} 292}
293#endif
278 294
279static void destroy_compound_page(struct page *page, unsigned long order) 295static void destroy_compound_page(struct page *page, unsigned long order)
280{ 296{
281 int i; 297 int i;
282 int nr_pages = 1 << order; 298 int nr_pages = 1 << order;
283 struct page *p = page + 1;
284 299
285 if (unlikely(compound_order(page) != order)) 300 if (unlikely(compound_order(page) != order))
286 bad_page(page); 301 bad_page(page);
@@ -288,9 +303,8 @@ static void destroy_compound_page(struct page *page, unsigned long order)
288 if (unlikely(!PageHead(page))) 303 if (unlikely(!PageHead(page)))
289 bad_page(page); 304 bad_page(page);
290 __ClearPageHead(page); 305 __ClearPageHead(page);
291 for (i = 1; i < nr_pages; i++, p++) { 306 for (i = 1; i < nr_pages; i++) {
292 if (unlikely((i & (MAX_ORDER_NR_PAGES - 1)) == 0)) 307 struct page *p = page + i;
293 p = pfn_to_page(page_to_pfn(page) + i);
294 308
295 if (unlikely(!PageTail(p) | 309 if (unlikely(!PageTail(p) |
296 (p->first_page != page))) 310 (p->first_page != page)))
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index b70a7fec1ff6..5e0ffd967452 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -130,10 +130,11 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
130 if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE) 130 if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
131 break; 131 break;
132 } 132 }
133 if (pfn < end_pfn) 133 page = __first_valid_page(start_pfn, end_pfn - start_pfn);
134 if ((pfn < end_pfn) || !page)
134 return -EBUSY; 135 return -EBUSY;
135 /* Check all pages are free or Marked as ISOLATED */ 136 /* Check all pages are free or Marked as ISOLATED */
136 zone = page_zone(pfn_to_page(pfn)); 137 zone = page_zone(page);
137 spin_lock_irqsave(&zone->lock, flags); 138 spin_lock_irqsave(&zone->lock, flags);
138 ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn); 139 ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn);
139 spin_unlock_irqrestore(&zone->lock, flags); 140 spin_unlock_irqrestore(&zone->lock, flags);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index a91b5f8fcaf6..a13ea6401ae7 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -64,7 +64,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
64 unsigned long pfn = pte_pfn(*pte); 64 unsigned long pfn = pte_pfn(*pte);
65 int actual_node = early_pfn_to_nid(pfn); 65 int actual_node = early_pfn_to_nid(pfn);
66 66
67 if (actual_node != node) 67 if (node_distance(actual_node, node) > LOCAL_DISTANCE)
68 printk(KERN_WARNING "[%lx-%lx] potential offnode " 68 printk(KERN_WARNING "[%lx-%lx] potential offnode "
69 "page_structs\n", start, end - 1); 69 "page_structs\n", start, end - 1);
70} 70}
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index f1cc03bbf6ac..66fad3fc02b1 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -178,7 +178,7 @@ static int vmap_page_range(unsigned long addr, unsigned long end,
178static inline int is_vmalloc_or_module_addr(const void *x) 178static inline int is_vmalloc_or_module_addr(const void *x)
179{ 179{
180 /* 180 /*
181 * x86-64 and sparc64 put modules in a special place, 181 * ARM, x86-64 and sparc64 put modules in a special place,
182 * and fall back on vmalloc() if that fails. Others 182 * and fall back on vmalloc() if that fails. Others
183 * just put it in the vmalloc space. 183 * just put it in the vmalloc space.
184 */ 184 */