aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/memory_hotplug.c9
-rw-r--r--mm/page_cgroup.c56
-rw-r--r--mm/slub.c6
-rw-r--r--mm/sparse.c2
-rw-r--r--mm/vmalloc.c20
-rw-r--r--mm/vmscan.c2
6 files changed, 63 insertions, 32 deletions
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index b5b2b15085a8..b17371185468 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -189,7 +189,7 @@ static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
189 pgdat->node_start_pfn; 189 pgdat->node_start_pfn;
190} 190}
191 191
192static int __add_zone(struct zone *zone, unsigned long phys_start_pfn) 192static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
193{ 193{
194 struct pglist_data *pgdat = zone->zone_pgdat; 194 struct pglist_data *pgdat = zone->zone_pgdat;
195 int nr_pages = PAGES_PER_SECTION; 195 int nr_pages = PAGES_PER_SECTION;
@@ -216,7 +216,7 @@ static int __add_zone(struct zone *zone, unsigned long phys_start_pfn)
216 return 0; 216 return 0;
217} 217}
218 218
219static int __add_section(struct zone *zone, unsigned long phys_start_pfn) 219static int __meminit __add_section(struct zone *zone, unsigned long phys_start_pfn)
220{ 220{
221 int nr_pages = PAGES_PER_SECTION; 221 int nr_pages = PAGES_PER_SECTION;
222 int ret; 222 int ret;
@@ -273,7 +273,7 @@ static int __remove_section(struct zone *zone, struct mem_section *ms)
273 * call this function after deciding the zone to which to 273 * call this function after deciding the zone to which to
274 * add the new pages. 274 * add the new pages.
275 */ 275 */
276int __add_pages(struct zone *zone, unsigned long phys_start_pfn, 276int __ref __add_pages(struct zone *zone, unsigned long phys_start_pfn,
277 unsigned long nr_pages) 277 unsigned long nr_pages)
278{ 278{
279 unsigned long i; 279 unsigned long i;
@@ -470,7 +470,8 @@ static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
470} 470}
471 471
472 472
473int add_memory(int nid, u64 start, u64 size) 473/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
474int __ref add_memory(int nid, u64 start, u64 size)
474{ 475{
475 pg_data_t *pgdat = NULL; 476 pg_data_t *pgdat = NULL;
476 int new_pgdat = 0; 477 int new_pgdat = 0;
diff --git a/mm/page_cgroup.c b/mm/page_cgroup.c
index 1223d927904d..0b3cbf090a67 100644
--- a/mm/page_cgroup.c
+++ b/mm/page_cgroup.c
@@ -21,7 +21,7 @@ static unsigned long total_usage;
21#if !defined(CONFIG_SPARSEMEM) 21#if !defined(CONFIG_SPARSEMEM)
22 22
23 23
24void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) 24void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
25{ 25{
26 pgdat->node_page_cgroup = NULL; 26 pgdat->node_page_cgroup = NULL;
27} 27}
@@ -97,7 +97,8 @@ struct page_cgroup *lookup_page_cgroup(struct page *page)
97 return section->page_cgroup + pfn; 97 return section->page_cgroup + pfn;
98} 98}
99 99
100int __meminit init_section_page_cgroup(unsigned long pfn) 100/* __alloc_bootmem...() is protected by !slab_available() */
101int __init_refok init_section_page_cgroup(unsigned long pfn)
101{ 102{
102 struct mem_section *section; 103 struct mem_section *section;
103 struct page_cgroup *base, *pc; 104 struct page_cgroup *base, *pc;
@@ -106,19 +107,29 @@ int __meminit init_section_page_cgroup(unsigned long pfn)
106 107
107 section = __pfn_to_section(pfn); 108 section = __pfn_to_section(pfn);
108 109
109 if (section->page_cgroup) 110 if (!section->page_cgroup) {
110 return 0; 111 nid = page_to_nid(pfn_to_page(pfn));
111 112 table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
112 nid = page_to_nid(pfn_to_page(pfn)); 113 if (slab_is_available()) {
113 114 base = kmalloc_node(table_size, GFP_KERNEL, nid);
114 table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION; 115 if (!base)
115 if (slab_is_available()) { 116 base = vmalloc_node(table_size, nid);
116 base = kmalloc_node(table_size, GFP_KERNEL, nid); 117 } else {
117 if (!base) 118 base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
118 base = vmalloc_node(table_size, nid); 119 table_size,
119 } else {
120 base = __alloc_bootmem_node_nopanic(NODE_DATA(nid), table_size,
121 PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); 120 PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
121 }
122 } else {
123 /*
124 * We don't have to allocate page_cgroup again, but
125 * address of memmap may be changed. So, we have to initialize
126 * again.
127 */
128 base = section->page_cgroup + pfn;
129 table_size = 0;
130 /* check address of memmap is changed or not. */
131 if (base->page == pfn_to_page(pfn))
132 return 0;
122 } 133 }
123 134
124 if (!base) { 135 if (!base) {
@@ -158,7 +169,7 @@ void __free_page_cgroup(unsigned long pfn)
158 } 169 }
159} 170}
160 171
161int online_page_cgroup(unsigned long start_pfn, 172int __meminit online_page_cgroup(unsigned long start_pfn,
162 unsigned long nr_pages, 173 unsigned long nr_pages,
163 int nid) 174 int nid)
164{ 175{
@@ -183,7 +194,7 @@ int online_page_cgroup(unsigned long start_pfn,
183 return -ENOMEM; 194 return -ENOMEM;
184} 195}
185 196
186int offline_page_cgroup(unsigned long start_pfn, 197int __meminit offline_page_cgroup(unsigned long start_pfn,
187 unsigned long nr_pages, int nid) 198 unsigned long nr_pages, int nid)
188{ 199{
189 unsigned long start, end, pfn; 200 unsigned long start, end, pfn;
@@ -197,7 +208,7 @@ int offline_page_cgroup(unsigned long start_pfn,
197 208
198} 209}
199 210
200static int page_cgroup_callback(struct notifier_block *self, 211static int __meminit page_cgroup_callback(struct notifier_block *self,
201 unsigned long action, void *arg) 212 unsigned long action, void *arg)
202{ 213{
203 struct memory_notify *mn = arg; 214 struct memory_notify *mn = arg;
@@ -207,18 +218,23 @@ static int page_cgroup_callback(struct notifier_block *self,
207 ret = online_page_cgroup(mn->start_pfn, 218 ret = online_page_cgroup(mn->start_pfn,
208 mn->nr_pages, mn->status_change_nid); 219 mn->nr_pages, mn->status_change_nid);
209 break; 220 break;
210 case MEM_CANCEL_ONLINE:
211 case MEM_OFFLINE: 221 case MEM_OFFLINE:
212 offline_page_cgroup(mn->start_pfn, 222 offline_page_cgroup(mn->start_pfn,
213 mn->nr_pages, mn->status_change_nid); 223 mn->nr_pages, mn->status_change_nid);
214 break; 224 break;
225 case MEM_CANCEL_ONLINE:
215 case MEM_GOING_OFFLINE: 226 case MEM_GOING_OFFLINE:
216 break; 227 break;
217 case MEM_ONLINE: 228 case MEM_ONLINE:
218 case MEM_CANCEL_OFFLINE: 229 case MEM_CANCEL_OFFLINE:
219 break; 230 break;
220 } 231 }
221 ret = notifier_from_errno(ret); 232
233 if (ret)
234 ret = notifier_from_errno(ret);
235 else
236 ret = NOTIFY_OK;
237
222 return ret; 238 return ret;
223} 239}
224 240
@@ -248,7 +264,7 @@ void __init page_cgroup_init(void)
248 " want\n"); 264 " want\n");
249} 265}
250 266
251void __init pgdat_page_cgroup_init(struct pglist_data *pgdat) 267void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
252{ 268{
253 return; 269 return;
254} 270}
diff --git a/mm/slub.c b/mm/slub.c
index 7ad489af9561..749588a50a5a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2931,8 +2931,10 @@ static int slab_memory_callback(struct notifier_block *self,
2931 case MEM_CANCEL_OFFLINE: 2931 case MEM_CANCEL_OFFLINE:
2932 break; 2932 break;
2933 } 2933 }
2934 2934 if (ret)
2935 ret = notifier_from_errno(ret); 2935 ret = notifier_from_errno(ret);
2936 else
2937 ret = NOTIFY_OK;
2936 return ret; 2938 return ret;
2937} 2939}
2938 2940
diff --git a/mm/sparse.c b/mm/sparse.c
index 39db301b920d..083f5b63e7a8 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -570,7 +570,7 @@ static void free_section_usemap(struct page *memmap, unsigned long *usemap)
570 * set. If this is <=0, then that means that the passed-in 570 * set. If this is <=0, then that means that the passed-in
571 * map was not consumed and must be freed. 571 * map was not consumed and must be freed.
572 */ 572 */
573int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 573int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
574 int nr_pages) 574 int nr_pages)
575{ 575{
576 unsigned long section_nr = pfn_to_section_nr(start_pfn); 576 unsigned long section_nr = pfn_to_section_nr(start_pfn);
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 30f826d484f0..f3f6e0758562 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -77,7 +77,6 @@ static void vunmap_page_range(unsigned long addr, unsigned long end)
77 77
78 BUG_ON(addr >= end); 78 BUG_ON(addr >= end);
79 pgd = pgd_offset_k(addr); 79 pgd = pgd_offset_k(addr);
80 flush_cache_vunmap(addr, end);
81 do { 80 do {
82 next = pgd_addr_end(addr, end); 81 next = pgd_addr_end(addr, end);
83 if (pgd_none_or_clear_bad(pgd)) 82 if (pgd_none_or_clear_bad(pgd))
@@ -543,9 +542,10 @@ static void purge_vmap_area_lazy(void)
543} 542}
544 543
545/* 544/*
546 * Free and unmap a vmap area 545 * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
546 * called for the correct range previously.
547 */ 547 */
548static void free_unmap_vmap_area(struct vmap_area *va) 548static void free_unmap_vmap_area_noflush(struct vmap_area *va)
549{ 549{
550 va->flags |= VM_LAZY_FREE; 550 va->flags |= VM_LAZY_FREE;
551 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr); 551 atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
@@ -553,6 +553,15 @@ static void free_unmap_vmap_area(struct vmap_area *va)
553 try_purge_vmap_area_lazy(); 553 try_purge_vmap_area_lazy();
554} 554}
555 555
556/*
557 * Free and unmap a vmap area
558 */
559static void free_unmap_vmap_area(struct vmap_area *va)
560{
561 flush_cache_vunmap(va->va_start, va->va_end);
562 free_unmap_vmap_area_noflush(va);
563}
564
556static struct vmap_area *find_vmap_area(unsigned long addr) 565static struct vmap_area *find_vmap_area(unsigned long addr)
557{ 566{
558 struct vmap_area *va; 567 struct vmap_area *va;
@@ -734,7 +743,7 @@ static void free_vmap_block(struct vmap_block *vb)
734 spin_unlock(&vmap_block_tree_lock); 743 spin_unlock(&vmap_block_tree_lock);
735 BUG_ON(tmp != vb); 744 BUG_ON(tmp != vb);
736 745
737 free_unmap_vmap_area(vb->va); 746 free_unmap_vmap_area_noflush(vb->va);
738 call_rcu(&vb->rcu_head, rcu_free_vb); 747 call_rcu(&vb->rcu_head, rcu_free_vb);
739} 748}
740 749
@@ -796,6 +805,9 @@ static void vb_free(const void *addr, unsigned long size)
796 805
797 BUG_ON(size & ~PAGE_MASK); 806 BUG_ON(size & ~PAGE_MASK);
798 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); 807 BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
808
809 flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
810
799 order = get_order(size); 811 order = get_order(size);
800 812
801 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); 813 offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 7ea1440b53db..62e7f62fb559 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1248,6 +1248,7 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
1248 list_add(&page->lru, &l_inactive); 1248 list_add(&page->lru, &l_inactive);
1249 } 1249 }
1250 1250
1251 spin_lock_irq(&zone->lru_lock);
1251 /* 1252 /*
1252 * Count referenced pages from currently used mappings as 1253 * Count referenced pages from currently used mappings as
1253 * rotated, even though they are moved to the inactive list. 1254 * rotated, even though they are moved to the inactive list.
@@ -1263,7 +1264,6 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
1263 1264
1264 pgmoved = 0; 1265 pgmoved = 0;
1265 lru = LRU_BASE + file * LRU_FILE; 1266 lru = LRU_BASE + file * LRU_FILE;
1266 spin_lock_irq(&zone->lru_lock);
1267 while (!list_empty(&l_inactive)) { 1267 while (!list_empty(&l_inactive)) {
1268 page = lru_to_page(&l_inactive); 1268 page = lru_to_page(&l_inactive);
1269 prefetchw_prev_lru_page(page, &l_inactive, flags); 1269 prefetchw_prev_lru_page(page, &l_inactive, flags);