aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/hugetlb.c67
1 files changed, 61 insertions, 6 deletions
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 6f978218c2c8..3a790651475a 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -32,6 +32,7 @@ static unsigned int surplus_huge_pages_node[MAX_NUMNODES];
32static gfp_t htlb_alloc_mask = GFP_HIGHUSER; 32static gfp_t htlb_alloc_mask = GFP_HIGHUSER;
33unsigned long hugepages_treat_as_movable; 33unsigned long hugepages_treat_as_movable;
34int hugetlb_dynamic_pool; 34int hugetlb_dynamic_pool;
35unsigned long nr_overcommit_huge_pages;
35static int hugetlb_next_nid; 36static int hugetlb_next_nid;
36 37
37/* 38/*
@@ -227,22 +228,62 @@ static struct page *alloc_buddy_huge_page(struct vm_area_struct *vma,
227 unsigned long address) 228 unsigned long address)
228{ 229{
229 struct page *page; 230 struct page *page;
231 unsigned int nid;
230 232
231 /* Check if the dynamic pool is enabled */ 233 /* Check if the dynamic pool is enabled */
232 if (!hugetlb_dynamic_pool) 234 if (!hugetlb_dynamic_pool)
233 return NULL; 235 return NULL;
234 236
237 /*
238 * Assume we will successfully allocate the surplus page to
239 * prevent racing processes from causing the surplus to exceed
240 * overcommit
241 *
242 * This however introduces a different race, where a process B
243 * tries to grow the static hugepage pool while alloc_pages() is
244 * called by process A. B will only examine the per-node
245 * counters in determining if surplus huge pages can be
246 * converted to normal huge pages in adjust_pool_surplus(). A
247 * won't be able to increment the per-node counter, until the
248 * lock is dropped by B, but B doesn't drop hugetlb_lock until
249 * no more huge pages can be converted from surplus to normal
250 * state (and doesn't try to convert again). Thus, we have a
251 * case where a surplus huge page exists, the pool is grown, and
252 * the surplus huge page still exists after, even though it
253 * should just have been converted to a normal huge page. This
254 * does not leak memory, though, as the hugepage will be freed
255 * once it is out of use. It also does not allow the counters to
256 * go out of whack in adjust_pool_surplus() as we don't modify
257 * the node values until we've gotten the hugepage and only the
258 * per-node value is checked there.
259 */
260 spin_lock(&hugetlb_lock);
261 if (surplus_huge_pages >= nr_overcommit_huge_pages) {
262 spin_unlock(&hugetlb_lock);
263 return NULL;
264 } else {
265 nr_huge_pages++;
266 surplus_huge_pages++;
267 }
268 spin_unlock(&hugetlb_lock);
269
235 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|__GFP_NOWARN, 270 page = alloc_pages(htlb_alloc_mask|__GFP_COMP|__GFP_NOWARN,
236 HUGETLB_PAGE_ORDER); 271 HUGETLB_PAGE_ORDER);
272
273 spin_lock(&hugetlb_lock);
237 if (page) { 274 if (page) {
275 nid = page_to_nid(page);
238 set_compound_page_dtor(page, free_huge_page); 276 set_compound_page_dtor(page, free_huge_page);
239 spin_lock(&hugetlb_lock); 277 /*
240 nr_huge_pages++; 278 * We incremented the global counters already
241 nr_huge_pages_node[page_to_nid(page)]++; 279 */
242 surplus_huge_pages++; 280 nr_huge_pages_node[nid]++;
243 surplus_huge_pages_node[page_to_nid(page)]++; 281 surplus_huge_pages_node[nid]++;
244 spin_unlock(&hugetlb_lock); 282 } else {
283 nr_huge_pages--;
284 surplus_huge_pages--;
245 } 285 }
286 spin_unlock(&hugetlb_lock);
246 287
247 return page; 288 return page;
248} 289}
@@ -481,6 +522,12 @@ static unsigned long set_max_huge_pages(unsigned long count)
481 * Increase the pool size 522 * Increase the pool size
482 * First take pages out of surplus state. Then make up the 523 * First take pages out of surplus state. Then make up the
483 * remaining difference by allocating fresh huge pages. 524 * remaining difference by allocating fresh huge pages.
525 *
526 * We might race with alloc_buddy_huge_page() here and be unable
527 * to convert a surplus huge page to a normal huge page. That is
528 * not critical, though, it just means the overall size of the
529 * pool might be one hugepage larger than it needs to be, but
530 * within all the constraints specified by the sysctls.
484 */ 531 */
485 spin_lock(&hugetlb_lock); 532 spin_lock(&hugetlb_lock);
486 while (surplus_huge_pages && count > persistent_huge_pages) { 533 while (surplus_huge_pages && count > persistent_huge_pages) {
@@ -509,6 +556,14 @@ static unsigned long set_max_huge_pages(unsigned long count)
509 * to keep enough around to satisfy reservations). Then place 556 * to keep enough around to satisfy reservations). Then place
510 * pages into surplus state as needed so the pool will shrink 557 * pages into surplus state as needed so the pool will shrink
511 * to the desired size as pages become free. 558 * to the desired size as pages become free.
559 *
560 * By placing pages into the surplus state independent of the
561 * overcommit value, we are allowing the surplus pool size to
562 * exceed overcommit. There are few sane options here. Since
563 * alloc_buddy_huge_page() is checking the global counter,
564 * though, we'll note that we're not allowed to exceed surplus
565 * and won't grow the pool anywhere else. Not until one of the
566 * sysctls are changed, or the surplus pages go out of use.
512 */ 567 */
513 min_count = resv_huge_pages + nr_huge_pages - free_huge_pages; 568 min_count = resv_huge_pages + nr_huge_pages - free_huge_pages;
514 min_count = max(count, min_count); 569 min_count = max(count, min_count);