diff options
-rw-r--r-- | fs/proc/page.c | 1 | ||||
-rw-r--r-- | include/linux/hugetlb.h | 7 | ||||
-rw-r--r-- | mm/hugetlb.c | 98 | ||||
-rw-r--r-- | mm/internal.h | 5 | ||||
-rw-r--r-- | mm/page_alloc.c | 17 |
5 files changed, 73 insertions, 55 deletions
diff --git a/fs/proc/page.c b/fs/proc/page.c index e9983837d08d..38dd88b7ce84 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c | |||
@@ -6,6 +6,7 @@ | |||
6 | #include <linux/mmzone.h> | 6 | #include <linux/mmzone.h> |
7 | #include <linux/proc_fs.h> | 7 | #include <linux/proc_fs.h> |
8 | #include <linux/seq_file.h> | 8 | #include <linux/seq_file.h> |
9 | #include <linux/hugetlb.h> | ||
9 | #include <asm/uaccess.h> | 10 | #include <asm/uaccess.h> |
10 | #include "internal.h" | 11 | #include "internal.h" |
11 | 12 | ||
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 03be7f29ca01..a05a5ef33391 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h | |||
@@ -11,6 +11,8 @@ | |||
11 | 11 | ||
12 | struct ctl_table; | 12 | struct ctl_table; |
13 | 13 | ||
14 | int PageHuge(struct page *page); | ||
15 | |||
14 | static inline int is_vm_hugetlb_page(struct vm_area_struct *vma) | 16 | static inline int is_vm_hugetlb_page(struct vm_area_struct *vma) |
15 | { | 17 | { |
16 | return vma->vm_flags & VM_HUGETLB; | 18 | return vma->vm_flags & VM_HUGETLB; |
@@ -61,6 +63,11 @@ void hugetlb_change_protection(struct vm_area_struct *vma, | |||
61 | 63 | ||
62 | #else /* !CONFIG_HUGETLB_PAGE */ | 64 | #else /* !CONFIG_HUGETLB_PAGE */ |
63 | 65 | ||
66 | static inline int PageHuge(struct page *page) | ||
67 | { | ||
68 | return 0; | ||
69 | } | ||
70 | |||
64 | static inline int is_vm_hugetlb_page(struct vm_area_struct *vma) | 71 | static inline int is_vm_hugetlb_page(struct vm_area_struct *vma) |
65 | { | 72 | { |
66 | return 0; | 73 | return 0; |
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7b9b6015b2ec..a56e6f3ce979 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c | |||
@@ -578,41 +578,6 @@ static void free_huge_page(struct page *page) | |||
578 | hugetlb_put_quota(mapping, 1); | 578 | hugetlb_put_quota(mapping, 1); |
579 | } | 579 | } |
580 | 580 | ||
581 | /* | ||
582 | * Increment or decrement surplus_huge_pages. Keep node-specific counters | ||
583 | * balanced by operating on them in a round-robin fashion. | ||
584 | * Returns 1 if an adjustment was made. | ||
585 | */ | ||
586 | static int adjust_pool_surplus(struct hstate *h, int delta) | ||
587 | { | ||
588 | static int prev_nid; | ||
589 | int nid = prev_nid; | ||
590 | int ret = 0; | ||
591 | |||
592 | VM_BUG_ON(delta != -1 && delta != 1); | ||
593 | do { | ||
594 | nid = next_node(nid, node_online_map); | ||
595 | if (nid == MAX_NUMNODES) | ||
596 | nid = first_node(node_online_map); | ||
597 | |||
598 | /* To shrink on this node, there must be a surplus page */ | ||
599 | if (delta < 0 && !h->surplus_huge_pages_node[nid]) | ||
600 | continue; | ||
601 | /* Surplus cannot exceed the total number of pages */ | ||
602 | if (delta > 0 && h->surplus_huge_pages_node[nid] >= | ||
603 | h->nr_huge_pages_node[nid]) | ||
604 | continue; | ||
605 | |||
606 | h->surplus_huge_pages += delta; | ||
607 | h->surplus_huge_pages_node[nid] += delta; | ||
608 | ret = 1; | ||
609 | break; | ||
610 | } while (nid != prev_nid); | ||
611 | |||
612 | prev_nid = nid; | ||
613 | return ret; | ||
614 | } | ||
615 | |||
616 | static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) | 581 | static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) |
617 | { | 582 | { |
618 | set_compound_page_dtor(page, free_huge_page); | 583 | set_compound_page_dtor(page, free_huge_page); |
@@ -623,6 +588,34 @@ static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) | |||
623 | put_page(page); /* free it into the hugepage allocator */ | 588 | put_page(page); /* free it into the hugepage allocator */ |
624 | } | 589 | } |
625 | 590 | ||
591 | static void prep_compound_gigantic_page(struct page *page, unsigned long order) | ||
592 | { | ||
593 | int i; | ||
594 | int nr_pages = 1 << order; | ||
595 | struct page *p = page + 1; | ||
596 | |||
597 | /* we rely on prep_new_huge_page to set the destructor */ | ||
598 | set_compound_order(page, order); | ||
599 | __SetPageHead(page); | ||
600 | for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { | ||
601 | __SetPageTail(p); | ||
602 | p->first_page = page; | ||
603 | } | ||
604 | } | ||
605 | |||
606 | int PageHuge(struct page *page) | ||
607 | { | ||
608 | compound_page_dtor *dtor; | ||
609 | |||
610 | if (!PageCompound(page)) | ||
611 | return 0; | ||
612 | |||
613 | page = compound_head(page); | ||
614 | dtor = get_compound_page_dtor(page); | ||
615 | |||
616 | return dtor == free_huge_page; | ||
617 | } | ||
618 | |||
626 | static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) | 619 | static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid) |
627 | { | 620 | { |
628 | struct page *page; | 621 | struct page *page; |
@@ -1140,6 +1133,41 @@ static inline void try_to_free_low(struct hstate *h, unsigned long count) | |||
1140 | } | 1133 | } |
1141 | #endif | 1134 | #endif |
1142 | 1135 | ||
1136 | /* | ||
1137 | * Increment or decrement surplus_huge_pages. Keep node-specific counters | ||
1138 | * balanced by operating on them in a round-robin fashion. | ||
1139 | * Returns 1 if an adjustment was made. | ||
1140 | */ | ||
1141 | static int adjust_pool_surplus(struct hstate *h, int delta) | ||
1142 | { | ||
1143 | static int prev_nid; | ||
1144 | int nid = prev_nid; | ||
1145 | int ret = 0; | ||
1146 | |||
1147 | VM_BUG_ON(delta != -1 && delta != 1); | ||
1148 | do { | ||
1149 | nid = next_node(nid, node_online_map); | ||
1150 | if (nid == MAX_NUMNODES) | ||
1151 | nid = first_node(node_online_map); | ||
1152 | |||
1153 | /* To shrink on this node, there must be a surplus page */ | ||
1154 | if (delta < 0 && !h->surplus_huge_pages_node[nid]) | ||
1155 | continue; | ||
1156 | /* Surplus cannot exceed the total number of pages */ | ||
1157 | if (delta > 0 && h->surplus_huge_pages_node[nid] >= | ||
1158 | h->nr_huge_pages_node[nid]) | ||
1159 | continue; | ||
1160 | |||
1161 | h->surplus_huge_pages += delta; | ||
1162 | h->surplus_huge_pages_node[nid] += delta; | ||
1163 | ret = 1; | ||
1164 | break; | ||
1165 | } while (nid != prev_nid); | ||
1166 | |||
1167 | prev_nid = nid; | ||
1168 | return ret; | ||
1169 | } | ||
1170 | |||
1143 | #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) | 1171 | #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages) |
1144 | static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count) | 1172 | static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count) |
1145 | { | 1173 | { |
diff --git a/mm/internal.h b/mm/internal.h index 4b1672a8cf76..b4ac332e8072 100644 --- a/mm/internal.h +++ b/mm/internal.h | |||
@@ -16,9 +16,6 @@ | |||
16 | void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, | 16 | void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, |
17 | unsigned long floor, unsigned long ceiling); | 17 | unsigned long floor, unsigned long ceiling); |
18 | 18 | ||
19 | extern void prep_compound_page(struct page *page, unsigned long order); | ||
20 | extern void prep_compound_gigantic_page(struct page *page, unsigned long order); | ||
21 | |||
22 | static inline void set_page_count(struct page *page, int v) | 19 | static inline void set_page_count(struct page *page, int v) |
23 | { | 20 | { |
24 | atomic_set(&page->_count, v); | 21 | atomic_set(&page->_count, v); |
@@ -51,6 +48,8 @@ extern void putback_lru_page(struct page *page); | |||
51 | */ | 48 | */ |
52 | extern unsigned long highest_memmap_pfn; | 49 | extern unsigned long highest_memmap_pfn; |
53 | extern void __free_pages_bootmem(struct page *page, unsigned int order); | 50 | extern void __free_pages_bootmem(struct page *page, unsigned int order); |
51 | extern void prep_compound_page(struct page *page, unsigned long order); | ||
52 | |||
54 | 53 | ||
55 | /* | 54 | /* |
56 | * function for dealing with page's order in buddy system. | 55 | * function for dealing with page's order in buddy system. |
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 8ca06d87dc1f..131655cdb6b2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
@@ -300,23 +300,6 @@ void prep_compound_page(struct page *page, unsigned long order) | |||
300 | } | 300 | } |
301 | } | 301 | } |
302 | 302 | ||
303 | #ifdef CONFIG_HUGETLBFS | ||
304 | void prep_compound_gigantic_page(struct page *page, unsigned long order) | ||
305 | { | ||
306 | int i; | ||
307 | int nr_pages = 1 << order; | ||
308 | struct page *p = page + 1; | ||
309 | |||
310 | set_compound_page_dtor(page, free_compound_page); | ||
311 | set_compound_order(page, order); | ||
312 | __SetPageHead(page); | ||
313 | for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { | ||
314 | __SetPageTail(p); | ||
315 | p->first_page = page; | ||
316 | } | ||
317 | } | ||
318 | #endif | ||
319 | |||
320 | static int destroy_compound_page(struct page *page, unsigned long order) | 303 | static int destroy_compound_page(struct page *page, unsigned long order) |
321 | { | 304 | { |
322 | int i; | 305 | int i; |