aboutsummaryrefslogtreecommitdiffstats
path: root/mm/swap.c
diff options
context:
space:
mode:
authorAndrea Arcangeli <aarcange@redhat.com>2011-01-13 18:46:32 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2011-01-13 20:32:39 -0500
commit9180706344487700b40da9eca5dedd3d11cb33b4 (patch)
treeeb0347efe7e40adc78a271752c7382aa67875d6a /mm/swap.c
parente9da73d67729b58bba256123e2b4651e0d8a01ac (diff)
thp: alter compound get_page/put_page
Alter compound get_page/put_page to keep references on subpages too, in order to allow __split_huge_page_refcount to split an hugepage even while subpages have been pinned by one of the get_user_pages() variants. Signed-off-by: Andrea Arcangeli <aarcange@redhat.com> Acked-by: Rik van Riel <riel@redhat.com> Acked-by: Mel Gorman <mel@csn.ul.ie> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm/swap.c')
-rw-r--r--mm/swap.c90
1 files changed, 83 insertions, 7 deletions
diff --git a/mm/swap.c b/mm/swap.c
index 3f4854205b1..33f5292fe13 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -56,17 +56,93 @@ static void __page_cache_release(struct page *page)
56 del_page_from_lru(zone, page); 56 del_page_from_lru(zone, page);
57 spin_unlock_irqrestore(&zone->lru_lock, flags); 57 spin_unlock_irqrestore(&zone->lru_lock, flags);
58 } 58 }
59}
60
61static void __put_single_page(struct page *page)
62{
63 __page_cache_release(page);
59 free_hot_cold_page(page, 0); 64 free_hot_cold_page(page, 0);
60} 65}
61 66
62static void put_compound_page(struct page *page) 67static void __put_compound_page(struct page *page)
63{ 68{
64 page = compound_head(page); 69 compound_page_dtor *dtor;
65 if (put_page_testzero(page)) { 70
66 compound_page_dtor *dtor; 71 __page_cache_release(page);
72 dtor = get_compound_page_dtor(page);
73 (*dtor)(page);
74}
67 75
68 dtor = get_compound_page_dtor(page); 76static void put_compound_page(struct page *page)
69 (*dtor)(page); 77{
78 if (unlikely(PageTail(page))) {
79 /* __split_huge_page_refcount can run under us */
80 struct page *page_head = page->first_page;
81 smp_rmb();
82 /*
83 * If PageTail is still set after smp_rmb() we can be sure
84 * that the page->first_page we read wasn't a dangling pointer.
85 * See __split_huge_page_refcount() smp_wmb().
86 */
87 if (likely(PageTail(page) && get_page_unless_zero(page_head))) {
88 unsigned long flags;
89 /*
90 * Verify that our page_head wasn't converted
91 * to a a regular page before we got a
92 * reference on it.
93 */
94 if (unlikely(!PageHead(page_head))) {
95 /* PageHead is cleared after PageTail */
96 smp_rmb();
97 VM_BUG_ON(PageTail(page));
98 goto out_put_head;
99 }
100 /*
101 * Only run compound_lock on a valid PageHead,
102 * after having it pinned with
103 * get_page_unless_zero() above.
104 */
105 smp_mb();
106 /* page_head wasn't a dangling pointer */
107 flags = compound_lock_irqsave(page_head);
108 if (unlikely(!PageTail(page))) {
109 /* __split_huge_page_refcount run before us */
110 compound_unlock_irqrestore(page_head, flags);
111 VM_BUG_ON(PageHead(page_head));
112 out_put_head:
113 if (put_page_testzero(page_head))
114 __put_single_page(page_head);
115 out_put_single:
116 if (put_page_testzero(page))
117 __put_single_page(page);
118 return;
119 }
120 VM_BUG_ON(page_head != page->first_page);
121 /*
122 * We can release the refcount taken by
123 * get_page_unless_zero now that
124 * split_huge_page_refcount is blocked on the
125 * compound_lock.
126 */
127 if (put_page_testzero(page_head))
128 VM_BUG_ON(1);
129 /* __split_huge_page_refcount will wait now */
130 VM_BUG_ON(atomic_read(&page->_count) <= 0);
131 atomic_dec(&page->_count);
132 VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
133 compound_unlock_irqrestore(page_head, flags);
134 if (put_page_testzero(page_head))
135 __put_compound_page(page_head);
136 } else {
137 /* page_head is a dangling pointer */
138 VM_BUG_ON(PageTail(page));
139 goto out_put_single;
140 }
141 } else if (put_page_testzero(page)) {
142 if (PageHead(page))
143 __put_compound_page(page);
144 else
145 __put_single_page(page);
70 } 146 }
71} 147}
72 148
@@ -75,7 +151,7 @@ void put_page(struct page *page)
75 if (unlikely(PageCompound(page))) 151 if (unlikely(PageCompound(page)))
76 put_compound_page(page); 152 put_compound_page(page);
77 else if (put_page_testzero(page)) 153 else if (put_page_testzero(page))
78 __page_cache_release(page); 154 __put_single_page(page);
79} 155}
80EXPORT_SYMBOL(put_page); 156EXPORT_SYMBOL(put_page);
81 157