diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/internal.h | 3 | ||||
| -rw-r--r-- | mm/swap.c | 33 |
2 files changed, 29 insertions, 7 deletions
diff --git a/mm/internal.h b/mm/internal.h index 684f7aa9692a..a85a3ab1f7ef 100644 --- a/mm/internal.h +++ b/mm/internal.h | |||
| @@ -51,7 +51,8 @@ static inline void __get_page_tail_foll(struct page *page, | |||
| 51 | VM_BUG_ON(page_mapcount(page) < 0); | 51 | VM_BUG_ON(page_mapcount(page) < 0); |
| 52 | if (get_page_head) | 52 | if (get_page_head) |
| 53 | atomic_inc(&page->first_page->_count); | 53 | atomic_inc(&page->first_page->_count); |
| 54 | atomic_inc(&page->_mapcount); | 54 | if (compound_tail_refcounted(page->first_page)) |
| 55 | atomic_inc(&page->_mapcount); | ||
| 55 | } | 56 | } |
| 56 | 57 | ||
| 57 | /* | 58 | /* |
| @@ -88,8 +88,9 @@ static void put_compound_page(struct page *page) | |||
| 88 | 88 | ||
| 89 | /* | 89 | /* |
| 90 | * THP can not break up slab pages so avoid taking | 90 | * THP can not break up slab pages so avoid taking |
| 91 | * compound_lock(). Slab performs non-atomic bit ops | 91 | * compound_lock() and skip the tail page refcounting |
| 92 | * on page->flags for better performance. In | 92 | * (in _mapcount) too. Slab performs non-atomic bit |
| 93 | * ops on page->flags for better performance. In | ||
| 93 | * particular slab_unlock() in slub used to be a hot | 94 | * particular slab_unlock() in slub used to be a hot |
| 94 | * path. It is still hot on arches that do not support | 95 | * path. It is still hot on arches that do not support |
| 95 | * this_cpu_cmpxchg_double(). | 96 | * this_cpu_cmpxchg_double(). |
| @@ -102,7 +103,7 @@ static void put_compound_page(struct page *page) | |||
| 102 | * PageTail clear after smp_rmb() and we'll treat it | 103 | * PageTail clear after smp_rmb() and we'll treat it |
| 103 | * as a single page. | 104 | * as a single page. |
| 104 | */ | 105 | */ |
| 105 | if (PageSlab(page_head) || PageHeadHuge(page_head)) { | 106 | if (!__compound_tail_refcounted(page_head)) { |
| 106 | /* | 107 | /* |
| 107 | * If "page" is a THP tail, we must read the tail page | 108 | * If "page" is a THP tail, we must read the tail page |
| 108 | * flags after the head page flags. The | 109 | * flags after the head page flags. The |
| @@ -117,10 +118,30 @@ static void put_compound_page(struct page *page) | |||
| 117 | * cannot race here. | 118 | * cannot race here. |
| 118 | */ | 119 | */ |
| 119 | VM_BUG_ON(!PageHead(page_head)); | 120 | VM_BUG_ON(!PageHead(page_head)); |
| 120 | VM_BUG_ON(page_mapcount(page) <= 0); | 121 | VM_BUG_ON(page_mapcount(page) != 0); |
| 121 | atomic_dec(&page->_mapcount); | 122 | if (put_page_testzero(page_head)) { |
| 122 | if (put_page_testzero(page_head)) | 123 | /* |
| 124 | * If this is the tail of a | ||
| 125 | * slab compound page, the | ||
| 126 | * tail pin must not be the | ||
| 127 | * last reference held on the | ||
| 128 | * page, because the PG_slab | ||
| 129 | * cannot be cleared before | ||
| 130 | * all tail pins (which skips | ||
| 131 | * the _mapcount tail | ||
| 132 | * refcounting) have been | ||
| 133 | * released. For hugetlbfs the | ||
| 134 | * tail pin may be the last | ||
| 135 | * reference on the page | ||
| 136 | * instead, because | ||
| 137 | * PageHeadHuge will not go | ||
| 138 | * away until the compound | ||
| 139 | * page enters the buddy | ||
| 140 | * allocator. | ||
| 141 | */ | ||
| 142 | VM_BUG_ON(PageSlab(page_head)); | ||
| 123 | __put_compound_page(page_head); | 143 | __put_compound_page(page_head); |
| 144 | } | ||
| 124 | return; | 145 | return; |
| 125 | } else | 146 | } else |
| 126 | /* | 147 | /* |