diff options
-rw-r--r-- | mm/huge_memory.c | 38 |
1 files changed, 11 insertions, 27 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 40f17c34b415..6f022f505e88 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -1701,64 +1701,49 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte) | |||
1701 | } | 1701 | } |
1702 | } | 1702 | } |
1703 | 1703 | ||
1704 | static void release_all_pte_pages(pte_t *pte) | ||
1705 | { | ||
1706 | release_pte_pages(pte, pte + HPAGE_PMD_NR); | ||
1707 | } | ||
1708 | |||
1709 | static int __collapse_huge_page_isolate(struct vm_area_struct *vma, | 1704 | static int __collapse_huge_page_isolate(struct vm_area_struct *vma, |
1710 | unsigned long address, | 1705 | unsigned long address, |
1711 | pte_t *pte) | 1706 | pte_t *pte) |
1712 | { | 1707 | { |
1713 | struct page *page; | 1708 | struct page *page; |
1714 | pte_t *_pte; | 1709 | pte_t *_pte; |
1715 | int referenced = 0, isolated = 0, none = 0; | 1710 | int referenced = 0, none = 0; |
1716 | for (_pte = pte; _pte < pte+HPAGE_PMD_NR; | 1711 | for (_pte = pte; _pte < pte+HPAGE_PMD_NR; |
1717 | _pte++, address += PAGE_SIZE) { | 1712 | _pte++, address += PAGE_SIZE) { |
1718 | pte_t pteval = *_pte; | 1713 | pte_t pteval = *_pte; |
1719 | if (pte_none(pteval)) { | 1714 | if (pte_none(pteval)) { |
1720 | if (++none <= khugepaged_max_ptes_none) | 1715 | if (++none <= khugepaged_max_ptes_none) |
1721 | continue; | 1716 | continue; |
1722 | else { | 1717 | else |
1723 | release_pte_pages(pte, _pte); | ||
1724 | goto out; | 1718 | goto out; |
1725 | } | ||
1726 | } | 1719 | } |
1727 | if (!pte_present(pteval) || !pte_write(pteval)) { | 1720 | if (!pte_present(pteval) || !pte_write(pteval)) |
1728 | release_pte_pages(pte, _pte); | ||
1729 | goto out; | 1721 | goto out; |
1730 | } | ||
1731 | page = vm_normal_page(vma, address, pteval); | 1722 | page = vm_normal_page(vma, address, pteval); |
1732 | if (unlikely(!page)) { | 1723 | if (unlikely(!page)) |
1733 | release_pte_pages(pte, _pte); | ||
1734 | goto out; | 1724 | goto out; |
1735 | } | 1725 | |
1736 | VM_BUG_ON(PageCompound(page)); | 1726 | VM_BUG_ON(PageCompound(page)); |
1737 | BUG_ON(!PageAnon(page)); | 1727 | BUG_ON(!PageAnon(page)); |
1738 | VM_BUG_ON(!PageSwapBacked(page)); | 1728 | VM_BUG_ON(!PageSwapBacked(page)); |
1739 | 1729 | ||
1740 | /* cannot use mapcount: can't collapse if there's a gup pin */ | 1730 | /* cannot use mapcount: can't collapse if there's a gup pin */ |
1741 | if (page_count(page) != 1) { | 1731 | if (page_count(page) != 1) |
1742 | release_pte_pages(pte, _pte); | ||
1743 | goto out; | 1732 | goto out; |
1744 | } | ||
1745 | /* | 1733 | /* |
1746 | * We can do it before isolate_lru_page because the | 1734 | * We can do it before isolate_lru_page because the |
1747 | * page can't be freed from under us. NOTE: PG_lock | 1735 | * page can't be freed from under us. NOTE: PG_lock |
1748 | * is needed to serialize against split_huge_page | 1736 | * is needed to serialize against split_huge_page |
1749 | * when invoked from the VM. | 1737 | * when invoked from the VM. |
1750 | */ | 1738 | */ |
1751 | if (!trylock_page(page)) { | 1739 | if (!trylock_page(page)) |
1752 | release_pte_pages(pte, _pte); | ||
1753 | goto out; | 1740 | goto out; |
1754 | } | ||
1755 | /* | 1741 | /* |
1756 | * Isolate the page to avoid collapsing an hugepage | 1742 | * Isolate the page to avoid collapsing an hugepage |
1757 | * currently in use by the VM. | 1743 | * currently in use by the VM. |
1758 | */ | 1744 | */ |
1759 | if (isolate_lru_page(page)) { | 1745 | if (isolate_lru_page(page)) { |
1760 | unlock_page(page); | 1746 | unlock_page(page); |
1761 | release_pte_pages(pte, _pte); | ||
1762 | goto out; | 1747 | goto out; |
1763 | } | 1748 | } |
1764 | /* 0 stands for page_is_file_cache(page) == false */ | 1749 | /* 0 stands for page_is_file_cache(page) == false */ |
@@ -1771,12 +1756,11 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, | |||
1771 | mmu_notifier_test_young(vma->vm_mm, address)) | 1756 | mmu_notifier_test_young(vma->vm_mm, address)) |
1772 | referenced = 1; | 1757 | referenced = 1; |
1773 | } | 1758 | } |
1774 | if (unlikely(!referenced)) | 1759 | if (likely(referenced)) |
1775 | release_all_pte_pages(pte); | 1760 | return 1; |
1776 | else | ||
1777 | isolated = 1; | ||
1778 | out: | 1761 | out: |
1779 | return isolated; | 1762 | release_pte_pages(pte, _pte); |
1763 | return 0; | ||
1780 | } | 1764 | } |
1781 | 1765 | ||
1782 | static void __collapse_huge_page_copy(pte_t *pte, struct page *page, | 1766 | static void __collapse_huge_page_copy(pte_t *pte, struct page *page, |