aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/huge_memory.c18
-rw-r--r--mm/pgtable-generic.c5
2 files changed, 12 insertions, 11 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index dab90fd67298..6b785e17b679 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -730,7 +730,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
730 entry = mk_huge_pmd(page, vma); 730 entry = mk_huge_pmd(page, vma);
731 page_add_new_anon_rmap(page, vma, haddr); 731 page_add_new_anon_rmap(page, vma, haddr);
732 set_pmd_at(mm, haddr, pmd, entry); 732 set_pmd_at(mm, haddr, pmd, entry);
733 pgtable_trans_huge_deposit(mm, pgtable); 733 pgtable_trans_huge_deposit(mm, pmd, pgtable);
734 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR); 734 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
735 mm->nr_ptes++; 735 mm->nr_ptes++;
736 spin_unlock(&mm->page_table_lock); 736 spin_unlock(&mm->page_table_lock);
@@ -772,7 +772,7 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
772 entry = pmd_wrprotect(entry); 772 entry = pmd_wrprotect(entry);
773 entry = pmd_mkhuge(entry); 773 entry = pmd_mkhuge(entry);
774 set_pmd_at(mm, haddr, pmd, entry); 774 set_pmd_at(mm, haddr, pmd, entry);
775 pgtable_trans_huge_deposit(mm, pgtable); 775 pgtable_trans_huge_deposit(mm, pmd, pgtable);
776 mm->nr_ptes++; 776 mm->nr_ptes++;
777 return true; 777 return true;
778} 778}
@@ -917,7 +917,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
917 pmdp_set_wrprotect(src_mm, addr, src_pmd); 917 pmdp_set_wrprotect(src_mm, addr, src_pmd);
918 pmd = pmd_mkold(pmd_wrprotect(pmd)); 918 pmd = pmd_mkold(pmd_wrprotect(pmd));
919 set_pmd_at(dst_mm, addr, dst_pmd, pmd); 919 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
920 pgtable_trans_huge_deposit(dst_mm, pgtable); 920 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
921 dst_mm->nr_ptes++; 921 dst_mm->nr_ptes++;
922 922
923 ret = 0; 923 ret = 0;
@@ -987,7 +987,7 @@ static int do_huge_pmd_wp_zero_page_fallback(struct mm_struct *mm,
987 pmdp_clear_flush(vma, haddr, pmd); 987 pmdp_clear_flush(vma, haddr, pmd);
988 /* leave pmd empty until pte is filled */ 988 /* leave pmd empty until pte is filled */
989 989
990 pgtable = pgtable_trans_huge_withdraw(mm); 990 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
991 pmd_populate(mm, &_pmd, pgtable); 991 pmd_populate(mm, &_pmd, pgtable);
992 992
993 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 993 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
@@ -1085,7 +1085,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
1085 pmdp_clear_flush(vma, haddr, pmd); 1085 pmdp_clear_flush(vma, haddr, pmd);
1086 /* leave pmd empty until pte is filled */ 1086 /* leave pmd empty until pte is filled */
1087 1087
1088 pgtable = pgtable_trans_huge_withdraw(mm); 1088 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1089 pmd_populate(mm, &_pmd, pgtable); 1089 pmd_populate(mm, &_pmd, pgtable);
1090 1090
1091 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 1091 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
@@ -1360,7 +1360,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1360 struct page *page; 1360 struct page *page;
1361 pgtable_t pgtable; 1361 pgtable_t pgtable;
1362 pmd_t orig_pmd; 1362 pmd_t orig_pmd;
1363 pgtable = pgtable_trans_huge_withdraw(tlb->mm); 1363 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
1364 orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd); 1364 orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd);
1365 tlb_remove_pmd_tlb_entry(tlb, pmd, addr); 1365 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1366 if (is_huge_zero_pmd(orig_pmd)) { 1366 if (is_huge_zero_pmd(orig_pmd)) {
@@ -1693,7 +1693,7 @@ static int __split_huge_page_map(struct page *page,
1693 pmd = page_check_address_pmd(page, mm, address, 1693 pmd = page_check_address_pmd(page, mm, address,
1694 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG); 1694 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
1695 if (pmd) { 1695 if (pmd) {
1696 pgtable = pgtable_trans_huge_withdraw(mm); 1696 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1697 pmd_populate(mm, &_pmd, pgtable); 1697 pmd_populate(mm, &_pmd, pgtable);
1698 1698
1699 haddr = address; 1699 haddr = address;
@@ -2363,7 +2363,7 @@ static void collapse_huge_page(struct mm_struct *mm,
2363 page_add_new_anon_rmap(new_page, vma, address); 2363 page_add_new_anon_rmap(new_page, vma, address);
2364 set_pmd_at(mm, address, pmd, _pmd); 2364 set_pmd_at(mm, address, pmd, _pmd);
2365 update_mmu_cache_pmd(vma, address, pmd); 2365 update_mmu_cache_pmd(vma, address, pmd);
2366 pgtable_trans_huge_deposit(mm, pgtable); 2366 pgtable_trans_huge_deposit(mm, pmd, pgtable);
2367 spin_unlock(&mm->page_table_lock); 2367 spin_unlock(&mm->page_table_lock);
2368 2368
2369 *hpage = NULL; 2369 *hpage = NULL;
@@ -2669,7 +2669,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2669 pmdp_clear_flush(vma, haddr, pmd); 2669 pmdp_clear_flush(vma, haddr, pmd);
2670 /* leave pmd empty until pte is filled */ 2670 /* leave pmd empty until pte is filled */
2671 2671
2672 pgtable = pgtable_trans_huge_withdraw(mm); 2672 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2673 pmd_populate(mm, &_pmd, pgtable); 2673 pmd_populate(mm, &_pmd, pgtable);
2674 2674
2675 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { 2675 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c
index 0c8323fe6c8f..e1a6e4fab016 100644
--- a/mm/pgtable-generic.c
+++ b/mm/pgtable-generic.c
@@ -124,7 +124,8 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
124 124
125#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT 125#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
126#ifdef CONFIG_TRANSPARENT_HUGEPAGE 126#ifdef CONFIG_TRANSPARENT_HUGEPAGE
127void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable) 127void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
128 pgtable_t pgtable)
128{ 129{
129 assert_spin_locked(&mm->page_table_lock); 130 assert_spin_locked(&mm->page_table_lock);
130 131
@@ -141,7 +142,7 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
141#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW 142#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
142#ifdef CONFIG_TRANSPARENT_HUGEPAGE 143#ifdef CONFIG_TRANSPARENT_HUGEPAGE
143/* no "address" argument so destroys page coloring of some arch */ 144/* no "address" argument so destroys page coloring of some arch */
144pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm) 145pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
145{ 146{
146 pgtable_t pgtable; 147 pgtable_t pgtable;
147 148