diff options
author | Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | 2014-11-02 10:45:27 -0500 |
---|---|---|
committer | Benjamin Herrenschmidt <benh@kernel.crashing.org> | 2014-12-01 22:10:10 -0500 |
commit | f1581bf14bc40b4a14bf10358eac0b22173b3313 (patch) | |
tree | 1b335a0b05ee0d0dfe412a594c4b300238a853d4 | |
parent | c4f3eb5fc527b749d6fb0d47ffdcfe83706dbe2f (diff) |
powerpc/mm/thp: Remove code duplication
Rename invalidate_old_hpte to flush_hash_hugepage and use that in
other places.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
-rw-r--r-- | arch/powerpc/include/asm/tlbflush.h | 3 | ||||
-rw-r--r-- | arch/powerpc/mm/hash_utils_64.c | 52 | ||||
-rw-r--r-- | arch/powerpc/mm/hugepage-hash64.c | 54 | ||||
-rw-r--r-- | arch/powerpc/mm/pgtable_64.c | 64 |
4 files changed, 65 insertions, 108 deletions
diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h index cd7c2719d3ef..19550d346fea 100644 --- a/arch/powerpc/include/asm/tlbflush.h +++ b/arch/powerpc/include/asm/tlbflush.h | |||
@@ -127,7 +127,8 @@ static inline void arch_leave_lazy_mmu_mode(void) | |||
127 | extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, | 127 | extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, |
128 | int ssize, int local); | 128 | int ssize, int local); |
129 | extern void flush_hash_range(unsigned long number, int local); | 129 | extern void flush_hash_range(unsigned long number, int local); |
130 | 130 | extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr, | |
131 | pmd_t *pmdp, unsigned int psize, int ssize); | ||
131 | 132 | ||
132 | static inline void local_flush_tlb_mm(struct mm_struct *mm) | 133 | static inline void local_flush_tlb_mm(struct mm_struct *mm) |
133 | { | 134 | { |
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index f01027731e23..6c2076c65d7c 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c | |||
@@ -1315,6 +1315,58 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize, | |||
1315 | #endif | 1315 | #endif |
1316 | } | 1316 | } |
1317 | 1317 | ||
1318 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
1319 | void flush_hash_hugepage(unsigned long vsid, unsigned long addr, | ||
1320 | pmd_t *pmdp, unsigned int psize, int ssize) | ||
1321 | { | ||
1322 | int i, max_hpte_count, valid; | ||
1323 | unsigned long s_addr; | ||
1324 | unsigned char *hpte_slot_array; | ||
1325 | unsigned long hidx, shift, vpn, hash, slot; | ||
1326 | |||
1327 | s_addr = addr & HPAGE_PMD_MASK; | ||
1328 | hpte_slot_array = get_hpte_slot_array(pmdp); | ||
1329 | /* | ||
1330 | * IF we try to do a HUGE PTE update after a withdraw is done. | ||
1331 | * we will find the below NULL. This happens when we do | ||
1332 | * split_huge_page_pmd | ||
1333 | */ | ||
1334 | if (!hpte_slot_array) | ||
1335 | return; | ||
1336 | |||
1337 | if (ppc_md.hugepage_invalidate) | ||
1338 | return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array, | ||
1339 | psize, ssize); | ||
1340 | /* | ||
1341 | * No bluk hpte removal support, invalidate each entry | ||
1342 | */ | ||
1343 | shift = mmu_psize_defs[psize].shift; | ||
1344 | max_hpte_count = HPAGE_PMD_SIZE >> shift; | ||
1345 | for (i = 0; i < max_hpte_count; i++) { | ||
1346 | /* | ||
1347 | * 8 bits per each hpte entries | ||
1348 | * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit] | ||
1349 | */ | ||
1350 | valid = hpte_valid(hpte_slot_array, i); | ||
1351 | if (!valid) | ||
1352 | continue; | ||
1353 | hidx = hpte_hash_index(hpte_slot_array, i); | ||
1354 | |||
1355 | /* get the vpn */ | ||
1356 | addr = s_addr + (i * (1ul << shift)); | ||
1357 | vpn = hpt_vpn(addr, vsid, ssize); | ||
1358 | hash = hpt_hash(vpn, shift, ssize); | ||
1359 | if (hidx & _PTEIDX_SECONDARY) | ||
1360 | hash = ~hash; | ||
1361 | |||
1362 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | ||
1363 | slot += hidx & _PTEIDX_GROUP_IX; | ||
1364 | ppc_md.hpte_invalidate(slot, vpn, psize, | ||
1365 | MMU_PAGE_16M, ssize, 0); | ||
1366 | } | ||
1367 | } | ||
1368 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | ||
1369 | |||
1318 | void flush_hash_range(unsigned long number, int local) | 1370 | void flush_hash_range(unsigned long number, int local) |
1319 | { | 1371 | { |
1320 | if (ppc_md.flush_hash_range) | 1372 | if (ppc_md.flush_hash_range) |
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c index 5f5e6328c21c..1b3ad46a71b5 100644 --- a/arch/powerpc/mm/hugepage-hash64.c +++ b/arch/powerpc/mm/hugepage-hash64.c | |||
@@ -18,57 +18,6 @@ | |||
18 | #include <linux/mm.h> | 18 | #include <linux/mm.h> |
19 | #include <asm/machdep.h> | 19 | #include <asm/machdep.h> |
20 | 20 | ||
21 | static void invalidate_old_hpte(unsigned long vsid, unsigned long addr, | ||
22 | pmd_t *pmdp, unsigned int psize, int ssize) | ||
23 | { | ||
24 | int i, max_hpte_count, valid; | ||
25 | unsigned long s_addr; | ||
26 | unsigned char *hpte_slot_array; | ||
27 | unsigned long hidx, shift, vpn, hash, slot; | ||
28 | |||
29 | s_addr = addr & HPAGE_PMD_MASK; | ||
30 | hpte_slot_array = get_hpte_slot_array(pmdp); | ||
31 | /* | ||
32 | * IF we try to do a HUGE PTE update after a withdraw is done. | ||
33 | * we will find the below NULL. This happens when we do | ||
34 | * split_huge_page_pmd | ||
35 | */ | ||
36 | if (!hpte_slot_array) | ||
37 | return; | ||
38 | |||
39 | if (ppc_md.hugepage_invalidate) | ||
40 | return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array, | ||
41 | psize, ssize); | ||
42 | /* | ||
43 | * No bluk hpte removal support, invalidate each entry | ||
44 | */ | ||
45 | shift = mmu_psize_defs[psize].shift; | ||
46 | max_hpte_count = HPAGE_PMD_SIZE >> shift; | ||
47 | for (i = 0; i < max_hpte_count; i++) { | ||
48 | /* | ||
49 | * 8 bits per each hpte entries | ||
50 | * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit] | ||
51 | */ | ||
52 | valid = hpte_valid(hpte_slot_array, i); | ||
53 | if (!valid) | ||
54 | continue; | ||
55 | hidx = hpte_hash_index(hpte_slot_array, i); | ||
56 | |||
57 | /* get the vpn */ | ||
58 | addr = s_addr + (i * (1ul << shift)); | ||
59 | vpn = hpt_vpn(addr, vsid, ssize); | ||
60 | hash = hpt_hash(vpn, shift, ssize); | ||
61 | if (hidx & _PTEIDX_SECONDARY) | ||
62 | hash = ~hash; | ||
63 | |||
64 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | ||
65 | slot += hidx & _PTEIDX_GROUP_IX; | ||
66 | ppc_md.hpte_invalidate(slot, vpn, psize, | ||
67 | MMU_PAGE_16M, ssize, 0); | ||
68 | } | ||
69 | } | ||
70 | |||
71 | |||
72 | int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, | 21 | int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, |
73 | pmd_t *pmdp, unsigned long trap, int local, int ssize, | 22 | pmd_t *pmdp, unsigned long trap, int local, int ssize, |
74 | unsigned int psize) | 23 | unsigned int psize) |
@@ -145,7 +94,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, | |||
145 | * hash page table entries. | 94 | * hash page table entries. |
146 | */ | 95 | */ |
147 | if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO)) | 96 | if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO)) |
148 | invalidate_old_hpte(vsid, ea, pmdp, MMU_PAGE_64K, ssize); | 97 | flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K, |
98 | ssize); | ||
149 | } | 99 | } |
150 | 100 | ||
151 | valid = hpte_valid(hpte_slot_array, index); | 101 | valid = hpte_valid(hpte_slot_array, index); |
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 87ff0c1908a9..c175c990580e 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c | |||
@@ -739,29 +739,13 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, | |||
739 | void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, | 739 | void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, |
740 | pmd_t *pmdp, unsigned long old_pmd) | 740 | pmd_t *pmdp, unsigned long old_pmd) |
741 | { | 741 | { |
742 | int ssize, i; | 742 | int ssize; |
743 | unsigned long s_addr; | 743 | unsigned int psize; |
744 | int max_hpte_count; | 744 | unsigned long vsid; |
745 | unsigned int psize, valid; | ||
746 | unsigned char *hpte_slot_array; | ||
747 | unsigned long hidx, vpn, vsid, hash, shift, slot; | ||
748 | |||
749 | /* | ||
750 | * Flush all the hptes mapping this hugepage | ||
751 | */ | ||
752 | s_addr = addr & HPAGE_PMD_MASK; | ||
753 | hpte_slot_array = get_hpte_slot_array(pmdp); | ||
754 | /* | ||
755 | * IF we try to do a HUGE PTE update after a withdraw is done. | ||
756 | * we will find the below NULL. This happens when we do | ||
757 | * split_huge_page_pmd | ||
758 | */ | ||
759 | if (!hpte_slot_array) | ||
760 | return; | ||
761 | 745 | ||
762 | /* get the base page size,vsid and segment size */ | 746 | /* get the base page size,vsid and segment size */ |
763 | #ifdef CONFIG_DEBUG_VM | 747 | #ifdef CONFIG_DEBUG_VM |
764 | psize = get_slice_psize(mm, s_addr); | 748 | psize = get_slice_psize(mm, addr); |
765 | BUG_ON(psize == MMU_PAGE_16M); | 749 | BUG_ON(psize == MMU_PAGE_16M); |
766 | #endif | 750 | #endif |
767 | if (old_pmd & _PAGE_COMBO) | 751 | if (old_pmd & _PAGE_COMBO) |
@@ -769,46 +753,16 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, | |||
769 | else | 753 | else |
770 | psize = MMU_PAGE_64K; | 754 | psize = MMU_PAGE_64K; |
771 | 755 | ||
772 | if (!is_kernel_addr(s_addr)) { | 756 | if (!is_kernel_addr(addr)) { |
773 | ssize = user_segment_size(s_addr); | 757 | ssize = user_segment_size(addr); |
774 | vsid = get_vsid(mm->context.id, s_addr, ssize); | 758 | vsid = get_vsid(mm->context.id, addr, ssize); |
775 | WARN_ON(vsid == 0); | 759 | WARN_ON(vsid == 0); |
776 | } else { | 760 | } else { |
777 | vsid = get_kernel_vsid(s_addr, mmu_kernel_ssize); | 761 | vsid = get_kernel_vsid(addr, mmu_kernel_ssize); |
778 | ssize = mmu_kernel_ssize; | 762 | ssize = mmu_kernel_ssize; |
779 | } | 763 | } |
780 | 764 | ||
781 | if (ppc_md.hugepage_invalidate) | 765 | return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize); |
782 | return ppc_md.hugepage_invalidate(vsid, s_addr, | ||
783 | hpte_slot_array, | ||
784 | psize, ssize); | ||
785 | /* | ||
786 | * No bluk hpte removal support, invalidate each entry | ||
787 | */ | ||
788 | shift = mmu_psize_defs[psize].shift; | ||
789 | max_hpte_count = HPAGE_PMD_SIZE >> shift; | ||
790 | for (i = 0; i < max_hpte_count; i++) { | ||
791 | /* | ||
792 | * 8 bits per each hpte entries | ||
793 | * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit] | ||
794 | */ | ||
795 | valid = hpte_valid(hpte_slot_array, i); | ||
796 | if (!valid) | ||
797 | continue; | ||
798 | hidx = hpte_hash_index(hpte_slot_array, i); | ||
799 | |||
800 | /* get the vpn */ | ||
801 | addr = s_addr + (i * (1ul << shift)); | ||
802 | vpn = hpt_vpn(addr, vsid, ssize); | ||
803 | hash = hpt_hash(vpn, shift, ssize); | ||
804 | if (hidx & _PTEIDX_SECONDARY) | ||
805 | hash = ~hash; | ||
806 | |||
807 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | ||
808 | slot += hidx & _PTEIDX_GROUP_IX; | ||
809 | ppc_md.hpte_invalidate(slot, vpn, psize, | ||
810 | MMU_PAGE_16M, ssize, 0); | ||
811 | } | ||
812 | } | 766 | } |
813 | 767 | ||
814 | static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot) | 768 | static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot) |