aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>2014-11-02 10:45:27 -0500
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2014-12-01 22:10:10 -0500
commitf1581bf14bc40b4a14bf10358eac0b22173b3313 (patch)
tree1b335a0b05ee0d0dfe412a594c4b300238a853d4
parentc4f3eb5fc527b749d6fb0d47ffdcfe83706dbe2f (diff)
powerpc/mm/thp: Remove code duplication
Rename invalidate_old_hpte to flush_hash_hugepage and use that in other places. Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
-rw-r--r--arch/powerpc/include/asm/tlbflush.h3
-rw-r--r--arch/powerpc/mm/hash_utils_64.c52
-rw-r--r--arch/powerpc/mm/hugepage-hash64.c54
-rw-r--r--arch/powerpc/mm/pgtable_64.c64
4 files changed, 65 insertions, 108 deletions
diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h
index cd7c2719d3ef..19550d346fea 100644
--- a/arch/powerpc/include/asm/tlbflush.h
+++ b/arch/powerpc/include/asm/tlbflush.h
@@ -127,7 +127,8 @@ static inline void arch_leave_lazy_mmu_mode(void)
127extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, 127extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
128 int ssize, int local); 128 int ssize, int local);
129extern void flush_hash_range(unsigned long number, int local); 129extern void flush_hash_range(unsigned long number, int local);
130 130extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
131 pmd_t *pmdp, unsigned int psize, int ssize);
131 132
132static inline void local_flush_tlb_mm(struct mm_struct *mm) 133static inline void local_flush_tlb_mm(struct mm_struct *mm)
133{ 134{
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index f01027731e23..6c2076c65d7c 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1315,6 +1315,58 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
1315#endif 1315#endif
1316} 1316}
1317 1317
1318#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1319void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
1320 pmd_t *pmdp, unsigned int psize, int ssize)
1321{
1322 int i, max_hpte_count, valid;
1323 unsigned long s_addr;
1324 unsigned char *hpte_slot_array;
1325 unsigned long hidx, shift, vpn, hash, slot;
1326
1327 s_addr = addr & HPAGE_PMD_MASK;
1328 hpte_slot_array = get_hpte_slot_array(pmdp);
1329 /*
1330 * IF we try to do a HUGE PTE update after a withdraw is done.
1331 * we will find the below NULL. This happens when we do
1332 * split_huge_page_pmd
1333 */
1334 if (!hpte_slot_array)
1335 return;
1336
1337 if (ppc_md.hugepage_invalidate)
1338 return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
1339 psize, ssize);
1340 /*
1341 * No bluk hpte removal support, invalidate each entry
1342 */
1343 shift = mmu_psize_defs[psize].shift;
1344 max_hpte_count = HPAGE_PMD_SIZE >> shift;
1345 for (i = 0; i < max_hpte_count; i++) {
1346 /*
1347 * 8 bits per each hpte entries
1348 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]
1349 */
1350 valid = hpte_valid(hpte_slot_array, i);
1351 if (!valid)
1352 continue;
1353 hidx = hpte_hash_index(hpte_slot_array, i);
1354
1355 /* get the vpn */
1356 addr = s_addr + (i * (1ul << shift));
1357 vpn = hpt_vpn(addr, vsid, ssize);
1358 hash = hpt_hash(vpn, shift, ssize);
1359 if (hidx & _PTEIDX_SECONDARY)
1360 hash = ~hash;
1361
1362 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
1363 slot += hidx & _PTEIDX_GROUP_IX;
1364 ppc_md.hpte_invalidate(slot, vpn, psize,
1365 MMU_PAGE_16M, ssize, 0);
1366 }
1367}
1368#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1369
1318void flush_hash_range(unsigned long number, int local) 1370void flush_hash_range(unsigned long number, int local)
1319{ 1371{
1320 if (ppc_md.flush_hash_range) 1372 if (ppc_md.flush_hash_range)
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index 5f5e6328c21c..1b3ad46a71b5 100644
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -18,57 +18,6 @@
18#include <linux/mm.h> 18#include <linux/mm.h>
19#include <asm/machdep.h> 19#include <asm/machdep.h>
20 20
21static void invalidate_old_hpte(unsigned long vsid, unsigned long addr,
22 pmd_t *pmdp, unsigned int psize, int ssize)
23{
24 int i, max_hpte_count, valid;
25 unsigned long s_addr;
26 unsigned char *hpte_slot_array;
27 unsigned long hidx, shift, vpn, hash, slot;
28
29 s_addr = addr & HPAGE_PMD_MASK;
30 hpte_slot_array = get_hpte_slot_array(pmdp);
31 /*
32 * IF we try to do a HUGE PTE update after a withdraw is done.
33 * we will find the below NULL. This happens when we do
34 * split_huge_page_pmd
35 */
36 if (!hpte_slot_array)
37 return;
38
39 if (ppc_md.hugepage_invalidate)
40 return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
41 psize, ssize);
42 /*
43 * No bluk hpte removal support, invalidate each entry
44 */
45 shift = mmu_psize_defs[psize].shift;
46 max_hpte_count = HPAGE_PMD_SIZE >> shift;
47 for (i = 0; i < max_hpte_count; i++) {
48 /*
49 * 8 bits per each hpte entries
50 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]
51 */
52 valid = hpte_valid(hpte_slot_array, i);
53 if (!valid)
54 continue;
55 hidx = hpte_hash_index(hpte_slot_array, i);
56
57 /* get the vpn */
58 addr = s_addr + (i * (1ul << shift));
59 vpn = hpt_vpn(addr, vsid, ssize);
60 hash = hpt_hash(vpn, shift, ssize);
61 if (hidx & _PTEIDX_SECONDARY)
62 hash = ~hash;
63
64 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
65 slot += hidx & _PTEIDX_GROUP_IX;
66 ppc_md.hpte_invalidate(slot, vpn, psize,
67 MMU_PAGE_16M, ssize, 0);
68 }
69}
70
71
72int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, 21int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
73 pmd_t *pmdp, unsigned long trap, int local, int ssize, 22 pmd_t *pmdp, unsigned long trap, int local, int ssize,
74 unsigned int psize) 23 unsigned int psize)
@@ -145,7 +94,8 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
145 * hash page table entries. 94 * hash page table entries.
146 */ 95 */
147 if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO)) 96 if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO))
148 invalidate_old_hpte(vsid, ea, pmdp, MMU_PAGE_64K, ssize); 97 flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K,
98 ssize);
149 } 99 }
150 100
151 valid = hpte_valid(hpte_slot_array, index); 101 valid = hpte_valid(hpte_slot_array, index);
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 87ff0c1908a9..c175c990580e 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -739,29 +739,13 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
739void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, 739void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
740 pmd_t *pmdp, unsigned long old_pmd) 740 pmd_t *pmdp, unsigned long old_pmd)
741{ 741{
742 int ssize, i; 742 int ssize;
743 unsigned long s_addr; 743 unsigned int psize;
744 int max_hpte_count; 744 unsigned long vsid;
745 unsigned int psize, valid;
746 unsigned char *hpte_slot_array;
747 unsigned long hidx, vpn, vsid, hash, shift, slot;
748
749 /*
750 * Flush all the hptes mapping this hugepage
751 */
752 s_addr = addr & HPAGE_PMD_MASK;
753 hpte_slot_array = get_hpte_slot_array(pmdp);
754 /*
755 * IF we try to do a HUGE PTE update after a withdraw is done.
756 * we will find the below NULL. This happens when we do
757 * split_huge_page_pmd
758 */
759 if (!hpte_slot_array)
760 return;
761 745
762 /* get the base page size,vsid and segment size */ 746 /* get the base page size,vsid and segment size */
763#ifdef CONFIG_DEBUG_VM 747#ifdef CONFIG_DEBUG_VM
764 psize = get_slice_psize(mm, s_addr); 748 psize = get_slice_psize(mm, addr);
765 BUG_ON(psize == MMU_PAGE_16M); 749 BUG_ON(psize == MMU_PAGE_16M);
766#endif 750#endif
767 if (old_pmd & _PAGE_COMBO) 751 if (old_pmd & _PAGE_COMBO)
@@ -769,46 +753,16 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
769 else 753 else
770 psize = MMU_PAGE_64K; 754 psize = MMU_PAGE_64K;
771 755
772 if (!is_kernel_addr(s_addr)) { 756 if (!is_kernel_addr(addr)) {
773 ssize = user_segment_size(s_addr); 757 ssize = user_segment_size(addr);
774 vsid = get_vsid(mm->context.id, s_addr, ssize); 758 vsid = get_vsid(mm->context.id, addr, ssize);
775 WARN_ON(vsid == 0); 759 WARN_ON(vsid == 0);
776 } else { 760 } else {
777 vsid = get_kernel_vsid(s_addr, mmu_kernel_ssize); 761 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
778 ssize = mmu_kernel_ssize; 762 ssize = mmu_kernel_ssize;
779 } 763 }
780 764
781 if (ppc_md.hugepage_invalidate) 765 return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize);
782 return ppc_md.hugepage_invalidate(vsid, s_addr,
783 hpte_slot_array,
784 psize, ssize);
785 /*
786 * No bluk hpte removal support, invalidate each entry
787 */
788 shift = mmu_psize_defs[psize].shift;
789 max_hpte_count = HPAGE_PMD_SIZE >> shift;
790 for (i = 0; i < max_hpte_count; i++) {
791 /*
792 * 8 bits per each hpte entries
793 * 000| [ secondary group (one bit) | hidx (3 bits) | valid bit]
794 */
795 valid = hpte_valid(hpte_slot_array, i);
796 if (!valid)
797 continue;
798 hidx = hpte_hash_index(hpte_slot_array, i);
799
800 /* get the vpn */
801 addr = s_addr + (i * (1ul << shift));
802 vpn = hpt_vpn(addr, vsid, ssize);
803 hash = hpt_hash(vpn, shift, ssize);
804 if (hidx & _PTEIDX_SECONDARY)
805 hash = ~hash;
806
807 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
808 slot += hidx & _PTEIDX_GROUP_IX;
809 ppc_md.hpte_invalidate(slot, vpn, psize,
810 MMU_PAGE_16M, ssize, 0);
811 }
812} 766}
813 767
814static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot) 768static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)