2 files changed, 34 insertions, 14 deletions

diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S
index 9bc0a9c2b9bc..e64ce3eec36e 100644
--- a/arch/powerpc/mm/hash_low_64.S
+++ b/arch/powerpc/mm/hash_low_64.S
@@ -445,9 +445,12 @@ END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 
 htab_insert_pte:
         /* real page number in r5, PTE RPN value + index */
-        rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
+        andis.  r0,r31,_PAGE_4K_PFN@h
+        srdi    r5,r31,PTE_RPN_SHIFT
+        bne-    htab_special_pfn
         sldi    r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
         add     r5,r5,r25
+htab_special_pfn:
         sldi    r5,r5,HW_PAGE_SHIFT
 
         /* Calculate primary group hash */
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 3c7fe2c65b5a..aae085317018 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -573,6 +573,27 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
         return pp;
 }
 
+/*
+ * Demote a segment to using 4k pages.
+ * For now this makes the whole process use 4k pages.
+ */
+void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
+{
+#ifdef CONFIG_PPC_64K_PAGES
+        if (mm->context.user_psize == MMU_PAGE_4K)
+                return;
+        mm->context.user_psize = MMU_PAGE_4K;
+        mm->context.sllp = SLB_VSID_USER | mmu_psize_defs[MMU_PAGE_4K].sllp;
+        get_paca()->context = mm->context;
+        slb_flush_and_rebolt();
+#ifdef CONFIG_SPE_BASE
+        spu_flush_all_slbs(mm);
+#endif
+#endif
+}
+
+EXPORT_SYMBOL_GPL(demote_segment_4k);
+
 /* Result code is:
  *  0 - handled
  *  1 - normal page fault
@@ -665,15 +686,19 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 #ifndef CONFIG_PPC_64K_PAGES
         rc = __hash_page_4K(ea, access, vsid, ptep, trap, local);
 #else
+        /* If _PAGE_4K_PFN is set, make sure this is a 4k segment */
+        if (pte_val(*ptep) & _PAGE_4K_PFN) {
+                demote_segment_4k(mm, ea);
+                psize = MMU_PAGE_4K;
+        }
+
         if (mmu_ci_restrictions) {
                 /* If this PTE is non-cacheable, switch to 4k */
                 if (psize == MMU_PAGE_64K &&
                     (pte_val(*ptep) & _PAGE_NO_CACHE)) {
                         if (user_region) {
+                                demote_segment_4k(mm, ea);
                                 psize = MMU_PAGE_4K;
-                                mm->context.user_psize = MMU_PAGE_4K;
-                                mm->context.sllp = SLB_VSID_USER |
-                                        mmu_psize_defs[MMU_PAGE_4K].sllp;
                         } else if (ea < VMALLOC_END) {
                                 /*
                                  * some driver did a non-cacheable mapping
@@ -756,16 +781,8 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
         if (mmu_ci_restrictions) {
                 /* If this PTE is non-cacheable, switch to 4k */
                 if (mm->context.user_psize == MMU_PAGE_64K &&
-                    (pte_val(*ptep) & _PAGE_NO_CACHE)) {
-                        mm->context.user_psize = MMU_PAGE_4K;
-                        mm->context.sllp = SLB_VSID_USER |
-                                mmu_psize_defs[MMU_PAGE_4K].sllp;
-                        get_paca()->context = mm->context;
-                        slb_flush_and_rebolt();
-#ifdef CONFIG_SPE_BASE
-                        spu_flush_all_slbs(mm);
-#endif
-                }
+                    (pte_val(*ptep) & _PAGE_NO_CACHE))
+                        demote_segment_4k(mm, ea);
         }
         if (mm->context.user_psize == MMU_PAGE_64K)
                 __hash_page_64K(ea, access, vsid, ptep, trap, local);