1 files changed, 129 insertions, 49 deletions

diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index ffc1e00f7a22..6a2aead5b0e5 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -39,7 +39,7 @@
 
 DEFINE_RAW_SPINLOCK(native_tlbie_lock);
 
-static inline void __tlbie(unsigned long vpn, int psize, int ssize)
+static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
 {
         unsigned long va;
         unsigned int penc;
@@ -61,17 +61,31 @@ static inline void __tlbie(unsigned long vpn, int psize, int ssize)
 
         switch (psize) {
         case MMU_PAGE_4K:
+                /* clear out bits after (52) [0....52.....63] */
+                va &= ~((1ul << (64 - 52)) - 1);
                 va |= ssize << 8;
+                va |= mmu_psize_defs[apsize].sllp << 6;
                 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
                              : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
                              : "memory");
                 break;
         default:
                 /* We need 14 to 14 + i bits of va */
-                penc = mmu_psize_defs[psize].penc;
-                va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
+                penc = mmu_psize_defs[psize].penc[apsize];
+                va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
                 va |= penc << 12;
                 va |= ssize << 8;
+                /* Add AVAL part */
+                if (psize != apsize) {
+                        /*
+                         * MPSS, 64K base page size and 16MB parge page size
+                         * We don't need all the bits, but rest of the bits
+                         * must be ignored by the processor.
+                         * vpn cover upto 65 bits of va. (0...65) and we need
+                         * 58..64 bits of va.
+                         */
+                        va |= (vpn & 0xfe);
+                }
                 va |= 1; /* L */
                 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
                              : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
@@ -80,7 +94,7 @@ static inline void __tlbie(unsigned long vpn, int psize, int ssize)
         }
 }
 
-static inline void __tlbiel(unsigned long vpn, int psize, int ssize)
+static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
 {
         unsigned long va;
         unsigned int penc;
@@ -96,16 +110,30 @@ static inline void __tlbiel(unsigned long vpn, int psize, int ssize)
 
         switch (psize) {
         case MMU_PAGE_4K:
+                /* clear out bits after(52) [0....52.....63] */
+                va &= ~((1ul << (64 - 52)) - 1);
                 va |= ssize << 8;
+                va |= mmu_psize_defs[apsize].sllp << 6;
                 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
                              : : "r"(va) : "memory");
                 break;
         default:
                 /* We need 14 to 14 + i bits of va */
-                penc = mmu_psize_defs[psize].penc;
-                va &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
+                penc = mmu_psize_defs[psize].penc[apsize];
+                va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
                 va |= penc << 12;
                 va |= ssize << 8;
+                /* Add AVAL part */
+                if (psize != apsize) {
+                        /*
+                         * MPSS, 64K base page size and 16MB parge page size
+                         * We don't need all the bits, but rest of the bits
+                         * must be ignored by the processor.
+                         * vpn cover upto 65 bits of va. (0...65) and we need
+                         * 58..64 bits of va.
+                         */
+                        va |= (vpn & 0xfe);
+                }
                 va |= 1; /* L */
                 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
                              : : "r"(va) : "memory");
@@ -114,7 +142,8 @@ static inline void __tlbiel(unsigned long vpn, int psize, int ssize)
 
 }
 
-static inline void tlbie(unsigned long vpn, int psize, int ssize, int local)
+static inline void tlbie(unsigned long vpn, int psize, int apsize,
+                         int ssize, int local)
 {
         unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL);
         int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
@@ -125,10 +154,10 @@ static inline void tlbie(unsigned long vpn, int psize, int ssize, int local)
                 raw_spin_lock(&native_tlbie_lock);
         asm volatile("ptesync": : :"memory");
         if (use_local) {
-                __tlbiel(vpn, psize, ssize);
+                __tlbiel(vpn, psize, apsize, ssize);
                 asm volatile("ptesync": : :"memory");
         } else {
-                __tlbie(vpn, psize, ssize);
+                __tlbie(vpn, psize, apsize, ssize);
                 asm volatile("eieio; tlbsync; ptesync": : :"memory");
         }
         if (lock_tlbie && !use_local)
@@ -156,7 +185,7 @@ static inline void native_unlock_hpte(struct hash_pte *hptep)
 
 static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
                         unsigned long pa, unsigned long rflags,
-                        unsigned long vflags, int psize, int ssize)
+                        unsigned long vflags, int psize, int apsize, int ssize)
 {
         struct hash_pte *hptep = htab_address + hpte_group;
         unsigned long hpte_v, hpte_r;
@@ -183,8 +212,8 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
         if (i == HPTES_PER_GROUP)
                 return -1;
 
-        hpte_v = hpte_encode_v(vpn, psize, ssize) | vflags | HPTE_V_VALID;
-        hpte_r = hpte_encode_r(pa, psize) | rflags;
+        hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
+        hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
 
         if (!(vflags & HPTE_V_BOLTED)) {
                 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
@@ -244,6 +273,51 @@ static long native_hpte_remove(unsigned long hpte_group)
         return i;
 }
 
+static inline int __hpte_actual_psize(unsigned int lp, int psize)
+{
+        int i, shift;
+        unsigned int mask;
+
+        /* start from 1 ignoring MMU_PAGE_4K */
+        for (i = 1; i < MMU_PAGE_COUNT; i++) {
+
+                /* invalid penc */
+                if (mmu_psize_defs[psize].penc[i] == -1)
+                        continue;
+                /*
+                 * encoding bits per actual page size
+                 *        PTE LP     actual page size
+                 *    rrrr rrrz         >=8KB
+                 *    rrrr rrzz         >=16KB
+                 *    rrrr rzzz         >=32KB
+                 *    rrrr zzzz         >=64KB
+                 * .......
+                 */
+                shift = mmu_psize_defs[i].shift - LP_SHIFT;
+                if (shift > LP_BITS)
+                        shift = LP_BITS;
+                mask = (1 << shift) - 1;
+                if ((lp & mask) == mmu_psize_defs[psize].penc[i])
+                        return i;
+        }
+        return -1;
+}
+
+static inline int hpte_actual_psize(struct hash_pte *hptep, int psize)
+{
+        /* Look at the 8 bit LP value */
+        unsigned int lp = (hptep->r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
+
+        if (!(hptep->v & HPTE_V_VALID))
+                return -1;
+
+        /* First check if it is large page */
+        if (!(hptep->v & HPTE_V_LARGE))
+                return MMU_PAGE_4K;
+
+        return __hpte_actual_psize(lp, psize);
+}
+
 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
                                  unsigned long vpn, int psize, int ssize,
                                  int local)
@@ -251,8 +325,9 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
         struct hash_pte *hptep = htab_address + slot;
         unsigned long hpte_v, want_v;
         int ret = 0;
+        int actual_psize;
 
-        want_v = hpte_encode_v(vpn, psize, ssize);
+        want_v = hpte_encode_avpn(vpn, psize, ssize);
 
         DBG_LOW("    update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
                 vpn, want_v & HPTE_V_AVPN, slot, newpp);
@@ -260,9 +335,13 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
         native_lock_hpte(hptep);
 
         hpte_v = hptep->v;
-
+        actual_psize = hpte_actual_psize(hptep, psize);
+        if (actual_psize < 0) {
+                native_unlock_hpte(hptep);
+                return -1;
+        }
         /* Even if we miss, we need to invalidate the TLB */
-        if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
+        if (!HPTE_V_COMPARE(hpte_v, want_v)) {
                 DBG_LOW(" -> miss\n");
                 ret = -1;
         } else {
@@ -274,7 +353,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
         native_unlock_hpte(hptep);
 
         /* Ensure it is out of the tlb too. */
-        tlbie(vpn, psize, ssize, local);
+        tlbie(vpn, psize, actual_psize, ssize, local);
 
         return ret;
 }
@@ -288,7 +367,7 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
         unsigned long want_v, hpte_v;
 
         hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
-        want_v = hpte_encode_v(vpn, psize, ssize);
+        want_v = hpte_encode_avpn(vpn, psize, ssize);
 
         /* Bolted mappings are only ever in the primary group */
         slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
@@ -315,6 +394,7 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
 static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
                                        int psize, int ssize)
 {
+        int actual_psize;
         unsigned long vpn;
         unsigned long vsid;
         long slot;
@@ -327,13 +407,16 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
         if (slot == -1)
                 panic("could not find page to bolt\n");
         hptep = htab_address + slot;
+        actual_psize = hpte_actual_psize(hptep, psize);
+        if (actual_psize < 0)
+                return;
 
         /* Update the HPTE */
         hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
                 (newpp & (HPTE_R_PP | HPTE_R_N));
 
         /* Ensure it is out of the tlb too. */
-        tlbie(vpn, psize, ssize, 0);
+        tlbie(vpn, psize, actual_psize, ssize, 0);
 }
 
 static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
@@ -343,64 +426,60 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
         unsigned long hpte_v;
         unsigned long want_v;
         unsigned long flags;
+        int actual_psize;
 
         local_irq_save(flags);
 
         DBG_LOW("    invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
 
-        want_v = hpte_encode_v(vpn, psize, ssize);
+        want_v = hpte_encode_avpn(vpn, psize, ssize);
         native_lock_hpte(hptep);
         hpte_v = hptep->v;
 
+        actual_psize = hpte_actual_psize(hptep, psize);
+        if (actual_psize < 0) {
+                native_unlock_hpte(hptep);
+                local_irq_restore(flags);
+                return;
+        }
         /* Even if we miss, we need to invalidate the TLB */
-        if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
+        if (!HPTE_V_COMPARE(hpte_v, want_v))
                 native_unlock_hpte(hptep);
         else
                 /* Invalidate the hpte. NOTE: this also unlocks it */
                 hptep->v = 0;
 
         /* Invalidate the TLB */
-        tlbie(vpn, psize, ssize, local);
+        tlbie(vpn, psize, actual_psize, ssize, local);
 
         local_irq_restore(flags);
 }
 
-#define LP_SHIFT        12
-#define LP_BITS         8
-#define LP_MASK(i)      ((0xFF >> (i)) << LP_SHIFT)
-
 static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
-                        int *psize, int *ssize, unsigned long *vpn)
+                        int *psize, int *apsize, int *ssize, unsigned long *vpn)
 {
         unsigned long avpn, pteg, vpi;
-        unsigned long hpte_r = hpte->r;
         unsigned long hpte_v = hpte->v;
         unsigned long vsid, seg_off;
-        int i, size, shift, penc;
+        int size, a_size, shift;
+        /* Look at the 8 bit LP value */
+        unsigned int lp = (hpte->r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
 
-        if (!(hpte_v & HPTE_V_LARGE))
-                size = MMU_PAGE_4K;
-        else {
-                for (i = 0; i < LP_BITS; i++) {
-                        if ((hpte_r & LP_MASK(i+1)) == LP_MASK(i+1))
-                                break;
-                }
-                penc = LP_MASK(i+1) >> LP_SHIFT;
+        if (!(hpte_v & HPTE_V_LARGE)) {
+                size   = MMU_PAGE_4K;
+                a_size = MMU_PAGE_4K;
+        } else {
                 for (size = 0; size < MMU_PAGE_COUNT; size++) {
 
-                        /* 4K pages are not represented by LP */
-                        if (size == MMU_PAGE_4K)
-                                continue;
-
                         /* valid entries have a shift value */
                         if (!mmu_psize_defs[size].shift)
                                 continue;
 
-                        if (penc == mmu_psize_defs[size].penc)
+                        a_size = __hpte_actual_psize(lp, size);
+                        if (a_size != -1)
                                 break;
                 }
         }
-
         /* This works for all page sizes, and for 256M and 1T segments */
         *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
         shift = mmu_psize_defs[size].shift;
@@ -433,7 +512,8 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
         default:
                 *vpn = size = 0;
         }
-        *psize = size;
+        *psize  = size;
+        *apsize = a_size;
 }
 
 /*
@@ -451,7 +531,7 @@ static void native_hpte_clear(void)
         struct hash_pte *hptep = htab_address;
         unsigned long hpte_v;
         unsigned long pteg_count;
-        int psize, ssize;
+        int psize, apsize, ssize;
 
         pteg_count = htab_hash_mask + 1;
 
@@ -477,9 +557,9 @@ static void native_hpte_clear(void)
                  * already hold the native_tlbie_lock.
                  */
                 if (hpte_v & HPTE_V_VALID) {
-                        hpte_decode(hptep, slot, &psize, &ssize, &vpn);
+                        hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
                         hptep->v = 0;
-                        __tlbie(vpn, psize, ssize);
+                        __tlbie(vpn, psize, apsize, ssize);
                 }
         }
 
@@ -520,7 +600,7 @@ static void native_flush_hash_range(unsigned long number, int local)
                         slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                         slot += hidx & _PTEIDX_GROUP_IX;
                         hptep = htab_address + slot;
-                        want_v = hpte_encode_v(vpn, psize, ssize);
+                        want_v = hpte_encode_avpn(vpn, psize, ssize);
                         native_lock_hpte(hptep);
                         hpte_v = hptep->v;
                         if (!HPTE_V_COMPARE(hpte_v, want_v) ||
@@ -540,7 +620,7 @@ static void native_flush_hash_range(unsigned long number, int local)
 
                         pte_iterate_hashed_subpages(pte, psize,
                                                     vpn, index, shift) {
-                                __tlbiel(vpn, psize, ssize);
+                                __tlbiel(vpn, psize, psize, ssize);
                         } pte_iterate_hashed_end();
                 }
                 asm volatile("ptesync":::"memory");
@@ -557,7 +637,7 @@ static void native_flush_hash_range(unsigned long number, int local)
 
                         pte_iterate_hashed_subpages(pte, psize,
                                                     vpn, index, shift) {
-                                __tlbie(vpn, psize, ssize);
+                                __tlbie(vpn, psize, psize, ssize);
                         } pte_iterate_hashed_end();
                 }
                 asm volatile("eieio; tlbsync; ptesync":::"memory");