powerpc/mm/thp: Use tlbiel if possible

If we know that user address space has never executed on other cpus
we could use tlbiel.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

7 files changed, 37 insertions, 13 deletions

diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index 15c9150a58cc..e5c0919acca4 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -60,7 +60,7 @@ struct machdep_calls {
         void            (*hugepage_invalidate)(unsigned long vsid,
                                                unsigned long addr,
                                                unsigned char *hpte_slot_array,
-                                               int psize, int ssize);
+                                               int psize, int ssize, int local);
         /* special for kexec, to be called in real mode, linear mapping is
          * destroyed as well */
         void            (*hpte_clear_all)(void);
diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h
index 19550d346fea..4d3ecd8d8929 100644
--- a/arch/powerpc/include/asm/tlbflush.h
+++ b/arch/powerpc/include/asm/tlbflush.h
@@ -128,7 +128,8 @@ extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
                             int ssize, int local);
 extern void flush_hash_range(unsigned long number, int local);
 extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
-                                pmd_t *pmdp, unsigned int psize, int ssize);
+                                pmd_t *pmdp, unsigned int psize, int ssize,
+                                int local);
 
 static inline void local_flush_tlb_mm(struct mm_struct *mm)
 {
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 558e50bac6f7..13700911b522 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -425,7 +425,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
 static void native_hugepage_invalidate(unsigned long vsid,
                                        unsigned long addr,
                                        unsigned char *hpte_slot_array,
-                                       int psize, int ssize)
+                                       int psize, int ssize, int local)
 {
         int i;
         struct hash_pte *hptep;
@@ -471,7 +471,7 @@ static void native_hugepage_invalidate(unsigned long vsid,
                  * instruction compares entry_VA in tlb with the VA specified
                  * here
                  */
-                tlbie(vpn, psize, actual_psize, ssize, 0);
+                tlbie(vpn, psize, actual_psize, ssize, local);
         }
         local_irq_restore(flags);
 }
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 6c2076c65d7c..68211d398fdb 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1317,7 +1317,7 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
-                         pmd_t *pmdp, unsigned int psize, int ssize)
+                         pmd_t *pmdp, unsigned int psize, int ssize, int local)
 {
         int i, max_hpte_count, valid;
         unsigned long s_addr;
@@ -1334,9 +1334,11 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
         if (!hpte_slot_array)
                 return;
 
-        if (ppc_md.hugepage_invalidate)
-                return ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
-                                                  psize, ssize);
+        if (ppc_md.hugepage_invalidate) {
+                ppc_md.hugepage_invalidate(vsid, s_addr, hpte_slot_array,
+                                           psize, ssize, local);
+                goto tm_abort;
+        }
         /*
          * No bluk hpte removal support, invalidate each entry
          */
@@ -1362,8 +1364,24 @@ void flush_hash_hugepage(unsigned long vsid, unsigned long addr,
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
                 slot += hidx & _PTEIDX_GROUP_IX;
                 ppc_md.hpte_invalidate(slot, vpn, psize,
-                                       MMU_PAGE_16M, ssize, 0);
+                                       MMU_PAGE_16M, ssize, local);
+        }
+tm_abort:
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+        /* Transactions are not aborted by tlbiel, only tlbie.
+         * Without, syncing a page back to a block device w/ PIO could pick up
+         * transactional data (bad!) so we force an abort here.  Before the
+         * sync the page will be made read-only, which will flush_hash_page.
+         * BIG ISSUE here: if the kernel uses a page from userspace without
+         * unmapping it first, it may see the speculated version.
+         */
+        if (local && cpu_has_feature(CPU_FTR_TM) &&
+            current->thread.regs &&
+            MSR_TM_ACTIVE(current->thread.regs->msr)) {
+                tm_enable();
+                tm_abort(TM_CAUSE_TLBI);
         }
+#endif
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c
index 1b3ad46a71b5..3a648cd363ae 100644
--- a/arch/powerpc/mm/hugepage-hash64.c
+++ b/arch/powerpc/mm/hugepage-hash64.c
@@ -95,7 +95,7 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid,
                  */
                 if ((old_pmd & _PAGE_HASHPTE) && !(old_pmd & _PAGE_COMBO))
                         flush_hash_hugepage(vsid, ea, pmdp, MMU_PAGE_64K,
-                                            ssize);
+                                            ssize, local);
         }
 
         valid = hpte_valid(hpte_slot_array, index);
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index c175c990580e..eea9fa1f8ae7 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -739,9 +739,10 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
                             pmd_t *pmdp, unsigned long old_pmd)
 {
-        int ssize;
+        int ssize, local = 0;
         unsigned int psize;
         unsigned long vsid;
+        const struct cpumask *tmp;
 
         /* get the base page size,vsid and segment size */
 #ifdef CONFIG_DEBUG_VM
@@ -762,7 +763,11 @@ void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr,
                 ssize = mmu_kernel_ssize;
         }
 
-        return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize);
+        tmp = cpumask_of(smp_processor_id());
+        if (cpumask_equal(mm_cpumask(mm), tmp))
+                local = 1;
+
+        return flush_hash_hugepage(vsid, addr, pmdp, psize, ssize, local);
 }
 
 static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index d214a012b026..832f221840f2 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -442,7 +442,7 @@ static void __pSeries_lpar_hugepage_invalidate(unsigned long *slot,
 static void pSeries_lpar_hugepage_invalidate(unsigned long vsid,
                                              unsigned long addr,
                                              unsigned char *hpte_slot_array,
-                                             int psize, int ssize)
+                                             int psize, int ssize, int local)
 {
         int i, index = 0;
         unsigned long s_addr = addr;