4 files changed, 78 insertions, 32 deletions

diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 229951ffc351..8726779e1409 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -32,6 +32,7 @@
 #include <linux/perf_event.h>
 #include <linux/magic.h>
 #include <linux/ratelimit.h>
+#include <linux/context_tracking.h>
 
 #include <asm/firmware.h>
 #include <asm/page.h>
@@ -196,6 +197,7 @@ static int mm_fault_error(struct pt_regs *regs, unsigned long addr, int fault)
 int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
                             unsigned long error_code)
 {
+        enum ctx_state prev_state = exception_enter();
         struct vm_area_struct * vma;
         struct mm_struct *mm = current->mm;
         unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
@@ -204,6 +206,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
         int trap = TRAP(regs);
         int is_exec = trap == 0x400;
         int fault;
+        int rc = 0;
 
 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE))
         /*
@@ -230,28 +233,30 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
          * look at it
          */
         if (error_code & ICSWX_DSI_UCT) {
-                int rc = acop_handle_fault(regs, address, error_code);
+                rc = acop_handle_fault(regs, address, error_code);
                 if (rc)
-                        return rc;
+                        goto bail;
         }
 #endif /* CONFIG_PPC_ICSWX */
 
         if (notify_page_fault(regs))
-                return 0;
+                goto bail;
 
         if (unlikely(debugger_fault_handler(regs)))
-                return 0;
+                goto bail;
 
         /* On a kernel SLB miss we can only check for a valid exception entry */
-        if (!user_mode(regs) && (address >= TASK_SIZE))
-                return SIGSEGV;
+        if (!user_mode(regs) && (address >= TASK_SIZE)) {
+                rc = SIGSEGV;
+                goto bail;
+        }
 
 #if !(defined(CONFIG_4xx) || defined(CONFIG_BOOKE) || \
                              defined(CONFIG_PPC_BOOK3S_64))
         if (error_code & DSISR_DABRMATCH) {
                 /* breakpoint match */
                 do_break(regs, address, error_code);
-                return 0;
+                goto bail;
         }
 #endif
 
@@ -260,8 +265,10 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
                 local_irq_enable();
 
         if (in_atomic() || mm == NULL) {
-                if (!user_mode(regs))
-                        return SIGSEGV;
+                if (!user_mode(regs)) {
+                        rc = SIGSEGV;
+                        goto bail;
+                }
                 /* in_atomic() in user mode is really bad,
                    as is current->mm == NULL. */
                 printk(KERN_EMERG "Page fault in user mode with "
@@ -417,9 +424,11 @@ good_area:
          */
         fault = handle_mm_fault(mm, vma, address, flags);
         if (unlikely(fault & (VM_FAULT_RETRY|VM_FAULT_ERROR))) {
-                int rc = mm_fault_error(regs, address, fault);
+                rc = mm_fault_error(regs, address, fault);
                 if (rc >= MM_FAULT_RETURN)
-                        return rc;
+                        goto bail;
+                else
+                        rc = 0;
         }
 
         /*
@@ -454,7 +463,7 @@ good_area:
         }
 
         up_read(&mm->mmap_sem);
-        return 0;
+        goto bail;
 
 bad_area:
         up_read(&mm->mmap_sem);
@@ -463,7 +472,7 @@ bad_area_nosemaphore:
         /* User mode accesses cause a SIGSEGV */
         if (user_mode(regs)) {
                 _exception(SIGSEGV, regs, code, address);
-                return 0;
+                goto bail;
         }
 
         if (is_exec && (error_code & DSISR_PROTFAULT))
@@ -471,7 +480,11 @@ bad_area_nosemaphore:
                                    " page (%lx) - exploit attempt? (uid: %d)\n",
                                    address, from_kuid(&init_user_ns, current_uid()));
 
-        return SIGSEGV;
+        rc = SIGSEGV;
+
+bail:
+        exception_exit(prev_state);
+        return rc;
 
 }
 
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 6a2aead5b0e5..4c122c3f1623 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -336,11 +336,18 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
 
         hpte_v = hptep->v;
         actual_psize = hpte_actual_psize(hptep, psize);
+        /*
+         * We need to invalidate the TLB always because hpte_remove doesn't do
+         * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
+         * random entry from it. When we do that we don't invalidate the TLB
+         * (hpte_remove) because we assume the old translation is still
+         * technically "valid".
+         */
         if (actual_psize < 0) {
-                native_unlock_hpte(hptep);
-                return -1;
+                actual_psize = psize;
+                ret = -1;
+                goto err_out;
         }
-        /* Even if we miss, we need to invalidate the TLB */
         if (!HPTE_V_COMPARE(hpte_v, want_v)) {
                 DBG_LOW(" -> miss\n");
                 ret = -1;
@@ -350,6 +357,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
                 hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
                         (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C));
         }
+err_out:
         native_unlock_hpte(hptep);
 
         /* Ensure it is out of the tlb too. */
@@ -409,7 +417,7 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
         hptep = htab_address + slot;
         actual_psize = hpte_actual_psize(hptep, psize);
         if (actual_psize < 0)
-                return;
+                actual_psize = psize;
 
         /* Update the HPTE */
         hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
@@ -437,21 +445,27 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
         hpte_v = hptep->v;
 
         actual_psize = hpte_actual_psize(hptep, psize);
+        /*
+         * We need to invalidate the TLB always because hpte_remove doesn't do
+         * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
+         * random entry from it. When we do that we don't invalidate the TLB
+         * (hpte_remove) because we assume the old translation is still
+         * technically "valid".
+         */
         if (actual_psize < 0) {
+                actual_psize = psize;
                 native_unlock_hpte(hptep);
-                local_irq_restore(flags);
-                return;
+                goto err_out;
         }
-        /* Even if we miss, we need to invalidate the TLB */
         if (!HPTE_V_COMPARE(hpte_v, want_v))
                 native_unlock_hpte(hptep);
         else
                 /* Invalidate the hpte. NOTE: this also unlocks it */
                 hptep->v = 0;
 
+err_out:
         /* Invalidate the TLB */
         tlbie(vpn, psize, actual_psize, ssize, local);
-
         local_irq_restore(flags);
 }
 
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 88ac0eeaadde..e303a6d74e3a 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -33,6 +33,7 @@
 #include <linux/init.h>
 #include <linux/signal.h>
 #include <linux/memblock.h>
+#include <linux/context_tracking.h>
 
 #include <asm/processor.h>
 #include <asm/pgtable.h>
@@ -954,6 +955,7 @@ void hash_failure_debug(unsigned long ea, unsigned long access,
  */
 int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 {
+        enum ctx_state prev_state = exception_enter();
         pgd_t *pgdir;
         unsigned long vsid;
         struct mm_struct *mm;
@@ -973,7 +975,8 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
                 mm = current->mm;
                 if (! mm) {
                         DBG_LOW(" user region with no mm !\n");
-                        return 1;
+                        rc = 1;
+                        goto bail;
                 }
                 psize = get_slice_psize(mm, ea);
                 ssize = user_segment_size(ea);
@@ -992,19 +995,23 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
                 /* Not a valid range
                  * Send the problem up to do_page_fault 
                  */
-                return 1;
+                rc = 1;
+                goto bail;
         }
         DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);
 
         /* Bad address. */
         if (!vsid) {
                 DBG_LOW("Bad address!\n");
-                return 1;
+                rc = 1;
+                goto bail;
         }
         /* Get pgdir */
         pgdir = mm->pgd;
-        if (pgdir == NULL)
-                return 1;
+        if (pgdir == NULL) {
+                rc = 1;
+                goto bail;
+        }
 
         /* Check CPU locality */
         tmp = cpumask_of(smp_processor_id());
@@ -1027,7 +1034,8 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
         ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugeshift);
         if (ptep == NULL || !pte_present(*ptep)) {
                 DBG_LOW(" no PTE !\n");
-                return 1;
+                rc = 1;
+                goto bail;
         }
 
         /* Add _PAGE_PRESENT to the required access perm */
@@ -1038,13 +1046,16 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
          */
         if (access & ~pte_val(*ptep)) {
                 DBG_LOW(" no access !\n");
-                return 1;
+                rc = 1;
+                goto bail;
         }
 
 #ifdef CONFIG_HUGETLB_PAGE
-        if (hugeshift)
-                return __hash_page_huge(ea, access, vsid, ptep, trap, local,
+        if (hugeshift) {
+                rc = __hash_page_huge(ea, access, vsid, ptep, trap, local,
                                         ssize, hugeshift, psize);
+                goto bail;
+        }
 #endif /* CONFIG_HUGETLB_PAGE */
 
 #ifndef CONFIG_PPC_64K_PAGES
@@ -1124,6 +1135,9 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
                 pte_val(*(ptep + PTRS_PER_PTE)));
 #endif
         DBG_LOW(" -> rc=%d\n", rc);
+
+bail:
+        exception_exit(prev_state);
         return rc;
 }
 EXPORT_SYMBOL_GPL(hash_page);
@@ -1259,6 +1273,8 @@ void flush_hash_range(unsigned long number, int local)
  */
 void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
 {
+        enum ctx_state prev_state = exception_enter();
+
         if (user_mode(regs)) {
 #ifdef CONFIG_PPC_SUBPAGE_PROT
                 if (rc == -2)
@@ -1268,6 +1284,8 @@ void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc)
                         _exception(SIGBUS, regs, BUS_ADRERR, address);
         } else
                 bad_page_fault(regs, address, SIGBUS);
+
+        exception_exit(prev_state);
 }
 
 long hpte_insert_repeating(unsigned long hash, unsigned long vpn,
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index c2787bf779ca..a90b9c458990 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -215,7 +215,8 @@ static void __meminit vmemmap_create_mapping(unsigned long start,
                                              unsigned long phys)
 {
         int  mapped = htab_bolt_mapping(start, start + page_size, phys,
-                                        PAGE_KERNEL, mmu_vmemmap_psize,
+                                        pgprot_val(PAGE_KERNEL),
+                                        mmu_vmemmap_psize,
                                         mmu_kernel_ssize);
         BUG_ON(mapped < 0);
 }