aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/include/asm/pgtable.h
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-02-10 11:02:37 -0500
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-02-11 00:00:10 -0500
commit8d30c14cab30d405a05f2aaceda1e9ad57800f36 (patch)
treef3c0f11b3ce157601719119b2fe4b6a869828ae8 /arch/powerpc/include/asm/pgtable.h
parent4b7ad3593634c593d0e891ea415f9cf1bbcfcbd2 (diff)
powerpc/mm: Rework I$/D$ coherency (v3)
This patch reworks the way we do I and D cache coherency on PowerPC. The "old" way was split in 3 different parts depending on the processor type: - Hash with per-page exec support (64-bit and >= POWER4 only) does it at hashing time, by preventing exec on unclean pages and cleaning pages on exec faults. - Everything without per-page exec support (32-bit hash, 8xx, and 64-bit < POWER4) does it for all page going to user space in update_mmu_cache(). - Embedded with per-page exec support does it from do_page_fault() on exec faults, in a way similar to what the hash code does. That leads to confusion, and bugs. For example, the method using update_mmu_cache() is racy on SMP where another processor can see the new PTE and hash it in before we have cleaned the cache, and then blow trying to execute. This is hard to hit but I think it has bitten us in the past. Also, it's inefficient for embedded where we always end up having to do at least one more page fault. This reworks the whole thing by moving the cache sync into two main call sites, though we keep different behaviours depending on the HW capability. The call sites are set_pte_at() which is now made out of line, and ptep_set_access_flags() which joins the former in pgtable.c The base idea for Embedded with per-page exec support, is that we now do the flush at set_pte_at() time when coming from an exec fault, which allows us to avoid the double fault problem completely (we can even improve the situation more by implementing TLB preload in update_mmu_cache() but that's for later). If for some reason we didn't do it there and we try to execute, we'll hit the page fault, which will do a minor fault, which will hit ptep_set_access_flags() to do things like update _PAGE_ACCESSED or _PAGE_DIRTY if needed, we just make this guys also perform the I/D cache sync for exec faults now. This second path is the catch all for things that weren't cleaned at set_pte_at() time. For cpus without per-pag exec support, we always do the sync at set_pte_at(), thus guaranteeing that when the PTE is visible to other processors, the cache is clean. For the 64-bit hash with per-page exec support case, we keep the old mechanism for now. I'll look into changing it later, once I've reworked a bit how we use _PAGE_EXEC. This is also a first step for adding _PAGE_EXEC support for embedded platforms Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Diffstat (limited to 'arch/powerpc/include/asm/pgtable.h')
-rw-r--r--arch/powerpc/include/asm/pgtable.h84
1 files changed, 84 insertions, 0 deletions
diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index 07f55e601696..5c1c4880723c 100644
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -6,7 +6,17 @@
6#include <asm/processor.h> /* For TASK_SIZE */ 6#include <asm/processor.h> /* For TASK_SIZE */
7#include <asm/mmu.h> 7#include <asm/mmu.h>
8#include <asm/page.h> 8#include <asm/page.h>
9
9struct mm_struct; 10struct mm_struct;
11
12#ifdef CONFIG_DEBUG_VM
13extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
14#else /* CONFIG_DEBUG_VM */
15static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
16{
17}
18#endif /* !CONFIG_DEBUG_VM */
19
10#endif /* !__ASSEMBLY__ */ 20#endif /* !__ASSEMBLY__ */
11 21
12#if defined(CONFIG_PPC64) 22#if defined(CONFIG_PPC64)
@@ -17,6 +27,80 @@ struct mm_struct;
17 27
18#ifndef __ASSEMBLY__ 28#ifndef __ASSEMBLY__
19 29
30/* Insert a PTE, top-level function is out of line. It uses an inline
31 * low level function in the respective pgtable-* files
32 */
33extern void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
34 pte_t pte);
35
36/* This low level function performs the actual PTE insertion
37 * Setting the PTE depends on the MMU type and other factors. It's
38 * an horrible mess that I'm not going to try to clean up now but
39 * I'm keeping it in one place rather than spread around
40 */
41static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
42 pte_t *ptep, pte_t pte, int percpu)
43{
44#if defined(CONFIG_PPC_STD_MMU_32) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT)
45 /* First case is 32-bit Hash MMU in SMP mode with 32-bit PTEs. We use the
46 * helper pte_update() which does an atomic update. We need to do that
47 * because a concurrent invalidation can clear _PAGE_HASHPTE. If it's a
48 * per-CPU PTE such as a kmap_atomic, we do a simple update preserving
49 * the hash bits instead (ie, same as the non-SMP case)
50 */
51 if (percpu)
52 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
53 | (pte_val(pte) & ~_PAGE_HASHPTE));
54 else
55 pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte));
56
57#elif defined(CONFIG_PPC32) && defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP)
58 /* Second case is 32-bit with 64-bit PTE in SMP mode. In this case, we
59 * can just store as long as we do the two halves in the right order
60 * with a barrier in between. This is possible because we take care,
61 * in the hash code, to pre-invalidate if the PTE was already hashed,
62 * which synchronizes us with any concurrent invalidation.
63 * In the percpu case, we also fallback to the simple update preserving
64 * the hash bits
65 */
66 if (percpu) {
67 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
68 | (pte_val(pte) & ~_PAGE_HASHPTE));
69 return;
70 }
71#if _PAGE_HASHPTE != 0
72 if (pte_val(*ptep) & _PAGE_HASHPTE)
73 flush_hash_entry(mm, ptep, addr);
74#endif
75 __asm__ __volatile__("\
76 stw%U0%X0 %2,%0\n\
77 eieio\n\
78 stw%U0%X0 %L2,%1"
79 : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4))
80 : "r" (pte) : "memory");
81
82#elif defined(CONFIG_PPC_STD_MMU_32)
83 /* Third case is 32-bit hash table in UP mode, we need to preserve
84 * the _PAGE_HASHPTE bit since we may not have invalidated the previous
85 * translation in the hash yet (done in a subsequent flush_tlb_xxx())
86 * and see we need to keep track that this PTE needs invalidating
87 */
88 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
89 | (pte_val(pte) & ~_PAGE_HASHPTE));
90
91#else
92 /* Anything else just stores the PTE normally. That covers all 64-bit
93 * cases, and 32-bit non-hash with 64-bit PTEs in UP mode
94 */
95 *ptep = pte;
96#endif
97}
98
99
100#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
101extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
102 pte_t *ptep, pte_t entry, int dirty);
103
20/* 104/*
21 * Macro to mark a page protection value as "uncacheable". 105 * Macro to mark a page protection value as "uncacheable".
22 */ 106 */