aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/include/asm/pgtable-ppc32.h
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-02-10 11:02:37 -0500
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2009-02-11 00:00:10 -0500
commit8d30c14cab30d405a05f2aaceda1e9ad57800f36 (patch)
treef3c0f11b3ce157601719119b2fe4b6a869828ae8 /arch/powerpc/include/asm/pgtable-ppc32.h
parent4b7ad3593634c593d0e891ea415f9cf1bbcfcbd2 (diff)
powerpc/mm: Rework I$/D$ coherency (v3)
This patch reworks the way we do I and D cache coherency on PowerPC. The "old" way was split in 3 different parts depending on the processor type: - Hash with per-page exec support (64-bit and >= POWER4 only) does it at hashing time, by preventing exec on unclean pages and cleaning pages on exec faults. - Everything without per-page exec support (32-bit hash, 8xx, and 64-bit < POWER4) does it for all page going to user space in update_mmu_cache(). - Embedded with per-page exec support does it from do_page_fault() on exec faults, in a way similar to what the hash code does. That leads to confusion, and bugs. For example, the method using update_mmu_cache() is racy on SMP where another processor can see the new PTE and hash it in before we have cleaned the cache, and then blow trying to execute. This is hard to hit but I think it has bitten us in the past. Also, it's inefficient for embedded where we always end up having to do at least one more page fault. This reworks the whole thing by moving the cache sync into two main call sites, though we keep different behaviours depending on the HW capability. The call sites are set_pte_at() which is now made out of line, and ptep_set_access_flags() which joins the former in pgtable.c The base idea for Embedded with per-page exec support, is that we now do the flush at set_pte_at() time when coming from an exec fault, which allows us to avoid the double fault problem completely (we can even improve the situation more by implementing TLB preload in update_mmu_cache() but that's for later). If for some reason we didn't do it there and we try to execute, we'll hit the page fault, which will do a minor fault, which will hit ptep_set_access_flags() to do things like update _PAGE_ACCESSED or _PAGE_DIRTY if needed, we just make this guys also perform the I/D cache sync for exec faults now. This second path is the catch all for things that weren't cleaned at set_pte_at() time. For cpus without per-pag exec support, we always do the sync at set_pte_at(), thus guaranteeing that when the PTE is visible to other processors, the cache is clean. For the 64-bit hash with per-page exec support case, we keep the old mechanism for now. I'll look into changing it later, once I've reworked a bit how we use _PAGE_EXEC. This is also a first step for adding _PAGE_EXEC support for embedded platforms Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Diffstat (limited to 'arch/powerpc/include/asm/pgtable-ppc32.h')
-rw-r--r--arch/powerpc/include/asm/pgtable-ppc32.h56
1 files changed, 5 insertions, 51 deletions
diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h
index f69a4d977729..211c90df4763 100644
--- a/arch/powerpc/include/asm/pgtable-ppc32.h
+++ b/arch/powerpc/include/asm/pgtable-ppc32.h
@@ -429,6 +429,8 @@ extern int icache_44x_need_flush;
429#define PMD_PAGE_SIZE(pmd) bad_call_to_PMD_PAGE_SIZE() 429#define PMD_PAGE_SIZE(pmd) bad_call_to_PMD_PAGE_SIZE()
430#endif 430#endif
431 431
432#define _PAGE_HPTEFLAGS _PAGE_HASHPTE
433
432#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY) 434#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY)
433 435
434 436
@@ -667,44 +669,6 @@ static inline unsigned long long pte_update(pte_t *p,
667#endif /* CONFIG_PTE_64BIT */ 669#endif /* CONFIG_PTE_64BIT */
668 670
669/* 671/*
670 * set_pte stores a linux PTE into the linux page table.
671 * On machines which use an MMU hash table we avoid changing the
672 * _PAGE_HASHPTE bit.
673 */
674
675static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
676 pte_t *ptep, pte_t pte)
677{
678#if (_PAGE_HASHPTE != 0) && defined(CONFIG_SMP) && !defined(CONFIG_PTE_64BIT)
679 pte_update(ptep, ~_PAGE_HASHPTE, pte_val(pte) & ~_PAGE_HASHPTE);
680#elif defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP)
681#if _PAGE_HASHPTE != 0
682 if (pte_val(*ptep) & _PAGE_HASHPTE)
683 flush_hash_entry(mm, ptep, addr);
684#endif
685 __asm__ __volatile__("\
686 stw%U0%X0 %2,%0\n\
687 eieio\n\
688 stw%U0%X0 %L2,%1"
689 : "=m" (*ptep), "=m" (*((unsigned char *)ptep+4))
690 : "r" (pte) : "memory");
691#else
692 *ptep = __pte((pte_val(*ptep) & _PAGE_HASHPTE)
693 | (pte_val(pte) & ~_PAGE_HASHPTE));
694#endif
695}
696
697
698static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
699 pte_t *ptep, pte_t pte)
700{
701#if defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP) && defined(CONFIG_DEBUG_VM)
702 WARN_ON(pte_present(*ptep));
703#endif
704 __set_pte_at(mm, addr, ptep, pte);
705}
706
707/*
708 * 2.6 calls this without flushing the TLB entry; this is wrong 672 * 2.6 calls this without flushing the TLB entry; this is wrong
709 * for our hash-based implementation, we fix that up here. 673 * for our hash-based implementation, we fix that up here.
710 */ 674 */
@@ -744,24 +708,14 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
744} 708}
745 709
746 710
747#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 711static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry)
748static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
749{ 712{
750 unsigned long bits = pte_val(entry) & 713 unsigned long bits = pte_val(entry) &
751 (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW); 714 (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW |
715 _PAGE_HWEXEC | _PAGE_EXEC);
752 pte_update(ptep, 0, bits); 716 pte_update(ptep, 0, bits);
753} 717}
754 718
755#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \
756({ \
757 int __changed = !pte_same(*(__ptep), __entry); \
758 if (__changed) { \
759 __ptep_set_access_flags(__ptep, __entry, __dirty); \
760 flush_tlb_page_nohash(__vma, __address); \
761 } \
762 __changed; \
763})
764
765#define __HAVE_ARCH_PTE_SAME 719#define __HAVE_ARCH_PTE_SAME
766#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0) 720#define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
767 721