From c26c20b823d48addbde9cb5709d80655c6fadf18 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:21:35 +0000 Subject: ARM: make_coherent: split adjust_pte() in two adjust_pte() walks the page tables, and do_adjust_pte() does the page table manipulation. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 52 +++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 56ee15321b00..074e6bb54eb3 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -36,28 +36,12 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; * Therefore those configurations which might call adjust_pte (those * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock. */ -static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep) { - pgd_t *pgd; - pmd_t *pmd; - pte_t *pte, entry; + pte_t entry = *ptep; int ret; - pgd = pgd_offset(vma->vm_mm, address); - if (pgd_none(*pgd)) - goto no_pgd; - if (pgd_bad(*pgd)) - goto bad_pgd; - - pmd = pmd_offset(pgd, address); - if (pmd_none(*pmd)) - goto no_pmd; - if (pmd_bad(*pmd)) - goto bad_pmd; - - pte = pte_offset_map(pmd, address); - entry = *pte; - /* * If this page is present, it's actually being shared. */ @@ -74,10 +58,38 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) (pfn << PAGE_SHIFT) + PAGE_SIZE); pte_val(entry) &= ~L_PTE_MT_MASK; pte_val(entry) |= shared_pte_mask; - set_pte_at(vma->vm_mm, address, pte, entry); + set_pte_at(vma->vm_mm, address, ptep, entry); flush_tlb_page(vma, address); } + + return ret; +} + +static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int ret; + + pgd = pgd_offset(vma->vm_mm, address); + if (pgd_none(*pgd)) + goto no_pgd; + if (pgd_bad(*pgd)) + goto bad_pgd; + + pmd = pmd_offset(pgd, address); + if (pmd_none(*pmd)) + goto no_pmd; + if (pmd_bad(*pmd)) + goto bad_pmd; + + pte = pte_offset_map(pmd, address); + + ret = do_adjust_pte(vma, address, pte); + pte_unmap(pte); + return ret; bad_pgd: -- cgit v1.2.2 From f8a85f1164a33e3eb5b421b137ced793ed53ee33 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:23:44 +0000 Subject: ARM: make_coherent: convert adjust_pte() to use p*d_none_or_clear_bad() Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 074e6bb54eb3..7a8efe1b37d8 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -73,16 +73,12 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) int ret; pgd = pgd_offset(vma->vm_mm, address); - if (pgd_none(*pgd)) - goto no_pgd; - if (pgd_bad(*pgd)) - goto bad_pgd; + if (pgd_none_or_clear_bad(pgd)) + return 0; pmd = pmd_offset(pgd, address); - if (pmd_none(*pmd)) - goto no_pmd; - if (pmd_bad(*pmd)) - goto bad_pmd; + if (pmd_none_or_clear_bad(pmd)) + return 0; pte = pte_offset_map(pmd, address); @@ -91,18 +87,6 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) pte_unmap(pte); return ret; - -bad_pgd: - pgd_ERROR(*pgd); - pgd_clear(pgd); -no_pgd: - return 0; - -bad_pmd: - pmd_ERROR(*pmd); - pmd_clear(pmd); -no_pmd: - return 0; } static void -- cgit v1.2.2 From 56dd47098abe1fdde598a8d8b7c04d775506f456 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:24:34 +0000 Subject: ARM: make_coherent: fix problems with highpte, part 1 update_mmu_cache() is called with a page table already mapped. We call make_coherent(), which then calls adjust_pte() which wants to map other page tables. This causes kmap_atomic() to BUG() because the slot its trying to use is already taken. Since do_adjust_pte() modifies the page tables, we are also missing any form of locking, so we're risking corrupting the page tables. Fix this by using pte_offset_map_nested(), and taking the pte page table lock around do_adjust_pte(). Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 7a8efe1b37d8..8e9bc517132e 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -67,6 +67,7 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, static int adjust_pte(struct vm_area_struct *vma, unsigned long address) { + spinlock_t *ptl; pgd_t *pgd; pmd_t *pmd; pte_t *pte; @@ -80,11 +81,19 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) if (pmd_none_or_clear_bad(pmd)) return 0; - pte = pte_offset_map(pmd, address); + /* + * This is called while another page table is mapped, so we + * must use the nested version. This also means we need to + * open-code the spin-locking. + */ + ptl = pte_lockptr(vma->vm_mm, pmd); + pte = pte_offset_map_nested(pmd, address); + spin_lock(ptl); ret = do_adjust_pte(vma, address, pte); - pte_unmap(pte); + spin_unlock(ptl); + pte_unmap_nested(pte); return ret; } -- cgit v1.2.2 From ed42acaef1a9d51631a31b55e9ed52d400430492 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:31:38 +0000 Subject: ARM: make_coherent: avoid recalculating the pfn for the modified page We already know the pfn for the page to be modified in make_coherent, so let's stop recalculating it unnecessarily. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 8e9bc517132e..ae88f2c3a6df 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -37,7 +37,7 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock. */ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep) + unsigned long pfn, pte_t *ptep) { pte_t entry = *ptep; int ret; @@ -52,7 +52,6 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, * fault (ie, is old), we can safely ignore any issues. */ if (ret && (pte_val(entry) & L_PTE_MT_MASK) != shared_pte_mask) { - unsigned long pfn = pte_pfn(entry); flush_cache_page(vma, address, pfn); outer_flush_range((pfn << PAGE_SHIFT), (pfn << PAGE_SHIFT) + PAGE_SIZE); @@ -65,7 +64,8 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } -static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +static int adjust_pte(struct vm_area_struct *vma, unsigned long address, + unsigned long pfn) { spinlock_t *ptl; pgd_t *pgd; @@ -90,7 +90,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) pte = pte_offset_map_nested(pmd, address); spin_lock(ptl); - ret = do_adjust_pte(vma, address, pte); + ret = do_adjust_pte(vma, address, pfn, pte); spin_unlock(ptl); pte_unmap_nested(pte); @@ -127,11 +127,11 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne if (!(mpnt->vm_flags & VM_MAYSHARE)) continue; offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; - aliases += adjust_pte(mpnt, mpnt->vm_start + offset); + aliases += adjust_pte(mpnt, mpnt->vm_start + offset, pfn); } flush_dcache_mmap_unlock(mapping); if (aliases) - adjust_pte(vma, addr); + adjust_pte(vma, addr, pfn); else flush_cache_page(vma, addr, pfn); } -- cgit v1.2.2 From 4b3073e1c53a256275f1079c0fbfbe85883d9275 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:40:18 +0000 Subject: MM: Pass a PTE pointer to update_mmu_cache() rather than the PTE itself On VIVT ARM, when we have multiple shared mappings of the same file in the same MM, we need to ensure that we have coherency across all copies. We do this via make_coherent() by making the pages uncacheable. This used to work fine, until we allowed highmem with highpte - we now have a page table which is mapped as required, and is not available for modification via update_mmu_cache(). Ralf Beache suggested getting rid of the PTE value passed to update_mmu_cache(): On MIPS update_mmu_cache() calls __update_tlb() which walks pagetables to construct a pointer to the pte again. Passing a pte_t * is much more elegant. Maybe we might even replace the pte argument with the pte_t? Ben Herrenschmidt would also like the pte pointer for PowerPC: Passing the ptep in there is exactly what I want. I want that -instead- of the PTE value, because I have issue on some ppc cases, for I$/D$ coherency, where set_pte_at() may decide to mask out the _PAGE_EXEC. So, pass in the mapped page table pointer into update_mmu_cache(), and remove the PTE value, updating all implementations and call sites to suit. Includes a fix from Stephen Rothwell: sparc: fix fallout from update_mmu_cache API change Signed-off-by: Stephen Rothwell Acked-by: Benjamin Herrenschmidt Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index ae88f2c3a6df..c45f9bb318ad 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -149,9 +149,10 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne * * Note that the pte lock will be held. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep) { - unsigned long pfn = pte_pfn(pte); + unsigned long pfn = pte_pfn(*ptep); struct address_space *mapping; struct page *page; -- cgit v1.2.2 From ae1402022edbeef3991f1e4bae8fa99558be291b Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:43:57 +0000 Subject: ARM: make_coherent(): fix problems with highpte, part 2 update_mmu_cache() is called with the page table for the faulted-in page still mapped. We need to modify the PTE for this page to ensure coherency with other shared mappings when multiple shared mappings exist within a MM. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm/fault-armv.c') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index c45f9bb318ad..c9b97e9836a2 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -99,7 +99,8 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, } static void -make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn) +make_coherent(struct address_space *mapping, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned long pfn) { struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *mpnt; @@ -131,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne } flush_dcache_mmap_unlock(mapping); if (aliases) - adjust_pte(vma, addr, pfn); + do_adjust_pte(vma, addr, pfn, ptep); else flush_cache_page(vma, addr, pfn); } @@ -174,7 +175,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, #endif if (mapping) { if (cache_is_vivt()) - make_coherent(mapping, vma, addr, pfn); + make_coherent(mapping, vma, addr, ptep, pfn); else if (vma->vm_flags & VM_EXEC) __flush_icache_all(); } -- cgit v1.2.2