From f00a75c094c340c4e7435665816c3273c870e849 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 5 Oct 2009 15:17:45 +0100 Subject: ARM: Pass VMA to copy_user_highpage() implementations Our copy_user_highpage() implementations may require cache maintainence. Ensure that implementations have all necessary details to perform this maintainence. Signed-off-by: Russell King --- arch/arm/mm/copypage-feroceon.c | 2 +- arch/arm/mm/copypage-v3.c | 2 +- arch/arm/mm/copypage-v4mc.c | 2 +- arch/arm/mm/copypage-v4wb.c | 2 +- arch/arm/mm/copypage-v4wt.c | 2 +- arch/arm/mm/copypage-v6.c | 4 ++-- arch/arm/mm/copypage-xsc3.c | 2 +- arch/arm/mm/copypage-xscale.c | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c index 70997d5bee2d..e2e8d29c548b 100644 --- a/arch/arm/mm/copypage-feroceon.c +++ b/arch/arm/mm/copypage-feroceon.c @@ -68,7 +68,7 @@ feroceon_copy_user_page(void *kto, const void *kfrom) } void feroceon_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; diff --git a/arch/arm/mm/copypage-v3.c b/arch/arm/mm/copypage-v3.c index de9c06854ad7..f72303e1d804 100644 --- a/arch/arm/mm/copypage-v3.c +++ b/arch/arm/mm/copypage-v3.c @@ -38,7 +38,7 @@ v3_copy_user_page(void *kto, const void *kfrom) } void v3_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 7370a7142b04..598c51ad5071 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -69,7 +69,7 @@ mc_copy_user_page(void *from, void *to) } void v4_mc_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto = kmap_atomic(to, KM_USER1); diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c index 9ab098414227..e9920f68b76b 100644 --- a/arch/arm/mm/copypage-v4wb.c +++ b/arch/arm/mm/copypage-v4wb.c @@ -48,7 +48,7 @@ v4wb_copy_user_page(void *kto, const void *kfrom) } void v4wb_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; diff --git a/arch/arm/mm/copypage-v4wt.c b/arch/arm/mm/copypage-v4wt.c index 300efafd6643..172e6a55458e 100644 --- a/arch/arm/mm/copypage-v4wt.c +++ b/arch/arm/mm/copypage-v4wt.c @@ -44,7 +44,7 @@ v4wt_copy_user_page(void *kto, const void *kfrom) } void v4wt_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 4127a7bddfe5..334d5602770e 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -34,7 +34,7 @@ static DEFINE_SPINLOCK(v6_lock); * attack the kernel's existing mapping of these pages. */ static void v6_copy_user_highpage_nonaliasing(struct page *to, - struct page *from, unsigned long vaddr) + struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; @@ -73,7 +73,7 @@ static void discard_old_kernel_data(void *kto) * Copy the page, taking account of the cache colour. */ static void v6_copy_user_highpage_aliasing(struct page *to, - struct page *from, unsigned long vaddr) + struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { unsigned int offset = CACHE_COLOUR(vaddr); unsigned long kfrom, kto; diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c index bc4525f5ab23..18ae05d5829c 100644 --- a/arch/arm/mm/copypage-xsc3.c +++ b/arch/arm/mm/copypage-xsc3.c @@ -71,7 +71,7 @@ xsc3_mc_copy_user_page(void *kto, const void *kfrom) } void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto, *kfrom; diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 76824d3e966a..9920c0ae2096 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -91,7 +91,7 @@ mc_copy_user_page(void *from, void *to) } void xscale_mc_copy_user_highpage(struct page *to, struct page *from, - unsigned long vaddr) + unsigned long vaddr, struct vm_area_struct *vma) { void *kto = kmap_atomic(to, KM_USER1); -- cgit v1.2.2 From 2725898fc9bb2121ac0fb1b5e4faf4fc09014729 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 5 Oct 2009 15:34:22 +0100 Subject: ARM: Flush user mapping on VIVT processors when copying a page Steven Walter writes: > I've been tracking down an instance of userspace data corruption, > and I believe I have found a window during fork where data can be > lost. The corruption is occurring on an ARMv5 system with VIVT > caches. Here's the scenario in question. Thread A is forking, > Thread B is running in userspace: > > Thread A: flush_cache_mm() (dup_mmap) > Thread B: writes to a page in the above mm > Thread A: pte_wrprotect() the above page (copy_one_pte) > Thread B: writes to the same page again > > During thread B's second write, he'll take a fault and enter the > do_wp_page() case. We'll end up calling copy_page(), which notably > uses the kernel virtual addresses for the old and new pages. This > means that the new page does not necessarily have the data from the > first write. Now there are two conflicting copies of the same > cache-line in dcache. If the userspace cache-line flushes before > the kernel cache-line, we lose the changes made during the first > write. do_wp_page does call flush_dcache_page on the newly-copied > page, but there's still a window where the CPU could flush the > userspace cache-line before then. Resolve this by flushing the user mapping before copying the page on processors with a writeback VIVT cache. Note: this does have a performance impact, and so needs further consideration before being merged - can we optimize out some of the cache flushes if, eg, we know that the page isn't yet mapped? Thread: Signed-off-by: Russell King --- arch/arm/mm/copypage-feroceon.c | 1 + arch/arm/mm/copypage-v4wb.c | 1 + arch/arm/mm/copypage-xsc3.c | 1 + 3 files changed, 3 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-feroceon.c b/arch/arm/mm/copypage-feroceon.c index e2e8d29c548b..5eb4fd93893d 100644 --- a/arch/arm/mm/copypage-feroceon.c +++ b/arch/arm/mm/copypage-feroceon.c @@ -74,6 +74,7 @@ void feroceon_copy_user_highpage(struct page *to, struct page *from, kto = kmap_atomic(to, KM_USER0); kfrom = kmap_atomic(from, KM_USER1); + flush_cache_page(vma, vaddr, page_to_pfn(from)); feroceon_copy_user_page(kto, kfrom); kunmap_atomic(kfrom, KM_USER1); kunmap_atomic(kto, KM_USER0); diff --git a/arch/arm/mm/copypage-v4wb.c b/arch/arm/mm/copypage-v4wb.c index e9920f68b76b..7c2eb55cd4a9 100644 --- a/arch/arm/mm/copypage-v4wb.c +++ b/arch/arm/mm/copypage-v4wb.c @@ -54,6 +54,7 @@ void v4wb_copy_user_highpage(struct page *to, struct page *from, kto = kmap_atomic(to, KM_USER0); kfrom = kmap_atomic(from, KM_USER1); + flush_cache_page(vma, vaddr, page_to_pfn(from)); v4wb_copy_user_page(kto, kfrom); kunmap_atomic(kfrom, KM_USER1); kunmap_atomic(kto, KM_USER0); diff --git a/arch/arm/mm/copypage-xsc3.c b/arch/arm/mm/copypage-xsc3.c index 18ae05d5829c..747ad4140fc7 100644 --- a/arch/arm/mm/copypage-xsc3.c +++ b/arch/arm/mm/copypage-xsc3.c @@ -77,6 +77,7 @@ void xsc3_mc_copy_user_highpage(struct page *to, struct page *from, kto = kmap_atomic(to, KM_USER0); kfrom = kmap_atomic(from, KM_USER1); + flush_cache_page(vma, vaddr, page_to_pfn(from)); xsc3_mc_copy_user_page(kto, kfrom); kunmap_atomic(kfrom, KM_USER1); kunmap_atomic(kto, KM_USER0); -- cgit v1.2.2 From 345a32296b1f9f6121379e0240915e0e2be2dbf5 Mon Sep 17 00:00:00 2001 From: Mikael Pettersson Date: Thu, 29 Oct 2009 11:46:56 -0700 Subject: iop: implement sched_clock() This adds a better sched_clock() to the IOP platform, implemented using its new clocksource support. Tested on n2100, compile-tested for all plat-iop machines. [dan.j.williams@intel.com: allow early cp6 access] Signed-off-by: Mikael Pettersson Signed-off-by: Dan Williams --- arch/arm/mm/proc-xsc3.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index 2028f3702881..fab134e29826 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -396,7 +396,7 @@ __xsc3_setup: orr r4, r4, #0x18 @ cache the page table in L2 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer - mov r0, #0 @ don't allow CP access + mov r0, #1 << 6 @ cp6 access for early sched_clock mcr p15, 0, r0, c15, c1, 0 @ write CP access register mrc p15, 0, r0, c1, c0, 1 @ get auxiliary control reg -- cgit v1.2.2 From 394168389c5770accf1d255fdfe45846ec121585 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 19 Nov 2009 11:30:30 +0100 Subject: ARM: 5791/1: ARM: MM: use 64bytes of L1 cache on plat S5PC1xx Samsung S5PC1xx SoCs are based on ARM Coretex8, which has 64 bytes of L1 cache line size. Enable proper handling of L1 cache on these SoCs. Signed-off-by: Kyungmin Park Signed-off-by: Marek Szyprowski Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index e993140edd88..9cf7706e0be0 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -777,5 +777,5 @@ config CACHE_XSC3L2 config ARM_L1_CACHE_SHIFT int - default 6 if ARCH_OMAP3 + default 6 if ARCH_OMAP3 || ARCH_S5PC1XX default 5 -- cgit v1.2.2 From 13ccf3ad99a45052664f2c1a6c64899f9d778152 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 15:07:04 +0000 Subject: ARM: dma-mapping: split out vmregion code from dma coherent mapping code Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/Makefile | 2 +- arch/arm/mm/dma-mapping.c | 132 +++++----------------------------------------- arch/arm/mm/vmregion.c | 131 +++++++++++++++++++++++++++++++++++++++++++++ arch/arm/mm/vmregion.h | 29 ++++++++++ 4 files changed, 174 insertions(+), 120 deletions(-) create mode 100644 arch/arm/mm/vmregion.c create mode 100644 arch/arm/mm/vmregion.h (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 055cb2aa8134..42352e75742b 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -6,7 +6,7 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ iomap.o obj-$(CONFIG_MMU) += fault-armv.o flush.o ioremap.o mmap.o \ - pgd.o mmu.o + pgd.o mmu.o vmregion.o ifneq ($(CONFIG_MMU),y) obj-y += nommu.o diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index b9590a7085ca..c54f1acf92c8 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -68,106 +68,16 @@ static u64 get_coherent_dma_mask(struct device *dev) * These are the page tables (2MB each) covering uncached, DMA consistent allocations */ static pte_t *consistent_pte[NUM_CONSISTENT_PTES]; -static DEFINE_SPINLOCK(consistent_lock); -/* - * VM region handling support. - * - * This should become something generic, handling VM region allocations for - * vmalloc and similar (ioremap, module space, etc). - * - * I envisage vmalloc()'s supporting vm_struct becoming: - * - * struct vm_struct { - * struct vm_region region; - * unsigned long flags; - * struct page **pages; - * unsigned int nr_pages; - * unsigned long phys_addr; - * }; - * - * get_vm_area() would then call vm_region_alloc with an appropriate - * struct vm_region head (eg): - * - * struct vm_region vmalloc_head = { - * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list), - * .vm_start = VMALLOC_START, - * .vm_end = VMALLOC_END, - * }; - * - * However, vmalloc_head.vm_start is variable (typically, it is dependent on - * the amount of RAM found at boot time.) I would imagine that get_vm_area() - * would have to initialise this each time prior to calling vm_region_alloc(). - */ -struct arm_vm_region { - struct list_head vm_list; - unsigned long vm_start; - unsigned long vm_end; - struct page *vm_pages; - int vm_active; -}; +#include "vmregion.h" -static struct arm_vm_region consistent_head = { +static struct arm_vmregion_head consistent_head = { + .vm_lock = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock), .vm_list = LIST_HEAD_INIT(consistent_head.vm_list), .vm_start = CONSISTENT_BASE, .vm_end = CONSISTENT_END, }; -static struct arm_vm_region * -arm_vm_region_alloc(struct arm_vm_region *head, size_t size, gfp_t gfp) -{ - unsigned long addr = head->vm_start, end = head->vm_end - size; - unsigned long flags; - struct arm_vm_region *c, *new; - - new = kmalloc(sizeof(struct arm_vm_region), gfp); - if (!new) - goto out; - - spin_lock_irqsave(&consistent_lock, flags); - - list_for_each_entry(c, &head->vm_list, vm_list) { - if ((addr + size) < addr) - goto nospc; - if ((addr + size) <= c->vm_start) - goto found; - addr = c->vm_end; - if (addr > end) - goto nospc; - } - - found: - /* - * Insert this entry _before_ the one we found. - */ - list_add_tail(&new->vm_list, &c->vm_list); - new->vm_start = addr; - new->vm_end = addr + size; - new->vm_active = 1; - - spin_unlock_irqrestore(&consistent_lock, flags); - return new; - - nospc: - spin_unlock_irqrestore(&consistent_lock, flags); - kfree(new); - out: - return NULL; -} - -static struct arm_vm_region *arm_vm_region_find(struct arm_vm_region *head, unsigned long addr) -{ - struct arm_vm_region *c; - - list_for_each_entry(c, &head->vm_list, vm_list) { - if (c->vm_active && c->vm_start == addr) - goto out; - } - c = NULL; - out: - return c; -} - #ifdef CONFIG_HUGETLB_PAGE #error ARM Coherent DMA allocator does not (yet) support huge TLB #endif @@ -177,7 +87,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) { struct page *page; - struct arm_vm_region *c; + struct arm_vmregion *c; unsigned long order; u64 mask = get_coherent_dma_mask(dev); u64 limit; @@ -191,13 +101,9 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, if (!mask) goto no_page; - /* - * Sanity check the allocation size. - */ size = PAGE_ALIGN(size); limit = (mask + 1) & ~mask; - if ((limit && size >= limit) || - size >= (CONSISTENT_END - CONSISTENT_BASE)) { + if (limit && size >= limit) { printk(KERN_WARNING "coherent allocation too big " "(requested %#x mask %#llx)\n", size, mask); goto no_page; @@ -226,7 +132,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, /* * Allocate a virtual address in the consistent mapping region. */ - c = arm_vm_region_alloc(&consistent_head, size, + c = arm_vmregion_alloc(&consistent_head, size, gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); if (c) { pte_t *pte; @@ -349,15 +255,12 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma, { int ret = -ENXIO; #ifdef CONFIG_MMU - unsigned long flags, user_size, kern_size; - struct arm_vm_region *c; + unsigned long user_size, kern_size; + struct arm_vmregion *c; user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - spin_lock_irqsave(&consistent_lock, flags); - c = arm_vm_region_find(&consistent_head, (unsigned long)cpu_addr); - spin_unlock_irqrestore(&consistent_lock, flags); - + c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); if (c) { unsigned long off = vma->vm_pgoff; @@ -399,8 +302,8 @@ EXPORT_SYMBOL(dma_mmap_writecombine); #ifdef CONFIG_MMU void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) { - struct arm_vm_region *c; - unsigned long flags, addr; + struct arm_vmregion *c; + unsigned long addr; pte_t *ptep; int idx; u32 off; @@ -417,14 +320,10 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr size = PAGE_ALIGN(size); - spin_lock_irqsave(&consistent_lock, flags); - c = arm_vm_region_find(&consistent_head, (unsigned long)cpu_addr); + c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); if (!c) goto no_area; - c->vm_active = 0; - spin_unlock_irqrestore(&consistent_lock, flags); - if ((c->vm_end - c->vm_start) != size) { printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", __func__, c->vm_end - c->vm_start, size); @@ -470,15 +369,10 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr flush_tlb_kernel_range(c->vm_start, c->vm_end); - spin_lock_irqsave(&consistent_lock, flags); - list_del(&c->vm_list); - spin_unlock_irqrestore(&consistent_lock, flags); - - kfree(c); + arm_vmregion_free(&consistent_head, c); return; no_area: - spin_unlock_irqrestore(&consistent_lock, flags); printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", __func__, cpu_addr); dump_stack(); diff --git a/arch/arm/mm/vmregion.c b/arch/arm/mm/vmregion.c new file mode 100644 index 000000000000..19e09bdb1b8a --- /dev/null +++ b/arch/arm/mm/vmregion.c @@ -0,0 +1,131 @@ +#include +#include +#include + +#include "vmregion.h" + +/* + * VM region handling support. + * + * This should become something generic, handling VM region allocations for + * vmalloc and similar (ioremap, module space, etc). + * + * I envisage vmalloc()'s supporting vm_struct becoming: + * + * struct vm_struct { + * struct vmregion region; + * unsigned long flags; + * struct page **pages; + * unsigned int nr_pages; + * unsigned long phys_addr; + * }; + * + * get_vm_area() would then call vmregion_alloc with an appropriate + * struct vmregion head (eg): + * + * struct vmregion vmalloc_head = { + * .vm_list = LIST_HEAD_INIT(vmalloc_head.vm_list), + * .vm_start = VMALLOC_START, + * .vm_end = VMALLOC_END, + * }; + * + * However, vmalloc_head.vm_start is variable (typically, it is dependent on + * the amount of RAM found at boot time.) I would imagine that get_vm_area() + * would have to initialise this each time prior to calling vmregion_alloc(). + */ + +struct arm_vmregion * +arm_vmregion_alloc(struct arm_vmregion_head *head, size_t size, gfp_t gfp) +{ + unsigned long addr = head->vm_start, end = head->vm_end - size; + unsigned long flags; + struct arm_vmregion *c, *new; + + if (head->vm_end - head->vm_start < size) { + printk(KERN_WARNING "%s: allocation too big (requested %#x)\n", + __func__, size); + goto out; + } + + new = kmalloc(sizeof(struct arm_vmregion), gfp); + if (!new) + goto out; + + spin_lock_irqsave(&head->vm_lock, flags); + + list_for_each_entry(c, &head->vm_list, vm_list) { + if ((addr + size) < addr) + goto nospc; + if ((addr + size) <= c->vm_start) + goto found; + addr = c->vm_end; + if (addr > end) + goto nospc; + } + + found: + /* + * Insert this entry _before_ the one we found. + */ + list_add_tail(&new->vm_list, &c->vm_list); + new->vm_start = addr; + new->vm_end = addr + size; + new->vm_active = 1; + + spin_unlock_irqrestore(&head->vm_lock, flags); + return new; + + nospc: + spin_unlock_irqrestore(&head->vm_lock, flags); + kfree(new); + out: + return NULL; +} + +static struct arm_vmregion *__arm_vmregion_find(struct arm_vmregion_head *head, unsigned long addr) +{ + struct arm_vmregion *c; + + list_for_each_entry(c, &head->vm_list, vm_list) { + if (c->vm_active && c->vm_start == addr) + goto out; + } + c = NULL; + out: + return c; +} + +struct arm_vmregion *arm_vmregion_find(struct arm_vmregion_head *head, unsigned long addr) +{ + struct arm_vmregion *c; + unsigned long flags; + + spin_lock_irqsave(&head->vm_lock, flags); + c = __arm_vmregion_find(head, addr); + spin_unlock_irqrestore(&head->vm_lock, flags); + return c; +} + +struct arm_vmregion *arm_vmregion_find_remove(struct arm_vmregion_head *head, unsigned long addr) +{ + struct arm_vmregion *c; + unsigned long flags; + + spin_lock_irqsave(&head->vm_lock, flags); + c = __arm_vmregion_find(head, addr); + if (c) + c->vm_active = 0; + spin_unlock_irqrestore(&head->vm_lock, flags); + return c; +} + +void arm_vmregion_free(struct arm_vmregion_head *head, struct arm_vmregion *c) +{ + unsigned long flags; + + spin_lock_irqsave(&head->vm_lock, flags); + list_del(&c->vm_list); + spin_unlock_irqrestore(&head->vm_lock, flags); + + kfree(c); +} diff --git a/arch/arm/mm/vmregion.h b/arch/arm/mm/vmregion.h new file mode 100644 index 000000000000..6b2cdbdf3a85 --- /dev/null +++ b/arch/arm/mm/vmregion.h @@ -0,0 +1,29 @@ +#ifndef VMREGION_H +#define VMREGION_H + +#include +#include + +struct page; + +struct arm_vmregion_head { + spinlock_t vm_lock; + struct list_head vm_list; + unsigned long vm_start; + unsigned long vm_end; +}; + +struct arm_vmregion { + struct list_head vm_list; + unsigned long vm_start; + unsigned long vm_end; + struct page *vm_pages; + int vm_active; +}; + +struct arm_vmregion *arm_vmregion_alloc(struct arm_vmregion_head *, size_t, gfp_t); +struct arm_vmregion *arm_vmregion_find(struct arm_vmregion_head *, unsigned long); +struct arm_vmregion *arm_vmregion_find_remove(struct arm_vmregion_head *, unsigned long); +void arm_vmregion_free(struct arm_vmregion_head *, struct arm_vmregion *); + +#endif -- cgit v1.2.2 From 7a9a32a9533fa01de911e1d056142ddd27360782 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 15:31:07 +0000 Subject: ARM: dma-mapping: functions to allocate/free a coherent buffer Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 110 +++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 44 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index c54f1acf92c8..dab2d7f04adf 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -63,6 +63,68 @@ static u64 get_coherent_dma_mask(struct device *dev) return mask; } +/* + * Allocate a DMA buffer for 'dev' of size 'size' using the + * specified gfp mask. Note that 'size' must be page aligned. + */ +static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) +{ + unsigned long order = get_order(size); + struct page *page, *p, *e; + void *ptr; + u64 mask = get_coherent_dma_mask(dev); + +#ifdef CONFIG_DMA_API_DEBUG + u64 limit = (mask + 1) & ~mask; + if (limit && size >= limit) { + dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", + size, mask); + return NULL; + } +#endif + + if (!mask) + return NULL; + + if (mask < 0xffffffffULL) + gfp |= GFP_DMA; + + page = alloc_pages(gfp, order); + if (!page) + return NULL; + + /* + * Now split the huge page and free the excess pages + */ + split_page(page, order); + for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) + __free_page(p); + + /* + * Ensure that the allocated pages are zeroed, and that any data + * lurking in the kernel direct-mapped region is invalidated. + */ + ptr = page_address(page); + memset(ptr, 0, size); + dmac_flush_range(ptr, ptr + size); + outer_flush_range(__pa(ptr), __pa(ptr) + size); + + return page; +} + +/* + * Free a DMA buffer. 'size' must be page aligned. + */ +static void __dma_free_buffer(struct page *page, size_t size) +{ + struct page *e = page + (size >> PAGE_SHIFT); + + while (page < e) { + __free_page(page); + page++; + } +} + #ifdef CONFIG_MMU /* * These are the page tables (2MB each) covering uncached, DMA consistent allocations @@ -88,9 +150,6 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, { struct page *page; struct arm_vmregion *c; - unsigned long order; - u64 mask = get_coherent_dma_mask(dev); - u64 limit; if (!consistent_pte[0]) { printk(KERN_ERR "%s: not initialised\n", __func__); @@ -98,37 +157,12 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, return NULL; } - if (!mask) - goto no_page; - size = PAGE_ALIGN(size); - limit = (mask + 1) & ~mask; - if (limit && size >= limit) { - printk(KERN_WARNING "coherent allocation too big " - "(requested %#x mask %#llx)\n", size, mask); - goto no_page; - } - - order = get_order(size); - - if (mask < 0xffffffffULL) - gfp |= GFP_DMA; - page = alloc_pages(gfp, order); + page = __dma_alloc_buffer(dev, size, gfp); if (!page) goto no_page; - /* - * Invalidate any data that might be lurking in the - * kernel direct-mapped region for device DMA. - */ - { - void *ptr = page_address(page); - memset(ptr, 0, size); - dmac_flush_range(ptr, ptr + size); - outer_flush_range(__pa(ptr), __pa(ptr) + size); - } - /* * Allocate a virtual address in the consistent mapping region. */ @@ -136,15 +170,12 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); if (c) { pte_t *pte; - struct page *end = page + (1 << order); int idx = CONSISTENT_PTE_INDEX(c->vm_start); u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); pte = consistent_pte[idx] + off; c->vm_pages = page; - split_page(page, order); - /* * Set the "dma handle" */ @@ -167,19 +198,11 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, } } while (size -= PAGE_SIZE); - /* - * Free the otherwise unused pages. - */ - while (page < end) { - __free_page(page); - page++; - } - return (void *)c->vm_start; } if (page) - __free_pages(page, order); + __dma_free_buffer(page, size); no_page: *handle = ~0; return NULL; @@ -357,12 +380,9 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr * x86 does not mark the pages reserved... */ ClearPageReserved(page); - - __free_page(page); continue; } } - printk(KERN_CRIT "%s: bad page in kernel page table\n", __func__); } while (size -= PAGE_SIZE); @@ -370,6 +390,8 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr flush_tlb_kernel_range(c->vm_start, c->vm_end); arm_vmregion_free(&consistent_head, c); + + __dma_free_buffer(dma_to_page(dev, handle), size); return; no_area: -- cgit v1.2.2 From 3e82d012e9281a0b6388ff2356e8396b9d781e1c Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 15:38:12 +0000 Subject: ARM: dma-mapping: fix coherent arch dma_alloc_coherent() The coherent architecture dma_alloc_coherent was using kmalloc/kfree to manage the memory. dma_alloc_coherent() is expected to work with a granularity of a page, so this is wrong. Fix it by using the helper functions now provided. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index dab2d7f04adf..a67130873431 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -246,14 +246,16 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gf return memory; if (arch_is_coherent()) { - void *virt; + struct page *page; - virt = kmalloc(size, gfp); - if (!virt) + page = __dma_alloc_buffer(dev, PAGE_ALIGN(size), gfp); + if (!page) { + *handle = ~0; return NULL; - *handle = virt_to_dma(dev, virt); + } - return virt; + *handle = page_to_dma(dev, page); + return page_address(page); } return __dma_alloc(dev, size, handle, gfp, @@ -336,13 +338,13 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) return; + size = PAGE_ALIGN(size); + if (arch_is_coherent()) { - kfree(cpu_addr); + __dma_free_buffer(dma_to_page(dev, handle), size); return; } - size = PAGE_ALIGN(size); - c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); if (!c) goto no_area; -- cgit v1.2.2 From 04da56943b416dd9fe7058abf8d5b9153164b3e9 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 15:54:45 +0000 Subject: ARM: dma-mapping: fix nommu dma_alloc_coherent() The nommu version of dma_alloc_coherent was using kmalloc/kfree to manage the memory. dma_alloc_coherent() is expected to work with a granularity of a page, so this is wrong. Fix it by using the helper functions now provided. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index a67130873431..62b4240b34b3 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -212,24 +212,17 @@ static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) { - void *virt; - u64 mask = get_coherent_dma_mask(dev); - - if (!mask) - goto error; + struct page *page; - if (mask < 0xffffffffULL) - gfp |= GFP_DMA; - virt = kmalloc(size, gfp); - if (!virt) - goto error; + *handle = ~0; + size = PAGE_ALIGN(size); - *handle = virt_to_dma(dev, virt); - return virt; + page = __dma_alloc_buffer(dev, size, gfp); + if (!page) + return NULL; -error: - *handle = ~0; - return NULL; + *handle = page_to_dma(dev, page); + return page_address(page); } #endif /* CONFIG_MMU */ @@ -406,7 +399,7 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr { if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) return; - kfree(cpu_addr); + __dma_free_buffer(dma_to_page(dev, handle), PAGE_ALIGN(size)); } #endif /* CONFIG_MMU */ EXPORT_SYMBOL(dma_free_coherent); -- cgit v1.2.2 From 695ae0af5a52df09dffcc2ce2d625d56ef36ce14 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 16:31:39 +0000 Subject: ARM: dma-mapping: factor dma_free_coherent() common code We effectively have three implementations of dma_free_coherent() mixed up in the code; the incoherent MMU, coherent MMU and noMMU versions. The coherent MMU and noMMU versions are actually functionally identical. The incoherent MMU version is almost the same, but with the additional step of unmapping the secondary mapping. Separate out this additional step into __dma_free_remap() and simplify the resulting dma_free_coherent() code. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 141 ++++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 73 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 62b4240b34b3..6b24e5ed85c2 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -207,7 +207,70 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, *handle = ~0; return NULL; } + +static void __dma_free_remap(void *cpu_addr, size_t size) +{ + struct arm_vmregion *c; + unsigned long addr; + pte_t *ptep; + int idx; + u32 off; + + c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); + if (!c) { + printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", + __func__, cpu_addr); + dump_stack(); + return; + } + + if ((c->vm_end - c->vm_start) != size) { + printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", + __func__, c->vm_end - c->vm_start, size); + dump_stack(); + size = c->vm_end - c->vm_start; + } + + idx = CONSISTENT_PTE_INDEX(c->vm_start); + off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); + ptep = consistent_pte[idx] + off; + addr = c->vm_start; + do { + pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); + unsigned long pfn; + + ptep++; + addr += PAGE_SIZE; + off++; + if (off >= PTRS_PER_PTE) { + off = 0; + ptep = consistent_pte[++idx]; + } + + if (!pte_none(pte) && pte_present(pte)) { + pfn = pte_pfn(pte); + + if (pfn_valid(pfn)) { + struct page *page = pfn_to_page(pfn); + + /* + * x86 does not mark the pages reserved... + */ + ClearPageReserved(page); + continue; + } + } + printk(KERN_CRIT "%s: bad page in kernel page table\n", + __func__); + } while (size -= PAGE_SIZE); + + flush_tlb_kernel_range(c->vm_start, c->vm_end); + + arm_vmregion_free(&consistent_head, c); +} + #else /* !CONFIG_MMU */ + static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) @@ -224,6 +287,9 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, *handle = page_to_dma(dev, page); return page_address(page); } + +#define __dma_free_remap(addr, size) do { } while (0) + #endif /* CONFIG_MMU */ /* @@ -317,15 +383,8 @@ EXPORT_SYMBOL(dma_mmap_writecombine); * free a page as defined by the above mapping. * Must not be called with IRQs disabled. */ -#ifdef CONFIG_MMU void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) { - struct arm_vmregion *c; - unsigned long addr; - pte_t *ptep; - int idx; - u32 off; - WARN_ON(irqs_disabled()); if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) @@ -333,75 +392,11 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr size = PAGE_ALIGN(size); - if (arch_is_coherent()) { - __dma_free_buffer(dma_to_page(dev, handle), size); - return; - } - - c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); - if (!c) - goto no_area; - - if ((c->vm_end - c->vm_start) != size) { - printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n", - __func__, c->vm_end - c->vm_start, size); - dump_stack(); - size = c->vm_end - c->vm_start; - } - - idx = CONSISTENT_PTE_INDEX(c->vm_start); - off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); - ptep = consistent_pte[idx] + off; - addr = c->vm_start; - do { - pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); - unsigned long pfn; - - ptep++; - addr += PAGE_SIZE; - off++; - if (off >= PTRS_PER_PTE) { - off = 0; - ptep = consistent_pte[++idx]; - } - - if (!pte_none(pte) && pte_present(pte)) { - pfn = pte_pfn(pte); - - if (pfn_valid(pfn)) { - struct page *page = pfn_to_page(pfn); - - /* - * x86 does not mark the pages reserved... - */ - ClearPageReserved(page); - continue; - } - } - printk(KERN_CRIT "%s: bad page in kernel page table\n", - __func__); - } while (size -= PAGE_SIZE); - - flush_tlb_kernel_range(c->vm_start, c->vm_end); - - arm_vmregion_free(&consistent_head, c); + if (!arch_is_coherent()) + __dma_free_remap(cpu_addr, size); __dma_free_buffer(dma_to_page(dev, handle), size); - return; - - no_area: - printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", - __func__, cpu_addr); - dump_stack(); } -#else /* !CONFIG_MMU */ -void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) -{ - if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) - return; - __dma_free_buffer(dma_to_page(dev, handle), PAGE_ALIGN(size)); -} -#endif /* CONFIG_MMU */ EXPORT_SYMBOL(dma_free_coherent); /* -- cgit v1.2.2 From 88c58f3b92bc7c26439802c300d39b6377739d81 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 16:46:02 +0000 Subject: ARM: dma-mapping: move consistent_init into CONFIG_MMU section No point wrapping the contents of this function with #ifdef CONFIG_MMU when we can place it and the core_initcall() entirely within the existing conditional block. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 78 +++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 40 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 6b24e5ed85c2..19357f76ce89 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -144,6 +144,44 @@ static struct arm_vmregion_head consistent_head = { #error ARM Coherent DMA allocator does not (yet) support huge TLB #endif +/* + * Initialise the consistent memory allocation. + */ +static int __init consistent_init(void) +{ + int ret = 0; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int i = 0; + u32 base = CONSISTENT_BASE; + + do { + pgd = pgd_offset(&init_mm, base); + pmd = pmd_alloc(&init_mm, pgd, base); + if (!pmd) { + printk(KERN_ERR "%s: no pmd tables\n", __func__); + ret = -ENOMEM; + break; + } + WARN_ON(!pmd_none(*pmd)); + + pte = pte_alloc_kernel(pmd, base); + if (!pte) { + printk(KERN_ERR "%s: no pte tables\n", __func__); + ret = -ENOMEM; + break; + } + + consistent_pte[i++] = pte; + base += (1 << PGDIR_SHIFT); + } while (base < CONSISTENT_END); + + return ret; +} + +core_initcall(consistent_init); + static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) @@ -399,46 +437,6 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr } EXPORT_SYMBOL(dma_free_coherent); -/* - * Initialise the consistent memory allocation. - */ -static int __init consistent_init(void) -{ - int ret = 0; -#ifdef CONFIG_MMU - pgd_t *pgd; - pmd_t *pmd; - pte_t *pte; - int i = 0; - u32 base = CONSISTENT_BASE; - - do { - pgd = pgd_offset(&init_mm, base); - pmd = pmd_alloc(&init_mm, pgd, base); - if (!pmd) { - printk(KERN_ERR "%s: no pmd tables\n", __func__); - ret = -ENOMEM; - break; - } - WARN_ON(!pmd_none(*pmd)); - - pte = pte_alloc_kernel(pmd, base); - if (!pte) { - printk(KERN_ERR "%s: no pte tables\n", __func__); - ret = -ENOMEM; - break; - } - - consistent_pte[i++] = pte; - base += (1 << PGDIR_SHIFT); - } while (base < CONSISTENT_END); -#endif /* !CONFIG_MMU */ - - return ret; -} - -core_initcall(consistent_init); - /* * Make an area consistent for devices. * Note: Drivers should NOT use this function directly, as it will break -- cgit v1.2.2 From ebd7a845fa4332da3ebcbe8cf1b09bb43413420e Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 20:58:31 +0000 Subject: ARM: dma-mapping: clean up coherent arch dma allocation IXP23xx added support for dma_alloc_coherent() for DMA arches with an exception in dma_alloc_coherent(). This is a subset of what goes on in __dma_alloc(), and there is no reason why dma_alloc_writecombine() should not be given the same treatment (except, maybe, that IXP23xx doesn't use it.) We can better deal with this by moving the arch_is_coherent() test inside __dma_alloc() and killing the code duplication. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 19357f76ce89..3a8e527c774c 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -189,18 +189,24 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct page *page; struct arm_vmregion *c; - if (!consistent_pte[0]) { - printk(KERN_ERR "%s: not initialised\n", __func__); - dump_stack(); - return NULL; - } - size = PAGE_ALIGN(size); page = __dma_alloc_buffer(dev, size, gfp); if (!page) goto no_page; + if (arch_is_coherent()) { + *handle = page_to_dma(dev, page); + return page_address(page); + } + + if (!consistent_pte[0]) { + printk(KERN_ERR "%s: not initialised\n", __func__); + dump_stack(); + __dma_free_buffer(page, size); + return NULL; + } + /* * Allocate a virtual address in the consistent mapping region. */ @@ -342,19 +348,6 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gf if (dma_alloc_from_coherent(dev, size, handle, &memory)) return memory; - if (arch_is_coherent()) { - struct page *page; - - page = __dma_alloc_buffer(dev, PAGE_ALIGN(size), gfp); - if (!page) { - *handle = ~0; - return NULL; - } - - *handle = page_to_dma(dev, page); - return page_address(page); - } - return __dma_alloc(dev, size, handle, gfp, pgprot_noncached(pgprot_kernel)); } -- cgit v1.2.2 From 31ebf94435f74294523683867fe0b89000e61521 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 21:12:17 +0000 Subject: ARM: dma-mapping: Factor out noMMU dma buffer allocation code This entirely separates the DMA coherent buffer remapping code from the allocation code, and gets rid of the duplicate copy in the !MMU section. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 3a8e527c774c..707d81247630 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -183,27 +183,13 @@ static int __init consistent_init(void) core_initcall(consistent_init); static void * -__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, - pgprot_t prot) +__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) { - struct page *page; struct arm_vmregion *c; - size = PAGE_ALIGN(size); - - page = __dma_alloc_buffer(dev, size, gfp); - if (!page) - goto no_page; - - if (arch_is_coherent()) { - *handle = page_to_dma(dev, page); - return page_address(page); - } - if (!consistent_pte[0]) { printk(KERN_ERR "%s: not initialised\n", __func__); dump_stack(); - __dma_free_buffer(page, size); return NULL; } @@ -220,11 +206,6 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pte = consistent_pte[idx] + off; c->vm_pages = page; - /* - * Set the "dma handle" - */ - *handle = page_to_dma(dev, page); - do { BUG_ON(!pte_none(*pte)); @@ -244,11 +225,6 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, return (void *)c->vm_start; } - - if (page) - __dma_free_buffer(page, size); - no_page: - *handle = ~0; return NULL; } @@ -315,11 +291,17 @@ static void __dma_free_remap(void *cpu_addr, size_t size) #else /* !CONFIG_MMU */ +#define __dma_alloc_remap(page, size, gfp, prot) page_address(page) +#define __dma_free_remap(addr, size) do { } while (0) + +#endif /* CONFIG_MMU */ + static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, pgprot_t prot) { struct page *page; + void *addr; *handle = ~0; size = PAGE_ALIGN(size); @@ -328,13 +310,16 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, if (!page) return NULL; - *handle = page_to_dma(dev, page); - return page_address(page); -} + if (!arch_is_coherent()) + addr = __dma_alloc_remap(page, size, gfp, prot); + else + addr = page_address(page); -#define __dma_free_remap(addr, size) do { } while (0) + if (addr) + *handle = page_to_dma(dev, page); -#endif /* CONFIG_MMU */ + return addr; +} /* * Allocate DMA-coherent memory space and return both the kernel remapped -- cgit v1.2.2 From acaac256b3a14a09ab278409a72d119f2d75b02b Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 20 Nov 2009 18:19:52 +0000 Subject: ARM: dma-mapping: get rid of setting/clearing the reserved page bit It's unnecessary; x86 doesn't do it, and ALSA doesn't require it anymore. Signed-off-by: Russell King Acked-by: Greg Ungerer --- arch/arm/mm/dma-mapping.c | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 707d81247630..6fac793329c6 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -209,10 +209,6 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) do { BUG_ON(!pte_none(*pte)); - /* - * x86 does not mark the pages reserved... - */ - SetPageReserved(page); set_pte_ext(pte, mk_pte(page, prot), 0); page++; pte++; @@ -257,7 +253,6 @@ static void __dma_free_remap(void *cpu_addr, size_t size) addr = c->vm_start; do { pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); - unsigned long pfn; ptep++; addr += PAGE_SIZE; @@ -267,21 +262,9 @@ static void __dma_free_remap(void *cpu_addr, size_t size) ptep = consistent_pte[++idx]; } - if (!pte_none(pte) && pte_present(pte)) { - pfn = pte_pfn(pte); - - if (pfn_valid(pfn)) { - struct page *page = pfn_to_page(pfn); - - /* - * x86 does not mark the pages reserved... - */ - ClearPageReserved(page); - continue; - } - } - printk(KERN_CRIT "%s: bad page in kernel page table\n", - __func__); + if (pte_none(pte) || !pte_present(pte)) + printk(KERN_CRIT "%s: bad page in kernel page table\n", + __func__); } while (size -= PAGE_SIZE); flush_tlb_kernel_range(c->vm_start, c->vm_end); -- cgit v1.2.2 From 26a26d329688ab018e068b412b03d43d7c299f0a Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 20 Nov 2009 21:06:43 +0000 Subject: ARM: dma-mapping: switch ARMv7 DMA mappings to retain 'memory' attribute On ARMv7, it is invalid to map the same physical address multiple times with different memory types. Since system RAM is already mapped as 'memory', subsequent remapping of it must retain this attribute. However, DMA memory maps it as "strongly ordered". Fix this by introducing 'pgprot_dmacoherent()' which provides the necessary page table bits for DMA mappings. Signed-off-by: Russell King Acked-by: Greg Ungerer Reviewed-by: Catalin Marinas --- arch/arm/mm/dma-mapping.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 6fac793329c6..26325cb5d368 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -317,7 +317,7 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gf return memory; return __dma_alloc(dev, size, handle, gfp, - pgprot_noncached(pgprot_kernel)); + pgprot_dmacoherent(pgprot_kernel)); } EXPORT_SYMBOL(dma_alloc_coherent); @@ -365,7 +365,7 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma, int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); return dma_mmap(dev, vma, cpu_addr, dma_addr, size); } EXPORT_SYMBOL(dma_mmap_coherent); -- cgit v1.2.2 From edabd38e1a017e922e3e3b485ee3ddb4df433aa4 Mon Sep 17 00:00:00 2001 From: Saeed Bishara Date: Thu, 6 Aug 2009 15:12:43 +0300 Subject: ARM: add base support for Marvell Dove SoC The Marvell Dove (88AP510) is a high-performance, highly integrated, low power SoC with high-end ARM-compatible processor (known as PJ4), graphics processing unit, high-definition video decoding acceleration hardware, and a broad range of peripherals. Signed-off-by: Lennert Buytenhek Signed-off-by: Saeed Bishara Signed-off-by: Nicolas Pitre --- arch/arm/mm/Kconfig | 2 +- arch/arm/mm/proc-v6.S | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 9264d814cd7a..1549863d7b54 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -388,7 +388,7 @@ config CPU_FEROCEON_OLD_ID # ARMv6 config CPU_V6 - bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX + bool "Support ARM V6 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX || ARCH_DOVE select CPU_32v6 select CPU_ABRT_EV6 select CPU_PABRT_V6 diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 70f75d2e3ead..5485c821101c 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -130,9 +130,16 @@ ENTRY(cpu_v6_set_pte_ext) - + .type cpu_v6_name, #object cpu_v6_name: .asciz "ARMv6-compatible processor" + .size cpu_v6_name, . - cpu_v6_name + + .type cpu_pj4_name, #object +cpu_pj4_name: + .asciz "Marvell PJ4 processor" + .size cpu_pj4_name, . - cpu_pj4_name + .align __INIT @@ -241,3 +248,27 @@ __v6_proc_info: .long v6_user_fns .long v6_cache_fns .size __v6_proc_info, . - __v6_proc_info + + .type __pj4_v6_proc_info, #object +__pj4_v6_proc_info: + .long 0x560f5810 + .long 0xff0ffff0 + .long PMD_TYPE_SECT | \ + PMD_SECT_BUFFERABLE | \ + PMD_SECT_CACHEABLE | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ + .long PMD_TYPE_SECT | \ + PMD_SECT_XN | \ + PMD_SECT_AP_WRITE | \ + PMD_SECT_AP_READ + b __v6_setup + .long cpu_arch_name + .long cpu_elf_name + .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP + .long cpu_pj4_name + .long v6_processor_functions + .long v6wbi_tlb_fns + .long v6_user_fns + .long v6_cache_fns + .size __pj4_v6_proc_info, . - __pj4_v6_proc_info -- cgit v1.2.2 From 573a652fb0da50a1ff3fca2c67afd81138fd06d2 Mon Sep 17 00:00:00 2001 From: Lennert Buytenhek Date: Tue, 24 Nov 2009 19:33:52 +0200 Subject: ARM: Add Tauros2 L2 cache controller support Support for the Tauros2 L2 cache controller as used with the PJ1 and PJ4 CPUs. Signed-off-by: Lennert Buytenhek Signed-off-by: Saeed Bishara Signed-off-by: Nicolas Pitre --- arch/arm/mm/Kconfig | 9 ++ arch/arm/mm/Makefile | 2 +- arch/arm/mm/cache-tauros2.c | 263 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 arch/arm/mm/cache-tauros2.c (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 1549863d7b54..4958ef2c6254 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -764,6 +764,15 @@ config CACHE_L2X0 help This option enables the L2x0 PrimeCell. +config CACHE_TAUROS2 + bool "Enable the Tauros2 L2 cache controller" + depends on ARCH_DOVE + default y + select OUTER_CACHE + help + This option enables the Tauros2 L2 cache controller (as + found on PJ1/PJ4). + config CACHE_XSC3L2 bool "Enable the L2 cache on XScale3" depends on CPU_XSC3 diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 055cb2aa8134..06bcf2e73858 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -87,4 +87,4 @@ obj-$(CONFIG_CPU_V7) += proc-v7.o obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o - +obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o diff --git a/arch/arm/mm/cache-tauros2.c b/arch/arm/mm/cache-tauros2.c new file mode 100644 index 000000000000..50868651890f --- /dev/null +++ b/arch/arm/mm/cache-tauros2.c @@ -0,0 +1,263 @@ +/* + * arch/arm/mm/cache-tauros2.c - Tauros2 L2 cache controller support + * + * Copyright (C) 2008 Marvell Semiconductor + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + * + * References: + * - PJ1 CPU Core Datasheet, + * Document ID MV-S104837-01, Rev 0.7, January 24 2008. + * - PJ4 CPU Core Datasheet, + * Document ID MV-S105190-00, Rev 0.7, March 14 2008. + */ + +#include +#include +#include + + +/* + * When Tauros2 is used on a CPU that supports the v7 hierarchical + * cache operations, the cache handling code in proc-v7.S takes care + * of everything, including handling DMA coherency. + * + * So, we only need to register outer cache operations here if we're + * being used on a pre-v7 CPU, and we only need to build support for + * outer cache operations into the kernel image if the kernel has been + * configured to support a pre-v7 CPU. + */ +#if __LINUX_ARM_ARCH__ < 7 +/* + * Low-level cache maintenance operations. + */ +static inline void tauros2_clean_pa(unsigned long addr) +{ + __asm__("mcr p15, 1, %0, c7, c11, 3" : : "r" (addr)); +} + +static inline void tauros2_clean_inv_pa(unsigned long addr) +{ + __asm__("mcr p15, 1, %0, c7, c15, 3" : : "r" (addr)); +} + +static inline void tauros2_inv_pa(unsigned long addr) +{ + __asm__("mcr p15, 1, %0, c7, c7, 3" : : "r" (addr)); +} + + +/* + * Linux primitives. + * + * Note that the end addresses passed to Linux primitives are + * noninclusive. + */ +#define CACHE_LINE_SIZE 32 + +static void tauros2_inv_range(unsigned long start, unsigned long end) +{ + /* + * Clean and invalidate partial first cache line. + */ + if (start & (CACHE_LINE_SIZE - 1)) { + tauros2_clean_inv_pa(start & ~(CACHE_LINE_SIZE - 1)); + start = (start | (CACHE_LINE_SIZE - 1)) + 1; + } + + /* + * Clean and invalidate partial last cache line. + */ + if (end & (CACHE_LINE_SIZE - 1)) { + tauros2_clean_inv_pa(end & ~(CACHE_LINE_SIZE - 1)); + end &= ~(CACHE_LINE_SIZE - 1); + } + + /* + * Invalidate all full cache lines between 'start' and 'end'. + */ + while (start < end) { + tauros2_inv_pa(start); + start += CACHE_LINE_SIZE; + } + + dsb(); +} + +static void tauros2_clean_range(unsigned long start, unsigned long end) +{ + start &= ~(CACHE_LINE_SIZE - 1); + while (start < end) { + tauros2_clean_pa(start); + start += CACHE_LINE_SIZE; + } + + dsb(); +} + +static void tauros2_flush_range(unsigned long start, unsigned long end) +{ + start &= ~(CACHE_LINE_SIZE - 1); + while (start < end) { + tauros2_clean_inv_pa(start); + start += CACHE_LINE_SIZE; + } + + dsb(); +} +#endif + +static inline u32 __init read_extra_features(void) +{ + u32 u; + + __asm__("mrc p15, 1, %0, c15, c1, 0" : "=r" (u)); + + return u; +} + +static inline void __init write_extra_features(u32 u) +{ + __asm__("mcr p15, 1, %0, c15, c1, 0" : : "r" (u)); +} + +static void __init disable_l2_prefetch(void) +{ + u32 u; + + /* + * Read the CPU Extra Features register and verify that the + * Disable L2 Prefetch bit is set. + */ + u = read_extra_features(); + if (!(u & 0x01000000)) { + printk(KERN_INFO "Tauros2: Disabling L2 prefetch.\n"); + write_extra_features(u | 0x01000000); + } +} + +static inline int __init cpuid_scheme(void) +{ + extern int processor_id; + + return !!((processor_id & 0x000f0000) == 0x000f0000); +} + +static inline u32 __init read_mmfr3(void) +{ + u32 mmfr3; + + __asm__("mrc p15, 0, %0, c0, c1, 7\n" : "=r" (mmfr3)); + + return mmfr3; +} + +static inline u32 __init read_actlr(void) +{ + u32 actlr; + + __asm__("mrc p15, 0, %0, c1, c0, 1\n" : "=r" (actlr)); + + return actlr; +} + +static inline void __init write_actlr(u32 actlr) +{ + __asm__("mcr p15, 0, %0, c1, c0, 1\n" : : "r" (actlr)); +} + +void __init tauros2_init(void) +{ + extern int processor_id; + char *mode; + + disable_l2_prefetch(); + +#ifdef CONFIG_CPU_32v5 + if ((processor_id & 0xff0f0000) == 0x56050000) { + u32 feat; + + /* + * v5 CPUs with Tauros2 have the L2 cache enable bit + * located in the CPU Extra Features register. + */ + feat = read_extra_features(); + if (!(feat & 0x00400000)) { + printk(KERN_INFO "Tauros2: Enabling L2 cache.\n"); + write_extra_features(feat | 0x00400000); + } + + mode = "ARMv5"; + outer_cache.inv_range = tauros2_inv_range; + outer_cache.clean_range = tauros2_clean_range; + outer_cache.flush_range = tauros2_flush_range; + } +#endif + +#ifdef CONFIG_CPU_32v6 + /* + * Check whether this CPU lacks support for the v7 hierarchical + * cache ops. (PJ4 is in its v6 personality mode if the MMFR3 + * register indicates no support for the v7 hierarchical cache + * ops.) + */ + if (cpuid_scheme() && (read_mmfr3() & 0xf) == 0) { + /* + * When Tauros2 is used in an ARMv6 system, the L2 + * enable bit is in the ARMv6 ARM-mandated position + * (bit [26] of the System Control Register). + */ + if (!(get_cr() & 0x04000000)) { + printk(KERN_INFO "Tauros2: Enabling L2 cache.\n"); + adjust_cr(0x04000000, 0x04000000); + } + + mode = "ARMv6"; + outer_cache.inv_range = tauros2_inv_range; + outer_cache.clean_range = tauros2_clean_range; + outer_cache.flush_range = tauros2_flush_range; + } +#endif + +#ifdef CONFIG_CPU_32v7 + /* + * Check whether this CPU has support for the v7 hierarchical + * cache ops. (PJ4 is in its v7 personality mode if the MMFR3 + * register indicates support for the v7 hierarchical cache + * ops.) + * + * (Although strictly speaking there may exist CPUs that + * implement the v7 cache ops but are only ARMv6 CPUs (due to + * not complying with all of the other ARMv7 requirements), + * there are no real-life examples of Tauros2 being used on + * such CPUs as of yet.) + */ + if (cpuid_scheme() && (read_mmfr3() & 0xf) == 1) { + u32 actlr; + + /* + * When Tauros2 is used in an ARMv7 system, the L2 + * enable bit is located in the Auxiliary System Control + * Register (which is the only register allowed by the + * ARMv7 spec to contain fine-grained cache control bits). + */ + actlr = read_actlr(); + if (!(actlr & 0x00000002)) { + printk(KERN_INFO "Tauros2: Enabling L2 cache.\n"); + write_actlr(actlr | 0x00000002); + } + + mode = "ARMv7"; + } +#endif + + if (mode == NULL) { + printk(KERN_CRIT "Tauros2: Unable to detect CPU mode.\n"); + return; + } + + printk(KERN_INFO "Tauros2: L2 cache support initialised " + "in %s mode.\n", mode); +} -- cgit v1.2.2 From d0a92fd3b84bf707f6b32f31d0f09d2b7bb1ad67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Cech?= Date: Fri, 11 Sep 2009 13:57:02 +0200 Subject: [ARM] pxa/treo: add Palm Centro 685 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Čech Signed-off-by: Eric Miao --- arch/arm/mm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ea67be0223ac..3e8556b16a9f 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -881,7 +881,7 @@ void __init reserve_node_zero(pg_data_t *pgdat) BOOTMEM_EXCLUSIVE); } - if (machine_is_treo680()) { + if (machine_is_treo680() || machine_is_centro()) { reserve_bootmem_node(pgdat, 0xa0000000, 0x1000, BOOTMEM_EXCLUSIVE); reserve_bootmem_node(pgdat, 0xa2000000, 0x1000, -- cgit v1.2.2 From 2f0b192633f1fbf253b21c90938733491549edae Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 10:40:02 +0000 Subject: ARM: Avoid duplicated implementation for VIVT cache flushing We had two copies of the wrapper code for VIVT cache flushing - one in asm/cacheflush.h and one in arch/arm/mm/flush.c. Reduce this down to one common copy. Signed-off-by: Russell King --- arch/arm/mm/flush.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 7f294f307c83..a480f161a4bb 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -41,8 +41,7 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) void flush_cache_mm(struct mm_struct *mm) { if (cache_is_vivt()) { - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) - __cpuc_flush_user_all(); + vivt_flush_cache_mm(mm); return; } @@ -59,9 +58,7 @@ void flush_cache_mm(struct mm_struct *mm) void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (cache_is_vivt()) { - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) - __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end), - vma->vm_flags); + vivt_flush_cache_range(vma, start, end); return; } @@ -78,10 +75,7 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) { if (cache_is_vivt()) { - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { - unsigned long addr = user_addr & PAGE_MASK; - __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); - } + vivt_flush_cache_page(vma, user_addr, pfn); return; } @@ -94,10 +88,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, unsigned long len, int write) { if (cache_is_vivt()) { - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { - unsigned long addr = (unsigned long)kaddr; - __cpuc_coherent_kern_range(addr, addr + len); - } + vivt_flush_ptrace_access(vma, page, uaddr, kaddr, len, write); return; } -- cgit v1.2.2 From b7dc0b2cfc6e9bc7270915c642a8a8e999b6095e Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 11:25:50 +0000 Subject: ARM: Avoid evaluating page_address() multiple times page_address() is a function call rather than a macro, and so: if (page_address(page)) do_something(page_address(page)); results in two calls to this function. This is unnecessary; remove the duplication. Signed-off-by: Russell King --- arch/arm/mm/flush.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index a480f161a4bb..43474d8752a6 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -111,6 +111,8 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, void __flush_dcache_page(struct address_space *mapping, struct page *page) { + void *addr = page_address(page); + /* * Writeback any data associated with the kernel mapping of this * page. This ensures that data in the physical page is mutually @@ -121,9 +123,9 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) * kmap_atomic() doesn't set the page virtual address, and * kunmap_atomic() takes care of cache flushing already. */ - if (page_address(page)) + if (addr) #endif - __cpuc_flush_dcache_page(page_address(page)); + __cpuc_flush_dcache_page(addr); /* * If this is a page cache page, and we have an aliasing VIPT cache, -- cgit v1.2.2 From 421fe93cc4b06b2f5e875cbe0f692800d4862ee5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 10:23:04 +0000 Subject: ARM: ZERO_PAGE: Avoid flush_dcache_page() for zero page The zero page is read-only, and has its cache state cleared during boot. No further maintanence for this page is required. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 7 +++++++ arch/arm/mm/flush.c | 11 ++++++++++- arch/arm/mm/mmu.c | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index d0d17b6a3703..4fbc7de8b4ac 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -151,7 +151,14 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) if (!pfn_valid(pfn)) return; + /* + * The zero page is never written to, so never has any dirty + * cache lines, and therefore never needs to be flushed. + */ page = pfn_to_page(pfn); + if (page == ZERO_PAGE(0)) + return; + mapping = page_mapping(page); #ifndef CONFIG_SMP if (test_and_clear_bit(PG_dcache_dirty, &page->flags)) diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 43474d8752a6..82f4b06bf6b4 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -189,7 +189,16 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p */ void flush_dcache_page(struct page *page) { - struct address_space *mapping = page_mapping(page); + struct address_space *mapping; + + /* + * The zero page is never written to, so never has any dirty + * cache lines, and therefore never needs to be flushed. + */ + if (page == ZERO_PAGE(0)) + return; + + mapping = page_mapping(page); #ifndef CONFIG_SMP if (!PageHighMem(page) && mapping && !mapping_mapped(mapping)) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ea67be0223ac..2427cdcd9098 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1036,7 +1036,7 @@ void __init paging_init(struct machine_desc *mdesc) */ zero_page = alloc_bootmem_low_pages(PAGE_SIZE); empty_zero_page = virt_to_page(zero_page); - flush_dcache_page(empty_zero_page); + __flush_dcache_page(NULL, empty_zero_page); } /* -- cgit v1.2.2 From 48371cd3f4226275c529bb8675a99572db19cc7c Mon Sep 17 00:00:00 2001 From: Srinidhi Kasagar Date: Wed, 2 Dec 2009 06:18:03 +0100 Subject: ARM: 5845/1: l2x0: check whether l2x0 already enabled If running in non-secure mode accessing some registers of l2x0 will fault. So check if l2x0 is already enabled, if so do not access those secure registers. Signed-off-by: srinidhi kasagar Acked-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/cache-l2x0.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index b480f1d3591f..747f9a9021bb 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -99,18 +99,25 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) l2x0_base = base; - /* disable L2X0 */ - writel(0, l2x0_base + L2X0_CTRL); + /* + * Check if l2x0 controller is already enabled. + * If you are booting from non-secure mode + * accessing the below registers will fault. + */ + if (!(readl(l2x0_base + L2X0_CTRL) & 1)) { - aux = readl(l2x0_base + L2X0_AUX_CTRL); - aux &= aux_mask; - aux |= aux_val; - writel(aux, l2x0_base + L2X0_AUX_CTRL); + /* l2x0 controller is disabled */ - l2x0_inv_all(); + aux = readl(l2x0_base + L2X0_AUX_CTRL); + aux &= aux_mask; + aux |= aux_val; + writel(aux, l2x0_base + L2X0_AUX_CTRL); - /* enable L2X0 */ - writel(1, l2x0_base + L2X0_CTRL); + l2x0_inv_all(); + + /* enable L2X0 */ + writel(1, l2x0_base + L2X0_CTRL); + } outer_cache.inv_range = l2x0_inv_range; outer_cache.clean_range = l2x0_clean_range; -- cgit v1.2.2 From 7b0a1003e780193e2a11c27f5059c26b65f60679 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 24 Oct 2009 14:11:59 +0100 Subject: ARM: Reduce __flush_dcache_page() visibility Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 2 ++ arch/arm/mm/mm.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 4fbc7de8b4ac..729602291958 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -23,6 +23,8 @@ #include #include +#include "mm.h" + static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; /* diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index c4f6f05198e0..a888363398f8 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -24,6 +24,8 @@ struct mem_type { const struct mem_type *get_mem_type(unsigned int type); +extern void __flush_dcache_page(struct address_space *mapping, struct page *page); + #endif struct map_desc; -- cgit v1.2.2 From 2df341edf6b8a2db7f414d00faeadccbdd9844ab Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 24 Oct 2009 22:58:40 +0100 Subject: ARM: Move __flush_icache_all() out of flush_pfn_alias() Signed-off-by: Russell King --- arch/arm/mm/flush.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 82f4b06bf6b4..302d66517488 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -35,7 +35,6 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) : : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES), "r" (zero) : "cc"); - __flush_icache_all(); } void flush_cache_mm(struct mm_struct *mm) @@ -79,8 +78,10 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig return; } - if (cache_is_vipt_aliasing()) + if (cache_is_vipt_aliasing()) { flush_pfn_alias(pfn, user_addr); + __flush_icache_all(); + } } void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, @@ -94,6 +95,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, if (cache_is_vipt_aliasing()) { flush_pfn_alias(page_to_pfn(page), uaddr); + __flush_icache_all(); return; } @@ -132,9 +134,11 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) * we only need to do one flush - which would be at the relevant * userspace colour, which is congruent with page->index. */ - if (mapping && cache_is_vipt_aliasing()) + if (mapping && cache_is_vipt_aliasing()) { flush_pfn_alias(page_to_pfn(page), page->index << PAGE_CACHE_SHIFT); + __flush_icache_all(); + } } static void __flush_dcache_aliases(struct address_space *mapping, struct page *page) @@ -244,6 +248,7 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l * userspace address only. */ flush_pfn_alias(pfn, vmaddr); + __flush_icache_all(); } /* -- cgit v1.2.2 From f91fb05d826a43063fa0aa2ec30c23d3993a208d Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 24 Oct 2009 23:05:34 +0100 Subject: ARM: Remove __flush_icache_all() from __flush_dcache_page() Both call sites for __flush_dcache_page() end up calling __flush_icache_all() themselves, so having __flush_dcache_page() do this as well is wasteful. Remove the duplicated icache flushing. Signed-off-by: Russell King --- arch/arm/mm/flush.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 302d66517488..dc66f867bec4 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -134,11 +134,9 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) * we only need to do one flush - which would be at the relevant * userspace colour, which is congruent with page->index. */ - if (mapping && cache_is_vipt_aliasing()) { + if (mapping && cache_is_vipt_aliasing()) flush_pfn_alias(page_to_pfn(page), page->index << PAGE_CACHE_SHIFT); - __flush_icache_all(); - } } static void __flush_dcache_aliases(struct address_space *mapping, struct page *page) -- cgit v1.2.2 From 115b22474eb1905da2f606a057da3455833333d3 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 24 Nov 2009 18:54:07 +0100 Subject: ARM: 5794/1: Flush the D-cache during copy_user_highpage() The I and D caches for copy-on-write pages on processors with write-allocate caches become incoherent causing problems on application relying on CoW for text pages (dynamic linker relocating symbols in a text page). This patch flushes the D-cache for such pages. Cc: Nicolas Pitre Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/copypage-v6.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 4127a7bddfe5..841f355319bf 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -41,6 +41,14 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, kfrom = kmap_atomic(from, KM_USER0); kto = kmap_atomic(to, KM_USER1); copy_page(kto, kfrom); +#ifdef CONFIG_HIGHMEM + /* + * kmap_atomic() doesn't set the page virtual address, and + * kunmap_atomic() takes care of cache flushing already. + */ + if (page_address(to) != NULL) +#endif + __cpuc_flush_dcache_page(kto); kunmap_atomic(kto, KM_USER1); kunmap_atomic(kfrom, KM_USER0); } -- cgit v1.2.2 From 9e95922b1016ac941db7edcf6b6088b3c2e916c8 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 13:35:13 +0000 Subject: ARM: I-cache: Add invalidation for VIVT ASID tagged caches Signed-off-by: Russell King --- arch/arm/mm/flush.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index dc66f867bec4..9770e27dd581 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -69,6 +69,9 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned : "cc"); __flush_icache_all(); } + + if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) + __flush_icache_all(); } void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) @@ -82,6 +85,9 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig flush_pfn_alias(pfn, user_addr); __flush_icache_all(); } + + if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) + __flush_icache_all(); } void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, -- cgit v1.2.2 From ea201dbb78651c71c56e440b8b3132906bc7456d Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 14:31:40 +0000 Subject: ARM: I-cache: avoid flushing in flush_cache_mm() flush_cache_mm() is called in two cases: 1. when a process exits, just before the page tables are torn down. We can allow the stale lines to evict themselves over time without causing any harm. 2. when a process forks, and we've allocated a new ASID. The instruction cache issues are dealt with as pages are brought into the new process address space. Flushing the I-cache here is therefore unnecessary. However, we must keep the VIPT aliasing D-cache flush to ensure that any dirty cache lines are not written back after the pages have been reallocated for some other use - which would result in corruption. Signed-off-by: Russell King --- arch/arm/mm/flush.c | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 9770e27dd581..f8feb5d919fe 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -50,7 +50,6 @@ void flush_cache_mm(struct mm_struct *mm) : : "r" (0) : "cc"); - __flush_icache_all(); } } -- cgit v1.2.2 From 6060e8df517847bf445ebc61de7d4d9c7faae990 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 25 Oct 2009 14:12:27 +0000 Subject: ARM: I-cache: flush executable mappings in flush_cache_range() Dirk Behme reported instability on ARM11 SMP (VIPT non-aliasing cache) caused by the dynamic linker changing protection on text pages to write GOT entries. The problem is due to an interaction between the write faulting code providing new anonymous pages which are incoherent with the I-cache due to write buffering, and the I-cache not having been invalidated. a4db94d plugs the hole with the data cache coherency. This patch provides the other half of the fix by flushing the I-cache in flush_cache_range() for VM_EXEC VMAs (which is what we have when the region is being made executable again.) This ensures that the I-cache will be up to date with the newly COW'd pages. Note: if users are writing instructions, then they still need to use the ARM sys_cacheflush API to ensure that the caches are correctly synchronized. Signed-off-by: Russell King --- arch/arm/mm/flush.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index f8feb5d919fe..329594e760cd 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -66,10 +66,9 @@ void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned : : "r" (0) : "cc"); - __flush_icache_all(); } - if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) + if (vma->vm_flags & VM_EXEC) __flush_icache_all(); } -- cgit v1.2.2 From f0e5d2c959d5f14dd3bc879c8f9390aa2c049423 Mon Sep 17 00:00:00 2001 From: Saeed Bishara Date: Sun, 6 Dec 2009 18:06:43 +0200 Subject: ARM: dove: fix the mm mmu flags of the pj4 procinfo ... to be the same as proc-v6 Signed-off-by: Saeed Bishara Signed-off-by: Nicolas Pitre --- arch/arm/mm/proc-v6.S | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 5485c821101c..395cc90c6613 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -254,10 +254,9 @@ __pj4_v6_proc_info: .long 0x560f5810 .long 0xff0ffff0 .long PMD_TYPE_SECT | \ - PMD_SECT_BUFFERABLE | \ - PMD_SECT_CACHEABLE | \ PMD_SECT_AP_WRITE | \ - PMD_SECT_AP_READ + PMD_SECT_AP_READ | \ + PMD_FLAGS .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ -- cgit v1.2.2 From e77414e0aad6a1b063ba5e5750c582c75327ea6a Mon Sep 17 00:00:00 2001 From: Al Viro Date: Sat, 5 Dec 2009 15:10:44 -0500 Subject: fix broken aliasing checks for MAP_FIXED on sparc32, mips, arm and sh We want addr - (pgoff << PAGE_SHIFT) consistently coloured... Acked-by: Paul Mundt Acked-by: Hugh Dickins Signed-off-by: Al Viro --- arch/arm/mm/mmap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index 2b7996401b0f..f5abc51c5a07 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -54,7 +54,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, * We enforce the MAP_FIXED case. */ if (flags & MAP_FIXED) { - if (aliasing && flags & MAP_SHARED && addr & (SHMLBA - 1)) + if (aliasing && flags & MAP_SHARED && + (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } -- cgit v1.2.2 From 0eb948dd7f7c3cec37440c16a6c738c8e75efcda Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 11:12:15 +0000 Subject: ARM: cache-l2x0: avoid taking spinlock for every iteration Taking the spinlock for every iteration is very expensive; instead, batch iterations up into 4K blocks, releasing and reacquiring the spinlock between each block. Signed-off-by: Russell King Acked-by: Catalin Marinas --- arch/arm/mm/cache-l2x0.c | 65 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 13 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index b480f1d3591f..c1b7bfff47f4 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -31,14 +31,10 @@ static DEFINE_SPINLOCK(l2x0_lock); static inline void sync_writel(unsigned long val, unsigned long reg, unsigned long complete_mask) { - unsigned long flags; - - spin_lock_irqsave(&l2x0_lock, flags); writel(val, l2x0_base + reg); /* wait for the operation to complete */ while (readl(l2x0_base + reg) & complete_mask) ; - spin_unlock_irqrestore(&l2x0_lock, flags); } static inline void cache_sync(void) @@ -48,15 +44,20 @@ static inline void cache_sync(void) static inline void l2x0_inv_all(void) { + unsigned long flags; + /* invalidate all ways */ + spin_lock_irqsave(&l2x0_lock, flags); sync_writel(0xff, L2X0_INV_WAY, 0xff); cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_inv_range(unsigned long start, unsigned long end) { - unsigned long addr; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); if (start & (CACHE_LINE_SIZE - 1)) { start &= ~(CACHE_LINE_SIZE - 1); sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1); @@ -68,29 +69,67 @@ static void l2x0_inv_range(unsigned long start, unsigned long end) sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1); } - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_INV_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + sync_writel(start, L2X0_INV_LINE_PA, 1); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_clean_range(unsigned long start, unsigned long end) { - unsigned long addr; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_CLEAN_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + sync_writel(start, L2X0_CLEAN_LINE_PA, 1); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_flush_range(unsigned long start, unsigned long end) { - unsigned long addr; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_CLEAN_INV_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) -- cgit v1.2.2 From 3d1074349b22c9653e746282564136c87668c2b8 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 19 Nov 2009 11:41:09 +0000 Subject: ARM: cache-l2x0: make better use of background cache handling There's no point having the hardware support background operations if we issue a cache operation, and then wait for it to complete before calculating the address of the next operation. We gain no advantage in the cache controller stalling the bus until completion. What we should be doing is using the 'wait' time productively by calculating the address of the next operation, and only then waiting for the previous operation to complete. This means that cache operations can occur in parallel with the CPU calculating the next address. Signed-off-by: Russell King Acked-by: Catalin Marinas --- arch/arm/mm/cache-l2x0.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index c1b7bfff47f4..ec85dda1e733 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -28,18 +28,18 @@ static void __iomem *l2x0_base; static DEFINE_SPINLOCK(l2x0_lock); -static inline void sync_writel(unsigned long val, unsigned long reg, - unsigned long complete_mask) +static inline void cache_wait(void __iomem *reg, unsigned long mask) { - writel(val, l2x0_base + reg); /* wait for the operation to complete */ - while (readl(l2x0_base + reg) & complete_mask) + while (readl(reg) & mask) ; } static inline void cache_sync(void) { - sync_writel(0, L2X0_CACHE_SYNC, 1); + void __iomem *base = l2x0_base; + writel(0, base + L2X0_CACHE_SYNC); + cache_wait(base + L2X0_CACHE_SYNC, 1); } static inline void l2x0_inv_all(void) @@ -48,32 +48,37 @@ static inline void l2x0_inv_all(void) /* invalidate all ways */ spin_lock_irqsave(&l2x0_lock, flags); - sync_writel(0xff, L2X0_INV_WAY, 0xff); + writel(0xff, l2x0_base + L2X0_INV_WAY); + cache_wait(l2x0_base + L2X0_INV_WAY, 0xff); cache_sync(); spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_inv_range(unsigned long start, unsigned long end) { + void __iomem *base = l2x0_base; unsigned long flags; spin_lock_irqsave(&l2x0_lock, flags); if (start & (CACHE_LINE_SIZE - 1)) { start &= ~(CACHE_LINE_SIZE - 1); - sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_INV_LINE_PA); start += CACHE_LINE_SIZE; } if (end & (CACHE_LINE_SIZE - 1)) { end &= ~(CACHE_LINE_SIZE - 1); - sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(end, base + L2X0_CLEAN_INV_LINE_PA); } while (start < end) { unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - sync_writel(start, L2X0_INV_LINE_PA, 1); + cache_wait(base + L2X0_INV_LINE_PA, 1); + writel(start, base + L2X0_INV_LINE_PA); start += CACHE_LINE_SIZE; } @@ -82,12 +87,14 @@ static void l2x0_inv_range(unsigned long start, unsigned long end) spin_lock_irqsave(&l2x0_lock, flags); } } + cache_wait(base + L2X0_INV_LINE_PA, 1); cache_sync(); spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_clean_range(unsigned long start, unsigned long end) { + void __iomem *base = l2x0_base; unsigned long flags; spin_lock_irqsave(&l2x0_lock, flags); @@ -96,7 +103,8 @@ static void l2x0_clean_range(unsigned long start, unsigned long end) unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - sync_writel(start, L2X0_CLEAN_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_LINE_PA); start += CACHE_LINE_SIZE; } @@ -105,12 +113,14 @@ static void l2x0_clean_range(unsigned long start, unsigned long end) spin_lock_irqsave(&l2x0_lock, flags); } } + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); cache_sync(); spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_flush_range(unsigned long start, unsigned long end) { + void __iomem *base = l2x0_base; unsigned long flags; spin_lock_irqsave(&l2x0_lock, flags); @@ -119,7 +129,8 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_INV_LINE_PA); start += CACHE_LINE_SIZE; } @@ -128,6 +139,7 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) spin_lock_irqsave(&l2x0_lock, flags); } } + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); cache_sync(); spin_unlock_irqrestore(&l2x0_lock, flags); } -- cgit v1.2.2 From ccaf5f05b218e5eb41e2f5cdfd26b18dce4a0218 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Fri, 11 Dec 2009 02:21:57 +0100 Subject: ARM: 5848/1: kill flush_ioremap_region() There is not enough users to warrant its existence, and it is actually an obstacle to progress with the new DMA API which cannot cover this case properly. To keep backward compatibility, let's perform the necessary custom cache maintenance locally in the only driver affected. Signed-off-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/mm/proc-syms.c | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c index ac5c80062b70..f604aa8acae9 100644 --- a/arch/arm/mm/proc-syms.c +++ b/arch/arm/mm/proc-syms.c @@ -28,7 +28,6 @@ EXPORT_SYMBOL(__cpuc_flush_user_all); EXPORT_SYMBOL(__cpuc_flush_user_range); EXPORT_SYMBOL(__cpuc_coherent_kern_range); EXPORT_SYMBOL(__cpuc_flush_dcache_page); -EXPORT_SYMBOL(dmac_inv_range); /* because of flush_ioremap_region() */ #else EXPORT_SYMBOL(cpu_cache); #endif -- cgit v1.2.2 From 2c9b9c8490b60428fa2d1c64042f7c7caed93940 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 26 Nov 2009 12:56:21 +0000 Subject: ARM: add size argument to __cpuc_flush_dcache_page ... and rename the function since it no longer operates on just pages. Signed-off-by: Russell King --- arch/arm/mm/cache-fa.S | 11 ++++++----- arch/arm/mm/cache-v3.S | 9 +++++---- arch/arm/mm/cache-v4.S | 9 +++++---- arch/arm/mm/cache-v4wb.S | 11 ++++++----- arch/arm/mm/cache-v4wt.S | 11 ++++++----- arch/arm/mm/cache-v6.S | 11 ++++++----- arch/arm/mm/cache-v7.S | 13 +++++++------ arch/arm/mm/flush.c | 4 ++-- arch/arm/mm/highmem.c | 2 +- arch/arm/mm/nommu.c | 2 +- arch/arm/mm/proc-arm1020.S | 11 ++++++----- arch/arm/mm/proc-arm1020e.S | 11 ++++++----- arch/arm/mm/proc-arm1022.S | 11 ++++++----- arch/arm/mm/proc-arm1026.S | 11 ++++++----- arch/arm/mm/proc-arm920.S | 11 ++++++----- arch/arm/mm/proc-arm922.S | 11 ++++++----- arch/arm/mm/proc-arm925.S | 11 ++++++----- arch/arm/mm/proc-arm926.S | 11 ++++++----- arch/arm/mm/proc-arm940.S | 9 +++++---- arch/arm/mm/proc-arm946.S | 11 ++++++----- arch/arm/mm/proc-feroceon.S | 15 ++++++++------- arch/arm/mm/proc-mohawk.S | 11 ++++++----- arch/arm/mm/proc-syms.c | 2 +- arch/arm/mm/proc-xsc3.S | 11 ++++++----- arch/arm/mm/proc-xscale.S | 13 +++++++------ 25 files changed, 132 insertions(+), 111 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index b63a8f7b95cf..a89444a3c016 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -127,15 +127,16 @@ ENTRY(fa_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(kaddr) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - size of region */ -ENTRY(fa_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(fa_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -213,7 +214,7 @@ ENTRY(fa_cache_fns) .long fa_flush_user_cache_range .long fa_coherent_kern_range .long fa_coherent_user_range - .long fa_flush_kern_dcache_page + .long fa_flush_kern_dcache_area .long fa_dma_inv_range .long fa_dma_clean_range .long fa_dma_flush_range diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S index 8a4abebc478a..2a482731ea36 100644 --- a/arch/arm/mm/cache-v3.S +++ b/arch/arm/mm/cache-v3.S @@ -72,14 +72,15 @@ ENTRY(v3_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *page, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v3_flush_kern_dcache_page) +ENTRY(v3_flush_kern_dcache_area) /* FALLTHROUGH */ /* @@ -129,7 +130,7 @@ ENTRY(v3_cache_fns) .long v3_flush_user_cache_range .long v3_coherent_kern_range .long v3_coherent_user_range - .long v3_flush_kern_dcache_page + .long v3_flush_kern_dcache_area .long v3_dma_inv_range .long v3_dma_clean_range .long v3_dma_flush_range diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index 3668611cb400..5c7da3e372e9 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -82,14 +82,15 @@ ENTRY(v4_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4_flush_kern_dcache_page) +ENTRY(v4_flush_kern_dcache_area) /* FALLTHROUGH */ /* @@ -141,7 +142,7 @@ ENTRY(v4_cache_fns) .long v4_flush_user_cache_range .long v4_coherent_kern_range .long v4_coherent_user_range - .long v4_flush_kern_dcache_page + .long v4_flush_kern_dcache_area .long v4_dma_inv_range .long v4_dma_clean_range .long v4_dma_flush_range diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index 2ebc1b3bf856..3dbedf1ec0e7 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -114,15 +114,16 @@ ENTRY(v4wb_flush_user_cache_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4wb_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(v4wb_flush_kern_dcache_area) + add r1, r0, r1 /* fall through */ /* @@ -224,7 +225,7 @@ ENTRY(v4wb_cache_fns) .long v4wb_flush_user_cache_range .long v4wb_coherent_kern_range .long v4wb_coherent_user_range - .long v4wb_flush_kern_dcache_page + .long v4wb_flush_kern_dcache_area .long v4wb_dma_inv_range .long v4wb_dma_clean_range .long v4wb_dma_flush_range diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index c54fa2cc40e6..b3b7410270b4 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -117,17 +117,18 @@ ENTRY(v4wt_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4wt_flush_kern_dcache_page) +ENTRY(v4wt_flush_kern_dcache_area) mov r2, #0 mcr p15, 0, r2, c7, c5, 0 @ invalidate I cache - add r1, r0, #PAGE_SZ + add r1, r0, r1 /* fallthrough */ /* @@ -180,7 +181,7 @@ ENTRY(v4wt_cache_fns) .long v4wt_flush_user_cache_range .long v4wt_coherent_kern_range .long v4wt_coherent_user_range - .long v4wt_flush_kern_dcache_page + .long v4wt_flush_kern_dcache_area .long v4wt_dma_inv_range .long v4wt_dma_clean_range .long v4wt_dma_flush_range diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 295e25dd6381..4ba0a24ce6f5 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -159,15 +159,16 @@ ENDPROC(v6_coherent_user_range) ENDPROC(v6_coherent_kern_range) /* - * v6_flush_kern_dcache_page(kaddr) + * v6_flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - region size */ -ENTRY(v6_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(v6_flush_kern_dcache_area) + add r1, r0, r1 1: #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line @@ -271,7 +272,7 @@ ENTRY(v6_cache_fns) .long v6_flush_user_cache_range .long v6_coherent_kern_range .long v6_coherent_user_range - .long v6_flush_kern_dcache_page + .long v6_flush_kern_dcache_area .long v6_dma_inv_range .long v6_dma_clean_range .long v6_dma_flush_range diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index e1bd9759617f..9073db849fb4 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -186,16 +186,17 @@ ENDPROC(v7_coherent_kern_range) ENDPROC(v7_coherent_user_range) /* - * v7_flush_kern_dcache_page(kaddr) + * v7_flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - region size */ -ENTRY(v7_flush_kern_dcache_page) +ENTRY(v7_flush_kern_dcache_area) dcache_line_size r2, r3 - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line add r0, r0, r2 @@ -203,7 +204,7 @@ ENTRY(v7_flush_kern_dcache_page) blo 1b dsb mov pc, lr -ENDPROC(v7_flush_kern_dcache_page) +ENDPROC(v7_flush_kern_dcache_area) /* * v7_dma_inv_range(start,end) @@ -279,7 +280,7 @@ ENTRY(v7_cache_fns) .long v7_flush_user_cache_range .long v7_coherent_kern_range .long v7_coherent_user_range - .long v7_flush_kern_dcache_page + .long v7_flush_kern_dcache_area .long v7_dma_inv_range .long v7_dma_clean_range .long v7_dma_flush_range diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 329594e760cd..6f3a4b7a3b82 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -131,7 +131,7 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) */ if (addr) #endif - __cpuc_flush_dcache_page(addr); + __cpuc_flush_dcache_area(addr, PAGE_SIZE); /* * If this is a page cache page, and we have an aliasing VIPT cache, @@ -258,5 +258,5 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l * in this mapping of the page. FIXME: this is overkill * since we actually ask for a write-back and invalidate. */ - __cpuc_flush_dcache_page(page_address(page)); + __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index 30f82fb5918c..2be1ec7c1b41 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -79,7 +79,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) unsigned int idx = type + KM_TYPE_NR * smp_processor_id(); if (kvaddr >= (void *)FIXADDR_START) { - __cpuc_flush_dcache_page((void *)vaddr); + __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); set_pte_ext(TOP_PTE(vaddr), __pte(0), 0); diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 900811cc9130..374a8311bc84 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -61,7 +61,7 @@ void setup_mm_for_reboot(char mode) void flush_dcache_page(struct page *page) { - __cpuc_flush_dcache_page(page_address(page)); + __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } EXPORT_SYMBOL(flush_dcache_page); diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index d9fb4b98c49f..8012e24282b2 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -231,17 +231,18 @@ ENTRY(arm1020_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1020_flush_kern_dcache_page) +ENTRY(arm1020_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry mcr p15, 0, ip, c7, c10, 4 @ drain WB add r0, r0, #CACHE_DLINESIZE @@ -335,7 +336,7 @@ ENTRY(arm1020_cache_fns) .long arm1020_flush_user_cache_range .long arm1020_coherent_kern_range .long arm1020_coherent_user_range - .long arm1020_flush_kern_dcache_page + .long arm1020_flush_kern_dcache_area .long arm1020_dma_inv_range .long arm1020_dma_clean_range .long arm1020_dma_flush_range diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index 7453b75dcea5..41fe25d234f5 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -225,17 +225,18 @@ ENTRY(arm1020e_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1020e_flush_kern_dcache_page) +ENTRY(arm1020e_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -321,7 +322,7 @@ ENTRY(arm1020e_cache_fns) .long arm1020e_flush_user_cache_range .long arm1020e_coherent_kern_range .long arm1020e_coherent_user_range - .long arm1020e_flush_kern_dcache_page + .long arm1020e_flush_kern_dcache_area .long arm1020e_dma_inv_range .long arm1020e_dma_clean_range .long arm1020e_dma_flush_range diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index 8eb72d75a8b6..20a5b1b31a70 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -214,17 +214,18 @@ ENTRY(arm1022_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1022_flush_kern_dcache_page) +ENTRY(arm1022_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -310,7 +311,7 @@ ENTRY(arm1022_cache_fns) .long arm1022_flush_user_cache_range .long arm1022_coherent_kern_range .long arm1022_coherent_user_range - .long arm1022_flush_kern_dcache_page + .long arm1022_flush_kern_dcache_area .long arm1022_dma_inv_range .long arm1022_dma_clean_range .long arm1022_dma_flush_range diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index 3b59f0d67139..96aedb10fcc4 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -208,17 +208,18 @@ ENTRY(arm1026_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1026_flush_kern_dcache_page) +ENTRY(arm1026_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -304,7 +305,7 @@ ENTRY(arm1026_cache_fns) .long arm1026_flush_user_cache_range .long arm1026_coherent_kern_range .long arm1026_coherent_user_range - .long arm1026_flush_kern_dcache_page + .long arm1026_flush_kern_dcache_area .long arm1026_dma_inv_range .long arm1026_dma_clean_range .long arm1026_dma_flush_range diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 2b7c197cc58d..471669e2d7cb 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -207,15 +207,16 @@ ENTRY(arm920_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm920_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm920_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -293,7 +294,7 @@ ENTRY(arm920_cache_fns) .long arm920_flush_user_cache_range .long arm920_coherent_kern_range .long arm920_coherent_user_range - .long arm920_flush_kern_dcache_page + .long arm920_flush_kern_dcache_area .long arm920_dma_inv_range .long arm920_dma_clean_range .long arm920_dma_flush_range diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index 06a1aa4e3398..ee111b00fa41 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -209,15 +209,16 @@ ENTRY(arm922_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm922_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm922_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -295,7 +296,7 @@ ENTRY(arm922_cache_fns) .long arm922_flush_user_cache_range .long arm922_coherent_kern_range .long arm922_coherent_user_range - .long arm922_flush_kern_dcache_page + .long arm922_flush_kern_dcache_area .long arm922_dma_inv_range .long arm922_dma_clean_range .long arm922_dma_flush_range diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index cb53435a85ae..8deb5bde58e4 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -251,15 +251,16 @@ ENTRY(arm925_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm925_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm925_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -346,7 +347,7 @@ ENTRY(arm925_cache_fns) .long arm925_flush_user_cache_range .long arm925_coherent_kern_range .long arm925_coherent_user_range - .long arm925_flush_kern_dcache_page + .long arm925_flush_kern_dcache_area .long arm925_dma_inv_range .long arm925_dma_clean_range .long arm925_dma_flush_range diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 1c4848704bb3..64db6e275a44 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -214,15 +214,16 @@ ENTRY(arm926_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm926_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm926_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -309,7 +310,7 @@ ENTRY(arm926_cache_fns) .long arm926_flush_user_cache_range .long arm926_coherent_kern_range .long arm926_coherent_user_range - .long arm926_flush_kern_dcache_page + .long arm926_flush_kern_dcache_area .long arm926_dma_inv_range .long arm926_dma_clean_range .long arm926_dma_flush_range diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index 5b0f8464c8f2..8196b9f401fb 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -141,14 +141,15 @@ ENTRY(arm940_coherent_user_range) /* FALLTHROUGH */ /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm940_flush_kern_dcache_page) +ENTRY(arm940_flush_kern_dcache_area) mov ip, #0 mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments 1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries @@ -238,7 +239,7 @@ ENTRY(arm940_cache_fns) .long arm940_flush_user_cache_range .long arm940_coherent_kern_range .long arm940_coherent_user_range - .long arm940_flush_kern_dcache_page + .long arm940_flush_kern_dcache_area .long arm940_dma_inv_range .long arm940_dma_clean_range .long arm940_dma_flush_range diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index 40c0449a139b..9a951239c86c 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -183,16 +183,17 @@ ENTRY(arm946_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size * (same as arm926) */ -ENTRY(arm946_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm946_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -280,7 +281,7 @@ ENTRY(arm946_cache_fns) .long arm946_flush_user_cache_range .long arm946_coherent_kern_range .long arm946_coherent_user_range - .long arm946_flush_kern_dcache_page + .long arm946_flush_kern_dcache_area .long arm946_dma_inv_range .long arm946_dma_clean_range .long arm946_dma_flush_range diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index d0d7795200fc..dbc39383e66a 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -226,16 +226,17 @@ ENTRY(feroceon_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ .align 5 -ENTRY(feroceon_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(feroceon_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -246,7 +247,7 @@ ENTRY(feroceon_flush_kern_dcache_page) mov pc, lr .align 5 -ENTRY(feroceon_range_flush_kern_dcache_page) +ENTRY(feroceon_range_flush_kern_dcache_area) mrs r2, cpsr add r1, r0, #PAGE_SZ - CACHE_DLINESIZE @ top addr is inclusive orr r3, r2, #PSR_I_BIT @@ -372,7 +373,7 @@ ENTRY(feroceon_cache_fns) .long feroceon_flush_user_cache_range .long feroceon_coherent_kern_range .long feroceon_coherent_user_range - .long feroceon_flush_kern_dcache_page + .long feroceon_flush_kern_dcache_area .long feroceon_dma_inv_range .long feroceon_dma_clean_range .long feroceon_dma_flush_range @@ -383,7 +384,7 @@ ENTRY(feroceon_range_cache_fns) .long feroceon_flush_user_cache_range .long feroceon_coherent_kern_range .long feroceon_coherent_user_range - .long feroceon_range_flush_kern_dcache_page + .long feroceon_range_flush_kern_dcache_area .long feroceon_range_dma_inv_range .long feroceon_range_dma_clean_range .long feroceon_range_dma_flush_range diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index 52b5fd74fbb3..9674d36cc97d 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -186,15 +186,16 @@ ENTRY(mohawk_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(mohawk_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(mohawk_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -273,7 +274,7 @@ ENTRY(mohawk_cache_fns) .long mohawk_flush_user_cache_range .long mohawk_coherent_kern_range .long mohawk_coherent_user_range - .long mohawk_flush_kern_dcache_page + .long mohawk_flush_kern_dcache_area .long mohawk_dma_inv_range .long mohawk_dma_clean_range .long mohawk_dma_flush_range diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c index f604aa8acae9..3e6210b4d6d4 100644 --- a/arch/arm/mm/proc-syms.c +++ b/arch/arm/mm/proc-syms.c @@ -27,7 +27,7 @@ EXPORT_SYMBOL(__cpuc_flush_kern_all); EXPORT_SYMBOL(__cpuc_flush_user_all); EXPORT_SYMBOL(__cpuc_flush_user_range); EXPORT_SYMBOL(__cpuc_coherent_kern_range); -EXPORT_SYMBOL(__cpuc_flush_dcache_page); +EXPORT_SYMBOL(__cpuc_flush_dcache_area); #else EXPORT_SYMBOL(cpu_cache); #endif diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index fab134e29826..96456f548798 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -226,15 +226,16 @@ ENTRY(xsc3_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache. * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(xsc3_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(xsc3_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean/invalidate L1 D line add r0, r0, #CACHELINESIZE cmp r0, r1 @@ -309,7 +310,7 @@ ENTRY(xsc3_cache_fns) .long xsc3_flush_user_cache_range .long xsc3_coherent_kern_range .long xsc3_coherent_user_range - .long xsc3_flush_kern_dcache_page + .long xsc3_flush_kern_dcache_area .long xsc3_dma_inv_range .long xsc3_dma_clean_range .long xsc3_dma_flush_range diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index f056c283682d..93df47265f2d 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -284,15 +284,16 @@ ENTRY(xscale_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(xscale_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(xscale_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry add r0, r0, #CACHELINESIZE @@ -368,7 +369,7 @@ ENTRY(xscale_cache_fns) .long xscale_flush_user_cache_range .long xscale_coherent_kern_range .long xscale_coherent_user_range - .long xscale_flush_kern_dcache_page + .long xscale_flush_kern_dcache_area .long xscale_dma_inv_range .long xscale_dma_clean_range .long xscale_dma_flush_range @@ -392,7 +393,7 @@ ENTRY(xscale_80200_A0_A1_cache_fns) .long xscale_flush_user_cache_range .long xscale_coherent_kern_range .long xscale_coherent_user_range - .long xscale_flush_kern_dcache_page + .long xscale_flush_kern_dcache_area .long xscale_dma_flush_range .long xscale_dma_clean_range .long xscale_dma_flush_range -- cgit v1.2.2 From 2ef7f3dbd7a70a48c3f09b498df528cb00ea03a4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 5 Nov 2009 13:29:36 +0000 Subject: ARM: Fix ptrace accesses Signed-off-by: Russell King --- arch/arm/mm/flush.c | 51 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 8 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 6f3a4b7a3b82..e34f095e2090 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -87,13 +88,26 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) __flush_icache_all(); } +#else +#define flush_pfn_alias(pfn,vaddr) do { } while (0) +#endif +#ifdef CONFIG_SMP +static void flush_ptrace_access_other(void *args) +{ + __flush_icache_all(); +} +#endif + +static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, - unsigned long uaddr, void *kaddr, - unsigned long len, int write) + unsigned long uaddr, void *kaddr, unsigned long len) { if (cache_is_vivt()) { - vivt_flush_ptrace_access(vma, page, uaddr, kaddr, len, write); + if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { + unsigned long addr = (unsigned long)kaddr; + __cpuc_coherent_kern_range(addr, addr + len); + } return; } @@ -104,16 +118,37 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, } /* VIPT non-aliasing cache */ - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)) && - vma->vm_flags & VM_EXEC) { + if (vma->vm_flags & VM_EXEC) { unsigned long addr = (unsigned long)kaddr; - /* only flushing the kernel mapping on non-aliasing VIPT */ __cpuc_coherent_kern_range(addr, addr + len); +#ifdef CONFIG_SMP + if (cache_ops_need_broadcast()) + smp_call_function(flush_ptrace_access_other, + NULL, 1); +#endif } } -#else -#define flush_pfn_alias(pfn,vaddr) do { } while (0) + +/* + * Copy user data from/to a page which is mapped into a different + * processes address space. Really, we want to allow our "user + * space" model to handle this. + * + * Note that this code needs to run on the current CPU. + */ +void copy_to_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long uaddr, void *dst, const void *src, + unsigned long len) +{ +#ifdef CONFIG_SMP + preempt_disable(); #endif + memcpy(dst, src, len); + flush_ptrace_access(vma, page, uaddr, dst, len); +#ifdef CONFIG_SMP + preempt_enable(); +#endif +} void __flush_dcache_page(struct address_space *mapping, struct page *page) { -- cgit v1.2.2 From 2395d66d09ce10c25b6756575c4aeb219760d1fc Mon Sep 17 00:00:00 2001 From: Anand Gadiyar Date: Fri, 18 Dec 2009 12:56:10 +0100 Subject: ARM: 5853/1: ARM: Fix build break on ARM v6 and v7 Commit 2c9b9c849 added an argument to __cpuc_flush_dcache_page and renamed it. Update a caller of the old function to fix this build error: CC arch/arm/mm/copypage-v6.o arch/arm/mm/copypage-v6.c: In function 'v6_copy_user_highpage_nonaliasing': arch/arm/mm/copypage-v6.c:51: error: implicit declaration of function '__cpuc_flush_dcache_page' make[1]: *** [arch/arm/mm/copypage-v6.o] Error 1 make: *** [arch/arm/mm] Error 2 Reported-by: Jinsung Yang Signed-off-by: Anand Gadiyar Signed-off-by: Russell King --- arch/arm/mm/copypage-v6.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 841f355319bf..0fa1319273de 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -48,7 +48,7 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, */ if (page_address(to) != NULL) #endif - __cpuc_flush_dcache_page(kto); + __cpuc_flush_dcache_area(kto, PAGE_SIZE); kunmap_atomic(kto, KM_USER1); kunmap_atomic(kfrom, KM_USER0); } -- cgit v1.2.2 From 4da8b8208eded0ba21e3ed7cee79e7454b2b973d Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:06:32 +0000 Subject: ARM: Kill CONFIG_CPU_32 26-bit ARM support was removed a long time ago, and this symbol has been defined to be 'y' ever since. As it's never disabled anymore, we can kill it without any side effects. Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 4 ---- 1 file changed, 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index dd4698c67cc3..baf638487a2d 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -1,9 +1,5 @@ comment "Processor Type" -config CPU_32 - bool - default y - # Select CPU types depending on the architecture selected. This selects # which CPUs we support in the kernel image, and the compiler instruction # optimiser behaviour. -- cgit v1.2.2 From 52e8bfd81af5492a14cc3a4641d4c80d05b66bdb Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 23 Dec 2009 19:54:31 +0000 Subject: ARM: Fix wrong shared bit for CPU write buffer bug test It is unpredictable to have the same memory mapped using different shared bit settings for ARMv6 and ARMv7 CPUs. Fix this for the CPU write buffer bug test. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 729602291958..56ee15321b00 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -207,9 +207,8 @@ void __init check_writebuffer_bugs(void) page = alloc_page(GFP_KERNEL); if (page) { unsigned long *p1, *p2; - pgprot_t prot = __pgprot(L_PTE_PRESENT|L_PTE_YOUNG| - L_PTE_DIRTY|L_PTE_WRITE| - L_PTE_MT_BUFFERABLE); + pgprot_t prot = __pgprot_modify(PAGE_KERNEL, + L_PTE_MT_MASK, L_PTE_MT_BUFFERABLE); p1 = vmap(&page, 1, VM_IOREMAP, prot); p2 = vmap(&page, 1, VM_IOREMAP, prot); -- cgit v1.2.2 From 6dc995a3da9adfa83d61ccf06aa1afa5f6ab764f Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 24 Dec 2009 10:16:21 +0000 Subject: ARM: fix PAGE_KERNEL PAGE_KERNEL should not be executable; any area marked executable can be prefetched into the instruction cache. We don't want vmalloc areas to be read in this way. Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 8c7fbd19a4b3..1708da82da96 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -453,8 +453,7 @@ static void __init build_mem_type_table(void) pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | - L_PTE_DIRTY | L_PTE_WRITE | - L_PTE_EXEC | kern_pgprot); + L_PTE_DIRTY | L_PTE_WRITE | kern_pgprot); mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; -- cgit v1.2.2 From dc8601a224d546bb321b058fc5ecabdb688a3582 Mon Sep 17 00:00:00 2001 From: Haojian Zhuang Date: Wed, 30 Dec 2009 02:27:24 -0500 Subject: [ARM] pxa: do not enable L2 after MMU is enabled Outer cache checked whether L2 is enabled or not. If L2 isn't enabled in XSC3, it would enable L2. This operation is evil that would make system hang. In XSC3 core document, these words are mentioned in below. "Following reset, the L2 Unified Cache Enable bit is cleared. To enable the L2 Cache, software may set the bit to a '1' before or at the same time as enabling the MMU. Enabling the L2 Cache after the MMU has been enabled or disabling the L2 Cache after the L2 Cache has been enabled, may result in unpredictable behavior of the processor." When outer cache is initialized, the MMU is already enabled. We couldn't enable L2 after MMU enabled. Signed-off-by: Haojian Zhuang Signed-off-by: Eric Miao --- arch/arm/mm/cache-xsc3l2.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-xsc3l2.c b/arch/arm/mm/cache-xsc3l2.c index 5d180cb0bd94..c3154928bccd 100644 --- a/arch/arm/mm/cache-xsc3l2.c +++ b/arch/arm/mm/cache-xsc3l2.c @@ -221,15 +221,14 @@ static int __init xsc3_l2_init(void) if (!cpu_is_xsc3() || !xsc3_l2_present()) return 0; - if (!(get_cr() & CR_L2)) { + if (get_cr() & CR_L2) { pr_info("XScale3 L2 cache enabled.\n"); - adjust_cr(CR_L2, CR_L2); xsc3_l2_inv_all(); - } - outer_cache.inv_range = xsc3_l2_inv_range; - outer_cache.clean_range = xsc3_l2_clean_range; - outer_cache.flush_range = xsc3_l2_flush_range; + outer_cache.inv_range = xsc3_l2_inv_range; + outer_cache.clean_range = xsc3_l2_clean_range; + outer_cache.flush_range = xsc3_l2_flush_range; + } return 0; } -- cgit v1.2.2 From 548c6af4627f4dcb24512a381193206e09bd6d31 Mon Sep 17 00:00:00 2001 From: Haojian Zhuang Date: Wed, 30 Dec 2009 10:02:57 -0500 Subject: [ARM] pxa: enable L2 if present in XSC3 Check whether L2 is present or not in XSC3. If it's present, enable L2 immediately. Disabling L2 after L2 is enabled that would result in unpredicatable behavior of XSC3 processor. Signed-off-by: Haojian Zhuang Signed-off-by: Eric Miao --- arch/arm/mm/proc-xsc3.S | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index 96456f548798..8e4f6dca8997 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -407,6 +407,13 @@ __xsc3_setup: adr r5, xsc3_crval ldmia r5, {r5, r6} + +#ifdef CONFIG_CACHE_XSC3L2 + mrc p15, 1, r0, c0, c0, 1 @ get L2 present information + ands r0, r0, #0xf8 + orrne r6, r6, #(1 << 26) @ enable L2 if present +#endif + mrc p15, 0, r0, c1, c0, 0 @ get control register bic r0, r0, r5 @ ..V. ..R. .... ..A. orr r0, r0, r6 @ ..VI Z..S .... .C.M (mmu) -- cgit v1.2.2 From 070f1f178c4377c09f72e414513aeacd8403f6d6 Mon Sep 17 00:00:00 2001 From: Bahadir Balban Date: Fri, 25 Dec 2009 14:25:48 +0100 Subject: ARM: 5858/1: Remove unused vma_vm_flags macro from v7wbi_flush_user_tlb_range Signed-off-by: Bahadir Balban Signed-off-by: Russell King --- arch/arm/mm/tlb-v7.S | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index a26a605b73bd..0cb1848bd876 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -40,7 +40,6 @@ ENTRY(v7wbi_flush_user_tlb_range) asid r3, r3 @ mask ASID orr r0, r3, r0, lsl #PAGE_SHIFT @ Create initial MVA mov r1, r1, lsl #PAGE_SHIFT - vma_vm_flags r2, r2 @ get vma->vm_flags 1: #ifdef CONFIG_SMP mcr p15, 0, r0, c8, c3, 1 @ TLB invalidate U MVA (shareable) -- cgit v1.2.2 From 4b529401c5089cf33f7165607cbc2fde43357bfb Mon Sep 17 00:00:00 2001 From: Andreas Fenkart Date: Fri, 8 Jan 2010 14:42:31 -0800 Subject: mm: make totalhigh_pages unsigned long Makes it consistent with the extern declaration, used when CONFIG_HIGHMEM is set Removes redundant casts in printout messages Signed-off-by: Andreas Fenkart Acked-by: Russell King Cc: Ralf Baechle Cc: David Howells Cc: Ingo Molnar Cc: Thomas Gleixner Cc: "H. Peter Anvin" Cc: Chen Liqin Cc: Lennox Wu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm/mm/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 52c40d155672..a04ffbbbe253 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -616,7 +616,7 @@ void __init mem_init(void) "%dK data, %dK init, %luK highmem)\n", nr_free_pages() << (PAGE_SHIFT-10), codesize >> 10, datasize >> 10, initsize >> 10, - (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))); + totalhigh_pages << (PAGE_SHIFT-10)); if (PAGE_SIZE >= 16384 && num_physpages <= 128) { extern int sysctl_overcommit_memory; -- cgit v1.2.2 From aff7b4f86737f1ae364bf5ece9a9b8586ddb2db4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 12 Jan 2010 19:02:05 +0000 Subject: ARM: Ensure ARMv6/7 mm files are built using appropriate assembler options A kernel with both ARMv6 and ARMv7 selected results in build errors. Fix this by specifying the proper architectures for these assembly files. Signed-off-by: Russell King --- arch/arm/mm/Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 827e238e5d4a..e8d34a80851c 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -27,6 +27,9 @@ obj-$(CONFIG_CPU_ABRT_EV5TJ) += abort-ev5tj.o obj-$(CONFIG_CPU_ABRT_EV6) += abort-ev6.o obj-$(CONFIG_CPU_ABRT_EV7) += abort-ev7.o +AFLAGS_abort-ev6.o :=-Wa,-march=armv6k +AFLAGS_abort-ev7.o :=-Wa,-march=armv7-a + obj-$(CONFIG_CPU_PABRT_LEGACY) += pabort-legacy.o obj-$(CONFIG_CPU_PABRT_V6) += pabort-v6.o obj-$(CONFIG_CPU_PABRT_V7) += pabort-v7.o @@ -39,6 +42,9 @@ obj-$(CONFIG_CPU_CACHE_V6) += cache-v6.o obj-$(CONFIG_CPU_CACHE_V7) += cache-v7.o obj-$(CONFIG_CPU_CACHE_FA) += cache-fa.o +AFLAGS_cache-v6.o :=-Wa,-march=armv6 +AFLAGS_cache-v7.o :=-Wa,-march=armv7-a + obj-$(CONFIG_CPU_COPY_V3) += copypage-v3.o obj-$(CONFIG_CPU_COPY_V4WT) += copypage-v4wt.o obj-$(CONFIG_CPU_COPY_V4WB) += copypage-v4wb.o @@ -58,6 +64,9 @@ obj-$(CONFIG_CPU_TLB_V6) += tlb-v6.o obj-$(CONFIG_CPU_TLB_V7) += tlb-v7.o obj-$(CONFIG_CPU_TLB_FA) += tlb-fa.o +AFLAGS_tlb-v6.o :=-Wa,-march=armv6 +AFLAGS_tlb-v7.o :=-Wa,-march=armv7-a + obj-$(CONFIG_CPU_ARM610) += proc-arm6_7.o obj-$(CONFIG_CPU_ARM710) += proc-arm6_7.o obj-$(CONFIG_CPU_ARM7TDMI) += proc-arm7tdmi.o @@ -84,6 +93,9 @@ obj-$(CONFIG_CPU_FEROCEON) += proc-feroceon.o obj-$(CONFIG_CPU_V6) += proc-v6.o obj-$(CONFIG_CPU_V7) += proc-v7.o +AFLAGS_proc-v6.o :=-Wa,-march=armv6 +AFLAGS_proc-v7.o :=-Wa,-march=armv7-a + obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o -- cgit v1.2.2 From ad3e6c0b1fad31282a8384903ed811671d840a9c Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 19 Jan 2010 16:42:12 +0100 Subject: ARM: 5885/1: arm: Flush TLB entries in setup_mm_for_reboot() We need to do that if we tinker with the MMU entries. This fixes the occasional bug with kexec where the new fails to uncompress with "crc error". Most likely at least kexec on v6 and v7 need this fix. Signed-off-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 1708da82da96..761ffede6a23 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1067,4 +1067,6 @@ void setup_mm_for_reboot(char mode) pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1))); flush_pmd_entry(pmd); } + + local_flush_tlb_all(); } -- cgit v1.2.2 From 1f667c690be3ab71036c436d8391105eee23f65b Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 19 Jan 2010 17:01:33 +0100 Subject: ARM: 5886/1: arm: Fix cpu_proc_fin() for proc-v7.S and make kexec work The comments in arm_machine_restart() suggest that cpu_proc_fin() will clean and disable cache and turn off interrupts. This does not seem to be implemented for proc-v7.S, implement it the same way as for proc-v6.S. This also makes kexec work for v7. Note that a related TLB and branch traget flush patch is also needed to avoid kexec "crc error". Note that there are still some issues that seem to be related to L2 cache being on and causing occasional uncompress "crc error" with kexec. Anyways, this gets kexec mostly working on V7 for now. Signed-off-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/proc-v7.S | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 3a285218fd15..d2a80747c6fe 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -45,7 +45,14 @@ ENTRY(cpu_v7_proc_init) ENDPROC(cpu_v7_proc_init) ENTRY(cpu_v7_proc_fin) - mov pc, lr + stmfd sp!, {lr} + cpsid if @ disable interrupts + bl v7_flush_kern_cache_all + mrc p15, 0, r0, c1, c0, 0 @ ctrl register + bic r0, r0, #0x1000 @ ...i............ + bic r0, r0, #0x0006 @ .............ca. + mcr p15, 0, r0, c1, c0, 0 @ disable caches + ldmfd sp!, {pc} ENDPROC(cpu_v7_proc_fin) /* -- cgit v1.2.2 From 2045124ffd1a5e46d157349016a2c50f19c8c91d Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 19 Jan 2010 23:42:08 +0100 Subject: ARM: 5888/1: arm: Update comments in cacheflush.h and remove unnecessary V6 and V7 comments The comments in cacheflush.h should follow what's in struct cpu_cache_fns. The comments for V6 and V7 are unnecessary. Signed-off-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/proc-v6.S | 2 -- arch/arm/mm/proc-v7.S | 2 -- 2 files changed, 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 395cc90c6613..7a5337ed7d68 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -59,8 +59,6 @@ ENTRY(cpu_v6_proc_fin) * to what would be the reset vector. * * - loc - location to jump to for soft reset - * - * It is assumed that: */ .align 5 ENTRY(cpu_v6_reset) diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index d2a80747c6fe..7aaf88a3b7aa 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -63,8 +63,6 @@ ENDPROC(cpu_v7_proc_fin) * to what would be the reset vector. * * - loc - location to jump to for soft reset - * - * It is assumed that: */ .align 5 ENTRY(cpu_v7_reset) -- cgit v1.2.2 From c26c20b823d48addbde9cb5709d80655c6fadf18 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:21:35 +0000 Subject: ARM: make_coherent: split adjust_pte() in two adjust_pte() walks the page tables, and do_adjust_pte() does the page table manipulation. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 52 +++++++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 56ee15321b00..074e6bb54eb3 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -36,28 +36,12 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; * Therefore those configurations which might call adjust_pte (those * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock. */ -static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, + pte_t *ptep) { - pgd_t *pgd; - pmd_t *pmd; - pte_t *pte, entry; + pte_t entry = *ptep; int ret; - pgd = pgd_offset(vma->vm_mm, address); - if (pgd_none(*pgd)) - goto no_pgd; - if (pgd_bad(*pgd)) - goto bad_pgd; - - pmd = pmd_offset(pgd, address); - if (pmd_none(*pmd)) - goto no_pmd; - if (pmd_bad(*pmd)) - goto bad_pmd; - - pte = pte_offset_map(pmd, address); - entry = *pte; - /* * If this page is present, it's actually being shared. */ @@ -74,10 +58,38 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) (pfn << PAGE_SHIFT) + PAGE_SIZE); pte_val(entry) &= ~L_PTE_MT_MASK; pte_val(entry) |= shared_pte_mask; - set_pte_at(vma->vm_mm, address, pte, entry); + set_pte_at(vma->vm_mm, address, ptep, entry); flush_tlb_page(vma, address); } + + return ret; +} + +static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + int ret; + + pgd = pgd_offset(vma->vm_mm, address); + if (pgd_none(*pgd)) + goto no_pgd; + if (pgd_bad(*pgd)) + goto bad_pgd; + + pmd = pmd_offset(pgd, address); + if (pmd_none(*pmd)) + goto no_pmd; + if (pmd_bad(*pmd)) + goto bad_pmd; + + pte = pte_offset_map(pmd, address); + + ret = do_adjust_pte(vma, address, pte); + pte_unmap(pte); + return ret; bad_pgd: -- cgit v1.2.2 From f8a85f1164a33e3eb5b421b137ced793ed53ee33 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:23:44 +0000 Subject: ARM: make_coherent: convert adjust_pte() to use p*d_none_or_clear_bad() Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 074e6bb54eb3..7a8efe1b37d8 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -73,16 +73,12 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) int ret; pgd = pgd_offset(vma->vm_mm, address); - if (pgd_none(*pgd)) - goto no_pgd; - if (pgd_bad(*pgd)) - goto bad_pgd; + if (pgd_none_or_clear_bad(pgd)) + return 0; pmd = pmd_offset(pgd, address); - if (pmd_none(*pmd)) - goto no_pmd; - if (pmd_bad(*pmd)) - goto bad_pmd; + if (pmd_none_or_clear_bad(pmd)) + return 0; pte = pte_offset_map(pmd, address); @@ -91,18 +87,6 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) pte_unmap(pte); return ret; - -bad_pgd: - pgd_ERROR(*pgd); - pgd_clear(pgd); -no_pgd: - return 0; - -bad_pmd: - pmd_ERROR(*pmd); - pmd_clear(pmd); -no_pmd: - return 0; } static void -- cgit v1.2.2 From 56dd47098abe1fdde598a8d8b7c04d775506f456 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:24:34 +0000 Subject: ARM: make_coherent: fix problems with highpte, part 1 update_mmu_cache() is called with a page table already mapped. We call make_coherent(), which then calls adjust_pte() which wants to map other page tables. This causes kmap_atomic() to BUG() because the slot its trying to use is already taken. Since do_adjust_pte() modifies the page tables, we are also missing any form of locking, so we're risking corrupting the page tables. Fix this by using pte_offset_map_nested(), and taking the pte page table lock around do_adjust_pte(). Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 7a8efe1b37d8..8e9bc517132e 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -67,6 +67,7 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, static int adjust_pte(struct vm_area_struct *vma, unsigned long address) { + spinlock_t *ptl; pgd_t *pgd; pmd_t *pmd; pte_t *pte; @@ -80,11 +81,19 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) if (pmd_none_or_clear_bad(pmd)) return 0; - pte = pte_offset_map(pmd, address); + /* + * This is called while another page table is mapped, so we + * must use the nested version. This also means we need to + * open-code the spin-locking. + */ + ptl = pte_lockptr(vma->vm_mm, pmd); + pte = pte_offset_map_nested(pmd, address); + spin_lock(ptl); ret = do_adjust_pte(vma, address, pte); - pte_unmap(pte); + spin_unlock(ptl); + pte_unmap_nested(pte); return ret; } -- cgit v1.2.2 From ed42acaef1a9d51631a31b55e9ed52d400430492 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:31:38 +0000 Subject: ARM: make_coherent: avoid recalculating the pfn for the modified page We already know the pfn for the page to be modified in make_coherent, so let's stop recalculating it unnecessarily. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 8e9bc517132e..ae88f2c3a6df 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -37,7 +37,7 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock. */ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, - pte_t *ptep) + unsigned long pfn, pte_t *ptep) { pte_t entry = *ptep; int ret; @@ -52,7 +52,6 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, * fault (ie, is old), we can safely ignore any issues. */ if (ret && (pte_val(entry) & L_PTE_MT_MASK) != shared_pte_mask) { - unsigned long pfn = pte_pfn(entry); flush_cache_page(vma, address, pfn); outer_flush_range((pfn << PAGE_SHIFT), (pfn << PAGE_SHIFT) + PAGE_SIZE); @@ -65,7 +64,8 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } -static int adjust_pte(struct vm_area_struct *vma, unsigned long address) +static int adjust_pte(struct vm_area_struct *vma, unsigned long address, + unsigned long pfn) { spinlock_t *ptl; pgd_t *pgd; @@ -90,7 +90,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address) pte = pte_offset_map_nested(pmd, address); spin_lock(ptl); - ret = do_adjust_pte(vma, address, pte); + ret = do_adjust_pte(vma, address, pfn, pte); spin_unlock(ptl); pte_unmap_nested(pte); @@ -127,11 +127,11 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne if (!(mpnt->vm_flags & VM_MAYSHARE)) continue; offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; - aliases += adjust_pte(mpnt, mpnt->vm_start + offset); + aliases += adjust_pte(mpnt, mpnt->vm_start + offset, pfn); } flush_dcache_mmap_unlock(mapping); if (aliases) - adjust_pte(vma, addr); + adjust_pte(vma, addr, pfn); else flush_cache_page(vma, addr, pfn); } -- cgit v1.2.2 From 4aba098c8d64329f0c4b24d12e1dc5398dd41a75 Mon Sep 17 00:00:00 2001 From: Russell King Date: Wed, 3 Feb 2010 15:48:03 +0000 Subject: ARM: Fix wrong register in proc-arm6_7.S data abort handler Signed-off-by: Russell King --- arch/arm/mm/proc-arm6_7.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/proc-arm6_7.S b/arch/arm/mm/proc-arm6_7.S index 3f9cd3d8f6d5..795dc615f43b 100644 --- a/arch/arm/mm/proc-arm6_7.S +++ b/arch/arm/mm/proc-arm6_7.S @@ -41,7 +41,7 @@ ENTRY(cpu_arm7_dcache_clean_area) ENTRY(cpu_arm7_data_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR - ldr r8, [r0] @ read arm instruction + ldr r8, [r2] @ read arm instruction tst r8, #1 << 20 @ L = 0 -> write? orreq r1, r1, #1 << 11 @ yes. and r7, r8, #15 << 24 -- cgit v1.2.2 From 7ada189f5c8627662c23f49b3e68463f86fc511e Mon Sep 17 00:00:00 2001 From: Jamie Iles Date: Tue, 2 Feb 2010 20:24:58 +0100 Subject: ARM: 5900/2: arm: enable support for software perf events The perf events subsystem allows counting of both hardware and software events. This patch implements the bare minimum for software performance events. Cc: Peter Zijlstra Cc: Ingo Molnar Signed-off-by: Jamie Iles Signed-off-by: Russell King --- arch/arm/mm/fault.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 10e06801afb3..9d40c341e07e 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -302,6 +303,12 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) fault = __do_page_fault(mm, addr, fsr, tsk); up_read(&mm->mmap_sem); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, addr); + if (fault & VM_FAULT_MAJOR) + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, regs, addr); + else if (fault & VM_FAULT_MINOR) + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, regs, addr); + /* * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR */ -- cgit v1.2.2 From 18eabe2347ae7a11b3db768695913724166dfb0e Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 31 Oct 2009 16:52:16 +0000 Subject: ARM: dma-mapping: introduce the idea of buffer ownership The DMA API has the notion of buffer ownership; make it explicit in the ARM implementation of this API. This gives us a set of hooks to allow us to deal with CPU cache issues arising from non-cache coherent DMA. Signed-off-by: Russell King Tested-By: Santosh Shilimkar Tested-By: Jamie Iles --- arch/arm/mm/dma-mapping.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 26325cb5d368..a316c9459526 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -573,8 +573,12 @@ void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int i; for_each_sg(sg, s, nents, i) { - dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0, - sg_dma_len(s), dir); + if (!dmabounce_sync_for_cpu(dev, sg_dma_address(s), 0, + sg_dma_len(s), dir)) + continue; + + __dma_page_dev_to_cpu(sg_page(s), s->offset, + s->length, dir); } } EXPORT_SYMBOL(dma_sync_sg_for_cpu); @@ -597,9 +601,8 @@ void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, sg_dma_len(s), dir)) continue; - if (!arch_is_coherent()) - dma_cache_maint_page(sg_page(s), s->offset, - s->length, dir); + __dma_page_cpu_to_dev(sg_page(s), s->offset, + s->length, dir); } } EXPORT_SYMBOL(dma_sync_sg_for_device); -- cgit v1.2.2 From 4ea0d7371e808628d11154b0d44140b70f05b998 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 24 Nov 2009 16:27:17 +0000 Subject: ARM: dma-mapping: push buffer ownership down into dma-mapping.c Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/dma-mapping.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index a316c9459526..bbf87880b915 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -404,7 +404,7 @@ EXPORT_SYMBOL(dma_free_coherent); * platforms with CONFIG_DMABOUNCE. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ -void dma_cache_maint(const void *start, size_t size, int direction) +static void dma_cache_maint(const void *start, size_t size, int direction) { void (*inner_op)(const void *, const void *); void (*outer_op)(unsigned long, unsigned long); @@ -431,7 +431,20 @@ void dma_cache_maint(const void *start, size_t size, int direction) inner_op(start, start + size); outer_op(__pa(start), __pa(start) + size); } -EXPORT_SYMBOL(dma_cache_maint); + +void ___dma_single_cpu_to_dev(const void *kaddr, size_t size, + enum dma_data_direction dir) +{ + dma_cache_maint(kaddr, size, dir); +} +EXPORT_SYMBOL(___dma_single_cpu_to_dev); + +void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, + enum dma_data_direction dir) +{ + /* nothing to do */ +} +EXPORT_SYMBOL(___dma_single_dev_to_cpu); static void dma_cache_maint_contiguous(struct page *page, unsigned long offset, size_t size, int direction) @@ -474,7 +487,7 @@ static void dma_cache_maint_contiguous(struct page *page, unsigned long offset, outer_op(paddr, paddr + size); } -void dma_cache_maint_page(struct page *page, unsigned long offset, +static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, int dir) { /* @@ -499,7 +512,20 @@ void dma_cache_maint_page(struct page *page, unsigned long offset, left -= len; } while (left); } -EXPORT_SYMBOL(dma_cache_maint_page); + +void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + dma_cache_maint_page(page, off, size, dir); +} +EXPORT_SYMBOL(___dma_page_cpu_to_dev); + +void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, + size_t size, enum dma_data_direction dir) +{ + /* nothing to do */ +} +EXPORT_SYMBOL(___dma_page_dev_to_cpu); /** * dma_map_sg - map a set of SG buffers for streaming mode DMA -- cgit v1.2.2 From 65af191a0414d0e1145f67c153e1b63d122dfbb4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 24 Nov 2009 17:53:33 +0000 Subject: ARM: dma-mapping: move selection of page ops out of dma_cache_maint_contiguous Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/dma-mapping.c | 59 ++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 29 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index bbf87880b915..77dc483e64c1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -447,48 +447,25 @@ void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, EXPORT_SYMBOL(___dma_single_dev_to_cpu); static void dma_cache_maint_contiguous(struct page *page, unsigned long offset, - size_t size, int direction) + size_t size, void (*op)(const void *, const void *)) { void *vaddr; - unsigned long paddr; - void (*inner_op)(const void *, const void *); - void (*outer_op)(unsigned long, unsigned long); - - switch (direction) { - case DMA_FROM_DEVICE: /* invalidate only */ - inner_op = dmac_inv_range; - outer_op = outer_inv_range; - break; - case DMA_TO_DEVICE: /* writeback only */ - inner_op = dmac_clean_range; - outer_op = outer_clean_range; - break; - case DMA_BIDIRECTIONAL: /* writeback and invalidate */ - inner_op = dmac_flush_range; - outer_op = outer_flush_range; - break; - default: - BUG(); - } if (!PageHighMem(page)) { vaddr = page_address(page) + offset; - inner_op(vaddr, vaddr + size); + op(vaddr, vaddr + size); } else { vaddr = kmap_high_get(page); if (vaddr) { vaddr += offset; - inner_op(vaddr, vaddr + size); + op(vaddr, vaddr + size); kunmap_high(page); } } - - paddr = page_to_phys(page) + offset; - outer_op(paddr, paddr + size); } static void dma_cache_maint_page(struct page *page, unsigned long offset, - size_t size, int dir) + size_t size, void (*op)(const void *, const void *)) { /* * A single sg entry may refer to multiple physically contiguous @@ -506,7 +483,7 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, } len = PAGE_SIZE - offset; } - dma_cache_maint_contiguous(page, offset, len, dir); + dma_cache_maint_contiguous(page, offset, len, op); offset = 0; page++; left -= len; @@ -516,7 +493,31 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, size_t size, enum dma_data_direction dir) { - dma_cache_maint_page(page, off, size, dir); + unsigned long paddr; + void (*inner_op)(const void *, const void *); + void (*outer_op)(unsigned long, unsigned long); + + switch (direction) { + case DMA_FROM_DEVICE: /* invalidate only */ + inner_op = dmac_inv_range; + outer_op = outer_inv_range; + break; + case DMA_TO_DEVICE: /* writeback only */ + inner_op = dmac_clean_range; + outer_op = outer_clean_range; + break; + case DMA_BIDIRECTIONAL: /* writeback and invalidate */ + inner_op = dmac_flush_range; + outer_op = outer_flush_range; + break; + default: + BUG(); + } + + dma_cache_maint_page(page, off, size, inner_op); + + paddr = page_to_phys(page) + off; + outer_op(paddr, paddr + size); } EXPORT_SYMBOL(___dma_page_cpu_to_dev); -- cgit v1.2.2 From 93f1d629e22b08642eb713ad96ac2cb9ade0641c Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 24 Nov 2009 14:41:01 +0000 Subject: ARM: dma-mapping: simplify dma_cache_maint_page dma_cache_maint_contiguous is now simple enough to live inside dma_cache_maint_page, so move it there. Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/dma-mapping.c | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 77dc483e64c1..0d68d2c83cda 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -446,24 +446,6 @@ void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, } EXPORT_SYMBOL(___dma_single_dev_to_cpu); -static void dma_cache_maint_contiguous(struct page *page, unsigned long offset, - size_t size, void (*op)(const void *, const void *)) -{ - void *vaddr; - - if (!PageHighMem(page)) { - vaddr = page_address(page) + offset; - op(vaddr, vaddr + size); - } else { - vaddr = kmap_high_get(page); - if (vaddr) { - vaddr += offset; - op(vaddr, vaddr + size); - kunmap_high(page); - } - } -} - static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, void (*op)(const void *, const void *)) { @@ -476,14 +458,26 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t left = size; do { size_t len = left; - if (PageHighMem(page) && len + offset > PAGE_SIZE) { - if (offset >= PAGE_SIZE) { - page += offset / PAGE_SIZE; - offset %= PAGE_SIZE; + void *vaddr; + + if (PageHighMem(page)) { + if (len + offset > PAGE_SIZE) { + if (offset >= PAGE_SIZE) { + page += offset / PAGE_SIZE; + offset %= PAGE_SIZE; + } + len = PAGE_SIZE - offset; + } + vaddr = kmap_high_get(page); + if (vaddr) { + vaddr += offset; + op(vaddr, vaddr + len); + kunmap_high(page); } - len = PAGE_SIZE - offset; + } else { + vaddr = page_address(page) + offset; + op(vaddr, vaddr + len); } - dma_cache_maint_contiguous(page, offset, len, op); offset = 0; page++; left -= len; -- cgit v1.2.2 From a9c9147eb9b1dba0ce567a41897c7773b4d1b0bc Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 26 Nov 2009 16:19:58 +0000 Subject: ARM: dma-mapping: provide per-cpu type map/unmap functions Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/cache-fa.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/cache-v3.S | 24 ++++++++++++++++++++++++ arch/arm/mm/cache-v4.S | 24 ++++++++++++++++++++++++ arch/arm/mm/cache-v4wb.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/cache-v4wt.S | 25 +++++++++++++++++++++++++ arch/arm/mm/cache-v6.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/cache-v7.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/dma-mapping.c | 29 ++++++++++++----------------- arch/arm/mm/proc-arm1020.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm1020e.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm1022.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm1026.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm920.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm922.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm925.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm926.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm940.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-arm946.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-feroceon.S | 42 ++++++++++++++++++++++++++++++++++++++++++ arch/arm/mm/proc-mohawk.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-xsc3.S | 26 ++++++++++++++++++++++++++ arch/arm/mm/proc-xscale.S | 41 +++++++++++++++++++++++++++++++++++++++++ 22 files changed, 584 insertions(+), 17 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index a89444a3c016..8ebffdd6fcff 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -205,6 +205,30 @@ ENTRY(fa_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain write buffer mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(fa_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq fa_dma_clean_range + bcs fa_dma_inv_range + b fa_dma_flush_range +ENDPROC(fa_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(fa_dma_unmap_area) + mov pc, lr +ENDPROC(fa_dma_unmap_area) + __INITDATA .type fa_cache_fns, #object @@ -215,6 +239,8 @@ ENTRY(fa_cache_fns) .long fa_coherent_kern_range .long fa_coherent_user_range .long fa_flush_kern_dcache_area + .long fa_dma_map_area + .long fa_dma_unmap_area .long fa_dma_inv_range .long fa_dma_clean_range .long fa_dma_flush_range diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S index 2a482731ea36..6df52dc014be 100644 --- a/arch/arm/mm/cache-v3.S +++ b/arch/arm/mm/cache-v3.S @@ -121,6 +121,28 @@ ENTRY(v3_dma_flush_range) ENTRY(v3_dma_clean_range) mov pc, lr +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v3_dma_unmap_area) + teq r2, #DMA_TO_DEVICE + bne v3_dma_inv_range + /* FALLTHROUGH */ + +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v3_dma_map_area) + mov pc, lr +ENDPROC(v3_dma_unmap_area) +ENDPROC(v3_dma_map_area) + __INITDATA .type v3_cache_fns, #object @@ -131,6 +153,8 @@ ENTRY(v3_cache_fns) .long v3_coherent_kern_range .long v3_coherent_user_range .long v3_flush_kern_dcache_area + .long v3_dma_map_area + .long v3_dma_unmap_area .long v3_dma_inv_range .long v3_dma_clean_range .long v3_dma_flush_range diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index 5c7da3e372e9..df3b423713b9 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -133,6 +133,28 @@ ENTRY(v4_dma_flush_range) ENTRY(v4_dma_clean_range) mov pc, lr +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4_dma_unmap_area) + teq r2, #DMA_TO_DEVICE + bne v4_dma_inv_range + /* FALLTHROUGH */ + +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4_dma_map_area) + mov pc, lr +ENDPROC(v4_dma_unmap_area) +ENDPROC(v4_dma_map_area) + __INITDATA .type v4_cache_fns, #object @@ -143,6 +165,8 @@ ENTRY(v4_cache_fns) .long v4_coherent_kern_range .long v4_coherent_user_range .long v4_flush_kern_dcache_area + .long v4_dma_map_area + .long v4_dma_unmap_area .long v4_dma_inv_range .long v4_dma_clean_range .long v4_dma_flush_range diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index 3dbedf1ec0e7..32e7a7448496 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -216,6 +216,30 @@ ENTRY(v4wb_dma_clean_range) .globl v4wb_dma_flush_range .set v4wb_dma_flush_range, v4wb_coherent_kern_range +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4wb_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq v4wb_dma_clean_range + bcs v4wb_dma_inv_range + b v4wb_dma_flush_range +ENDPROC(v4wb_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4wb_dma_unmap_area) + mov pc, lr +ENDPROC(v4wb_dma_unmap_area) + __INITDATA .type v4wb_cache_fns, #object @@ -226,6 +250,8 @@ ENTRY(v4wb_cache_fns) .long v4wb_coherent_kern_range .long v4wb_coherent_user_range .long v4wb_flush_kern_dcache_area + .long v4wb_dma_map_area + .long v4wb_dma_unmap_area .long v4wb_dma_inv_range .long v4wb_dma_clean_range .long v4wb_dma_flush_range diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index b3b7410270b4..3d8dad5b2650 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -172,6 +172,29 @@ ENTRY(v4wt_dma_clean_range) .globl v4wt_dma_flush_range .equ v4wt_dma_flush_range, v4wt_dma_inv_range +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4wt_dma_unmap_area) + add r1, r1, r0 + teq r2, #DMA_TO_DEVICE + bne v4wt_dma_inv_range + /* FALLTHROUGH */ + +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v4wt_dma_map_area) + mov pc, lr +ENDPROC(v4wt_dma_unmap_area) +ENDPROC(v4wt_dma_map_area) + __INITDATA .type v4wt_cache_fns, #object @@ -182,6 +205,8 @@ ENTRY(v4wt_cache_fns) .long v4wt_coherent_kern_range .long v4wt_coherent_user_range .long v4wt_flush_kern_dcache_area + .long v4wt_dma_map_area + .long v4wt_dma_unmap_area .long v4wt_dma_inv_range .long v4wt_dma_clean_range .long v4wt_dma_flush_range diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 4ba0a24ce6f5..6f926dd0e0f7 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -263,6 +263,30 @@ ENTRY(v6_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain write buffer mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v6_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq v6_dma_clean_range + bcs v6_dma_inv_range + b v6_dma_flush_range +ENDPROC(v6_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v6_dma_unmap_area) + mov pc, lr +ENDPROC(v6_dma_unmap_area) + __INITDATA .type v6_cache_fns, #object @@ -273,6 +297,8 @@ ENTRY(v6_cache_fns) .long v6_coherent_kern_range .long v6_coherent_user_range .long v6_flush_kern_dcache_area + .long v6_dma_map_area + .long v6_dma_unmap_area .long v6_dma_inv_range .long v6_dma_clean_range .long v6_dma_flush_range diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index 9073db849fb4..e30d8bc67182 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -271,6 +271,30 @@ ENTRY(v7_dma_flush_range) mov pc, lr ENDPROC(v7_dma_flush_range) +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v7_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq v7_dma_clean_range + bcs v7_dma_inv_range + b v7_dma_flush_range +ENDPROC(v7_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(v7_dma_unmap_area) + mov pc, lr +ENDPROC(v7_dma_unmap_area) + __INITDATA .type v7_cache_fns, #object @@ -281,6 +305,8 @@ ENTRY(v7_cache_fns) .long v7_coherent_kern_range .long v7_coherent_user_range .long v7_flush_kern_dcache_area + .long v7_dma_map_area + .long v7_dma_unmap_area .long v7_dma_inv_range .long v7_dma_clean_range .long v7_dma_flush_range diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 0d68d2c83cda..efa8efa33f5e 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -406,35 +406,31 @@ EXPORT_SYMBOL(dma_free_coherent); */ static void dma_cache_maint(const void *start, size_t size, int direction) { - void (*inner_op)(const void *, const void *); void (*outer_op)(unsigned long, unsigned long); - BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(start + size - 1)); - switch (direction) { case DMA_FROM_DEVICE: /* invalidate only */ - inner_op = dmac_inv_range; outer_op = outer_inv_range; break; case DMA_TO_DEVICE: /* writeback only */ - inner_op = dmac_clean_range; outer_op = outer_clean_range; break; case DMA_BIDIRECTIONAL: /* writeback and invalidate */ - inner_op = dmac_flush_range; outer_op = outer_flush_range; break; default: BUG(); } - inner_op(start, start + size); outer_op(__pa(start), __pa(start) + size); } void ___dma_single_cpu_to_dev(const void *kaddr, size_t size, enum dma_data_direction dir) { + BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); + + dmac_map_area(kaddr, size, dir); dma_cache_maint(kaddr, size, dir); } EXPORT_SYMBOL(___dma_single_cpu_to_dev); @@ -442,12 +438,15 @@ EXPORT_SYMBOL(___dma_single_cpu_to_dev); void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, enum dma_data_direction dir) { - /* nothing to do */ + BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); + + dmac_unmap_area(kaddr, size, dir); } EXPORT_SYMBOL(___dma_single_dev_to_cpu); static void dma_cache_maint_page(struct page *page, unsigned long offset, - size_t size, void (*op)(const void *, const void *)) + size_t size, enum dma_data_direction dir, + void (*op)(const void *, size_t, int)) { /* * A single sg entry may refer to multiple physically contiguous @@ -471,12 +470,12 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, vaddr = kmap_high_get(page); if (vaddr) { vaddr += offset; - op(vaddr, vaddr + len); + op(vaddr, len, dir); kunmap_high(page); } } else { vaddr = page_address(page) + offset; - op(vaddr, vaddr + len); + op(vaddr, len, dir); } offset = 0; page++; @@ -488,27 +487,23 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, size_t size, enum dma_data_direction dir) { unsigned long paddr; - void (*inner_op)(const void *, const void *); void (*outer_op)(unsigned long, unsigned long); switch (direction) { case DMA_FROM_DEVICE: /* invalidate only */ - inner_op = dmac_inv_range; outer_op = outer_inv_range; break; case DMA_TO_DEVICE: /* writeback only */ - inner_op = dmac_clean_range; outer_op = outer_clean_range; break; case DMA_BIDIRECTIONAL: /* writeback and invalidate */ - inner_op = dmac_flush_range; outer_op = outer_flush_range; break; default: BUG(); } - dma_cache_maint_page(page, off, size, inner_op); + dma_cache_maint_page(page, off, size, dir, dmac_map_area); paddr = page_to_phys(page) + off; outer_op(paddr, paddr + size); @@ -518,7 +513,7 @@ EXPORT_SYMBOL(___dma_page_cpu_to_dev); void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, size_t size, enum dma_data_direction dir) { - /* nothing to do */ + dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); } EXPORT_SYMBOL(___dma_page_dev_to_cpu); diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index 8012e24282b2..c85f5eb42634 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -330,6 +330,30 @@ ENTRY(arm1020_dma_flush_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1020_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm1020_dma_clean_range + bcs arm1020_dma_inv_range + b arm1020_dma_flush_range +ENDPROC(arm1020_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1020_dma_unmap_area) + mov pc, lr +ENDPROC(arm1020_dma_unmap_area) + ENTRY(arm1020_cache_fns) .long arm1020_flush_kern_cache_all .long arm1020_flush_user_cache_all @@ -337,6 +361,8 @@ ENTRY(arm1020_cache_fns) .long arm1020_coherent_kern_range .long arm1020_coherent_user_range .long arm1020_flush_kern_dcache_area + .long arm1020_dma_map_area + .long arm1020_dma_unmap_area .long arm1020_dma_inv_range .long arm1020_dma_clean_range .long arm1020_dma_flush_range diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index 41fe25d234f5..5a3cf7620a2c 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -316,6 +316,30 @@ ENTRY(arm1020e_dma_flush_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1020e_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm1020e_dma_clean_range + bcs arm1020e_dma_inv_range + b arm1020e_dma_flush_range +ENDPROC(arm1020e_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1020e_dma_unmap_area) + mov pc, lr +ENDPROC(arm1020e_dma_unmap_area) + ENTRY(arm1020e_cache_fns) .long arm1020e_flush_kern_cache_all .long arm1020e_flush_user_cache_all @@ -323,6 +347,8 @@ ENTRY(arm1020e_cache_fns) .long arm1020e_coherent_kern_range .long arm1020e_coherent_user_range .long arm1020e_flush_kern_dcache_area + .long arm1020e_dma_map_area + .long arm1020e_dma_unmap_area .long arm1020e_dma_inv_range .long arm1020e_dma_clean_range .long arm1020e_dma_flush_range diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index 20a5b1b31a70..fec8f5878438 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -305,6 +305,30 @@ ENTRY(arm1022_dma_flush_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1022_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm1022_dma_clean_range + bcs arm1022_dma_inv_range + b arm1022_dma_flush_range +ENDPROC(arm1022_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1022_dma_unmap_area) + mov pc, lr +ENDPROC(arm1022_dma_unmap_area) + ENTRY(arm1022_cache_fns) .long arm1022_flush_kern_cache_all .long arm1022_flush_user_cache_all @@ -312,6 +336,8 @@ ENTRY(arm1022_cache_fns) .long arm1022_coherent_kern_range .long arm1022_coherent_user_range .long arm1022_flush_kern_dcache_area + .long arm1022_dma_map_area + .long arm1022_dma_unmap_area .long arm1022_dma_inv_range .long arm1022_dma_clean_range .long arm1022_dma_flush_range diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index 96aedb10fcc4..9ece6f666497 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -299,6 +299,30 @@ ENTRY(arm1026_dma_flush_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1026_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm1026_dma_clean_range + bcs arm1026_dma_inv_range + b arm1026_dma_flush_range +ENDPROC(arm1026_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm1026_dma_unmap_area) + mov pc, lr +ENDPROC(arm1026_dma_unmap_area) + ENTRY(arm1026_cache_fns) .long arm1026_flush_kern_cache_all .long arm1026_flush_user_cache_all @@ -306,6 +330,8 @@ ENTRY(arm1026_cache_fns) .long arm1026_coherent_kern_range .long arm1026_coherent_user_range .long arm1026_flush_kern_dcache_area + .long arm1026_dma_map_area + .long arm1026_dma_unmap_area .long arm1026_dma_inv_range .long arm1026_dma_clean_range .long arm1026_dma_flush_range diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 471669e2d7cb..6f6ab2747da6 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -288,6 +288,30 @@ ENTRY(arm920_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm920_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm920_dma_clean_range + bcs arm920_dma_inv_range + b arm920_dma_flush_range +ENDPROC(arm920_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm920_dma_unmap_area) + mov pc, lr +ENDPROC(arm920_dma_unmap_area) + ENTRY(arm920_cache_fns) .long arm920_flush_kern_cache_all .long arm920_flush_user_cache_all @@ -295,6 +319,8 @@ ENTRY(arm920_cache_fns) .long arm920_coherent_kern_range .long arm920_coherent_user_range .long arm920_flush_kern_dcache_area + .long arm920_dma_map_area + .long arm920_dma_unmap_area .long arm920_dma_inv_range .long arm920_dma_clean_range .long arm920_dma_flush_range diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index ee111b00fa41..4e4396b121ca 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -290,6 +290,30 @@ ENTRY(arm922_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm922_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm922_dma_clean_range + bcs arm922_dma_inv_range + b arm922_dma_flush_range +ENDPROC(arm922_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm922_dma_unmap_area) + mov pc, lr +ENDPROC(arm922_dma_unmap_area) + ENTRY(arm922_cache_fns) .long arm922_flush_kern_cache_all .long arm922_flush_user_cache_all @@ -297,6 +321,8 @@ ENTRY(arm922_cache_fns) .long arm922_coherent_kern_range .long arm922_coherent_user_range .long arm922_flush_kern_dcache_area + .long arm922_dma_map_area + .long arm922_dma_unmap_area .long arm922_dma_inv_range .long arm922_dma_clean_range .long arm922_dma_flush_range diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index 8deb5bde58e4..7c01c5d1108c 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -341,6 +341,30 @@ ENTRY(arm925_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm925_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm925_dma_clean_range + bcs arm925_dma_inv_range + b arm925_dma_flush_range +ENDPROC(arm925_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm925_dma_unmap_area) + mov pc, lr +ENDPROC(arm925_dma_unmap_area) + ENTRY(arm925_cache_fns) .long arm925_flush_kern_cache_all .long arm925_flush_user_cache_all @@ -348,6 +372,8 @@ ENTRY(arm925_cache_fns) .long arm925_coherent_kern_range .long arm925_coherent_user_range .long arm925_flush_kern_dcache_area + .long arm925_dma_map_area + .long arm925_dma_unmap_area .long arm925_dma_inv_range .long arm925_dma_clean_range .long arm925_dma_flush_range diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 64db6e275a44..72a01a4b80ab 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -304,6 +304,30 @@ ENTRY(arm926_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm926_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm926_dma_clean_range + bcs arm926_dma_inv_range + b arm926_dma_flush_range +ENDPROC(arm926_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm926_dma_unmap_area) + mov pc, lr +ENDPROC(arm926_dma_unmap_area) + ENTRY(arm926_cache_fns) .long arm926_flush_kern_cache_all .long arm926_flush_user_cache_all @@ -311,6 +335,8 @@ ENTRY(arm926_cache_fns) .long arm926_coherent_kern_range .long arm926_coherent_user_range .long arm926_flush_kern_dcache_area + .long arm926_dma_map_area + .long arm926_dma_unmap_area .long arm926_dma_inv_range .long arm926_dma_clean_range .long arm926_dma_flush_range diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index 8196b9f401fb..6bb58fca7270 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -233,6 +233,30 @@ ENTRY(arm940_dma_flush_range) mcr p15, 0, ip, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm940_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm940_dma_clean_range + bcs arm940_dma_inv_range + b arm940_dma_flush_range +ENDPROC(arm940_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm940_dma_unmap_area) + mov pc, lr +ENDPROC(arm940_dma_unmap_area) + ENTRY(arm940_cache_fns) .long arm940_flush_kern_cache_all .long arm940_flush_user_cache_all @@ -240,6 +264,8 @@ ENTRY(arm940_cache_fns) .long arm940_coherent_kern_range .long arm940_coherent_user_range .long arm940_flush_kern_dcache_area + .long arm940_dma_map_area + .long arm940_dma_unmap_area .long arm940_dma_inv_range .long arm940_dma_clean_range .long arm940_dma_flush_range diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index 9a951239c86c..ac0f9ba719d7 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -275,6 +275,30 @@ ENTRY(arm946_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm946_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq arm946_dma_clean_range + bcs arm946_dma_inv_range + b arm946_dma_flush_range +ENDPROC(arm946_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(arm946_dma_unmap_area) + mov pc, lr +ENDPROC(arm946_dma_unmap_area) + ENTRY(arm946_cache_fns) .long arm946_flush_kern_cache_all .long arm946_flush_user_cache_all @@ -282,6 +306,8 @@ ENTRY(arm946_cache_fns) .long arm946_coherent_kern_range .long arm946_coherent_user_range .long arm946_flush_kern_dcache_area + .long arm946_dma_map_area + .long arm946_dma_unmap_area .long arm946_dma_inv_range .long arm946_dma_clean_range .long arm946_dma_flush_range diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index dbc39383e66a..97e1d784f152 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -367,6 +367,44 @@ ENTRY(feroceon_range_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(feroceon_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq feroceon_dma_clean_range + bcs feroceon_dma_inv_range + b feroceon_dma_flush_range +ENDPROC(feroceon_dma_map_area) + +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(feroceon_range_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq feroceon_range_dma_clean_range + bcs feroceon_range_dma_inv_range + b feroceon_range_dma_flush_range +ENDPROC(feroceon_range_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(feroceon_dma_unmap_area) + mov pc, lr +ENDPROC(feroceon_dma_unmap_area) + ENTRY(feroceon_cache_fns) .long feroceon_flush_kern_cache_all .long feroceon_flush_user_cache_all @@ -374,6 +412,8 @@ ENTRY(feroceon_cache_fns) .long feroceon_coherent_kern_range .long feroceon_coherent_user_range .long feroceon_flush_kern_dcache_area + .long feroceon_dma_map_area + .long feroceon_dma_unmap_area .long feroceon_dma_inv_range .long feroceon_dma_clean_range .long feroceon_dma_flush_range @@ -385,6 +425,8 @@ ENTRY(feroceon_range_cache_fns) .long feroceon_coherent_kern_range .long feroceon_coherent_user_range .long feroceon_range_flush_kern_dcache_area + .long feroceon_range_dma_map_area + .long feroceon_dma_unmap_area .long feroceon_range_dma_inv_range .long feroceon_range_dma_clean_range .long feroceon_range_dma_flush_range diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index 9674d36cc97d..55b7fbec6548 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -268,6 +268,30 @@ ENTRY(mohawk_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ drain WB mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(mohawk_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq mohawk_dma_clean_range + bcs mohawk_dma_inv_range + b mohawk_dma_flush_range +ENDPROC(mohawk_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(mohawk_dma_unmap_area) + mov pc, lr +ENDPROC(mohawk_dma_unmap_area) + ENTRY(mohawk_cache_fns) .long mohawk_flush_kern_cache_all .long mohawk_flush_user_cache_all @@ -275,6 +299,8 @@ ENTRY(mohawk_cache_fns) .long mohawk_coherent_kern_range .long mohawk_coherent_user_range .long mohawk_flush_kern_dcache_area + .long mohawk_dma_map_area + .long mohawk_dma_unmap_area .long mohawk_dma_inv_range .long mohawk_dma_clean_range .long mohawk_dma_flush_range diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index 96456f548798..4e4ce889b3e6 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -304,6 +304,30 @@ ENTRY(xsc3_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ data write barrier mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(xsc3_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq xsc3_dma_clean_range + bcs xsc3_dma_inv_range + b xsc3_dma_flush_range +ENDPROC(xsc3_dma_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(xsc3_dma_unmap_area) + mov pc, lr +ENDPROC(xsc3_dma_unmap_area) + ENTRY(xsc3_cache_fns) .long xsc3_flush_kern_cache_all .long xsc3_flush_user_cache_all @@ -311,6 +335,8 @@ ENTRY(xsc3_cache_fns) .long xsc3_coherent_kern_range .long xsc3_coherent_user_range .long xsc3_flush_kern_dcache_area + .long xsc3_dma_map_area + .long xsc3_dma_unmap_area .long xsc3_dma_inv_range .long xsc3_dma_clean_range .long xsc3_dma_flush_range diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index 93df47265f2d..a7999f94bf27 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -363,6 +363,43 @@ ENTRY(xscale_dma_flush_range) mcr p15, 0, r0, c7, c10, 4 @ Drain Write (& Fill) Buffer mov pc, lr +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(xscale_dma_map_area) + add r1, r1, r0 + cmp r2, #DMA_TO_DEVICE + beq xscale_dma_clean_range + bcs xscale_dma_inv_range + b xscale_dma_flush_range +ENDPROC(xscale_dma_map_area) + +/* + * dma_map_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(xscale_dma_a0_map_area) + add r1, r1, r0 + teq r2, #DMA_TO_DEVICE + beq xscale_dma_clean_range + b xscale_dma_flush_range +ENDPROC(xscsale_dma_a0_map_area) + +/* + * dma_unmap_area(start, size, dir) + * - start - kernel virtual start address + * - size - size of region + * - dir - DMA direction + */ +ENTRY(xscale_dma_unmap_area) + mov pc, lr +ENDPROC(xscale_dma_unmap_area) + ENTRY(xscale_cache_fns) .long xscale_flush_kern_cache_all .long xscale_flush_user_cache_all @@ -370,6 +407,8 @@ ENTRY(xscale_cache_fns) .long xscale_coherent_kern_range .long xscale_coherent_user_range .long xscale_flush_kern_dcache_area + .long xscale_dma_map_area + .long xscale_dma_unmap_area .long xscale_dma_inv_range .long xscale_dma_clean_range .long xscale_dma_flush_range @@ -394,6 +433,8 @@ ENTRY(xscale_80200_A0_A1_cache_fns) .long xscale_coherent_kern_range .long xscale_coherent_user_range .long xscale_flush_kern_dcache_area + .long xscale_dma_a0_map_area + .long xscale_dma_unmap_area .long xscale_dma_flush_range .long xscale_dma_clean_range .long xscale_dma_flush_range -- cgit v1.2.2 From 702b94bff3c50542a6e4ab9a4f4cef093262fe65 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 26 Nov 2009 16:24:19 +0000 Subject: ARM: dma-mapping: remove dmac_clean_range and dmac_inv_range These are now unused, and so can be removed. Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/cache-fa.S | 6 ++---- arch/arm/mm/cache-v3.S | 29 +---------------------------- arch/arm/mm/cache-v4.S | 29 +---------------------------- arch/arm/mm/cache-v4wb.S | 6 ++---- arch/arm/mm/cache-v4wt.S | 15 +-------------- arch/arm/mm/cache-v6.S | 6 ++---- arch/arm/mm/cache-v7.S | 6 ++---- arch/arm/mm/proc-arm1020.S | 6 ++---- arch/arm/mm/proc-arm1020e.S | 6 ++---- arch/arm/mm/proc-arm1022.S | 6 ++---- arch/arm/mm/proc-arm1026.S | 6 ++---- arch/arm/mm/proc-arm920.S | 6 ++---- arch/arm/mm/proc-arm922.S | 6 ++---- arch/arm/mm/proc-arm925.S | 6 ++---- arch/arm/mm/proc-arm926.S | 6 ++---- arch/arm/mm/proc-arm940.S | 6 ++---- arch/arm/mm/proc-arm946.S | 6 ++---- arch/arm/mm/proc-feroceon.S | 12 ++++-------- arch/arm/mm/proc-mohawk.S | 6 ++---- arch/arm/mm/proc-xsc3.S | 6 ++---- arch/arm/mm/proc-xscale.S | 8 ++------ 21 files changed, 41 insertions(+), 148 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index 8ebffdd6fcff..7148e53e6078 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -157,7 +157,7 @@ ENTRY(fa_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_dma_inv_range) +fa_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 bic r0, r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D entry @@ -180,7 +180,7 @@ ENTRY(fa_dma_inv_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(fa_dma_clean_range) +fa_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -241,7 +241,5 @@ ENTRY(fa_cache_fns) .long fa_flush_kern_dcache_area .long fa_dma_map_area .long fa_dma_unmap_area - .long fa_dma_inv_range - .long fa_dma_clean_range .long fa_dma_flush_range .size fa_cache_fns, . - fa_cache_fns diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S index 6df52dc014be..c2ff3c599fee 100644 --- a/arch/arm/mm/cache-v3.S +++ b/arch/arm/mm/cache-v3.S @@ -83,20 +83,6 @@ ENTRY(v3_coherent_user_range) ENTRY(v3_flush_kern_dcache_area) /* FALLTHROUGH */ -/* - * dma_inv_range(start, end) - * - * Invalidate (discard) the specified virtual address range. - * May not write back any entries. If 'start' or 'end' - * are not cache line aligned, those lines must be written - * back. - * - * - start - virtual start address - * - end - virtual end address - */ -ENTRY(v3_dma_inv_range) - /* FALLTHROUGH */ - /* * dma_flush_range(start, end) * @@ -108,17 +94,6 @@ ENTRY(v3_dma_inv_range) ENTRY(v3_dma_flush_range) mov r0, #0 mcr p15, 0, r0, c7, c0, 0 @ flush ID cache - /* FALLTHROUGH */ - -/* - * dma_clean_range(start, end) - * - * Clean (write back) the specified virtual address range. - * - * - start - virtual start address - * - end - virtual end address - */ -ENTRY(v3_dma_clean_range) mov pc, lr /* @@ -129,7 +104,7 @@ ENTRY(v3_dma_clean_range) */ ENTRY(v3_dma_unmap_area) teq r2, #DMA_TO_DEVICE - bne v3_dma_inv_range + bne v3_dma_flush_range /* FALLTHROUGH */ /* @@ -155,7 +130,5 @@ ENTRY(v3_cache_fns) .long v3_flush_kern_dcache_area .long v3_dma_map_area .long v3_dma_unmap_area - .long v3_dma_inv_range - .long v3_dma_clean_range .long v3_dma_flush_range .size v3_cache_fns, . - v3_cache_fns diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index df3b423713b9..4810f7e3e813 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -93,20 +93,6 @@ ENTRY(v4_coherent_user_range) ENTRY(v4_flush_kern_dcache_area) /* FALLTHROUGH */ -/* - * dma_inv_range(start, end) - * - * Invalidate (discard) the specified virtual address range. - * May not write back any entries. If 'start' or 'end' - * are not cache line aligned, those lines must be written - * back. - * - * - start - virtual start address - * - end - virtual end address - */ -ENTRY(v4_dma_inv_range) - /* FALLTHROUGH */ - /* * dma_flush_range(start, end) * @@ -120,17 +106,6 @@ ENTRY(v4_dma_flush_range) mov r0, #0 mcr p15, 0, r0, c7, c7, 0 @ flush ID cache #endif - /* FALLTHROUGH */ - -/* - * dma_clean_range(start, end) - * - * Clean (write back) the specified virtual address range. - * - * - start - virtual start address - * - end - virtual end address - */ -ENTRY(v4_dma_clean_range) mov pc, lr /* @@ -141,7 +116,7 @@ ENTRY(v4_dma_clean_range) */ ENTRY(v4_dma_unmap_area) teq r2, #DMA_TO_DEVICE - bne v4_dma_inv_range + bne v4_dma_flush_range /* FALLTHROUGH */ /* @@ -167,7 +142,5 @@ ENTRY(v4_cache_fns) .long v4_flush_kern_dcache_area .long v4_dma_map_area .long v4_dma_unmap_area - .long v4_dma_inv_range - .long v4_dma_clean_range .long v4_dma_flush_range .size v4_cache_fns, . - v4_cache_fns diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index 32e7a7448496..df8368afa102 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -173,7 +173,7 @@ ENTRY(v4wb_coherent_user_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wb_dma_inv_range) +v4wb_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 bic r0, r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -194,7 +194,7 @@ ENTRY(v4wb_dma_inv_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wb_dma_clean_range) +v4wb_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -252,7 +252,5 @@ ENTRY(v4wb_cache_fns) .long v4wb_flush_kern_dcache_area .long v4wb_dma_map_area .long v4wb_dma_unmap_area - .long v4wb_dma_inv_range - .long v4wb_dma_clean_range .long v4wb_dma_flush_range .size v4wb_cache_fns, . - v4wb_cache_fns diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index 3d8dad5b2650..45c70312f43b 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -142,23 +142,12 @@ ENTRY(v4wt_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(v4wt_dma_inv_range) +v4wt_dma_inv_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 blo 1b - /* FALLTHROUGH */ - -/* - * dma_clean_range(start, end) - * - * Clean the specified virtual address range. - * - * - start - virtual start address - * - end - virtual end address - */ -ENTRY(v4wt_dma_clean_range) mov pc, lr /* @@ -207,7 +196,5 @@ ENTRY(v4wt_cache_fns) .long v4wt_flush_kern_dcache_area .long v4wt_dma_map_area .long v4wt_dma_unmap_area - .long v4wt_dma_inv_range - .long v4wt_dma_clean_range .long v4wt_dma_flush_range .size v4wt_cache_fns, . - v4wt_cache_fns diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 6f926dd0e0f7..a11934e53fbd 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -195,7 +195,7 @@ ENTRY(v6_flush_kern_dcache_area) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v6_dma_inv_range) +v6_dma_inv_range: tst r0, #D_CACHE_LINE_SIZE - 1 bic r0, r0, #D_CACHE_LINE_SIZE - 1 #ifdef HARVARD_CACHE @@ -228,7 +228,7 @@ ENTRY(v6_dma_inv_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v6_dma_clean_range) +v6_dma_clean_range: bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: #ifdef HARVARD_CACHE @@ -299,7 +299,5 @@ ENTRY(v6_cache_fns) .long v6_flush_kern_dcache_area .long v6_dma_map_area .long v6_dma_unmap_area - .long v6_dma_inv_range - .long v6_dma_clean_range .long v6_dma_flush_range .size v6_cache_fns, . - v6_cache_fns diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index e30d8bc67182..b1cd0fd91207 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -216,7 +216,7 @@ ENDPROC(v7_flush_kern_dcache_area) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v7_dma_inv_range) +v7_dma_inv_range: dcache_line_size r2, r3 sub r3, r2, #1 tst r0, r3 @@ -240,7 +240,7 @@ ENDPROC(v7_dma_inv_range) * - start - virtual start address of region * - end - virtual end address of region */ -ENTRY(v7_dma_clean_range) +v7_dma_clean_range: dcache_line_size r2, r3 sub r3, r2, #1 bic r0, r0, r3 @@ -307,7 +307,5 @@ ENTRY(v7_cache_fns) .long v7_flush_kern_dcache_area .long v7_dma_map_area .long v7_dma_unmap_area - .long v7_dma_inv_range - .long v7_dma_clean_range .long v7_dma_flush_range .size v7_cache_fns, . - v7_cache_fns diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index c85f5eb42634..72507c630ceb 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -265,7 +265,7 @@ ENTRY(arm1020_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm1020_dma_inv_range) +arm1020_dma_inv_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE tst r0, #CACHE_DLINESIZE - 1 @@ -295,7 +295,7 @@ ENTRY(arm1020_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm1020_dma_clean_range) +arm1020_dma_clean_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -363,8 +363,6 @@ ENTRY(arm1020_cache_fns) .long arm1020_flush_kern_dcache_area .long arm1020_dma_map_area .long arm1020_dma_unmap_area - .long arm1020_dma_inv_range - .long arm1020_dma_clean_range .long arm1020_dma_flush_range .align 5 diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index 5a3cf7620a2c..d27829805609 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -258,7 +258,7 @@ ENTRY(arm1020e_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm1020e_dma_inv_range) +arm1020e_dma_inv_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE tst r0, #CACHE_DLINESIZE - 1 @@ -284,7 +284,7 @@ ENTRY(arm1020e_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm1020e_dma_clean_range) +arm1020e_dma_clean_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -349,8 +349,6 @@ ENTRY(arm1020e_cache_fns) .long arm1020e_flush_kern_dcache_area .long arm1020e_dma_map_area .long arm1020e_dma_unmap_area - .long arm1020e_dma_inv_range - .long arm1020e_dma_clean_range .long arm1020e_dma_flush_range .align 5 diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index fec8f5878438..ce13e4a827de 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -247,7 +247,7 @@ ENTRY(arm1022_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm1022_dma_inv_range) +arm1022_dma_inv_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE tst r0, #CACHE_DLINESIZE - 1 @@ -273,7 +273,7 @@ ENTRY(arm1022_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm1022_dma_clean_range) +arm1022_dma_clean_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -338,8 +338,6 @@ ENTRY(arm1022_cache_fns) .long arm1022_flush_kern_dcache_area .long arm1022_dma_map_area .long arm1022_dma_unmap_area - .long arm1022_dma_inv_range - .long arm1022_dma_clean_range .long arm1022_dma_flush_range .align 5 diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index 9ece6f666497..636672a29c6d 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -241,7 +241,7 @@ ENTRY(arm1026_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm1026_dma_inv_range) +arm1026_dma_inv_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE tst r0, #CACHE_DLINESIZE - 1 @@ -267,7 +267,7 @@ ENTRY(arm1026_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm1026_dma_clean_range) +arm1026_dma_clean_range: mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE bic r0, r0, #CACHE_DLINESIZE - 1 @@ -332,8 +332,6 @@ ENTRY(arm1026_cache_fns) .long arm1026_flush_kern_dcache_area .long arm1026_dma_map_area .long arm1026_dma_unmap_area - .long arm1026_dma_inv_range - .long arm1026_dma_clean_range .long arm1026_dma_flush_range .align 5 diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 6f6ab2747da6..8be81992645d 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -239,7 +239,7 @@ ENTRY(arm920_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm920_dma_inv_range) +arm920_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 bic r0, r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -262,7 +262,7 @@ ENTRY(arm920_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm920_dma_clean_range) +arm920_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -321,8 +321,6 @@ ENTRY(arm920_cache_fns) .long arm920_flush_kern_dcache_area .long arm920_dma_map_area .long arm920_dma_unmap_area - .long arm920_dma_inv_range - .long arm920_dma_clean_range .long arm920_dma_flush_range #endif diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index 4e4396b121ca..c0ff8e4b1074 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -241,7 +241,7 @@ ENTRY(arm922_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm922_dma_inv_range) +arm922_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 bic r0, r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -264,7 +264,7 @@ ENTRY(arm922_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm922_dma_clean_range) +arm922_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -323,8 +323,6 @@ ENTRY(arm922_cache_fns) .long arm922_flush_kern_dcache_area .long arm922_dma_map_area .long arm922_dma_unmap_area - .long arm922_dma_inv_range - .long arm922_dma_clean_range .long arm922_dma_flush_range #endif diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index 7c01c5d1108c..3c6cffe400f6 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -283,7 +283,7 @@ ENTRY(arm925_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm925_dma_inv_range) +arm925_dma_inv_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH tst r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -308,7 +308,7 @@ ENTRY(arm925_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm925_dma_clean_range) +arm925_dma_clean_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -374,8 +374,6 @@ ENTRY(arm925_cache_fns) .long arm925_flush_kern_dcache_area .long arm925_dma_map_area .long arm925_dma_unmap_area - .long arm925_dma_inv_range - .long arm925_dma_clean_range .long arm925_dma_flush_range ENTRY(cpu_arm925_dcache_clean_area) diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 72a01a4b80ab..75b707c9cce1 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -246,7 +246,7 @@ ENTRY(arm926_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(arm926_dma_inv_range) +arm926_dma_inv_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH tst r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -271,7 +271,7 @@ ENTRY(arm926_dma_inv_range) * * (same as v4wb) */ -ENTRY(arm926_dma_clean_range) +arm926_dma_clean_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -337,8 +337,6 @@ ENTRY(arm926_cache_fns) .long arm926_flush_kern_dcache_area .long arm926_dma_map_area .long arm926_dma_unmap_area - .long arm926_dma_inv_range - .long arm926_dma_clean_range .long arm926_dma_flush_range ENTRY(cpu_arm926_dcache_clean_area) diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index 6bb58fca7270..1af1657819eb 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -171,7 +171,7 @@ ENTRY(arm940_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm940_dma_inv_range) +arm940_dma_inv_range: mov ip, #0 mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments 1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries @@ -192,7 +192,7 @@ ENTRY(arm940_dma_inv_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(arm940_dma_clean_range) +arm940_dma_clean_range: ENTRY(cpu_arm940_dcache_clean_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH @@ -266,8 +266,6 @@ ENTRY(arm940_cache_fns) .long arm940_flush_kern_dcache_area .long arm940_dma_map_area .long arm940_dma_unmap_area - .long arm940_dma_inv_range - .long arm940_dma_clean_range .long arm940_dma_flush_range __INIT diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index ac0f9ba719d7..1664b6aaff79 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -215,7 +215,7 @@ ENTRY(arm946_flush_kern_dcache_area) * - end - virtual end address * (same as arm926) */ -ENTRY(arm946_dma_inv_range) +arm946_dma_inv_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH tst r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -240,7 +240,7 @@ ENTRY(arm946_dma_inv_range) * * (same as arm926) */ -ENTRY(arm946_dma_clean_range) +arm946_dma_clean_range: #ifndef CONFIG_CPU_DCACHE_WRITETHROUGH bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry @@ -308,8 +308,6 @@ ENTRY(arm946_cache_fns) .long arm946_flush_kern_dcache_area .long arm946_dma_map_area .long arm946_dma_unmap_area - .long arm946_dma_inv_range - .long arm946_dma_clean_range .long arm946_dma_flush_range diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index 97e1d784f152..53e632343849 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -274,7 +274,7 @@ ENTRY(feroceon_range_flush_kern_dcache_area) * (same as v4wb) */ .align 5 -ENTRY(feroceon_dma_inv_range) +feroceon_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 bic r0, r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -288,7 +288,7 @@ ENTRY(feroceon_dma_inv_range) mov pc, lr .align 5 -ENTRY(feroceon_range_dma_inv_range) +feroceon_range_dma_inv_range: mrs r2, cpsr tst r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -314,7 +314,7 @@ ENTRY(feroceon_range_dma_inv_range) * (same as v4wb) */ .align 5 -ENTRY(feroceon_dma_clean_range) +feroceon_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -324,7 +324,7 @@ ENTRY(feroceon_dma_clean_range) mov pc, lr .align 5 -ENTRY(feroceon_range_dma_clean_range) +feroceon_range_dma_clean_range: mrs r2, cpsr cmp r1, r0 subne r1, r1, #1 @ top address is inclusive @@ -414,8 +414,6 @@ ENTRY(feroceon_cache_fns) .long feroceon_flush_kern_dcache_area .long feroceon_dma_map_area .long feroceon_dma_unmap_area - .long feroceon_dma_inv_range - .long feroceon_dma_clean_range .long feroceon_dma_flush_range ENTRY(feroceon_range_cache_fns) @@ -427,8 +425,6 @@ ENTRY(feroceon_range_cache_fns) .long feroceon_range_flush_kern_dcache_area .long feroceon_range_dma_map_area .long feroceon_dma_unmap_area - .long feroceon_range_dma_inv_range - .long feroceon_range_dma_clean_range .long feroceon_range_dma_flush_range .align 5 diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index 55b7fbec6548..caa31154e7db 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -218,7 +218,7 @@ ENTRY(mohawk_flush_kern_dcache_area) * * (same as v4wb) */ -ENTRY(mohawk_dma_inv_range) +mohawk_dma_inv_range: tst r0, #CACHE_DLINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry tst r1, #CACHE_DLINESIZE - 1 @@ -241,7 +241,7 @@ ENTRY(mohawk_dma_inv_range) * * (same as v4wb) */ -ENTRY(mohawk_dma_clean_range) +mohawk_dma_clean_range: bic r0, r0, #CACHE_DLINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHE_DLINESIZE @@ -301,8 +301,6 @@ ENTRY(mohawk_cache_fns) .long mohawk_flush_kern_dcache_area .long mohawk_dma_map_area .long mohawk_dma_unmap_area - .long mohawk_dma_inv_range - .long mohawk_dma_clean_range .long mohawk_dma_flush_range ENTRY(cpu_mohawk_dcache_clean_area) diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index 4e4ce889b3e6..046b3d88955e 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -257,7 +257,7 @@ ENTRY(xsc3_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(xsc3_dma_inv_range) +xsc3_dma_inv_range: tst r0, #CACHELINESIZE - 1 bic r0, r0, #CACHELINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean L1 D line @@ -278,7 +278,7 @@ ENTRY(xsc3_dma_inv_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(xsc3_dma_clean_range) +xsc3_dma_clean_range: bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean L1 D line add r0, r0, #CACHELINESIZE @@ -337,8 +337,6 @@ ENTRY(xsc3_cache_fns) .long xsc3_flush_kern_dcache_area .long xsc3_dma_map_area .long xsc3_dma_unmap_area - .long xsc3_dma_inv_range - .long xsc3_dma_clean_range .long xsc3_dma_flush_range ENTRY(cpu_xsc3_dcache_clean_area) diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index a7999f94bf27..63037e2162f2 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -315,7 +315,7 @@ ENTRY(xscale_flush_kern_dcache_area) * - start - virtual start address * - end - virtual end address */ -ENTRY(xscale_dma_inv_range) +xscale_dma_inv_range: tst r0, #CACHELINESIZE - 1 bic r0, r0, #CACHELINESIZE - 1 mcrne p15, 0, r0, c7, c10, 1 @ clean D entry @@ -336,7 +336,7 @@ ENTRY(xscale_dma_inv_range) * - start - virtual start address * - end - virtual end address */ -ENTRY(xscale_dma_clean_range) +xscale_dma_clean_range: bic r0, r0, #CACHELINESIZE - 1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry add r0, r0, #CACHELINESIZE @@ -409,8 +409,6 @@ ENTRY(xscale_cache_fns) .long xscale_flush_kern_dcache_area .long xscale_dma_map_area .long xscale_dma_unmap_area - .long xscale_dma_inv_range - .long xscale_dma_clean_range .long xscale_dma_flush_range /* @@ -436,8 +434,6 @@ ENTRY(xscale_80200_A0_A1_cache_fns) .long xscale_dma_a0_map_area .long xscale_dma_unmap_area .long xscale_dma_flush_range - .long xscale_dma_clean_range - .long xscale_dma_flush_range ENTRY(cpu_xscale_dcache_clean_area) 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry -- cgit v1.2.2 From 2ffe2da3e71652d4f4cae19539b5c78c2a239136 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 31 Oct 2009 16:52:16 +0000 Subject: ARM: dma-mapping: fix for speculative prefetching ARMv6 and ARMv7 CPUs can perform speculative prefetching, which makes DMA cache coherency handling slightly more interesting. Rather than being able to rely upon the CPU not accessing the DMA buffer until DMA has completed, we now must expect that the cache could be loaded with possibly stale data from the DMA buffer. Where DMA involves data being transferred to the device, we clean the cache before handing it over for DMA, otherwise we invalidate the buffer to get rid of potential writebacks. On DMA Completion, if data was transferred from the device, we invalidate the buffer to get rid of any stale speculative prefetches. Signed-off-by: Russell King Tested-By: Santosh Shilimkar --- arch/arm/mm/cache-v6.S | 10 ++++--- arch/arm/mm/cache-v7.S | 10 ++++--- arch/arm/mm/dma-mapping.c | 68 +++++++++++++++++++++-------------------------- 3 files changed, 42 insertions(+), 46 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index a11934e53fbd..9d89c67a1cc3 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -271,10 +271,9 @@ ENTRY(v6_dma_flush_range) */ ENTRY(v6_dma_map_area) add r1, r1, r0 - cmp r2, #DMA_TO_DEVICE - beq v6_dma_clean_range - bcs v6_dma_inv_range - b v6_dma_flush_range + teq r2, #DMA_FROM_DEVICE + beq v6_dma_inv_range + b v6_dma_clean_range ENDPROC(v6_dma_map_area) /* @@ -284,6 +283,9 @@ ENDPROC(v6_dma_map_area) * - dir - DMA direction */ ENTRY(v6_dma_unmap_area) + add r1, r1, r0 + teq r2, #DMA_TO_DEVICE + bne v6_dma_inv_range mov pc, lr ENDPROC(v6_dma_unmap_area) diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index b1cd0fd91207..bcd64f265870 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -279,10 +279,9 @@ ENDPROC(v7_dma_flush_range) */ ENTRY(v7_dma_map_area) add r1, r1, r0 - cmp r2, #DMA_TO_DEVICE - beq v7_dma_clean_range - bcs v7_dma_inv_range - b v7_dma_flush_range + teq r2, #DMA_FROM_DEVICE + beq v7_dma_inv_range + b v7_dma_clean_range ENDPROC(v7_dma_map_area) /* @@ -292,6 +291,9 @@ ENDPROC(v7_dma_map_area) * - dir - DMA direction */ ENTRY(v7_dma_unmap_area) + add r1, r1, r0 + teq r2, #DMA_TO_DEVICE + bne v7_dma_inv_range mov pc, lr ENDPROC(v7_dma_unmap_area) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index efa8efa33f5e..64daef2173bd 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -404,34 +404,22 @@ EXPORT_SYMBOL(dma_free_coherent); * platforms with CONFIG_DMABOUNCE. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ -static void dma_cache_maint(const void *start, size_t size, int direction) -{ - void (*outer_op)(unsigned long, unsigned long); - - switch (direction) { - case DMA_FROM_DEVICE: /* invalidate only */ - outer_op = outer_inv_range; - break; - case DMA_TO_DEVICE: /* writeback only */ - outer_op = outer_clean_range; - break; - case DMA_BIDIRECTIONAL: /* writeback and invalidate */ - outer_op = outer_flush_range; - break; - default: - BUG(); - } - - outer_op(__pa(start), __pa(start) + size); -} - void ___dma_single_cpu_to_dev(const void *kaddr, size_t size, enum dma_data_direction dir) { + unsigned long paddr; + BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); dmac_map_area(kaddr, size, dir); - dma_cache_maint(kaddr, size, dir); + + paddr = __pa(kaddr); + if (dir == DMA_FROM_DEVICE) { + outer_inv_range(paddr, paddr + size); + } else { + outer_clean_range(paddr, paddr + size); + } + /* FIXME: non-speculating: flush on bidirectional mappings? */ } EXPORT_SYMBOL(___dma_single_cpu_to_dev); @@ -440,6 +428,13 @@ void ___dma_single_dev_to_cpu(const void *kaddr, size_t size, { BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1)); + /* FIXME: non-speculating: not required */ + /* don't bother invalidating if DMA to device */ + if (dir != DMA_TO_DEVICE) { + unsigned long paddr = __pa(kaddr); + outer_inv_range(paddr, paddr + size); + } + dmac_unmap_area(kaddr, size, dir); } EXPORT_SYMBOL(___dma_single_dev_to_cpu); @@ -487,32 +482,29 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off, size_t size, enum dma_data_direction dir) { unsigned long paddr; - void (*outer_op)(unsigned long, unsigned long); - - switch (direction) { - case DMA_FROM_DEVICE: /* invalidate only */ - outer_op = outer_inv_range; - break; - case DMA_TO_DEVICE: /* writeback only */ - outer_op = outer_clean_range; - break; - case DMA_BIDIRECTIONAL: /* writeback and invalidate */ - outer_op = outer_flush_range; - break; - default: - BUG(); - } dma_cache_maint_page(page, off, size, dir, dmac_map_area); paddr = page_to_phys(page) + off; - outer_op(paddr, paddr + size); + if (dir == DMA_FROM_DEVICE) { + outer_inv_range(paddr, paddr + size); + } else { + outer_clean_range(paddr, paddr + size); + } + /* FIXME: non-speculating: flush on bidirectional mappings? */ } EXPORT_SYMBOL(___dma_page_cpu_to_dev); void ___dma_page_dev_to_cpu(struct page *page, unsigned long off, size_t size, enum dma_data_direction dir) { + unsigned long paddr = page_to_phys(page) + off; + + /* FIXME: non-speculating: not required */ + /* don't bother invalidating if DMA to device */ + if (dir != DMA_TO_DEVICE) + outer_inv_range(paddr, paddr + size); + dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); } EXPORT_SYMBOL(___dma_page_dev_to_cpu); -- cgit v1.2.2 From 31aa8fd6fd30b0f36416df7d09619768d26b4332 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 11:10:03 +0000 Subject: ARM: Add caller information to ioremap This allows the procfs vmallocinfo file to show who created the ioremap regions. Note: __builtin_return_address(0) doesn't do what's expected if its used in an inline function, so we leave __arm_ioremap callers in such places alone. Signed-off-by: Russell King --- arch/arm/mm/ioremap.c | 57 ++++++++++++++++++++++++++++++--------------------- arch/arm/mm/nommu.c | 12 +++++++++++ 2 files changed, 46 insertions(+), 23 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 0ab75c60f7cf..28c8b950ef04 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -139,8 +139,8 @@ void __check_kvm_seq(struct mm_struct *mm) * which requires the new ioremap'd region to be referenced, the CPU will * reference the _old_ region. * - * Note that get_vm_area() allocates a guard 4K page, so we need to mask - * the size back to 1MB aligned or we will overflow in the loop below. + * Note that get_vm_area_caller() allocates a guard 4K page, so we need to + * mask the size back to 1MB aligned or we will overflow in the loop below. */ static void unmap_area_sections(unsigned long virt, unsigned long size) { @@ -254,22 +254,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, } #endif - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. Needed when the kernel wants to access high addresses - * directly. - * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. - * - * 'flags' are the extra L_PTE_ flags that you want to specify for this - * mapping. See for more information. - */ -void __iomem * -__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, - unsigned int mtype) +void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, + unsigned long offset, size_t size, unsigned int mtype, void *caller) { const struct mem_type *type; int err; @@ -291,7 +277,7 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, */ size = PAGE_ALIGN(offset + size); - area = get_vm_area(size, VM_IOREMAP); + area = get_vm_area_caller(size, VM_IOREMAP, caller); if (!area) return NULL; addr = (unsigned long)area->addr; @@ -318,10 +304,9 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, flush_cache_vmap(addr, addr + size); return (void __iomem *) (offset + addr); } -EXPORT_SYMBOL(__arm_ioremap_pfn); -void __iomem * -__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) +void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size, + unsigned int mtype, void *caller) { unsigned long last_addr; unsigned long offset = phys_addr & ~PAGE_MASK; @@ -334,7 +319,33 @@ __arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) if (!size || last_addr < phys_addr) return NULL; - return __arm_ioremap_pfn(pfn, offset, size, mtype); + return __arm_ioremap_pfn_caller(pfn, offset, size, mtype, + caller); +} + +/* + * Remap an arbitrary physical address space into the kernel virtual + * address space. Needed when the kernel wants to access high addresses + * directly. + * + * NOTE! We need to allow non-page-aligned mappings too: we will obviously + * have to convert them into an offset in a page-aligned mapping, but the + * caller shouldn't need to know that small detail. + */ +void __iomem * +__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, + unsigned int mtype) +{ + return __arm_ioremap_pfn_caller(pfn, offset, size, mtype, + __builtin_return_address(0)); +} +EXPORT_SYMBOL(__arm_ioremap_pfn); + +void __iomem * +__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) +{ + return __arm_ioremap_caller(phys_addr, size, mtype, + __builtin_return_address(0)); } EXPORT_SYMBOL(__arm_ioremap); diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 374a8311bc84..9bfeb6b9509a 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -74,6 +74,12 @@ void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, } EXPORT_SYMBOL(__arm_ioremap_pfn); +void __iomem *__arm_ioremap_pfn_caller(unsigned long pfn, unsigned long offset, + size_t size, unsigned int mtype, void *caller) +{ + return __arm_ioremap_pfn(pfn, offset, size, mtype); +} + void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size, unsigned int mtype) { @@ -81,6 +87,12 @@ void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size, } EXPORT_SYMBOL(__arm_ioremap); +void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size, + unsigned int mtype, void *caller) +{ + return __arm_ioremap(phys_addr, size, mtype); +} + void __iounmap(volatile void __iomem *addr) { } -- cgit v1.2.2 From e119bfff1f102f8d1505910cd6c09df55c776b43 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 10 Jan 2010 17:23:29 +0000 Subject: ARM: Move creation of /proc/cpu out of alignment.c Always creating this directory avoids other users having to jump through silly hoops when they want to share this directory. Signed-off-by: Russell King --- arch/arm/mm/alignment.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index b270d6228fe2..0c5eb6983cef 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -898,11 +898,7 @@ static int __init alignment_init(void) #ifdef CONFIG_PROC_FS struct proc_dir_entry *res; - res = proc_mkdir("cpu", NULL); - if (!res) - return -ENOMEM; - - res = create_proc_entry("alignment", S_IWUSR | S_IRUGO, res); + res = create_proc_entry("cpu/alignment", S_IWUSR | S_IRUGO, NULL); if (!res) return -ENOMEM; -- cgit v1.2.2 From 2b0d8c251b8876d530a6bf671eb5425838fa698a Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Mon, 11 Jan 2010 23:17:34 +0100 Subject: ARM: 5880/1: arm: use generic infrastructure for early params The ARM setup code includes its own parser for early params, there's also one in the generic init code. This patch removes __early_init (and related code) from arch/arm/kernel/setup.c, and changes users to the generic early_init macro instead. The generic macro takes a char * argument, rather than char **, so we need to update the parser functions a little. Signed-off-by: Jeremy Kerr Signed-off-by: Russell King --- arch/arm/mm/init.c | 12 +++++++----- arch/arm/mm/mmu.c | 41 +++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 25 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index a04ffbbbe253..a340569b991e 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -32,19 +32,21 @@ static unsigned long phys_initrd_start __initdata = 0; static unsigned long phys_initrd_size __initdata = 0; -static void __init early_initrd(char **p) +static int __init early_initrd(char *p) { unsigned long start, size; + char *endp; - start = memparse(*p, p); - if (**p == ',') { - size = memparse((*p) + 1, p); + start = memparse(p, &endp); + if (*endp == ',') { + size = memparse(endp + 1, NULL); phys_initrd_start = start; phys_initrd_size = size; } + return 0; } -__early_param("initrd=", early_initrd); +early_param("initrd", early_initrd); static int __init parse_tag_initrd(const struct tag *tag) { diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 1708da82da96..88f5d71248d9 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -100,18 +100,17 @@ static struct cachepolicy cache_policies[] __initdata = { * writebuffer to be turned off. (Note: the write * buffer should not be on and the cache off). */ -static void __init early_cachepolicy(char **p) +static int __init early_cachepolicy(char *p) { int i; for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { int len = strlen(cache_policies[i].policy); - if (memcmp(*p, cache_policies[i].policy, len) == 0) { + if (memcmp(p, cache_policies[i].policy, len) == 0) { cachepolicy = i; cr_alignment &= ~cache_policies[i].cr_mask; cr_no_alignment &= ~cache_policies[i].cr_mask; - *p += len; break; } } @@ -130,36 +129,37 @@ static void __init early_cachepolicy(char **p) } flush_cache_all(); set_cr(cr_alignment); + return 0; } -__early_param("cachepolicy=", early_cachepolicy); +early_param("cachepolicy", early_cachepolicy); -static void __init early_nocache(char **__unused) +static int __init early_nocache(char *__unused) { char *p = "buffered"; printk(KERN_WARNING "nocache is deprecated; use cachepolicy=%s\n", p); - early_cachepolicy(&p); + early_cachepolicy(p); + return 0; } -__early_param("nocache", early_nocache); +early_param("nocache", early_nocache); -static void __init early_nowrite(char **__unused) +static int __init early_nowrite(char *__unused) { char *p = "uncached"; printk(KERN_WARNING "nowb is deprecated; use cachepolicy=%s\n", p); - early_cachepolicy(&p); + early_cachepolicy(p); + return 0; } -__early_param("nowb", early_nowrite); +early_param("nowb", early_nowrite); -static void __init early_ecc(char **p) +static int __init early_ecc(char *p) { - if (memcmp(*p, "on", 2) == 0) { + if (memcmp(p, "on", 2) == 0) ecc_mask = PMD_PROTECTION; - *p += 2; - } else if (memcmp(*p, "off", 3) == 0) { + else if (memcmp(p, "off", 3) == 0) ecc_mask = 0; - *p += 3; - } + return 0; } -__early_param("ecc=", early_ecc); +early_param("ecc", early_ecc); static int __init noalign_setup(char *__unused) { @@ -670,9 +670,9 @@ static unsigned long __initdata vmalloc_reserve = SZ_128M; * bytes. This can be used to increase (or decrease) the vmalloc * area - the default is 128m. */ -static void __init early_vmalloc(char **arg) +static int __init early_vmalloc(char *arg) { - vmalloc_reserve = memparse(*arg, arg); + vmalloc_reserve = memparse(arg, NULL); if (vmalloc_reserve < SZ_16M) { vmalloc_reserve = SZ_16M; @@ -687,8 +687,9 @@ static void __init early_vmalloc(char **arg) "vmalloc area is too big, limiting to %luMB\n", vmalloc_reserve >> 20); } + return 0; } -__early_param("vmalloc=", early_vmalloc); +early_param("vmalloc", early_vmalloc); #define VMALLOC_MIN (void *)(VMALLOC_END - vmalloc_reserve) -- cgit v1.2.2 From 11805bcfa411c816b7c76fc40724be6733c74ffc Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 26 Jan 2010 19:09:42 +0100 Subject: ARM: 5905/1: ARM: Global ASID allocation on SMP The current ASID allocation algorithm doesn't ensure the notification of the other CPUs when the ASID rolls over. This may lead to two processes using the same ASID (but different generation) or multiple threads of the same process using different ASIDs. This patch adds the broadcasting of the ASID rollover event to the other CPUs. To avoid a race on multiple CPUs modifying "cpu_last_asid" during the handling of the broadcast, the ASID numbering now starts at "smp_processor_id() + 1". At rollover, the cpu_last_asid will be set to NR_CPUS. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/context.c | 124 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 110 insertions(+), 14 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index a9e22e31eaa1..b0ee9ba3cfab 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -10,12 +10,17 @@ #include #include #include +#include +#include #include #include static DEFINE_SPINLOCK(cpu_asid_lock); unsigned int cpu_last_asid = ASID_FIRST_VERSION; +#ifdef CONFIG_SMP +DEFINE_PER_CPU(struct mm_struct *, current_mm); +#endif /* * We fork()ed a process, and we need a new context for the child @@ -26,13 +31,109 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION; void __init_new_context(struct task_struct *tsk, struct mm_struct *mm) { mm->context.id = 0; + spin_lock_init(&mm->context.id_lock); } +static void flush_context(void) +{ + /* set the reserved ASID before flushing the TLB */ + asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (0)); + isb(); + local_flush_tlb_all(); + if (icache_is_vivt_asid_tagged()) { + __flush_icache_all(); + dsb(); + } +} + +#ifdef CONFIG_SMP + +static void set_mm_context(struct mm_struct *mm, unsigned int asid) +{ + unsigned long flags; + + /* + * Locking needed for multi-threaded applications where the + * same mm->context.id could be set from different CPUs during + * the broadcast. This function is also called via IPI so the + * mm->context.id_lock has to be IRQ-safe. + */ + spin_lock_irqsave(&mm->context.id_lock, flags); + if (likely((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) { + /* + * Old version of ASID found. Set the new one and + * reset mm_cpumask(mm). + */ + mm->context.id = asid; + cpumask_clear(mm_cpumask(mm)); + } + spin_unlock_irqrestore(&mm->context.id_lock, flags); + + /* + * Set the mm_cpumask(mm) bit for the current CPU. + */ + cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); +} + +/* + * Reset the ASID on the current CPU. This function call is broadcast + * from the CPU handling the ASID rollover and holding cpu_asid_lock. + */ +static void reset_context(void *info) +{ + unsigned int asid; + unsigned int cpu = smp_processor_id(); + struct mm_struct *mm = per_cpu(current_mm, cpu); + + /* + * Check if a current_mm was set on this CPU as it might still + * be in the early booting stages and using the reserved ASID. + */ + if (!mm) + return; + + smp_rmb(); + asid = cpu_last_asid + cpu + 1; + + flush_context(); + set_mm_context(mm, asid); + + /* set the new ASID */ + asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id)); + isb(); +} + +#else + +static inline void set_mm_context(struct mm_struct *mm, unsigned int asid) +{ + mm->context.id = asid; + cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id())); +} + +#endif + void __new_context(struct mm_struct *mm) { unsigned int asid; spin_lock(&cpu_asid_lock); +#ifdef CONFIG_SMP + /* + * Check the ASID again, in case the change was broadcast from + * another CPU before we acquired the lock. + */ + if (unlikely(((mm->context.id ^ cpu_last_asid) >> ASID_BITS) == 0)) { + cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); + spin_unlock(&cpu_asid_lock); + return; + } +#endif + /* + * At this point, it is guaranteed that the current mm (with + * an old ASID) isn't active on any other CPU since the ASIDs + * are changed simultaneously via IPI. + */ asid = ++cpu_last_asid; if (asid == 0) asid = cpu_last_asid = ASID_FIRST_VERSION; @@ -42,20 +143,15 @@ void __new_context(struct mm_struct *mm) * to start a new version and flush the TLB. */ if (unlikely((asid & ~ASID_MASK) == 0)) { - asid = ++cpu_last_asid; - /* set the reserved ASID before flushing the TLB */ - asm("mcr p15, 0, %0, c13, c0, 1 @ set reserved context ID\n" - : - : "r" (0)); - isb(); - flush_tlb_all(); - if (icache_is_vivt_asid_tagged()) { - __flush_icache_all(); - dsb(); - } + asid = cpu_last_asid + smp_processor_id() + 1; + flush_context(); +#ifdef CONFIG_SMP + smp_wmb(); + smp_call_function(reset_context, NULL, 1); +#endif + cpu_last_asid += NR_CPUS; } - spin_unlock(&cpu_asid_lock); - cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id())); - mm->context.id = asid; + set_mm_context(mm, asid); + spin_unlock(&cpu_asid_lock); } -- cgit v1.2.2 From 1a28e3d977860dc760909083df625b300f695680 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Mon, 1 Feb 2010 23:30:26 +0100 Subject: ARM: 5911/1: ARM: Select CPU_32v6K for CPU_V7 only if ARCH_OMAP2 is not selected Otherwise the kernel built with both CPU_V6 and CPU_V7 will not boot on omap2. Signed-off-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index baf638487a2d..4c2e90dc549c 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -399,7 +399,7 @@ config CPU_V6 config CPU_32v6K bool "Support ARM V6K processor extensions" if !SMP depends on CPU_V6 - default y if SMP && !ARCH_MX3 + default y if SMP && !(ARCH_MX3 || ARCH_OMAP2) help Say Y here if your ARMv6 processor supports the 'K' extension. This enables the kernel to use some instructions not present @@ -410,7 +410,7 @@ config CPU_32v6K # ARMv7 config CPU_V7 bool "Support ARM V7 processor" if ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX - select CPU_32v6K + select CPU_32v6K if !ARCH_OMAP2 select CPU_32v7 select CPU_ABRT_EV7 select CPU_PABRT_V7 -- cgit v1.2.2 From 424d6b145f863d012c540082d0c1afb5bb4dea48 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Thu, 4 Feb 2010 19:35:06 +0100 Subject: ARM: 5916/1: ARM: L2 : Add maintainace by line helper functions This patch adds the cache maintainance by line helper functions. Signed-off-by: Santosh Shilimkar Acked-by: Catalin Marinas Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/cache-l2x0.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index cb8fc6573b1b..1a14d18e5713 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -42,6 +42,27 @@ static inline void cache_sync(void) cache_wait(base + L2X0_CACHE_SYNC, 1); } +static inline void l2x0_clean_line(unsigned long addr) +{ + void __iomem *base = l2x0_base; + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); + writel(addr, base + L2X0_CLEAN_LINE_PA); +} + +static inline void l2x0_inv_line(unsigned long addr) +{ + void __iomem *base = l2x0_base; + cache_wait(base + L2X0_INV_LINE_PA, 1); + writel(addr, base + L2X0_INV_LINE_PA); +} + +static inline void l2x0_flush_line(unsigned long addr) +{ + void __iomem *base = l2x0_base; + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(addr, base + L2X0_CLEAN_INV_LINE_PA); +} + static inline void l2x0_inv_all(void) { unsigned long flags; @@ -62,23 +83,20 @@ static void l2x0_inv_range(unsigned long start, unsigned long end) spin_lock_irqsave(&l2x0_lock, flags); if (start & (CACHE_LINE_SIZE - 1)) { start &= ~(CACHE_LINE_SIZE - 1); - cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); - writel(start, base + L2X0_CLEAN_INV_LINE_PA); + l2x0_flush_line(start); start += CACHE_LINE_SIZE; } if (end & (CACHE_LINE_SIZE - 1)) { end &= ~(CACHE_LINE_SIZE - 1); - cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); - writel(end, base + L2X0_CLEAN_INV_LINE_PA); + l2x0_flush_line(end); } while (start < end) { unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - cache_wait(base + L2X0_INV_LINE_PA, 1); - writel(start, base + L2X0_INV_LINE_PA); + l2x0_inv_line(start); start += CACHE_LINE_SIZE; } @@ -103,8 +121,7 @@ static void l2x0_clean_range(unsigned long start, unsigned long end) unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - cache_wait(base + L2X0_CLEAN_LINE_PA, 1); - writel(start, base + L2X0_CLEAN_LINE_PA); + l2x0_clean_line(start); start += CACHE_LINE_SIZE; } @@ -129,8 +146,7 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) unsigned long blk_end = start + min(end - start, 4096UL); while (start < blk_end) { - cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); - writel(start, base + L2X0_CLEAN_INV_LINE_PA); + l2x0_flush_line(start); start += CACHE_LINE_SIZE; } -- cgit v1.2.2 From d309427e792ea750cdd312e7a92cf6047ae44962 Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Thu, 4 Feb 2010 19:37:09 +0100 Subject: ARM: 5917/1: OMAP4: Add L2 Cache support This patch adds L2 Cache support for OMAP4. External L2 cache is used in OMAP4 CC: Catalin Marinas Signed-off-by: Santosh Shilimkar Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 4c2e90dc549c..e859743024a0 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -754,7 +754,7 @@ config CACHE_FEROCEON_L2_WRITETHROUGH config CACHE_L2X0 bool "Enable the L2x0 outer cache controller" depends on REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP || MACH_REALVIEW_PB1176 || \ - REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK + REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK || ARCH_OMAP4 default y select OUTER_CACHE help -- cgit v1.2.2 From 9e65582a8e8715f883a34eea66e0643778ce878d Mon Sep 17 00:00:00 2001 From: Santosh Shilimkar Date: Thu, 4 Feb 2010 19:42:42 +0100 Subject: ARM: 5919/1: ARM: L2 : Errata 588369: Clean & Invalidate do not invalidate clean lines This patch implements the work-around for the errata 588369.The secure API is used to alter L2 debug register because of trust-zone. This version updated with comments from Russell and Catalin and generated against 2.6.33-rc6 mainline kernel. Detail comments can be found: http://www.spinics.net/lists/linux-omap/msg23431.html Signed-off-by: Woodruff Richard Signed-off-by: Santosh Shilimkar Acked-by: Catalin Marinas Acked-by: Tony Lindgren Signed-off-by: Russell King --- arch/arm/mm/cache-l2x0.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 1a14d18e5713..07334632d3e2 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -56,12 +56,42 @@ static inline void l2x0_inv_line(unsigned long addr) writel(addr, base + L2X0_INV_LINE_PA); } +#ifdef CONFIG_PL310_ERRATA_588369 +static void debug_writel(unsigned long val) +{ + extern void omap_smc1(u32 fn, u32 arg); + + /* + * Texas Instrument secure monitor api to modify the + * PL310 Debug Control Register. + */ + omap_smc1(0x100, val); +} + +static inline void l2x0_flush_line(unsigned long addr) +{ + void __iomem *base = l2x0_base; + + /* Clean by PA followed by Invalidate by PA */ + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); + writel(addr, base + L2X0_CLEAN_LINE_PA); + cache_wait(base + L2X0_INV_LINE_PA, 1); + writel(addr, base + L2X0_INV_LINE_PA); +} +#else + +/* Optimised out for non-errata case */ +static inline void debug_writel(unsigned long val) +{ +} + static inline void l2x0_flush_line(unsigned long addr) { void __iomem *base = l2x0_base; cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); writel(addr, base + L2X0_CLEAN_INV_LINE_PA); } +#endif static inline void l2x0_inv_all(void) { @@ -83,13 +113,17 @@ static void l2x0_inv_range(unsigned long start, unsigned long end) spin_lock_irqsave(&l2x0_lock, flags); if (start & (CACHE_LINE_SIZE - 1)) { start &= ~(CACHE_LINE_SIZE - 1); + debug_writel(0x03); l2x0_flush_line(start); + debug_writel(0x00); start += CACHE_LINE_SIZE; } if (end & (CACHE_LINE_SIZE - 1)) { end &= ~(CACHE_LINE_SIZE - 1); + debug_writel(0x03); l2x0_flush_line(end); + debug_writel(0x00); } while (start < end) { @@ -145,10 +179,12 @@ static void l2x0_flush_range(unsigned long start, unsigned long end) while (start < end) { unsigned long blk_end = start + min(end - start, 4096UL); + debug_writel(0x03); while (start < blk_end) { l2x0_flush_line(start); start += CACHE_LINE_SIZE; } + debug_writel(0x00); if (blk_end < end) { spin_unlock_irqrestore(&l2x0_lock, flags); -- cgit v1.2.2 From db9ef1af4879c121c354ad2f653f185f1d50fd89 Mon Sep 17 00:00:00 2001 From: Fenkart/Bostandzhyan Date: Sun, 7 Feb 2010 21:45:47 +0100 Subject: ARM: 5926/1: Add "Virtual kernel memory..." printout. Code based on parisc and x86_32. Tested-by: H Hartley Sweeten Signed-off-by: Andreas Fenkart Signed-off-by: Russell King --- arch/arm/mm/init.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 9 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index a340569b991e..e8e3a74ac5b5 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -562,7 +563,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi) */ void __init mem_init(void) { - unsigned int codesize, datasize, initsize; + unsigned long reserved_pages, free_pages; int i, node; #ifndef CONFIG_DISCONTIGMEM @@ -598,6 +599,33 @@ void __init mem_init(void) totalram_pages += totalhigh_pages; #endif + reserved_pages = free_pages = 0; + + for_each_online_node(node) { + pg_data_t *n = NODE_DATA(node); + struct page *map = pgdat_page_nr(n, 0) - n->node_start_pfn; + + for_each_nodebank(i, &meminfo, node) { + struct membank *bank = &meminfo.bank[i]; + unsigned int pfn1, pfn2; + struct page *page, *end; + + pfn1 = bank_pfn_start(bank); + pfn2 = bank_pfn_end(bank); + + page = map + pfn1; + end = map + pfn2; + + do { + if (PageReserved(page)) + reserved_pages++; + else if (!page_count(page)) + free_pages++; + page++; + } while (page < end); + } + } + /* * Since our memory may not be contiguous, calculate the * real number of pages we have in this system @@ -610,16 +638,48 @@ void __init mem_init(void) } printk(" = %luMB total\n", num_physpages >> (20 - PAGE_SHIFT)); - codesize = _etext - _text; - datasize = _end - _data; - initsize = __init_end - __init_begin; - - printk(KERN_NOTICE "Memory: %luKB available (%dK code, " - "%dK data, %dK init, %luK highmem)\n", - nr_free_pages() << (PAGE_SHIFT-10), codesize >> 10, - datasize >> 10, initsize >> 10, + printk(KERN_NOTICE "Memory: %luk/%luk available, %luk reserved, %luK highmem\n", + nr_free_pages() << (PAGE_SHIFT-10), + free_pages << (PAGE_SHIFT-10), + reserved_pages << (PAGE_SHIFT-10), totalhigh_pages << (PAGE_SHIFT-10)); +#define MLK(b, t) b, t, ((t) - (b)) >> 10 +#define MLM(b, t) b, t, ((t) - (b)) >> 20 +#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) + + printk(KERN_NOTICE "Virtual kernel memory layout:\n" + " vector : 0x%08lx - 0x%08lx (%4ld kB)\n" + " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" + " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" + " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" +#ifdef CONFIG_HIGHMEM + " pkmap : 0x%08lx - 0x%08lx (%4ld MB)\n" +#endif + " modules : 0x%08lx - 0x%08lx (%4ld MB)\n" + " .init : 0x%p" " - 0x%p" " (%4d kB)\n" + " .text : 0x%p" " - 0x%p" " (%4d kB)\n" + " .data : 0x%p" " - 0x%p" " (%4d kB)\n", + + MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) + + (PAGE_SIZE)), + MLK(FIXADDR_START, FIXADDR_TOP), + MLM(VMALLOC_START, (unsigned long)VMALLOC_END), + MLM(PAGE_OFFSET, (unsigned long)high_memory), +#ifdef CONFIG_HIGHMEM + MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) * + (PAGE_SIZE)), +#endif + MLM(MODULES_VADDR, MODULES_END), + + MLK_ROUNDUP(__init_begin, __init_end), + MLK_ROUNDUP(_text, _etext), + MLK_ROUNDUP(_data, _edata)); + +#undef MLK +#undef MLM +#undef MLK_ROUNDUP + if (PAGE_SIZE >= 16384 && num_physpages <= 128) { extern int sysctl_overcommit_memory; /* -- cgit v1.2.2 From a7bd08c82e4f74387a39eeebb942712f23967420 Mon Sep 17 00:00:00 2001 From: Fenkart/Bostandzhyan Date: Sun, 7 Feb 2010 21:46:33 +0100 Subject: ARM: 5927/1: Make delimiters of DMA area globally visibly. Adds DMA area to 'virtual memory map' startup message Tested-by: H Hartley Sweeten Signed-off-by: Andreas Fenkart Signed-off-by: Russell King --- arch/arm/mm/dma-mapping.c | 3 --- arch/arm/mm/init.c | 6 ++++++ 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 26325cb5d368..48eedab1609b 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -29,9 +29,6 @@ #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB" #endif -#define CONSISTENT_END (0xffe00000) -#define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE) - #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index e8e3a74ac5b5..bda481e6bc0f 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -651,6 +651,9 @@ void __init mem_init(void) printk(KERN_NOTICE "Virtual kernel memory layout:\n" " vector : 0x%08lx - 0x%08lx (%4ld kB)\n" " fixmap : 0x%08lx - 0x%08lx (%4ld kB)\n" +#ifdef CONFIG_MMU + " DMA : 0x%08lx - 0x%08lx (%4ld MB)\n" +#endif " vmalloc : 0x%08lx - 0x%08lx (%4ld MB)\n" " lowmem : 0x%08lx - 0x%08lx (%4ld MB)\n" #ifdef CONFIG_HIGHMEM @@ -664,6 +667,9 @@ void __init mem_init(void) MLK(UL(CONFIG_VECTORS_BASE), UL(CONFIG_VECTORS_BASE) + (PAGE_SIZE)), MLK(FIXADDR_START, FIXADDR_TOP), +#ifdef CONFIG_MMU + MLM(CONSISTENT_BASE, CONSISTENT_END), +#endif MLM(VMALLOC_START, (unsigned long)VMALLOC_END), MLM(PAGE_OFFSET, (unsigned long)high_memory), #ifdef CONFIG_HIGHMEM -- cgit v1.2.2 From c931b4f655a1b86c929384e674eb8c31795f3bd7 Mon Sep 17 00:00:00 2001 From: Fenkart/Bostandzhyan Date: Sun, 7 Feb 2010 21:47:17 +0100 Subject: ARM: 5928/1: Change type of VMALLOC_END to unsigned long. Makes it consistent with VMALLOC_START Tested-by: H Hartley Sweeten Signed-off-by: Andreas Fenkart Signed-off-by: Russell King --- arch/arm/mm/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index bda481e6bc0f..3a2077239474 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -670,7 +670,7 @@ void __init mem_init(void) #ifdef CONFIG_MMU MLM(CONSISTENT_BASE, CONSISTENT_END), #endif - MLM(VMALLOC_START, (unsigned long)VMALLOC_END), + MLM(VMALLOC_START, VMALLOC_END), MLM(PAGE_OFFSET, (unsigned long)high_memory), #ifdef CONFIG_HIGHMEM MLM(PKMAP_BASE, (PKMAP_BASE) + (LAST_PKMAP) * -- cgit v1.2.2 From a183927213df225bd93d21857b6aaafbb95e590d Mon Sep 17 00:00:00 2001 From: Fenkart/Bostandzhyan Date: Sun, 7 Feb 2010 21:47:58 +0100 Subject: ARM: 5929/1: Add checks to detect overlap of memory regions. Tested-by: H Hartley Sweeten Signed-off-by: Andreas Fenkart Signed-off-by: Russell King --- arch/arm/mm/init.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 3a2077239474..7829cb5425f5 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -686,6 +686,23 @@ void __init mem_init(void) #undef MLM #undef MLK_ROUNDUP + /* + * Check boundaries twice: Some fundamental inconsistencies can + * be detected at build time already. + */ +#ifdef CONFIG_MMU + BUILD_BUG_ON(VMALLOC_END > CONSISTENT_BASE); + BUG_ON(VMALLOC_END > CONSISTENT_BASE); + + BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); + BUG_ON(TASK_SIZE > MODULES_VADDR); +#endif + +#ifdef CONFIG_HIGHMEM + BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); + BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); +#endif + if (PAGE_SIZE >= 16384 && num_physpages <= 128) { extern int sysctl_overcommit_memory; /* -- cgit v1.2.2 From d944d549aa86e08cba080396513234cf048fee1f Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 20 Feb 2010 16:13:29 +0000 Subject: ARM: allow alignment fault mode to be configured at kernel boot Some glibc versions intentionally create lots of alignment faults in their gconv code, which if not fixed up, results in segfaults during boot. This can prevent systems booting properly. There is no clear hard-configurable default for this; the desired default depends on the nature of the userspace which is going to be booted. So, provide a way for the alignment fault handler to be configured via the kernel command line. Signed-off-by: Russell King --- arch/arm/mm/alignment.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index b270d6228fe2..62820eda84d9 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -11,6 +11,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include #include #include #include @@ -77,6 +78,8 @@ static unsigned long ai_dword; static unsigned long ai_multi; static int ai_usermode; +core_param(alignment, ai_usermode, int, 0600); + #define UM_WARN (1 << 0) #define UM_FIXUP (1 << 1) #define UM_SIGNAL (1 << 2) -- cgit v1.2.2 From 4b3073e1c53a256275f1079c0fbfbe85883d9275 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:40:18 +0000 Subject: MM: Pass a PTE pointer to update_mmu_cache() rather than the PTE itself On VIVT ARM, when we have multiple shared mappings of the same file in the same MM, we need to ensure that we have coherency across all copies. We do this via make_coherent() by making the pages uncacheable. This used to work fine, until we allowed highmem with highpte - we now have a page table which is mapped as required, and is not available for modification via update_mmu_cache(). Ralf Beache suggested getting rid of the PTE value passed to update_mmu_cache(): On MIPS update_mmu_cache() calls __update_tlb() which walks pagetables to construct a pointer to the pte again. Passing a pte_t * is much more elegant. Maybe we might even replace the pte argument with the pte_t? Ben Herrenschmidt would also like the pte pointer for PowerPC: Passing the ptep in there is exactly what I want. I want that -instead- of the PTE value, because I have issue on some ppc cases, for I$/D$ coherency, where set_pte_at() may decide to mask out the _PAGE_EXEC. So, pass in the mapped page table pointer into update_mmu_cache(), and remove the PTE value, updating all implementations and call sites to suit. Includes a fix from Stephen Rothwell: sparc: fix fallout from update_mmu_cache API change Signed-off-by: Stephen Rothwell Acked-by: Benjamin Herrenschmidt Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index ae88f2c3a6df..c45f9bb318ad 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -149,9 +149,10 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne * * Note that the pte lock will be held. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep) { - unsigned long pfn = pte_pfn(pte); + unsigned long pfn = pte_pfn(*ptep); struct address_space *mapping; struct page *page; -- cgit v1.2.2 From ae1402022edbeef3991f1e4bae8fa99558be291b Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 18 Dec 2009 16:43:57 +0000 Subject: ARM: make_coherent(): fix problems with highpte, part 2 update_mmu_cache() is called with the page table for the faulted-in page still mapped. We need to modify the PTE for this page to ensure coherency with other shared mappings when multiple shared mappings exist within a MM. Signed-off-by: Russell King --- arch/arm/mm/fault-armv.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index c45f9bb318ad..c9b97e9836a2 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -99,7 +99,8 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, } static void -make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn) +make_coherent(struct address_space *mapping, struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned long pfn) { struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *mpnt; @@ -131,7 +132,7 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne } flush_dcache_mmap_unlock(mapping); if (aliases) - adjust_pte(vma, addr, pfn); + do_adjust_pte(vma, addr, pfn, ptep); else flush_cache_page(vma, addr, pfn); } @@ -174,7 +175,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, #endif if (mapping) { if (cache_is_vivt()) - make_coherent(mapping, vma, addr, pfn); + make_coherent(mapping, vma, addr, ptep, pfn); else if (vma->vm_flags & VM_EXEC) __flush_icache_all(); } -- cgit v1.2.2 From d6d502fa4be1acd01971476fc732c95a4da16d90 Mon Sep 17 00:00:00 2001 From: Kukjin Kim Date: Mon, 22 Feb 2010 00:02:59 +0100 Subject: ARM: 5952/1: ARM: MM: Add ARM_L1_CACHE_SHIFT_6 for handle inside each ARCH Kconfig Add ARM_L1_CACHE_SHIFT_6 to arch/arm/Kconfig to allow CPUs with L1 cache lines which are 64bytes to indicate this without having to alter the arch/arm/mm/Kconfig entry each time. Update the mm Kconfig so that ARM_L1_CACHE_SHIFT default value uses this and change OMAP3 and S5PC1XX to select ARM_L1_CACHE_SHIFT_6. Acked-by: Ben Dooks Acked-by: Tony Lindgren Signed-off-by: Kukjin Kim Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index e859743024a0..c4ed9f93f646 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -779,5 +779,5 @@ config CACHE_XSC3L2 config ARM_L1_CACHE_SHIFT int - default 6 if ARCH_OMAP3 || ARCH_S5PC1XX + default 6 if ARM_L1_CACHE_SHIFT_6 default 5 -- cgit v1.2.2 From 319f551a0a167b49b5bbb4a9ff4802046a572bc5 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 24 Mar 2010 16:47:53 +0100 Subject: ARM: 5994/1: ARM: Add outer_cache_fns.sync function pointer (2/4) This patch introduces the outer_cache_fns.sync function pointer together with the OUTER_CACHE_SYNC config option that can be used to drain the write buffer of the outer cache. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index c4ed9f93f646..88a24de55aaf 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -736,6 +736,12 @@ config NEEDS_SYSCALL_FOR_CMPXCHG config OUTER_CACHE bool +config OUTER_CACHE_SYNC + bool + help + The outer cache has a outer_cache_fns.sync function pointer + that can be used to drain the write buffer of the outer cache. + config CACHE_FEROCEON_L2 bool "Enable the Feroceon L2 cache controller" depends on ARCH_KIRKWOOD || ARCH_MV78XX0 -- cgit v1.2.2 From 23107c542068b2b94390aa333f6b330af64961e4 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 24 Mar 2010 16:48:53 +0100 Subject: ARM: 5995/1: ARM: Add L2x0 outer_sync() support (3/4) The L2x0 cache controllers need to explicitly drain their write buffer even for Normal Noncacheable memory accesses. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 1 + arch/arm/mm/cache-l2x0.c | 10 ++++++++++ 2 files changed, 11 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 88a24de55aaf..55a2a00db77f 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -763,6 +763,7 @@ config CACHE_L2X0 REALVIEW_EB_A9MP || ARCH_MX35 || ARCH_MX31 || MACH_REALVIEW_PBX || ARCH_NOMADIK || ARCH_OMAP4 default y select OUTER_CACHE + select OUTER_CACHE_SYNC help This option enables the L2x0 PrimeCell. diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 07334632d3e2..21ad68ba22ba 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -93,6 +93,15 @@ static inline void l2x0_flush_line(unsigned long addr) } #endif +static void l2x0_cache_sync(void) +{ + unsigned long flags; + + spin_lock_irqsave(&l2x0_lock, flags); + cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); +} + static inline void l2x0_inv_all(void) { unsigned long flags; @@ -225,6 +234,7 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) outer_cache.inv_range = l2x0_inv_range; outer_cache.clean_range = l2x0_clean_range; outer_cache.flush_range = l2x0_flush_range; + outer_cache.sync = l2x0_cache_sync; printk(KERN_INFO "L2X0 cache controller enabled\n"); } -- cgit v1.2.2 From e7c5650f6067f65f8e961394f376d4862808d0d2 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 24 Mar 2010 16:49:54 +0100 Subject: ARM: 5996/1: ARM: Change the mandatory barriers implementation (4/4) The mandatory barriers (mb, rmb, wmb) are used even on uniprocessor systems for things like ordering Normal Non-cacheable memory accesses with DMA transfer (via Device memory writes). The current implementation uses dmb() for mb() and friends but this is not sufficient. The DMB only ensures the relative ordering of the observability of accesses by other processors or devices acting as masters. In case of DMA transfers started by writes to device memory, the relative ordering is not ensured because accesses to slave ports of a device are not considered observable by the DMB definition. A DSB is required for the data to reach the main memory (even if mapped as Normal Non-cacheable) before the device receives the notification to begin the transfer. Furthermore, some L2 cache controllers (like L2x0 or PL310) buffer stores to Normal Non-cacheable memory and this would need to be drained with the outer_sync() function call. The patch also allows platforms to define their own mandatory barriers implementation by selecting CONFIG_ARCH_HAS_BARRIERS and providing a mach/barriers.h file. Note that the SMP barriers are unchanged (being DMBs as before) since they are only guaranteed to work with Normal Cacheable memory. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/Kconfig | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 55a2a00db77f..5bd7c89a6045 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -788,3 +788,9 @@ config ARM_L1_CACHE_SHIFT int default 6 if ARM_L1_CACHE_SHIFT_6 default 5 + +config ARCH_HAS_BARRIERS + bool + help + This option allows the use of custom mandatory barriers + included via the mach/barriers.h file. -- cgit v1.2.2 From 5a0e3ad6af8660be21ca98a971cd00f331318c05 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 24 Mar 2010 17:04:11 +0900 Subject: include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h percpu.h is included by sched.h and module.h and thus ends up being included when building most .c files. percpu.h includes slab.h which in turn includes gfp.h making everything defined by the two files universally available and complicating inclusion dependencies. percpu.h -> slab.h dependency is about to be removed. Prepare for this change by updating users of gfp and slab facilities include those headers directly instead of assuming availability. As this conversion needs to touch large number of source files, the following script is used as the basis of conversion. http://userweb.kernel.org/~tj/misc/slabh-sweep.py The script does the followings. * Scan files for gfp and slab usages and update includes such that only the necessary includes are there. ie. if only gfp is used, gfp.h, if slab is used, slab.h. * When the script inserts a new include, it looks at the include blocks and try to put the new include such that its order conforms to its surrounding. It's put in the include block which contains core kernel includes, in the same order that the rest are ordered - alphabetical, Christmas tree, rev-Xmas-tree or at the end if there doesn't seem to be any matching order. * If the script can't find a place to put a new include (mostly because the file doesn't have fitting include block), it prints out an error message indicating which .h file needs to be added to the file. The conversion was done in the following steps. 1. The initial automatic conversion of all .c files updated slightly over 4000 files, deleting around 700 includes and adding ~480 gfp.h and ~3000 slab.h inclusions. The script emitted errors for ~400 files. 2. Each error was manually checked. Some didn't need the inclusion, some needed manual addition while adding it to implementation .h or embedding .c file was more appropriate for others. This step added inclusions to around 150 files. 3. The script was run again and the output was compared to the edits from #2 to make sure no file was left behind. 4. Several build tests were done and a couple of problems were fixed. e.g. lib/decompress_*.c used malloc/free() wrappers around slab APIs requiring slab.h to be added manually. 5. The script was run on all .h files but without automatically editing them as sprinkling gfp.h and slab.h inclusions around .h files could easily lead to inclusion dependency hell. Most gfp.h inclusion directives were ignored as stuff from gfp.h was usually wildly available and often used in preprocessor macros. Each slab.h inclusion directive was examined and added manually as necessary. 6. percpu.h was updated not to include slab.h. 7. Build test were done on the following configurations and failures were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my distributed build env didn't work with gcov compiles) and a few more options had to be turned off depending on archs to make things build (like ipr on powerpc/64 which failed due to missing writeq). * x86 and x86_64 UP and SMP allmodconfig and a custom test config. * powerpc and powerpc64 SMP allmodconfig * sparc and sparc64 SMP allmodconfig * ia64 SMP allmodconfig * s390 SMP allmodconfig * alpha SMP allmodconfig * um on x86_64 SMP allmodconfig 8. percpu.h modifications were reverted so that it could be applied as a separate patch and serve as bisection point. Given the fact that I had only a couple of failures from tests on step 6, I'm fairly confident about the coverage of this conversion patch. If there is a breakage, it's likely to be something in one of the arch headers which should be easily discoverable easily on most builds of the specific arch. Signed-off-by: Tejun Heo Guess-its-ok-by: Christoph Lameter Cc: Ingo Molnar Cc: Lee Schermerhorn --- arch/arm/mm/dma-mapping.c | 2 +- arch/arm/mm/fault-armv.c | 1 + arch/arm/mm/init.c | 1 + arch/arm/mm/pgd.c | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 0da7eccf7749..1351edc0b26f 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -11,7 +11,7 @@ */ #include #include -#include +#include #include #include #include diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index c9b97e9836a2..0d414c28eb2c 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 7829cb5425f5..83db12a68d56 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 2690146161ba..be5f58e153bf 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -8,6 +8,7 @@ * published by the Free Software Foundation. */ #include +#include #include #include -- cgit v1.2.2 From 85b3cce880a19e78286570d5fd004cc3cac06f57 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 9 Apr 2010 15:00:11 +0100 Subject: ARM: Fix ioremap_cached()/ioremap_wc() for SMP platforms Write combining/cached device mappings are not setting the shared bit, which could potentially cause problems on SMP systems since the cache lines won't participate in the cache coherency protocol. Signed-off-by: Russell King Tested-by: Santosh Shilimkar --- arch/arm/mm/mmu.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 9d4da6ac28eb..4223d086aa17 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -420,6 +420,10 @@ static void __init build_mem_type_table(void) user_pgprot |= L_PTE_SHARED; kern_pgprot |= L_PTE_SHARED; vecs_pgprot |= L_PTE_SHARED; + mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_S; + mem_types[MT_DEVICE_WC].prot_pte |= L_PTE_SHARED; + mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_S; + mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; #endif -- cgit v1.2.2 From 7e5a69e83ba7a0d5917ad830f417cba8b8d6aa72 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Mon, 29 Mar 2010 21:46:02 +0100 Subject: ARM: 6007/1: fix highmem with VIPT cache and DMA The VIVT cache of a highmem page is always flushed before the page is unmapped. This cache flush is explicit through flush_cache_kmaps() in flush_all_zero_pkmaps(), or through __cpuc_flush_dcache_area() in kunmap_atomic(). There is also an implicit flush of those highmem pages that were part of a process that just terminated making those pages free as the whole VIVT cache has to be flushed on every task switch. Hence unmapped highmem pages need no cache maintenance in that case. However unmapped pages may still be cached with a VIPT cache because the cache is tagged with physical addresses. There is no need for a whole cache flush during task switching for that reason, and despite the explicit cache flushes in flush_all_zero_pkmaps() and kunmap_atomic(), some highmem pages that were mapped in user space end up still cached even when they become unmapped. So, we do have to perform cache maintenance on those unmapped highmem pages in the context of DMA when using a VIPT cache. Unfortunately, it is not possible to perform that cache maintenance using physical addresses as all the L1 cache maintenance coprocessor functions accept virtual addresses only. Therefore we have no choice but to set up a temporary virtual mapping for that purpose. And of course the explicit cache flushing when unmapping a highmem page on a system with a VIPT cache now can go, which should increase performance. While at it, because the code in __flush_dcache_page() has to be modified anyway, let's also make sure the mapped highmem pages are pinned with kmap_high_get() for the duration of the cache maintenance operation. Because kunmap() does unmap highmem pages lazily, it was reported by Gary King that those pages ended up being unmapped during cache maintenance on SMP causing segmentation faults. Signed-off-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/mm/copypage-v6.c | 9 +---- arch/arm/mm/dma-mapping.c | 5 +++ arch/arm/mm/flush.c | 25 ++++++++------ arch/arm/mm/highmem.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 19 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 8bca4dea6dfa..f55fa1044f72 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -41,14 +41,7 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, kfrom = kmap_atomic(from, KM_USER0); kto = kmap_atomic(to, KM_USER1); copy_page(kto, kfrom); -#ifdef CONFIG_HIGHMEM - /* - * kmap_atomic() doesn't set the page virtual address, and - * kunmap_atomic() takes care of cache flushing already. - */ - if (page_address(to) != NULL) -#endif - __cpuc_flush_dcache_area(kto, PAGE_SIZE); + __cpuc_flush_dcache_area(kto, PAGE_SIZE); kunmap_atomic(kto, KM_USER1); kunmap_atomic(kfrom, KM_USER0); } diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 1351edc0b26f..13fa536d82e6 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -464,6 +464,11 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, vaddr += offset; op(vaddr, len, dir); kunmap_high(page); + } else if (cache_is_vipt()) { + pte_t saved_pte; + vaddr = kmap_high_l1_vipt(page, &saved_pte); + op(vaddr + offset, len, dir); + kunmap_high_l1_vipt(page, saved_pte); } } else { vaddr = page_address(page) + offset; diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index e34f095e2090..c6844cb9b508 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -152,21 +153,25 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, void __flush_dcache_page(struct address_space *mapping, struct page *page) { - void *addr = page_address(page); - /* * Writeback any data associated with the kernel mapping of this * page. This ensures that data in the physical page is mutually * coherent with the kernels mapping. */ -#ifdef CONFIG_HIGHMEM - /* - * kmap_atomic() doesn't set the page virtual address, and - * kunmap_atomic() takes care of cache flushing already. - */ - if (addr) -#endif - __cpuc_flush_dcache_area(addr, PAGE_SIZE); + if (!PageHighMem(page)) { + __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); + } else { + void *addr = kmap_high_get(page); + if (addr) { + __cpuc_flush_dcache_area(addr, PAGE_SIZE); + kunmap_high(page); + } else if (cache_is_vipt()) { + pte_t saved_pte; + addr = kmap_high_l1_vipt(page, &saved_pte); + __cpuc_flush_dcache_area(addr, PAGE_SIZE); + kunmap_high_l1_vipt(page, saved_pte); + } + } /* * If this is a page cache page, and we have an aliasing VIPT cache, diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index 2be1ec7c1b41..77b030f5ec09 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -79,7 +79,8 @@ void kunmap_atomic(void *kvaddr, enum km_type type) unsigned int idx = type + KM_TYPE_NR * smp_processor_id(); if (kvaddr >= (void *)FIXADDR_START) { - __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); + if (cache_is_vivt()) + __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); set_pte_ext(TOP_PTE(vaddr), __pte(0), 0); @@ -124,3 +125,87 @@ struct page *kmap_atomic_to_page(const void *ptr) pte = TOP_PTE(vaddr); return pte_page(*pte); } + +#ifdef CONFIG_CPU_CACHE_VIPT + +#include + +/* + * The VIVT cache of a highmem page is always flushed before the page + * is unmapped. Hence unmapped highmem pages need no cache maintenance + * in that case. + * + * However unmapped pages may still be cached with a VIPT cache, and + * it is not possible to perform cache maintenance on them using physical + * addresses unfortunately. So we have no choice but to set up a temporary + * virtual mapping for that purpose. + * + * Yet this VIPT cache maintenance may be triggered from DMA support + * functions which are possibly called from interrupt context. As we don't + * want to keep interrupt disabled all the time when such maintenance is + * taking place, we therefore allow for some reentrancy by preserving and + * restoring the previous fixmap entry before the interrupted context is + * resumed. If the reentrancy depth is 0 then there is no need to restore + * the previous fixmap, and leaving the current one in place allow it to + * be reused the next time without a TLB flush (common with DMA). + */ + +static DEFINE_PER_CPU(int, kmap_high_l1_vipt_depth); + +void *kmap_high_l1_vipt(struct page *page, pte_t *saved_pte) +{ + unsigned int idx, cpu = smp_processor_id(); + int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu); + unsigned long vaddr, flags; + pte_t pte, *ptep; + + idx = KM_L1_CACHE + KM_TYPE_NR * cpu; + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); + ptep = TOP_PTE(vaddr); + pte = mk_pte(page, kmap_prot); + + if (!in_interrupt()) + preempt_disable(); + + raw_local_irq_save(flags); + (*depth)++; + if (pte_val(*ptep) == pte_val(pte)) { + *saved_pte = pte; + } else { + *saved_pte = *ptep; + set_pte_ext(ptep, pte, 0); + local_flush_tlb_kernel_page(vaddr); + } + raw_local_irq_restore(flags); + + return (void *)vaddr; +} + +void kunmap_high_l1_vipt(struct page *page, pte_t saved_pte) +{ + unsigned int idx, cpu = smp_processor_id(); + int *depth = &per_cpu(kmap_high_l1_vipt_depth, cpu); + unsigned long vaddr, flags; + pte_t pte, *ptep; + + idx = KM_L1_CACHE + KM_TYPE_NR * cpu; + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); + ptep = TOP_PTE(vaddr); + pte = mk_pte(page, kmap_prot); + + BUG_ON(pte_val(*ptep) != pte_val(pte)); + BUG_ON(*depth <= 0); + + raw_local_irq_save(flags); + (*depth)--; + if (*depth != 0 && pte_val(pte) != pte_val(saved_pte)) { + set_pte_ext(ptep, saved_pte, 0); + local_flush_tlb_kernel_page(vaddr); + } + raw_local_irq_restore(flags); + + if (!in_interrupt()) + preempt_enable(); +} + +#endif /* CONFIG_CPU_CACHE_VIPT */ -- cgit v1.2.2 From 3f2d4f561fab4588344cc519fd323382ab950928 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Tue, 13 Apr 2010 07:01:46 +0100 Subject: ARM: 6052/1: kdump: make kexec work in interrupt context When crash happens in interrupt context there is no userspace context. We always use current->active_mm in those cases. Signed-off-by: Mika Westerberg Signed-off-by: Russell King --- arch/arm/mm/mmu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 4223d086aa17..241c24a1c18f 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -1054,10 +1054,12 @@ void setup_mm_for_reboot(char mode) pgd_t *pgd; int i; - if (current->mm && current->mm->pgd) - pgd = current->mm->pgd; - else - pgd = init_mm.pgd; + /* + * We need to access to user-mode page tables here. For kernel threads + * we don't have any user-mode mappings so we use the context that we + * "borrowed". + */ + pgd = current->active_mm->pgd; base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT; if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) -- cgit v1.2.2 From 4260415f6a3b92c5c986398d96c314df37a4ccbf Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 19 Apr 2010 10:15:03 +0100 Subject: ARM: fix build error in arch/arm/kernel/process.c /tmp/ccJ3ssZW.s: Assembler messages: /tmp/ccJ3ssZW.s:1952: Error: can't resolve `.text' {.text section} - `.LFB1077' This is caused because: .section .data .section .text .section .text .previous does not return us to the .text section, but the .data section; this makes use of .previous dangerous if the ordering of previous sections is not known. Fix up the other users of .previous; .pushsection and .popsection are a safer pairing to use than .section and .previous. Signed-off-by: Russell King --- arch/arm/mm/alignment.c | 24 ++++++++++++------------ arch/arm/mm/proc-sa1100.S | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index edddd66faac6..a2ab51fa73e2 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -166,15 +166,15 @@ union offset_union { THUMB( "1: "ins" %1, [%2]\n" ) \ THUMB( " add %2, %2, #1\n" ) \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ + " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, #1\n" \ " b 2b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ + " .popsection\n" \ + " .pushsection __ex_table,\"a\"\n" \ " .align 3\n" \ " .long 1b, 3b\n" \ - " .previous\n" \ + " .popsection\n" \ : "=r" (err), "=&r" (val), "=r" (addr) \ : "0" (err), "2" (addr)) @@ -226,16 +226,16 @@ union offset_union { " mov %1, %1, "NEXT_BYTE"\n" \ "2: "ins" %1, [%2]\n" \ "3:\n" \ - " .section .fixup,\"ax\"\n" \ + " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ "4: mov %0, #1\n" \ " b 3b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ + " .popsection\n" \ + " .pushsection __ex_table,\"a\"\n" \ " .align 3\n" \ " .long 1b, 4b\n" \ " .long 2b, 4b\n" \ - " .previous\n" \ + " .popsection\n" \ : "=r" (err), "=&r" (v), "=&r" (a) \ : "0" (err), "1" (v), "2" (a)); \ if (err) \ @@ -266,18 +266,18 @@ union offset_union { " mov %1, %1, "NEXT_BYTE"\n" \ "4: "ins" %1, [%2]\n" \ "5:\n" \ - " .section .fixup,\"ax\"\n" \ + " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ "6: mov %0, #1\n" \ " b 5b\n" \ - " .previous\n" \ - " .section __ex_table,\"a\"\n" \ + " .popsection\n" \ + " .pushsection __ex_table,\"a\"\n" \ " .align 3\n" \ " .long 1b, 6b\n" \ " .long 2b, 6b\n" \ " .long 3b, 6b\n" \ " .long 4b, 6b\n" \ - " .previous\n" \ + " .popsection\n" \ : "=r" (err), "=&r" (v), "=&r" (a) \ : "0" (err), "1" (v), "2" (a)); \ if (err) \ diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S index ee7700242c19..5c47760c2064 100644 --- a/arch/arm/mm/proc-sa1100.S +++ b/arch/arm/mm/proc-sa1100.S @@ -45,7 +45,7 @@ ENTRY(cpu_sa1100_proc_init) mcr p15, 0, r0, c9, c0, 5 @ Allow read-buffer operations from userland mov pc, lr - .previous + .section .text /* * cpu_sa1100_proc_fin() -- cgit v1.2.2 From ea056df7965fc46cfff28fd3808bf3ada23d5059 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 4 May 2010 17:27:43 +0100 Subject: ARM: 6093/1: Fix kernel memory printing for sparsemem The show_mem() and mem_init() function are assuming that the page map is contiguous and calculates the start and end page of a bank using (map + pfn). This fails with SPARSEMEM where pfn_to_page() must be used. Tested-by: Will Deacon Tested-by: Marek Vasut Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/init.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 83db12a68d56..0ed29bfeba1c 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -86,9 +86,6 @@ void show_mem(void) printk("Mem-info:\n"); show_free_areas(); for_each_online_node(node) { - pg_data_t *n = NODE_DATA(node); - struct page *map = pgdat_page_nr(n, 0) - n->node_start_pfn; - for_each_nodebank (i,mi,node) { struct membank *bank = &mi->bank[i]; unsigned int pfn1, pfn2; @@ -97,8 +94,8 @@ void show_mem(void) pfn1 = bank_pfn_start(bank); pfn2 = bank_pfn_end(bank); - page = map + pfn1; - end = map + pfn2; + page = pfn_to_page(pfn1); + end = pfn_to_page(pfn2 - 1) + 1; do { total++; @@ -603,9 +600,6 @@ void __init mem_init(void) reserved_pages = free_pages = 0; for_each_online_node(node) { - pg_data_t *n = NODE_DATA(node); - struct page *map = pgdat_page_nr(n, 0) - n->node_start_pfn; - for_each_nodebank(i, &meminfo, node) { struct membank *bank = &meminfo.bank[i]; unsigned int pfn1, pfn2; @@ -614,8 +608,8 @@ void __init mem_init(void) pfn1 = bank_pfn_start(bank); pfn2 = bank_pfn_end(bank); - page = map + pfn1; - end = map + pfn2; + page = pfn_to_page(pfn1); + end = pfn_to_page(pfn2 - 1) + 1; do { if (PageReserved(page)) -- cgit v1.2.2 From b1a9ceb2e003aab7b96e30d990c1092453a0ea44 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Thu, 6 May 2010 15:14:09 +0100 Subject: ARM: 6105/1: Fix the __arm_ioremap_caller() definition in nommu.c Commit 31aa8fd6 introduced the __arm_ioremap_caller() function but the nommu.c version did not have the _caller suffix. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/nommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 9bfeb6b9509a..f8791eed7594 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -87,8 +87,8 @@ void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size, } EXPORT_SYMBOL(__arm_ioremap); -void __iomem *__arm_ioremap(unsigned long phys_addr, size_t size, - unsigned int mtype, void *caller) +void __iomem *__arm_ioremap_caller(unsigned long phys_addr, size_t size, + unsigned int mtype, void *caller) { return __arm_ioremap(phys_addr, size, mtype); } -- cgit v1.2.2 From b5a07faadeb4e0cfd6dcee359e501d4755cab875 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Thu, 6 May 2010 15:15:28 +0100 Subject: ARM: 6106/1: Implement copy_to_user_page() for noMMU Commit 7959722 introduced calls to copy_(to|from)_user_page() from access_process_vm() in mm/nommu.c. The copy_to_user_page() was not implemented on noMMU ARM. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/nommu.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index f8791eed7594..33b327379f07 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -65,6 +65,15 @@ void flush_dcache_page(struct page *page) } EXPORT_SYMBOL(flush_dcache_page); +void copy_to_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long uaddr, void *dst, const void *src, + unsigned long len) +{ + memcpy(dst, src, len); + if (vma->vm_flags & VM_EXEC) + __cpuc_coherent_user_range(uaddr, uaddr + len); +} + void __iomem *__arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, unsigned int mtype) { -- cgit v1.2.2 From f4d6477f7f073b99220386d62f5bf54bec3482cc Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 7 May 2010 16:26:24 +0100 Subject: ARM: 6111/1: Implement read/write for ownership in the ARMv6 DMA cache ops The Snoop Control Unit on the ARM11MPCore hardware does not detect the cache operations and the dma_cache_maint*() functions may leave stale cache entries on other CPUs. The solution implemented in this patch performs a Read or Write For Ownership in the ARMv6 DMA cache maintenance functions. These LDR/STR instructions change the cache line state to shared or exclusive so that the cache maintenance operation has the desired effect. Tested-by: George G. Davis Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/cache-v6.S | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 9d89c67a1cc3..e46ecd847138 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -211,6 +211,9 @@ v6_dma_inv_range: mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line #endif 1: +#ifdef CONFIG_SMP + str r0, [r0] @ write for ownership +#endif #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c6, 1 @ invalidate D line #else @@ -231,6 +234,9 @@ v6_dma_inv_range: v6_dma_clean_range: bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: +#ifdef CONFIG_SMP + ldr r2, [r0] @ read for ownership +#endif #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c10, 1 @ clean D line #else @@ -251,6 +257,10 @@ v6_dma_clean_range: ENTRY(v6_dma_flush_range) bic r0, r0, #D_CACHE_LINE_SIZE - 1 1: +#ifdef CONFIG_SMP + ldr r2, [r0] @ read for ownership + str r2, [r0] @ write for ownership +#endif #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line #else @@ -273,7 +283,9 @@ ENTRY(v6_dma_map_area) add r1, r1, r0 teq r2, #DMA_FROM_DEVICE beq v6_dma_inv_range - b v6_dma_clean_range + teq r2, #DMA_TO_DEVICE + beq v6_dma_clean_range + b v6_dma_flush_range ENDPROC(v6_dma_map_area) /* @@ -283,9 +295,6 @@ ENDPROC(v6_dma_map_area) * - dir - DMA direction */ ENTRY(v6_dma_unmap_area) - add r1, r1, r0 - teq r2, #DMA_TO_DEVICE - bne v6_dma_inv_range mov pc, lr ENDPROC(v6_dma_unmap_area) -- cgit v1.2.2 From b8349b569aae661dea9d59d7d2ee587ccea3336c Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 7 May 2010 18:03:05 +0100 Subject: ARM: 6112/1: Use the Inner Shareable I-cache and BTB ops on ARMv7 SMP The standard I-cache Invalidate All (ICIALLU) and Branch Predication Invalidate All (BPIALL) operations are not automatically broadcast to the other CPUs in an ARMv7 MP system. The patch adds the Inner Shareable variants, ICIALLUIS and BPIALLIS, if ARMv7 and SMP. Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/mm/cache-v7.S | 4 ++++ arch/arm/mm/tlb-v7.S | 8 ++++++++ 2 files changed, 12 insertions(+) (limited to 'arch/arm/mm') diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index bcd64f265870..06a90dcfc60a 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -167,7 +167,11 @@ ENTRY(v7_coherent_user_range) cmp r0, r1 blo 1b mov r0, #0 +#ifdef CONFIG_SMP + mcr p15, 0, r0, c7, c1, 6 @ invalidate BTB Inner Shareable +#else mcr p15, 0, r0, c7, c5, 6 @ invalidate BTB +#endif dsb isb mov pc, lr diff --git a/arch/arm/mm/tlb-v7.S b/arch/arm/mm/tlb-v7.S index 0cb1848bd876..f3f288a9546d 100644 --- a/arch/arm/mm/tlb-v7.S +++ b/arch/arm/mm/tlb-v7.S @@ -50,7 +50,11 @@ ENTRY(v7wbi_flush_user_tlb_range) cmp r0, r1 blo 1b mov ip, #0 +#ifdef CONFIG_SMP + mcr p15, 0, ip, c7, c1, 6 @ flush BTAC/BTB Inner Shareable +#else mcr p15, 0, ip, c7, c5, 6 @ flush BTAC/BTB +#endif dsb mov pc, lr ENDPROC(v7wbi_flush_user_tlb_range) @@ -79,7 +83,11 @@ ENTRY(v7wbi_flush_kern_tlb_range) cmp r0, r1 blo 1b mov r2, #0 +#ifdef CONFIG_SMP + mcr p15, 0, r2, c7, c1, 6 @ flush BTAC/BTB Inner Shareable +#else mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB +#endif dsb isb mov pc, lr -- cgit v1.2.2