From 247055aa21ffef1c49dd64710d5e94c2aee19b58 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 13 Sep 2010 16:03:21 +0100 Subject: ARM: 6384/1: Remove the domain switching on ARMv6k/v7 CPUs This patch removes the domain switching functionality via the set_fs and __switch_to functions on cores that have a TLS register. Currently, the ioremap and vmalloc areas share the same level 1 page tables and therefore have the same domain (DOMAIN_KERNEL). When the kernel domain is modified from Client to Manager (via the __set_fs or in the __switch_to function), the XN (eXecute Never) bit is overridden and newer CPUs can speculatively prefetch the ioremap'ed memory. Linux performs the kernel domain switching to allow user-specific functions (copy_to/from_user, get/put_user etc.) to access kernel memory. In order for these functions to work with the kernel domain set to Client, the patch modifies the LDRT/STRT and related instructions to the LDR/STR ones. The user pages access rights are also modified for kernel read-only access rather than read/write so that the copy-on-write mechanism still works. CPU_USE_DOMAINS gets disabled only if the hardware has a TLS register (CPU_32v6K is defined) since writing the TLS value to the high vectors page isn't possible. The user addresses passed to the kernel are checked by the access_ok() function so that they do not point to the kernel space. Tested-by: Anton Vorontsov Cc: Tony Lindgren Signed-off-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/assembler.h | 13 +++++++------ arch/arm/include/asm/domain.h | 31 +++++++++++++++++++++++++++++-- arch/arm/include/asm/futex.h | 9 +++++---- arch/arm/include/asm/traps.h | 2 ++ arch/arm/include/asm/uaccess.h | 16 ++++++++-------- 5 files changed, 51 insertions(+), 20 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 062b58c029ab..4e84d09c9c1b 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -18,6 +18,7 @@ #endif #include +#include /* * Endian independent macros for shifting bytes within registers. @@ -206,12 +207,12 @@ */ #ifdef CONFIG_THUMB2_KERNEL - .macro usraccoff, instr, reg, ptr, inc, off, cond, abort + .macro usraccoff, instr, reg, ptr, inc, off, cond, abort, t=T() 9999: .if \inc == 1 - \instr\cond\()bt \reg, [\ptr, #\off] + \instr\cond\()b\()\t\().w \reg, [\ptr, #\off] .elseif \inc == 4 - \instr\cond\()t \reg, [\ptr, #\off] + \instr\cond\()\t\().w \reg, [\ptr, #\off] .else .error "Unsupported inc macro argument" .endif @@ -246,13 +247,13 @@ #else /* !CONFIG_THUMB2_KERNEL */ - .macro usracc, instr, reg, ptr, inc, cond, rept, abort + .macro usracc, instr, reg, ptr, inc, cond, rept, abort, t=T() .rept \rept 9999: .if \inc == 1 - \instr\cond\()bt \reg, [\ptr], #\inc + \instr\cond\()b\()\t \reg, [\ptr], #\inc .elseif \inc == 4 - \instr\cond\()t \reg, [\ptr], #\inc + \instr\cond\()\t \reg, [\ptr], #\inc .else .error "Unsupported inc macro argument" .endif diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index cc7ef4080711..af18ceaacf5d 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -45,13 +45,17 @@ */ #define DOMAIN_NOACCESS 0 #define DOMAIN_CLIENT 1 +#ifdef CONFIG_CPU_USE_DOMAINS #define DOMAIN_MANAGER 3 +#else +#define DOMAIN_MANAGER 1 +#endif #define domain_val(dom,type) ((type) << (2*(dom))) #ifndef __ASSEMBLY__ -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS #define set_domain(x) \ do { \ __asm__ __volatile__( \ @@ -74,5 +78,28 @@ #define modify_domain(dom,type) do { } while (0) #endif +/* + * Generate the T (user) versions of the LDR/STR and related + * instructions (inline assembly) + */ +#ifdef CONFIG_CPU_USE_DOMAINS +#define T(instr) #instr "t" +#else +#define T(instr) #instr #endif -#endif /* !__ASSEMBLY__ */ + +#else /* __ASSEMBLY__ */ + +/* + * Generate the T (user) versions of the LDR/STR and related + * instructions + */ +#ifdef CONFIG_CPU_USE_DOMAINS +#define T(instr) instr ## t +#else +#define T(instr) instr +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* !__ASM_PROC_DOMAIN_H */ diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 540a044153a5..b33fe7065b38 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -13,12 +13,13 @@ #include #include #include +#include #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ __asm__ __volatile__( \ - "1: ldrt %1, [%2]\n" \ + "1: " T(ldr) " %1, [%2]\n" \ " " insn "\n" \ - "2: strt %0, [%2]\n" \ + "2: " T(str) " %0, [%2]\n" \ " mov %0, #0\n" \ "3:\n" \ " .pushsection __ex_table,\"a\"\n" \ @@ -97,10 +98,10 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) pagefault_disable(); /* implies preempt_disable() */ __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" - "1: ldrt %0, [%3]\n" + "1: " T(ldr) " %0, [%3]\n" " teq %0, %1\n" " it eq @ explicit IT needed for the 2b label\n" - "2: streqt %2, [%3]\n" + "2: " T(streq) " %2, [%3]\n" "3:\n" " .pushsection __ex_table,\"a\"\n" " .align 3\n" diff --git a/arch/arm/include/asm/traps.h b/arch/arm/include/asm/traps.h index 491960bf4260..af5d5d1388c6 100644 --- a/arch/arm/include/asm/traps.h +++ b/arch/arm/include/asm/traps.h @@ -27,4 +27,6 @@ static inline int in_exception_text(unsigned long ptr) extern void __init early_trap_init(void); extern void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame); +extern void *vectors_page; + #endif diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 33e4a48fe103..b293616a1a1a 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -227,7 +227,7 @@ do { \ #define __get_user_asm_byte(x,addr,err) \ __asm__ __volatile__( \ - "1: ldrbt %1,[%2]\n" \ + "1: " T(ldrb) " %1,[%2],#0\n" \ "2:\n" \ " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ @@ -263,7 +263,7 @@ do { \ #define __get_user_asm_word(x,addr,err) \ __asm__ __volatile__( \ - "1: ldrt %1,[%2]\n" \ + "1: " T(ldr) " %1,[%2],#0\n" \ "2:\n" \ " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ @@ -308,7 +308,7 @@ do { \ #define __put_user_asm_byte(x,__pu_addr,err) \ __asm__ __volatile__( \ - "1: strbt %1,[%2]\n" \ + "1: " T(strb) " %1,[%2],#0\n" \ "2:\n" \ " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ @@ -341,7 +341,7 @@ do { \ #define __put_user_asm_word(x,__pu_addr,err) \ __asm__ __volatile__( \ - "1: strt %1,[%2]\n" \ + "1: " T(str) " %1,[%2],#0\n" \ "2:\n" \ " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ @@ -366,10 +366,10 @@ do { \ #define __put_user_asm_dword(x,__pu_addr,err) \ __asm__ __volatile__( \ - ARM( "1: strt " __reg_oper1 ", [%1], #4\n" ) \ - ARM( "2: strt " __reg_oper0 ", [%1]\n" ) \ - THUMB( "1: strt " __reg_oper1 ", [%1]\n" ) \ - THUMB( "2: strt " __reg_oper0 ", [%1, #4]\n" ) \ + ARM( "1: " T(str) " " __reg_oper1 ", [%1], #4\n" ) \ + ARM( "2: " T(str) " " __reg_oper0 ", [%1]\n" ) \ + THUMB( "1: " T(str) " " __reg_oper1 ", [%1]\n" ) \ + THUMB( "2: " T(str) " " __reg_oper0 ", [%1, #4]\n" ) \ "3:\n" \ " .pushsection .fixup,\"ax\"\n" \ " .align 2\n" \ -- cgit v1.2.2 From 6d803ba736abb5e122dede70a4720e4843dd6df4 Mon Sep 17 00:00:00 2001 From: Jean-Christop PLAGNIOL-VILLARD Date: Wed, 17 Nov 2010 10:04:33 +0100 Subject: ARM: 6483/1: arm & sh: factorised duplicated clkdev.c factorise some generic infrastructure to assist looking up struct clks for the ARM & SH architecture. as the code is identical at 99% put the arch specific code for allocation as example in asm/clkdev.h Signed-off-by: Jean-Christophe PLAGNIOL-VILLARD Acked-by: Paul Mundt Signed-off-by: Russell King --- arch/arm/include/asm/clkdev.h | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/clkdev.h b/arch/arm/include/asm/clkdev.h index b56c1389b6fa..765d33222369 100644 --- a/arch/arm/include/asm/clkdev.h +++ b/arch/arm/include/asm/clkdev.h @@ -12,23 +12,13 @@ #ifndef __ASM_CLKDEV_H #define __ASM_CLKDEV_H -struct clk; -struct device; +#include -struct clk_lookup { - struct list_head node; - const char *dev_id; - const char *con_id; - struct clk *clk; -}; +#include -struct clk_lookup *clkdev_alloc(struct clk *clk, const char *con_id, - const char *dev_fmt, ...); - -void clkdev_add(struct clk_lookup *cl); -void clkdev_drop(struct clk_lookup *cl); - -void clkdev_add_table(struct clk_lookup *, size_t); -int clk_add_alias(const char *, const char *, char *, struct device *); +static inline struct clk_lookup_alloc *__clkdev_alloc(size_t size) +{ + return kzalloc(size, GFP_KERNEL); +} #endif -- cgit v1.2.2 From ad3b6993b9c5482e8a2ec5aed181538c921fdcbd Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 09:42:08 +0000 Subject: ARM: SMP: pass an ipi number to smp_cross_call() This allows us to use smp_cross_call() to trigger a number of different software generated interrupts, rather than combining them all on one SGI. Recover the SGI number via do_IPI. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/smp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 3d05190797cb..da7e7ca53cc1 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -38,7 +38,7 @@ extern void show_ipi_list(struct seq_file *p); /* * Called from assembly code, this handles an IPI. */ -asmlinkage void do_IPI(struct pt_regs *regs); +asmlinkage void do_IPI(int ipinr, struct pt_regs *regs); /* * Setup the set of possible CPUs (via set_cpu_possible) @@ -53,7 +53,7 @@ extern void smp_store_cpu_info(unsigned int cpuid); /* * Raise an IPI cross call on CPUs in callmap. */ -extern void smp_cross_call(const struct cpumask *mask); +extern void smp_cross_call(const struct cpumask *mask, int ipi); /* * Boot a secondary CPU, and assign it the specified idle task. -- cgit v1.2.2 From daf8741675562197d4fb4c4e9d773f53494203a5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sat, 4 Dec 2010 17:08:32 +0000 Subject: ARM: implement support for read-mostly sections As our SMP implementation uses MESI protocols. Grouping together data which is mostly only read together means that we avoid unnecessary cache line bouncing when this code shares a cache line with other data. In other words, cache lines associated with read-mostly data are expected to spend most of their time in shared state. Signed-off-by: Russell King --- arch/arm/include/asm/cache.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/cache.h b/arch/arm/include/asm/cache.h index 9d6122096fbe..75fe66bc02b4 100644 --- a/arch/arm/include/asm/cache.h +++ b/arch/arm/include/asm/cache.h @@ -23,4 +23,6 @@ #define ARCH_SLAB_MINALIGN 8 #endif +#define __read_mostly __attribute__((__section__(".data..read_mostly"))) + #endif -- cgit v1.2.2 From ec405ea9fe5fdeb40824edba7082803b3e98f176 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 13:38:06 +0000 Subject: ARM: include local timer irq stats only when local timers configured Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hardirq.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 6d7485aff955..522fb23241ac 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -7,7 +7,9 @@ typedef struct { unsigned int __softirq_pending; +#ifdef CONFIG_LOCAL_TIMERS unsigned int local_timer_irqs; +#endif } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ -- cgit v1.2.2 From 46c48f222f568decb881a552caa1c8f9c96c521e Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 14:15:03 +0000 Subject: ARM: SMP: provide accessors for irq_stat data Provide __inc_irq_stat() and __get_irq_stat() to increment and read the irq stat counters. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hardirq.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 522fb23241ac..923eaa1ba83b 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -14,6 +14,9 @@ typedef struct { #include /* Standard mappings for irq_cpustat_t above */ +#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ +#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) + #if NR_IRQS > 512 #define HARDIRQ_BITS 10 #elif NR_IRQS > 256 -- cgit v1.2.2 From cab8c6f3053c1b147bba825844c8e208f8b3b9f4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 14:20:41 +0000 Subject: ARM: SMP: move ipi_count into irq_stat structure Move the ipi_count into irq_stat, which allows the ipi_data structure to be entirely removed. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hardirq.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 923eaa1ba83b..824b08aa3f7a 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -10,6 +10,9 @@ typedef struct { #ifdef CONFIG_LOCAL_TIMERS unsigned int local_timer_irqs; #endif +#ifdef CONFIG_SMP + unsigned int ipi_irqs; +#endif } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ -- cgit v1.2.2 From f13cd4170ee789f63b3c9585c1ae34e028bd549d Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 14:33:51 +0000 Subject: ARM: fix /proc/interrupts formatting As per x86, align the initial column according to how many IRQs we have. Also, provide an english explaination for the 'LOC:' and 'IPI:' lines. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/mach/irq.h | 2 +- arch/arm/include/asm/smp.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/mach/irq.h b/arch/arm/include/asm/mach/irq.h index ce3eee9fe26c..2bc47fb94d51 100644 --- a/arch/arm/include/asm/mach/irq.h +++ b/arch/arm/include/asm/mach/irq.h @@ -20,7 +20,7 @@ struct seq_file; extern unsigned int arch_nr_irqs; extern void (*init_arch_irq)(void); extern void init_FIQ(void); -extern int show_fiq_list(struct seq_file *, void *); +extern int show_fiq_list(struct seq_file *, int); /* * This is for easy migration, but should be changed in the source diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index da7e7ca53cc1..f93d0a637016 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -33,7 +33,7 @@ struct seq_file; /* * generate IPI list text */ -extern void show_ipi_list(struct seq_file *p); +extern void show_ipi_list(struct seq_file *, int); /* * Called from assembly code, this handles an IPI. @@ -97,6 +97,6 @@ extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); /* * show local interrupt info */ -extern void show_local_irqs(struct seq_file *); +extern void show_local_irqs(struct seq_file *, int); #endif /* ifndef __ASM_ARM_SMP_H */ -- cgit v1.2.2 From 4a88abd7b48e8ec8084b1252d0f5ebdab43c2508 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 14:40:29 +0000 Subject: ARM: SMP: provide individual IPI interrupt statistics This separates out the individual IPI interrupt counts from the total IPI count, which allows better visibility of what IPIs are being used for. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hardirq.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 824b08aa3f7a..c52e2507c9e0 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -5,13 +5,15 @@ #include #include +#define NR_IPI 5 + typedef struct { unsigned int __softirq_pending; #ifdef CONFIG_LOCAL_TIMERS unsigned int local_timer_irqs; #endif #ifdef CONFIG_SMP - unsigned int ipi_irqs; + unsigned int ipi_irqs[NR_IPI]; #endif } ____cacheline_aligned irq_cpustat_t; -- cgit v1.2.2 From b54992fe1b4bad7b7488d58b8696e4e8974fdab0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 15 Nov 2010 14:46:46 +0000 Subject: ARM: SMP: collect IPI and local timer IRQs for /proc/stat The IPI and local timer interrupts weren't being properly accounted for in /proc/stat. Collect them from the irq_stat structure, and return their sum. Reviewed-by: Catalin Marinas Signed-off-by: Russell King --- arch/arm/include/asm/hardirq.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index c52e2507c9e0..89ad1805e579 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -22,6 +22,14 @@ typedef struct { #define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ #define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) +#ifdef CONFIG_SMP +u64 smp_irq_stat_cpu(unsigned int cpu); +#else +#define smp_irq_stat_cpu(cpu) 0 +#endif + +#define arch_irq_stat_cpu smp_irq_stat_cpu + #if NR_IRQS > 512 #define HARDIRQ_BITS 10 #elif NR_IRQS > 256 -- cgit v1.2.2 From 05c74a6cbcfb416286a947668ba32f63d99fe74a Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 3 Dec 2010 11:09:48 +0000 Subject: ARM: SMP: consolidate the common parts of smp_prepare_cpus() There is a certain amount of smp_prepare_cpus() which doesn't belong in the platform support code - that is, code which is invariant to the SMP implementation. Move this code into arch/arm/kernel/smp.c, and add a platform_ prefix to the original function. Signed-off-by: Russell King --- arch/arm/include/asm/smp.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index f93d0a637016..96ed521f2408 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -45,10 +45,6 @@ asmlinkage void do_IPI(int ipinr, struct pt_regs *regs); */ extern void smp_init_cpus(void); -/* - * Move global data into per-processor storage. - */ -extern void smp_store_cpu_info(unsigned int cpuid); /* * Raise an IPI cross call on CPUs in callmap. @@ -72,6 +68,11 @@ asmlinkage void secondary_start_kernel(void); */ extern void platform_secondary_init(unsigned int cpu); +/* + * Initialize cpu_possible map, and enable coherency + */ +extern void platform_smp_prepare_cpus(unsigned int); + /* * Initial data for bringing up a secondary CPU. */ -- cgit v1.2.2 From 86e62b93368cffca9111996e3ed9e5b7bf6f0af3 Mon Sep 17 00:00:00 2001 From: Russell King Date: Tue, 30 Nov 2010 18:24:57 +0000 Subject: ARM: SMP: remove smp_mpidr.h With "ARM: CPU hotplug: remove bug checks in platform_cpu_die()", we now do not use hard_smp_processor_id(), we no longer need to read the hardware processor ID. Remove the include providing this function. Signed-off-by: Russell King --- arch/arm/include/asm/smp_mpidr.h | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 arch/arm/include/asm/smp_mpidr.h (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/smp_mpidr.h b/arch/arm/include/asm/smp_mpidr.h deleted file mode 100644 index 6a9307d64900..000000000000 --- a/arch/arm/include/asm/smp_mpidr.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef ASMARM_SMP_MIDR_H -#define ASMARM_SMP_MIDR_H - -#define hard_smp_processor_id() \ - ({ \ - unsigned int cpunum; \ - __asm__("\n" \ - "1: mrc p15, 0, %0, c0, c0, 5\n" \ - " .pushsection \".alt.smp.init\", \"a\"\n"\ - " .long 1b\n" \ - " mov %0, #0\n" \ - " .popsection" \ - : "=r" (cpunum)); \ - cpunum &= 0x0F; \ - }) - -#endif -- cgit v1.2.2 From ed3768a8d9dc2d345d4f27eb44ee1e4825056c08 Mon Sep 17 00:00:00 2001 From: Dave Martin Date: Wed, 1 Dec 2010 15:39:23 +0100 Subject: ARM: 6516/1: Allow SMP_ON_UP to work with Thumb-2 kernels. * __fixup_smp_on_up has been modified with support for the THUMB2_KERNEL case. For THUMB2_KERNEL only, fixups are split into halfwords in case of misalignment, since we can't rely on unaligned accesses working before turning the MMU on. No attempt is made to optimise the aligned case, since the number of fixups is typically small, and it seems best to keep the code as simple as possible. * Add a rotate in the fixup_smp code in order to support CPU_BIG_ENDIAN, as suggested by Nicolas Pitre. * Add an assembly-time sanity-check to ALT_UP() to ensure that the content really is the right size (4 bytes). (No check is done for ALT_SMP(). Possibly, this could be fixed by splitting the two uses ot ALT_SMP() (ALT_SMP...SMP_UP versus ALT_SMP...SMP_UP_B) into two macros. In the first case, ALT_SMP needs to expand to >= 4 bytes, not == 4.) * smp_mpidr.h (which implements ALT_SMP()/ALT_UP() manually due to macro limitations) has not been modified: the affected instruction (mov) has no 16-bit encoding, so the correct instruction size is satisfied in this case. * A "mode" parameter has been added to smp_dmb: smp_dmb arm @ assumes 4-byte instructions (for ARM code, e.g. kuser) smp_dmb @ uses W() to ensure 4-byte instructions for ALT_SMP() This avoids assembly failures due to use of W() inside smp_dmb, when assembling pure-ARM code in the vectors page. There might be a better way to achieve this. * Kconfig: make SMP_ON_UP depend on (!THUMB2_KERNEL || !BIG_ENDIAN) i.e., THUMB2_KERNEL is now supported, but only if !BIG_ENDIAN (The fixup code for Thumb-2 currently assumes little-endian order.) Tested using a single generic realview kernel on: ARM RealView PB-A8 (CONFIG_THUMB2_KERNEL={n,y}) ARM RealView PBX-A9 (SMP) Signed-off-by: Dave Martin Acked-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/include/asm/assembler.h | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 749bb6622404..72d3389e9c12 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -157,16 +157,24 @@ #ifdef CONFIG_SMP #define ALT_SMP(instr...) \ 9998: instr +/* + * Note: if you get assembler errors from ALT_UP() when building with + * CONFIG_THUMB2_KERNEL, you almost certainly need to use + * ALT_SMP( W(instr) ... ) + */ #define ALT_UP(instr...) \ .pushsection ".alt.smp.init", "a" ;\ .long 9998b ;\ - instr ;\ +9997: instr ;\ + .if . - 9997b != 4 ;\ + .error "ALT_UP() content must assemble to exactly 4 bytes";\ + .endif ;\ .popsection #define ALT_UP_B(label) \ .equ up_b_offset, label - 9998b ;\ .pushsection ".alt.smp.init", "a" ;\ .long 9998b ;\ - b . + up_b_offset ;\ + W(b) . + up_b_offset ;\ .popsection #else #define ALT_SMP(instr...) @@ -177,16 +185,24 @@ /* * SMP data memory barrier */ - .macro smp_dmb + .macro smp_dmb mode #ifdef CONFIG_SMP #if __LINUX_ARM_ARCH__ >= 7 + .ifeqs "\mode","arm" ALT_SMP(dmb) + .else + ALT_SMP(W(dmb)) + .endif #elif __LINUX_ARM_ARCH__ == 6 ALT_SMP(mcr p15, 0, r0, c7, c10, 5) @ dmb #else #error Incompatible SMP platform #endif + .ifeqs "\mode","arm" ALT_UP(nop) + .else + ALT_UP(W(nop)) + .endif #endif .endm -- cgit v1.2.2 From 10034aabca9032246762daaca3152f3e79380ea0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 20 Dec 2010 14:28:02 +0000 Subject: ARM: localtimer: clean up local timer on hot unplug When a CPU is hot unplugged, the generic tick code cleans up the clock event device, but fails to call down to the device's set_mode function to actually shut the device down. To work around this, we've historically had a local_timer_stop() callback out of the hotplug code. However, this adds needless complexity when we have the clock event device itself available. Explicitly call the clock event device's set_mode function with CLOCK_EVT_MODE_UNUSED, so that the hardware can be cleanly shutdown without any special external callbacks. When/if the generic code is fixed, percpu_timer_stop() can be killed off. Signed-off-by: Russell King --- arch/arm/include/asm/localtimer.h | 12 ------------ arch/arm/include/asm/smp_twd.h | 1 - 2 files changed, 13 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/localtimer.h b/arch/arm/include/asm/localtimer.h index 50c7e7cfd670..6bc63ab498ce 100644 --- a/arch/arm/include/asm/localtimer.h +++ b/arch/arm/include/asm/localtimer.h @@ -30,7 +30,6 @@ asmlinkage void do_local_timer(struct pt_regs *); #include "smp_twd.h" #define local_timer_ack() twd_timer_ack() -#define local_timer_stop() twd_timer_stop() #else @@ -40,11 +39,6 @@ asmlinkage void do_local_timer(struct pt_regs *); */ int local_timer_ack(void); -/* - * Stop a local timer interrupt. - */ -void local_timer_stop(void); - #endif /* @@ -52,12 +46,6 @@ void local_timer_stop(void); */ void local_timer_setup(struct clock_event_device *); -#else - -static inline void local_timer_stop(void) -{ -} - #endif #endif diff --git a/arch/arm/include/asm/smp_twd.h b/arch/arm/include/asm/smp_twd.h index 634f357be6bb..fed9981fba08 100644 --- a/arch/arm/include/asm/smp_twd.h +++ b/arch/arm/include/asm/smp_twd.h @@ -22,7 +22,6 @@ struct clock_event_device; extern void __iomem *twd_base; -void twd_timer_stop(void); int twd_timer_ack(void); void twd_timer_setup(struct clock_event_device *); -- cgit v1.2.2 From 521086412ee423fbdfc7da81f257239c43f707b4 Mon Sep 17 00:00:00 2001 From: eric miao Date: Mon, 13 Dec 2010 09:42:34 +0100 Subject: ARM: 6532/1: Allow machine to specify it's own IRQ handlers at run-time Normally different ARM platform has different way to decode the IRQ hardware status and demultiplex to the corresponding IRQ handler. This is highly optimized by macro irq_handler in entry-armv.S, and each machine defines their own macro to decode the IRQ number. However, this prevents multiple machine classes to be built into a single kernel. By allowing each machine to specify thier own handler, and making function pointer 'handle_arch_irq' to point to it at run time, this can be solved. And introduce CONFIG_MULTI_IRQ_HANDLER to allow both solutions to work. Comparing with the highly optimized macro of irq_handler, the new function must be written with care not to lose too much performance. And the IPI stuff on SMP is expected to move to the provided arch IRQ handler as well. The assembly code to invoke handle_arch_irq is optimized by Russell King. Signed-off-by: Eric Miao Acked-by: Nicolas Pitre Signed-off-by: Russell King --- arch/arm/include/asm/mach/arch.h | 3 +++ arch/arm/include/asm/mach/irq.h | 4 ++++ 2 files changed, 7 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index d97a964207fa..7d55356110d4 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -40,6 +40,9 @@ struct machine_desc { void (*init_irq)(void); struct sys_timer *timer; /* system tick timer */ void (*init_machine)(void); +#ifdef CONFIG_MULTI_IRQ_HANDLER + void (*handle_irq)(struct pt_regs *); +#endif }; /* diff --git a/arch/arm/include/asm/mach/irq.h b/arch/arm/include/asm/mach/irq.h index ce3eee9fe26c..6ecdad9a5c42 100644 --- a/arch/arm/include/asm/mach/irq.h +++ b/arch/arm/include/asm/mach/irq.h @@ -22,6 +22,10 @@ extern void (*init_arch_irq)(void); extern void init_FIQ(void); extern int show_fiq_list(struct seq_file *, void *); +#ifdef CONFIG_MULTI_IRQ_HANDLER +extern void (*handle_arch_irq)(struct pt_regs *); +#endif + /* * This is for easy migration, but should be changed in the source */ -- cgit v1.2.2 From cd544ce754ac2432ffcc0626ea802d2b30876b50 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Wed, 22 Dec 2010 13:20:08 +0100 Subject: ARM: 6538/1: Subarch IRQ handler macros V3 Per subarch interrupt handler macros V3. This patch breaks out code from the irq_handler macro into arch_irq_handler and arch_irq_handler_default. The macros are put in the header file "entry-macro-multi.S" The arch_irq_handler_default macro is designed to be used by irq_handler in entry-armv.S while arch_irq_handler is suitable for per-subarch use. Signed-off-by: Magnus Damm Signed-off-by: Russell King --- arch/arm/include/asm/entry-macro-multi.S | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 arch/arm/include/asm/entry-macro-multi.S (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S new file mode 100644 index 000000000000..44f21d4584e8 --- /dev/null +++ b/arch/arm/include/asm/entry-macro-multi.S @@ -0,0 +1,44 @@ +/* + * Interrupt handling. Preserves r7, r8, r9 + */ + .macro arch_irq_handler_default + get_irqnr_preamble r5, lr +1: get_irqnr_and_base r0, r6, r5, lr + movne r1, sp + @ + @ routine called with r0 = irq number, r1 = struct pt_regs * + @ + adrne lr, BSYM(1b) + bne asm_do_IRQ + +#ifdef CONFIG_SMP + /* + * XXX + * + * this macro assumes that irqstat (r6) and base (r5) are + * preserved from get_irqnr_and_base above + */ + ALT_SMP(test_for_ipi r0, r6, r5, lr) + ALT_UP_B(9997f) + movne r0, sp + adrne lr, BSYM(1b) + bne do_IPI + +#ifdef CONFIG_LOCAL_TIMERS + test_for_ltirq r0, r6, r5, lr + movne r0, sp + adrne lr, BSYM(1b) + bne do_local_timer +#endif +#endif +9997: + .endm + + .macro arch_irq_handler, symbol_name + .align 5 + .global \symbol_name +\symbol_name: + mov r4, lr + arch_irq_handler_default + mov pc, r4 + .endm -- cgit v1.2.2 From 8ff1443c5439ecee7472b80cf12ecfc337e6ee98 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 20 Dec 2010 10:18:36 +0000 Subject: ARM: simplify early machine init hooks Rather than storing each machine init hook separately, store a pointer to the machine description record and dereference this instead. This pointer is only available while the init sections are present, which is not a problem as we only use it from init code. Signed-off-by: Russell King --- arch/arm/include/asm/mach/arch.h | 5 +++++ arch/arm/include/asm/mach/irq.h | 2 -- arch/arm/include/asm/mach/time.h | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 7d55356110d4..69908ddbaf18 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -45,6 +45,11 @@ struct machine_desc { #endif }; +/* + * Current machine - only accessible during boot. + */ +extern struct machine_desc *machine_desc; + /* * Set of macros to define architecture features. This is built into * a table by the linker. diff --git a/arch/arm/include/asm/mach/irq.h b/arch/arm/include/asm/mach/irq.h index 6ecdad9a5c42..831e81453f81 100644 --- a/arch/arm/include/asm/mach/irq.h +++ b/arch/arm/include/asm/mach/irq.h @@ -17,8 +17,6 @@ struct seq_file; /* * This is internal. Do not use it. */ -extern unsigned int arch_nr_irqs; -extern void (*init_arch_irq)(void); extern void init_FIQ(void); extern int show_fiq_list(struct seq_file *, void *); diff --git a/arch/arm/include/asm/mach/time.h b/arch/arm/include/asm/mach/time.h index 35d408f6dccf..883f6be5117a 100644 --- a/arch/arm/include/asm/mach/time.h +++ b/arch/arm/include/asm/mach/time.h @@ -43,7 +43,6 @@ struct sys_timer { #endif }; -extern struct sys_timer *system_timer; extern void timer_tick(void); #endif -- cgit v1.2.2 From dec12e62c03d26bbc7a142f067215a3a43cce7d0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 16 Dec 2010 13:49:34 +0000 Subject: ARM: provide an early platform initialization hook This allows platforms to hook into the initialization early to setup things like scheduler clocks, etc. Signed-off-by: Russell King --- arch/arm/include/asm/mach/arch.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/mach/arch.h b/arch/arm/include/asm/mach/arch.h index 69908ddbaf18..3a0893a76a3b 100644 --- a/arch/arm/include/asm/mach/arch.h +++ b/arch/arm/include/asm/mach/arch.h @@ -37,6 +37,7 @@ struct machine_desc { struct meminfo *); void (*reserve)(void);/* reserve mem blocks */ void (*map_io)(void);/* IO mapping function */ + void (*init_early)(void); void (*init_irq)(void); struct sys_timer *timer; /* system tick timer */ void (*init_machine)(void); -- cgit v1.2.2 From 3d29005ab8d828e36108ecc2338612ce3acdd86f Mon Sep 17 00:00:00 2001 From: Shiraz Hashim Date: Thu, 23 Dec 2010 11:32:41 +0100 Subject: ARM: 6541/1: move sev definition to common system.h include file sev is used to send wakeup event to other cores in ARMv6K and above. This has been moved from platform specific part to standard common ARM header file (asm/system.h). Also introduced wfi() and wfe(). Signed-off-by: Shiraz Hashim Signed-off-by: Russell King --- arch/arm/include/asm/system.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h index 1120f18a6b17..9ab8d7e64737 100644 --- a/arch/arm/include/asm/system.h +++ b/arch/arm/include/asm/system.h @@ -119,6 +119,13 @@ extern unsigned int user_debug; #define vectors_high() (0) #endif +#if __LINUX_ARM_ARCH__ >= 7 || \ + (__LINUX_ARM_ARCH__ == 6 && defined(CONFIG_CPU_32v6K)) +#define sev() __asm__ __volatile__ ("sev" : : : "memory") +#define wfe() __asm__ __volatile__ ("wfe" : : : "memory") +#define wfi() __asm__ __volatile__ ("wfi" : : : "memory") +#endif + #if __LINUX_ARM_ARCH__ >= 7 #define isb() __asm__ __volatile__ ("isb" : : : "memory") #define dsb() __asm__ __volatile__ ("dsb" : : : "memory") -- cgit v1.2.2 From 9eedd96301cad8ab58ee8c1e579677d0a75c2ba1 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 3 Jan 2011 00:00:17 +0000 Subject: ARM: DMA: Replace page_to_dma()/dma_to_page() with pfn_to_dma()/dma_to_pfn() Replace the page_to_dma() and dma_to_page() macros with their PFN equivalents. This allows us to map parts of memory which do not have a struct page allocated to them to bus addresses. This will be used internally by dma_alloc_coherent()/dma_alloc_writecombine(). Build tested on Versatile, OMAP1, IOP13xx and KS8695. Tested-by: Janusz Krzysztofik Signed-off-by: Russell King --- arch/arm/include/asm/dma-mapping.h | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index c568da7dcae4..8f69b98f68fc 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -9,20 +9,24 @@ #include #include +#ifdef __arch_page_to_dma +#error Please update to __arch_pfn_to_dma +#endif + /* - * page_to_dma/dma_to_virt/virt_to_dma are architecture private functions - * used internally by the DMA-mapping API to provide DMA addresses. They - * must not be used by drivers. + * dma_to_pfn/pfn_to_dma/dma_to_virt/virt_to_dma are architecture private + * functions used internally by the DMA-mapping API to provide DMA + * addresses. They must not be used by drivers. */ -#ifndef __arch_page_to_dma -static inline dma_addr_t page_to_dma(struct device *dev, struct page *page) +#ifndef __arch_pfn_to_dma +static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn) { - return (dma_addr_t)__pfn_to_bus(page_to_pfn(page)); + return (dma_addr_t)__pfn_to_bus(pfn); } -static inline struct page *dma_to_page(struct device *dev, dma_addr_t addr) +static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) { - return pfn_to_page(__bus_to_pfn(addr)); + return __bus_to_pfn(addr); } static inline void *dma_to_virt(struct device *dev, dma_addr_t addr) @@ -35,14 +39,14 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) return (dma_addr_t)__virt_to_bus((unsigned long)(addr)); } #else -static inline dma_addr_t page_to_dma(struct device *dev, struct page *page) +static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn) { - return __arch_page_to_dma(dev, page); + return __arch_pfn_to_dma(dev, pfn); } -static inline struct page *dma_to_page(struct device *dev, dma_addr_t addr) +static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) { - return __arch_dma_to_page(dev, addr); + return __arch_dma_to_pfn(dev, addr); } static inline void *dma_to_virt(struct device *dev, dma_addr_t addr) @@ -368,7 +372,7 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, __dma_page_cpu_to_dev(page, offset, size, dir); - return page_to_dma(dev, page) + offset; + return pfn_to_dma(dev, page_to_pfn(page)) + offset; } /** @@ -408,8 +412,8 @@ static inline void dma_unmap_single(struct device *dev, dma_addr_t handle, static inline void dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { - __dma_page_dev_to_cpu(dma_to_page(dev, handle), handle & ~PAGE_MASK, - size, dir); + __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), + handle & ~PAGE_MASK, size, dir); } #endif /* CONFIG_DMABOUNCE */ -- cgit v1.2.2 From 24056f525051a9e186af28904b396320e18bf9a0 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 3 Jan 2011 11:29:28 +0000 Subject: ARM: DMA: add support for DMA debugging Add ARM support for the DMA debug infrastructure, which allows the DMA API usage to be debugged. Signed-off-by: Russell King --- arch/arm/include/asm/dma-mapping.h | 65 +++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 12 deletions(-) (limited to 'arch/arm/include') diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 8f69b98f68fc..4fff837363ed 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -297,13 +298,13 @@ extern int dma_needs_bounce(struct device*, dma_addr_t, size_t); /* * The DMA API, implemented by dmabounce.c. See below for descriptions. */ -extern dma_addr_t dma_map_single(struct device *, void *, size_t, +extern dma_addr_t __dma_map_single(struct device *, void *, size_t, enum dma_data_direction); -extern void dma_unmap_single(struct device *, dma_addr_t, size_t, +extern void __dma_unmap_single(struct device *, dma_addr_t, size_t, enum dma_data_direction); -extern dma_addr_t dma_map_page(struct device *, struct page *, +extern dma_addr_t __dma_map_page(struct device *, struct page *, unsigned long, size_t, enum dma_data_direction); -extern void dma_unmap_page(struct device *, dma_addr_t, size_t, +extern void __dma_unmap_page(struct device *, dma_addr_t, size_t, enum dma_data_direction); /* @@ -327,6 +328,34 @@ static inline int dmabounce_sync_for_device(struct device *d, dma_addr_t addr, } +static inline dma_addr_t __dma_map_single(struct device *dev, void *cpu_addr, + size_t size, enum dma_data_direction dir) +{ + __dma_single_cpu_to_dev(cpu_addr, size, dir); + return virt_to_dma(dev, cpu_addr); +} + +static inline dma_addr_t __dma_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir) +{ + __dma_page_cpu_to_dev(page, offset, size, dir); + return pfn_to_dma(dev, page_to_pfn(page)) + offset; +} + +static inline void __dma_unmap_single(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + __dma_single_dev_to_cpu(dma_to_virt(dev, handle), size, dir); +} + +static inline void __dma_unmap_page(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir) +{ + __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), + handle & ~PAGE_MASK, size, dir); +} +#endif /* CONFIG_DMABOUNCE */ + /** * dma_map_single - map a single buffer for streaming DMA * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices @@ -344,11 +373,16 @@ static inline int dmabounce_sync_for_device(struct device *d, dma_addr_t addr, static inline dma_addr_t dma_map_single(struct device *dev, void *cpu_addr, size_t size, enum dma_data_direction dir) { + dma_addr_t addr; + BUG_ON(!valid_dma_direction(dir)); - __dma_single_cpu_to_dev(cpu_addr, size, dir); + addr = __dma_map_single(dev, cpu_addr, size, dir); + debug_dma_map_page(dev, virt_to_page(cpu_addr), + (unsigned long)cpu_addr & ~PAGE_MASK, size, + dir, addr, true); - return virt_to_dma(dev, cpu_addr); + return addr; } /** @@ -368,11 +402,14 @@ static inline dma_addr_t dma_map_single(struct device *dev, void *cpu_addr, static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir) { + dma_addr_t addr; + BUG_ON(!valid_dma_direction(dir)); - __dma_page_cpu_to_dev(page, offset, size, dir); + addr = __dma_map_page(dev, page, offset, size, dir); + debug_dma_map_page(dev, page, offset, size, dir, addr, false); - return pfn_to_dma(dev, page_to_pfn(page)) + offset; + return addr; } /** @@ -392,7 +429,8 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, static inline void dma_unmap_single(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { - __dma_single_dev_to_cpu(dma_to_virt(dev, handle), size, dir); + debug_dma_unmap_page(dev, handle, size, dir, true); + __dma_unmap_single(dev, handle, size, dir); } /** @@ -412,10 +450,9 @@ static inline void dma_unmap_single(struct device *dev, dma_addr_t handle, static inline void dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { - __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), - handle & ~PAGE_MASK, size, dir); + debug_dma_unmap_page(dev, handle, size, dir, false); + __dma_unmap_page(dev, handle, size, dir); } -#endif /* CONFIG_DMABOUNCE */ /** * dma_sync_single_range_for_cpu @@ -441,6 +478,8 @@ static inline void dma_sync_single_range_for_cpu(struct device *dev, { BUG_ON(!valid_dma_direction(dir)); + debug_dma_sync_single_for_cpu(dev, handle + offset, size, dir); + if (!dmabounce_sync_for_cpu(dev, handle, offset, size, dir)) return; @@ -453,6 +492,8 @@ static inline void dma_sync_single_range_for_device(struct device *dev, { BUG_ON(!valid_dma_direction(dir)); + debug_dma_sync_single_for_device(dev, handle + offset, size, dir); + if (!dmabounce_sync_for_device(dev, handle, offset, size, dir)) return; -- cgit v1.2.2