aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm')
-rw-r--r--arch/arm/Kconfig16
-rw-r--r--arch/arm/lib/Makefile3
-rw-r--r--arch/arm/lib/uaccess_with_memcpy.c139
3 files changed, 158 insertions, 0 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 9d02cdb15b23..c63e65d436a3 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1085,6 +1085,22 @@ config ALIGNMENT_TRAP
1085 correct operation of some network protocols. With an IP-only 1085 correct operation of some network protocols. With an IP-only
1086 configuration it is safe to say N, otherwise say Y. 1086 configuration it is safe to say N, otherwise say Y.
1087 1087
1088config UACCESS_WITH_MEMCPY
1089 bool "Use kernel mem{cpy,set}() for {copy_to,clear}_user() (EXPERIMENTAL)"
1090 depends on MMU && EXPERIMENTAL
1091 default y if CPU_FEROCEON
1092 help
1093 Implement faster copy_to_user and clear_user methods for CPU
1094 cores where a 8-word STM instruction give significantly higher
1095 memory write throughput than a sequence of individual 32bit stores.
1096
1097 A possible side effect is a slight increase in scheduling latency
1098 between threads sharing the same address space if they invoke
1099 such copy operations with large buffers.
1100
1101 However, if the CPU data cache is using a write-allocate mode,
1102 this option is unlikely to provide any performance gain.
1103
1088endmenu 1104endmenu
1089 1105
1090menu "Boot options" 1106menu "Boot options"
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 866f84a586ff..030ba7219f48 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -29,6 +29,9 @@ else
29endif 29endif
30endif 30endif
31 31
32# using lib_ here won't override already available weak symbols
33obj-$(CONFIG_UACCESS_WITH_MEMCPY) += uaccess_with_memcpy.o
34
32lib-$(CONFIG_MMU) += $(mmu-y) 35lib-$(CONFIG_MMU) += $(mmu-y)
33 36
34ifeq ($(CONFIG_CPU_32v3),y) 37ifeq ($(CONFIG_CPU_32v3),y)
diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
new file mode 100644
index 000000000000..bf987b4a2571
--- /dev/null
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -0,0 +1,139 @@
1/*
2 * linux/arch/arm/lib/uaccess_with_memcpy.c
3 *
4 * Written by: Lennert Buytenhek and Nicolas Pitre
5 * Copyright (C) 2009 Marvell Semiconductor
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/kernel.h>
13#include <linux/ctype.h>
14#include <linux/uaccess.h>
15#include <linux/rwsem.h>
16#include <linux/mm.h>
17#include <linux/sched.h>
18#include <linux/hardirq.h> /* for in_atomic() */
19#include <asm/current.h>
20#include <asm/page.h>
21
22static int
23pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
24{
25 unsigned long addr = (unsigned long)_addr;
26 pgd_t *pgd;
27 pmd_t *pmd;
28 pte_t *pte;
29 spinlock_t *ptl;
30
31 pgd = pgd_offset(current->mm, addr);
32 if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
33 return 0;
34
35 pmd = pmd_offset(pgd, addr);
36 if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
37 return 0;
38
39 pte = pte_offset_map_lock(current->mm, pmd, addr, &ptl);
40 if (unlikely(!pte_present(*pte) || !pte_young(*pte) ||
41 !pte_write(*pte) || !pte_dirty(*pte))) {
42 pte_unmap_unlock(pte, ptl);
43 return 0;
44 }
45
46 *ptep = pte;
47 *ptlp = ptl;
48
49 return 1;
50}
51
52unsigned long
53__copy_to_user(void __user *to, const void *from, unsigned long n)
54{
55 int atomic;
56
57 if (n < 1024)
58 return __copy_to_user_std(to, from, n);
59
60 if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
61 memcpy((void *)to, from, n);
62 return 0;
63 }
64
65 /* the mmap semaphore is taken only if not in an atomic context */
66 atomic = in_atomic();
67
68 if (!atomic)
69 down_read(&current->mm->mmap_sem);
70 while (n) {
71 pte_t *pte;
72 spinlock_t *ptl;
73 int tocopy;
74
75 while (!pin_page_for_write(to, &pte, &ptl)) {
76 if (!atomic)
77 up_read(&current->mm->mmap_sem);
78 if (__put_user(0, (char __user *)to))
79 goto out;
80 if (!atomic)
81 down_read(&current->mm->mmap_sem);
82 }
83
84 tocopy = (~(unsigned long)to & ~PAGE_MASK) + 1;
85 if (tocopy > n)
86 tocopy = n;
87
88 memcpy((void *)to, from, tocopy);
89 to += tocopy;
90 from += tocopy;
91 n -= tocopy;
92
93 pte_unmap_unlock(pte, ptl);
94 }
95 if (!atomic)
96 up_read(&current->mm->mmap_sem);
97
98out:
99 return n;
100}
101
102unsigned long __clear_user(void __user *addr, unsigned long n)
103{
104 if (n < 256)
105 return __clear_user_std(addr, n);
106
107 if (unlikely(segment_eq(get_fs(), KERNEL_DS))) {
108 memset((void *)addr, 0, n);
109 return 0;
110 }
111
112 down_read(&current->mm->mmap_sem);
113 while (n) {
114 pte_t *pte;
115 spinlock_t *ptl;
116 int tocopy;
117
118 while (!pin_page_for_write(addr, &pte, &ptl)) {
119 up_read(&current->mm->mmap_sem);
120 if (__put_user(0, (char __user *)addr))
121 goto out;
122 down_read(&current->mm->mmap_sem);
123 }
124
125 tocopy = (~(unsigned long)addr & ~PAGE_MASK) + 1;
126 if (tocopy > n)
127 tocopy = n;
128
129 memset((void *)addr, 0, tocopy);
130 addr += tocopy;
131 n -= tocopy;
132
133 pte_unmap_unlock(pte, ptl);
134 }
135 up_read(&current->mm->mmap_sem);
136
137out:
138 return n;
139}