aboutsummaryrefslogtreecommitdiffstats
path: root/arch/mips/lib
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/lib')
-rw-r--r--arch/mips/lib/Makefile5
-rw-r--r--arch/mips/lib/iomap-pci.c74
-rw-r--r--arch/mips/lib/iomap.c253
-rw-r--r--arch/mips/lib/memcpy-inatomic.S436
-rw-r--r--arch/mips/lib/memset.S166
-rw-r--r--arch/mips/lib/uncached.c4
6 files changed, 884 insertions, 54 deletions
diff --git a/arch/mips/lib/Makefile b/arch/mips/lib/Makefile
index 989c900b8b14..2453ea244cb8 100644
--- a/arch/mips/lib/Makefile
+++ b/arch/mips/lib/Makefile
@@ -2,10 +2,11 @@
2# Makefile for MIPS-specific library files.. 2# Makefile for MIPS-specific library files..
3# 3#
4 4
5lib-y += csum_partial.o memcpy.o promlib.o \ 5lib-y += csum_partial.o memcpy.o memcpy-inatomic.o memset.o promlib.o \
6 strlen_user.o strncpy_user.o strnlen_user.o uncached.o 6 strlen_user.o strncpy_user.o strnlen_user.o uncached.o
7 7
8obj-y += iomap.o 8obj-y += iomap.o
9obj-$(CONFIG_PCI) += iomap-pci.o
9 10
10# libgcc-style stuff needed in the kernel 11# libgcc-style stuff needed in the kernel
11lib-y += ashldi3.o ashrdi3.o lshrdi3.o 12lib-y += ashldi3.o ashrdi3.o lshrdi3.o
diff --git a/arch/mips/lib/iomap-pci.c b/arch/mips/lib/iomap-pci.c
new file mode 100644
index 000000000000..c11b2494bb6e
--- /dev/null
+++ b/arch/mips/lib/iomap-pci.c
@@ -0,0 +1,74 @@
1/*
2 * Implement the default iomap interfaces
3 *
4 * (C) Copyright 2004 Linus Torvalds
5 * (C) Copyright 2006 Ralf Baechle <ralf@linux-mips.org>
6 * (C) Copyright 2007 MIPS Technologies, Inc.
7 * written by Ralf Baechle <ralf@linux-mips.org>
8 */
9#include <linux/pci.h>
10#include <linux/module.h>
11#include <asm/io.h>
12
13static void __iomem *ioport_map_pci(struct pci_dev *dev,
14 unsigned long port, unsigned int nr)
15{
16 struct pci_controller *ctrl = dev->bus->sysdata;
17 unsigned long base = ctrl->io_map_base;
18
19 /* This will eventually become a BUG_ON but for now be gentle */
20 if (unlikely(!ctrl->io_map_base)) {
21 struct pci_bus *bus = dev->bus;
22 char name[8];
23
24 while (bus->parent)
25 bus = bus->parent;
26
27 ctrl->io_map_base = base = mips_io_port_base;
28
29 sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number);
30 printk(KERN_WARNING "io_map_base of root PCI bus %s unset. "
31 "Trying to continue but you better\nfix this issue or "
32 "report it to linux-mips@linux-mips.org or your "
33 "vendor.\n", name);
34#ifdef CONFIG_PCI_DOMAINS
35 panic("To avoid data corruption io_map_base MUST be set with "
36 "multiple PCI domains.");
37#endif
38 }
39
40 return (void __iomem *) (ctrl->io_map_base + port);
41}
42
43/*
44 * Create a virtual mapping cookie for a PCI BAR (memory or IO)
45 */
46void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
47{
48 unsigned long start = pci_resource_start(dev, bar);
49 unsigned long len = pci_resource_len(dev, bar);
50 unsigned long flags = pci_resource_flags(dev, bar);
51
52 if (!len || !start)
53 return NULL;
54 if (maxlen && len > maxlen)
55 len = maxlen;
56 if (flags & IORESOURCE_IO)
57 return ioport_map_pci(dev, start, len);
58 if (flags & IORESOURCE_MEM) {
59 if (flags & IORESOURCE_CACHEABLE)
60 return ioremap(start, len);
61 return ioremap_nocache(start, len);
62 }
63 /* What? */
64 return NULL;
65}
66
67EXPORT_SYMBOL(pci_iomap);
68
69void pci_iounmap(struct pci_dev *dev, void __iomem * addr)
70{
71 iounmap(addr);
72}
73
74EXPORT_SYMBOL(pci_iounmap);
diff --git a/arch/mips/lib/iomap.c b/arch/mips/lib/iomap.c
index f4ac5bbcd81f..d51d5cb0a4a9 100644
--- a/arch/mips/lib/iomap.c
+++ b/arch/mips/lib/iomap.c
@@ -1,78 +1,227 @@
1/* 1/*
2 * iomap.c, Memory Mapped I/O routines for MIPS architecture. 2 * Implement the default iomap interfaces
3 * 3 *
4 * This code is based on lib/iomap.c, by Linus Torvalds. 4 * (C) Copyright 2004 Linus Torvalds
5 * 5 * (C) Copyright 2006 Ralf Baechle <ralf@linux-mips.org>
6 * Copyright (C) 2004-2005 Yoichi Yuasa <yoichi_yuasa@tripeaks.co.jp> 6 * (C) Copyright 2007 MIPS Technologies, Inc.
7 * 7 * written by Ralf Baechle <ralf@linux-mips.org>
8 * This program is free software; you can redistribute it and/or modify 8 */
9 * it under the terms of the GNU General Public License as published by 9#include <linux/pci.h>
10 * the Free Software Foundation; either version 2 of the License, or 10#include <linux/module.h>
11 * (at your option) any later version. 11#include <asm/io.h>
12
13/*
14 * Read/write from/to an (offsettable) iomem cookie. It might be a PIO
15 * access or a MMIO access, these functions don't care. The info is
16 * encoded in the hardware mapping set up by the mapping functions
17 * (or the cookie itself, depending on implementation and hw).
12 * 18 *
13 * This program is distributed in the hope that it will be useful, 19 * The generic routines don't assume any hardware mappings, and just
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 * encode the PIO/MMIO as part of the cookie. They coldly assume that
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 * the MMIO IO mappings are not in the low address range.
16 * GNU General Public License for more details.
17 * 22 *
18 * You should have received a copy of the GNU General Public License 23 * Architectures for which this is not true can't use this generic
19 * along with this program; if not, write to the Free Software 24 * implementation and should do their own copy.
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 */ 25 */
22#include <linux/ioport.h>
23#include <linux/module.h>
24#include <linux/pci.h>
25 26
26#include <asm/io.h> 27#define PIO_MASK 0x0ffffUL
27 28
28void __iomem *ioport_map(unsigned long port, unsigned int nr) 29unsigned int ioread8(void __iomem *addr)
29{ 30{
30 unsigned long end; 31 return readb(addr);
32}
31 33
32 end = port + nr - 1UL; 34EXPORT_SYMBOL(ioread8);
33 if (ioport_resource.start > port ||
34 ioport_resource.end < end || port > end)
35 return NULL;
36 35
37 return (void __iomem *)(mips_io_port_base + port); 36unsigned int ioread16(void __iomem *addr)
37{
38 return readw(addr);
38} 39}
39 40
40void ioport_unmap(void __iomem *addr) 41EXPORT_SYMBOL(ioread16);
42
43unsigned int ioread16be(void __iomem *addr)
41{ 44{
45 return be16_to_cpu(__raw_readw(addr));
42} 46}
43EXPORT_SYMBOL(ioport_map);
44EXPORT_SYMBOL(ioport_unmap);
45 47
46void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 48EXPORT_SYMBOL(ioread16be);
49
50unsigned int ioread32(void __iomem *addr)
47{ 51{
48 unsigned long start, len, flags; 52 return readl(addr);
53}
49 54
50 if (dev == NULL) 55EXPORT_SYMBOL(ioread32);
51 return NULL;
52 56
53 start = pci_resource_start(dev, bar); 57unsigned int ioread32be(void __iomem *addr)
54 len = pci_resource_len(dev, bar); 58{
55 if (!start || !len) 59 return be32_to_cpu(__raw_readl(addr));
56 return NULL; 60}
57 61
58 if (maxlen != 0 && len > maxlen) 62EXPORT_SYMBOL(ioread32be);
59 len = maxlen; 63
64void iowrite8(u8 val, void __iomem *addr)
65{
66 writeb(val, addr);
67}
60 68
61 flags = pci_resource_flags(dev, bar); 69EXPORT_SYMBOL(iowrite8);
62 if (flags & IORESOURCE_IO) 70
63 return ioport_map(start, len); 71void iowrite16(u16 val, void __iomem *addr)
64 if (flags & IORESOURCE_MEM) { 72{
65 if (flags & IORESOURCE_CACHEABLE) 73 writew(val, addr);
66 return ioremap_cachable(start, len); 74}
67 return ioremap_nocache(start, len); 75
76EXPORT_SYMBOL(iowrite16);
77
78void iowrite16be(u16 val, void __iomem *addr)
79{
80 __raw_writew(cpu_to_be16(val), addr);
81}
82
83EXPORT_SYMBOL(iowrite16be);
84
85void iowrite32(u32 val, void __iomem *addr)
86{
87 writel(val, addr);
88}
89
90EXPORT_SYMBOL(iowrite32);
91
92void iowrite32be(u32 val, void __iomem *addr)
93{
94 __raw_writel(cpu_to_be32(val), addr);
95}
96
97EXPORT_SYMBOL(iowrite32be);
98
99/*
100 * These are the "repeat MMIO read/write" functions.
101 * Note the "__raw" accesses, since we don't want to
102 * convert to CPU byte order. We write in "IO byte
103 * order" (we also don't have IO barriers).
104 */
105static inline void mmio_insb(void __iomem *addr, u8 *dst, int count)
106{
107 while (--count >= 0) {
108 u8 data = __raw_readb(addr);
109 *dst = data;
110 dst++;
68 } 111 }
112}
69 113
70 return NULL; 114static inline void mmio_insw(void __iomem *addr, u16 *dst, int count)
115{
116 while (--count >= 0) {
117 u16 data = __raw_readw(addr);
118 *dst = data;
119 dst++;
120 }
71} 121}
72 122
73void pci_iounmap(struct pci_dev *dev, void __iomem *addr) 123static inline void mmio_insl(void __iomem *addr, u32 *dst, int count)
74{ 124{
75 iounmap(addr); 125 while (--count >= 0) {
126 u32 data = __raw_readl(addr);
127 *dst = data;
128 dst++;
129 }
76} 130}
77EXPORT_SYMBOL(pci_iomap); 131
78EXPORT_SYMBOL(pci_iounmap); 132static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count)
133{
134 while (--count >= 0) {
135 __raw_writeb(*src, addr);
136 src++;
137 }
138}
139
140static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count)
141{
142 while (--count >= 0) {
143 __raw_writew(*src, addr);
144 src++;
145 }
146}
147
148static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count)
149{
150 while (--count >= 0) {
151 __raw_writel(*src, addr);
152 src++;
153 }
154}
155
156void ioread8_rep(void __iomem *addr, void *dst, unsigned long count)
157{
158 mmio_insb(addr, dst, count);
159}
160
161EXPORT_SYMBOL(ioread8_rep);
162
163void ioread16_rep(void __iomem *addr, void *dst, unsigned long count)
164{
165 mmio_insw(addr, dst, count);
166}
167
168EXPORT_SYMBOL(ioread16_rep);
169
170void ioread32_rep(void __iomem *addr, void *dst, unsigned long count)
171{
172 mmio_insl(addr, dst, count);
173}
174
175EXPORT_SYMBOL(ioread32_rep);
176
177void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count)
178{
179 mmio_outsb(addr, src, count);
180}
181
182EXPORT_SYMBOL(iowrite8_rep);
183
184void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count)
185{
186 mmio_outsw(addr, src, count);
187}
188
189EXPORT_SYMBOL(iowrite16_rep);
190
191void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count)
192{
193 mmio_outsl(addr, src, count);
194}
195
196EXPORT_SYMBOL(iowrite32_rep);
197
198/*
199 * Create a virtual mapping cookie for an IO port range
200 *
201 * This uses the same mapping are as the in/out family which has to be setup
202 * by the platform initialization code.
203 *
204 * Just to make matters somewhat more interesting on MIPS systems with
205 * multiple host bridge each will have it's own ioport address space.
206 */
207static void __iomem *ioport_map_legacy(unsigned long port, unsigned int nr)
208{
209 return (void __iomem *) (mips_io_port_base + port);
210}
211
212void __iomem *ioport_map(unsigned long port, unsigned int nr)
213{
214 if (port > PIO_MASK)
215 return NULL;
216
217 return ioport_map_legacy(port, nr);
218}
219
220EXPORT_SYMBOL(ioport_map);
221
222void ioport_unmap(void __iomem *addr)
223{
224 /* Nothing to do */
225}
226
227EXPORT_SYMBOL(ioport_unmap);
diff --git a/arch/mips/lib/memcpy-inatomic.S b/arch/mips/lib/memcpy-inatomic.S
new file mode 100644
index 000000000000..3a534b2baa0f
--- /dev/null
+++ b/arch/mips/lib/memcpy-inatomic.S
@@ -0,0 +1,436 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Unified implementation of memcpy, memmove and the __copy_user backend.
7 *
8 * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org)
9 * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
10 * Copyright (C) 2002 Broadcom, Inc.
11 * memcpy/copy_user author: Mark Vandevoorde
12 *
13 * Mnemonic names for arguments to memcpy/__copy_user
14 */
15
16/*
17 * Hack to resolve longstanding prefetch issue
18 *
19 * Prefetching may be fatal on some systems if we're prefetching beyond the
20 * end of memory on some systems. It's also a seriously bad idea on non
21 * dma-coherent systems.
22 */
23#if !defined(CONFIG_DMA_COHERENT) || !defined(CONFIG_DMA_IP27)
24#undef CONFIG_CPU_HAS_PREFETCH
25#endif
26#ifdef CONFIG_MIPS_MALTA
27#undef CONFIG_CPU_HAS_PREFETCH
28#endif
29
30#include <asm/asm.h>
31#include <asm/asm-offsets.h>
32#include <asm/regdef.h>
33
34#define dst a0
35#define src a1
36#define len a2
37
38/*
39 * Spec
40 *
41 * memcpy copies len bytes from src to dst and sets v0 to dst.
42 * It assumes that
43 * - src and dst don't overlap
44 * - src is readable
45 * - dst is writable
46 * memcpy uses the standard calling convention
47 *
48 * __copy_user copies up to len bytes from src to dst and sets a2 (len) to
49 * the number of uncopied bytes due to an exception caused by a read or write.
50 * __copy_user assumes that src and dst don't overlap, and that the call is
51 * implementing one of the following:
52 * copy_to_user
53 * - src is readable (no exceptions when reading src)
54 * copy_from_user
55 * - dst is writable (no exceptions when writing dst)
56 * __copy_user uses a non-standard calling convention; see
57 * include/asm-mips/uaccess.h
58 *
59 * When an exception happens on a load, the handler must
60 # ensure that all of the destination buffer is overwritten to prevent
61 * leaking information to user mode programs.
62 */
63
64/*
65 * Implementation
66 */
67
68/*
69 * The exception handler for loads requires that:
70 * 1- AT contain the address of the byte just past the end of the source
71 * of the copy,
72 * 2- src_entry <= src < AT, and
73 * 3- (dst - src) == (dst_entry - src_entry),
74 * The _entry suffix denotes values when __copy_user was called.
75 *
76 * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
77 * (2) is met by incrementing src by the number of bytes copied
78 * (3) is met by not doing loads between a pair of increments of dst and src
79 *
80 * The exception handlers for stores adjust len (if necessary) and return.
81 * These handlers do not need to overwrite any data.
82 *
83 * For __rmemcpy and memmove an exception is always a kernel bug, therefore
84 * they're not protected.
85 */
86
87#define EXC(inst_reg,addr,handler) \
889: inst_reg, addr; \
89 .section __ex_table,"a"; \
90 PTR 9b, handler; \
91 .previous
92
93/*
94 * Only on the 64-bit kernel we can made use of 64-bit registers.
95 */
96#ifdef CONFIG_64BIT
97#define USE_DOUBLE
98#endif
99
100#ifdef USE_DOUBLE
101
102#define LOAD ld
103#define LOADL ldl
104#define LOADR ldr
105#define STOREL sdl
106#define STORER sdr
107#define STORE sd
108#define ADD daddu
109#define SUB dsubu
110#define SRL dsrl
111#define SRA dsra
112#define SLL dsll
113#define SLLV dsllv
114#define SRLV dsrlv
115#define NBYTES 8
116#define LOG_NBYTES 3
117
118/*
119 * As we are sharing code base with the mips32 tree (which use the o32 ABI
120 * register definitions). We need to redefine the register definitions from
121 * the n64 ABI register naming to the o32 ABI register naming.
122 */
123#undef t0
124#undef t1
125#undef t2
126#undef t3
127#define t0 $8
128#define t1 $9
129#define t2 $10
130#define t3 $11
131#define t4 $12
132#define t5 $13
133#define t6 $14
134#define t7 $15
135
136#else
137
138#define LOAD lw
139#define LOADL lwl
140#define LOADR lwr
141#define STOREL swl
142#define STORER swr
143#define STORE sw
144#define ADD addu
145#define SUB subu
146#define SRL srl
147#define SLL sll
148#define SRA sra
149#define SLLV sllv
150#define SRLV srlv
151#define NBYTES 4
152#define LOG_NBYTES 2
153
154#endif /* USE_DOUBLE */
155
156#ifdef CONFIG_CPU_LITTLE_ENDIAN
157#define LDFIRST LOADR
158#define LDREST LOADL
159#define STFIRST STORER
160#define STREST STOREL
161#define SHIFT_DISCARD SLLV
162#else
163#define LDFIRST LOADL
164#define LDREST LOADR
165#define STFIRST STOREL
166#define STREST STORER
167#define SHIFT_DISCARD SRLV
168#endif
169
170#define FIRST(unit) ((unit)*NBYTES)
171#define REST(unit) (FIRST(unit)+NBYTES-1)
172#define UNIT(unit) FIRST(unit)
173
174#define ADDRMASK (NBYTES-1)
175
176 .text
177 .set noreorder
178 .set noat
179
180/*
181 * A combined memcpy/__copy_user
182 * __copy_user sets len to 0 for success; else to an upper bound of
183 * the number of uncopied bytes.
184 * memcpy sets v0 to dst.
185 */
186 .align 5
187LEAF(__copy_user_inatomic)
188 /*
189 * Note: dst & src may be unaligned, len may be 0
190 * Temps
191 */
192#define rem t8
193
194 /*
195 * The "issue break"s below are very approximate.
196 * Issue delays for dcache fills will perturb the schedule, as will
197 * load queue full replay traps, etc.
198 *
199 * If len < NBYTES use byte operations.
200 */
201 PREF( 0, 0(src) )
202 PREF( 1, 0(dst) )
203 sltu t2, len, NBYTES
204 and t1, dst, ADDRMASK
205 PREF( 0, 1*32(src) )
206 PREF( 1, 1*32(dst) )
207 bnez t2, copy_bytes_checklen
208 and t0, src, ADDRMASK
209 PREF( 0, 2*32(src) )
210 PREF( 1, 2*32(dst) )
211 bnez t1, dst_unaligned
212 nop
213 bnez t0, src_unaligned_dst_aligned
214 /*
215 * use delay slot for fall-through
216 * src and dst are aligned; need to compute rem
217 */
218both_aligned:
219 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
220 beqz t0, cleanup_both_aligned # len < 8*NBYTES
221 and rem, len, (8*NBYTES-1) # rem = len % (8*NBYTES)
222 PREF( 0, 3*32(src) )
223 PREF( 1, 3*32(dst) )
224 .align 4
2251:
226EXC( LOAD t0, UNIT(0)(src), l_exc)
227EXC( LOAD t1, UNIT(1)(src), l_exc_copy)
228EXC( LOAD t2, UNIT(2)(src), l_exc_copy)
229EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
230 SUB len, len, 8*NBYTES
231EXC( LOAD t4, UNIT(4)(src), l_exc_copy)
232EXC( LOAD t7, UNIT(5)(src), l_exc_copy)
233 STORE t0, UNIT(0)(dst)
234 STORE t1, UNIT(1)(dst)
235EXC( LOAD t0, UNIT(6)(src), l_exc_copy)
236EXC( LOAD t1, UNIT(7)(src), l_exc_copy)
237 ADD src, src, 8*NBYTES
238 ADD dst, dst, 8*NBYTES
239 STORE t2, UNIT(-6)(dst)
240 STORE t3, UNIT(-5)(dst)
241 STORE t4, UNIT(-4)(dst)
242 STORE t7, UNIT(-3)(dst)
243 STORE t0, UNIT(-2)(dst)
244 STORE t1, UNIT(-1)(dst)
245 PREF( 0, 8*32(src) )
246 PREF( 1, 8*32(dst) )
247 bne len, rem, 1b
248 nop
249
250 /*
251 * len == rem == the number of bytes left to copy < 8*NBYTES
252 */
253cleanup_both_aligned:
254 beqz len, done
255 sltu t0, len, 4*NBYTES
256 bnez t0, less_than_4units
257 and rem, len, (NBYTES-1) # rem = len % NBYTES
258 /*
259 * len >= 4*NBYTES
260 */
261EXC( LOAD t0, UNIT(0)(src), l_exc)
262EXC( LOAD t1, UNIT(1)(src), l_exc_copy)
263EXC( LOAD t2, UNIT(2)(src), l_exc_copy)
264EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
265 SUB len, len, 4*NBYTES
266 ADD src, src, 4*NBYTES
267 STORE t0, UNIT(0)(dst)
268 STORE t1, UNIT(1)(dst)
269 STORE t2, UNIT(2)(dst)
270 STORE t3, UNIT(3)(dst)
271 beqz len, done
272 ADD dst, dst, 4*NBYTES
273less_than_4units:
274 /*
275 * rem = len % NBYTES
276 */
277 beq rem, len, copy_bytes
278 nop
2791:
280EXC( LOAD t0, 0(src), l_exc)
281 ADD src, src, NBYTES
282 SUB len, len, NBYTES
283 STORE t0, 0(dst)
284 bne rem, len, 1b
285 ADD dst, dst, NBYTES
286
287 /*
288 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
289 * A loop would do only a byte at a time with possible branch
290 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
291 * because can't assume read-access to dst. Instead, use
292 * STREST dst, which doesn't require read access to dst.
293 *
294 * This code should perform better than a simple loop on modern,
295 * wide-issue mips processors because the code has fewer branches and
296 * more instruction-level parallelism.
297 */
298#define bits t2
299 beqz len, done
300 ADD t1, dst, len # t1 is just past last byte of dst
301 li bits, 8*NBYTES
302 SLL rem, len, 3 # rem = number of bits to keep
303EXC( LOAD t0, 0(src), l_exc)
304 SUB bits, bits, rem # bits = number of bits to discard
305 SHIFT_DISCARD t0, t0, bits
306 STREST t0, -1(t1)
307 jr ra
308 move len, zero
309dst_unaligned:
310 /*
311 * dst is unaligned
312 * t0 = src & ADDRMASK
313 * t1 = dst & ADDRMASK; T1 > 0
314 * len >= NBYTES
315 *
316 * Copy enough bytes to align dst
317 * Set match = (src and dst have same alignment)
318 */
319#define match rem
320EXC( LDFIRST t3, FIRST(0)(src), l_exc)
321 ADD t2, zero, NBYTES
322EXC( LDREST t3, REST(0)(src), l_exc_copy)
323 SUB t2, t2, t1 # t2 = number of bytes copied
324 xor match, t0, t1
325 STFIRST t3, FIRST(0)(dst)
326 beq len, t2, done
327 SUB len, len, t2
328 ADD dst, dst, t2
329 beqz match, both_aligned
330 ADD src, src, t2
331
332src_unaligned_dst_aligned:
333 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
334 PREF( 0, 3*32(src) )
335 beqz t0, cleanup_src_unaligned
336 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
337 PREF( 1, 3*32(dst) )
3381:
339/*
340 * Avoid consecutive LD*'s to the same register since some mips
341 * implementations can't issue them in the same cycle.
342 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
343 * are to the same unit (unless src is aligned, but it's not).
344 */
345EXC( LDFIRST t0, FIRST(0)(src), l_exc)
346EXC( LDFIRST t1, FIRST(1)(src), l_exc_copy)
347 SUB len, len, 4*NBYTES
348EXC( LDREST t0, REST(0)(src), l_exc_copy)
349EXC( LDREST t1, REST(1)(src), l_exc_copy)
350EXC( LDFIRST t2, FIRST(2)(src), l_exc_copy)
351EXC( LDFIRST t3, FIRST(3)(src), l_exc_copy)
352EXC( LDREST t2, REST(2)(src), l_exc_copy)
353EXC( LDREST t3, REST(3)(src), l_exc_copy)
354 PREF( 0, 9*32(src) ) # 0 is PREF_LOAD (not streamed)
355 ADD src, src, 4*NBYTES
356#ifdef CONFIG_CPU_SB1
357 nop # improves slotting
358#endif
359 STORE t0, UNIT(0)(dst)
360 STORE t1, UNIT(1)(dst)
361 STORE t2, UNIT(2)(dst)
362 STORE t3, UNIT(3)(dst)
363 PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
364 bne len, rem, 1b
365 ADD dst, dst, 4*NBYTES
366
367cleanup_src_unaligned:
368 beqz len, done
369 and rem, len, NBYTES-1 # rem = len % NBYTES
370 beq rem, len, copy_bytes
371 nop
3721:
373EXC( LDFIRST t0, FIRST(0)(src), l_exc)
374EXC( LDREST t0, REST(0)(src), l_exc_copy)
375 ADD src, src, NBYTES
376 SUB len, len, NBYTES
377 STORE t0, 0(dst)
378 bne len, rem, 1b
379 ADD dst, dst, NBYTES
380
381copy_bytes_checklen:
382 beqz len, done
383 nop
384copy_bytes:
385 /* 0 < len < NBYTES */
386#define COPY_BYTE(N) \
387EXC( lb t0, N(src), l_exc); \
388 SUB len, len, 1; \
389 beqz len, done; \
390 sb t0, N(dst)
391
392 COPY_BYTE(0)
393 COPY_BYTE(1)
394#ifdef USE_DOUBLE
395 COPY_BYTE(2)
396 COPY_BYTE(3)
397 COPY_BYTE(4)
398 COPY_BYTE(5)
399#endif
400EXC( lb t0, NBYTES-2(src), l_exc)
401 SUB len, len, 1
402 jr ra
403 sb t0, NBYTES-2(dst)
404done:
405 jr ra
406 nop
407 END(__copy_user_inatomic)
408
409l_exc_copy:
410 /*
411 * Copy bytes from src until faulting load address (or until a
412 * lb faults)
413 *
414 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
415 * may be more than a byte beyond the last address.
416 * Hence, the lb below may get an exception.
417 *
418 * Assumes src < THREAD_BUADDR($28)
419 */
420 LOAD t0, TI_TASK($28)
421 nop
422 LOAD t0, THREAD_BUADDR(t0)
4231:
424EXC( lb t1, 0(src), l_exc)
425 ADD src, src, 1
426 sb t1, 0(dst) # can't fault -- we're copy_from_user
427 bne src, t0, 1b
428 ADD dst, dst, 1
429l_exc:
430 LOAD t0, TI_TASK($28)
431 nop
432 LOAD t0, THREAD_BUADDR(t0) # t0 is just past last good address
433 nop
434 SUB len, AT, t0 # len number of uncopied bytes
435 jr ra
436 nop
diff --git a/arch/mips/lib/memset.S b/arch/mips/lib/memset.S
new file mode 100644
index 000000000000..3f8b8b3d0b23
--- /dev/null
+++ b/arch/mips/lib/memset.S
@@ -0,0 +1,166 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1998, 1999, 2000 by Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 */
9#include <asm/asm.h>
10#include <asm/asm-offsets.h>
11#include <asm/regdef.h>
12
13#if LONGSIZE == 4
14#define LONG_S_L swl
15#define LONG_S_R swr
16#else
17#define LONG_S_L sdl
18#define LONG_S_R sdr
19#endif
20
21#define EX(insn,reg,addr,handler) \
229: insn reg, addr; \
23 .section __ex_table,"a"; \
24 PTR 9b, handler; \
25 .previous
26
27 .macro f_fill64 dst, offset, val, fixup
28 EX(LONG_S, \val, (\offset + 0 * LONGSIZE)(\dst), \fixup)
29 EX(LONG_S, \val, (\offset + 1 * LONGSIZE)(\dst), \fixup)
30 EX(LONG_S, \val, (\offset + 2 * LONGSIZE)(\dst), \fixup)
31 EX(LONG_S, \val, (\offset + 3 * LONGSIZE)(\dst), \fixup)
32 EX(LONG_S, \val, (\offset + 4 * LONGSIZE)(\dst), \fixup)
33 EX(LONG_S, \val, (\offset + 5 * LONGSIZE)(\dst), \fixup)
34 EX(LONG_S, \val, (\offset + 6 * LONGSIZE)(\dst), \fixup)
35 EX(LONG_S, \val, (\offset + 7 * LONGSIZE)(\dst), \fixup)
36#if LONGSIZE == 4
37 EX(LONG_S, \val, (\offset + 8 * LONGSIZE)(\dst), \fixup)
38 EX(LONG_S, \val, (\offset + 9 * LONGSIZE)(\dst), \fixup)
39 EX(LONG_S, \val, (\offset + 10 * LONGSIZE)(\dst), \fixup)
40 EX(LONG_S, \val, (\offset + 11 * LONGSIZE)(\dst), \fixup)
41 EX(LONG_S, \val, (\offset + 12 * LONGSIZE)(\dst), \fixup)
42 EX(LONG_S, \val, (\offset + 13 * LONGSIZE)(\dst), \fixup)
43 EX(LONG_S, \val, (\offset + 14 * LONGSIZE)(\dst), \fixup)
44 EX(LONG_S, \val, (\offset + 15 * LONGSIZE)(\dst), \fixup)
45#endif
46 .endm
47
48/*
49 * memset(void *s, int c, size_t n)
50 *
51 * a0: start of area to clear
52 * a1: char to fill with
53 * a2: size of area to clear
54 */
55 .set noreorder
56 .align 5
57LEAF(memset)
58 beqz a1, 1f
59 move v0, a0 /* result */
60
61 andi a1, 0xff /* spread fillword */
62 LONG_SLL t1, a1, 8
63 or a1, t1
64 LONG_SLL t1, a1, 16
65#if LONGSIZE == 8
66 or a1, t1
67 LONG_SLL t1, a1, 32
68#endif
69 or a1, t1
701:
71
72FEXPORT(__bzero)
73 sltiu t0, a2, LONGSIZE /* very small region? */
74 bnez t0, small_memset
75 andi t0, a0, LONGMASK /* aligned? */
76
77 beqz t0, 1f
78 PTR_SUBU t0, LONGSIZE /* alignment in bytes */
79
80#ifdef __MIPSEB__
81 EX(LONG_S_L, a1, (a0), first_fixup) /* make word/dword aligned */
82#endif
83#ifdef __MIPSEL__
84 EX(LONG_S_R, a1, (a0), first_fixup) /* make word/dword aligned */
85#endif
86 PTR_SUBU a0, t0 /* long align ptr */
87 PTR_ADDU a2, t0 /* correct size */
88
891: ori t1, a2, 0x3f /* # of full blocks */
90 xori t1, 0x3f
91 beqz t1, memset_partial /* no block to fill */
92 andi t0, a2, 0x40-LONGSIZE
93
94 PTR_ADDU t1, a0 /* end address */
95 .set reorder
961: PTR_ADDIU a0, 64
97 f_fill64 a0, -64, a1, fwd_fixup
98 bne t1, a0, 1b
99 .set noreorder
100
101memset_partial:
102 PTR_LA t1, 2f /* where to start */
103#if LONGSIZE == 4
104 PTR_SUBU t1, t0
105#else
106 .set noat
107 LONG_SRL AT, t0, 1
108 PTR_SUBU t1, AT
109 .set noat
110#endif
111 jr t1
112 PTR_ADDU a0, t0 /* dest ptr */
113
114 .set push
115 .set noreorder
116 .set nomacro
117 f_fill64 a0, -64, a1, partial_fixup /* ... but first do longs ... */
1182: .set pop
119 andi a2, LONGMASK /* At most one long to go */
120
121 beqz a2, 1f
122 PTR_ADDU a0, a2 /* What's left */
123#ifdef __MIPSEB__
124 EX(LONG_S_R, a1, -1(a0), last_fixup)
125#endif
126#ifdef __MIPSEL__
127 EX(LONG_S_L, a1, -1(a0), last_fixup)
128#endif
1291: jr ra
130 move a2, zero
131
132small_memset:
133 beqz a2, 2f
134 PTR_ADDU t1, a0, a2
135
1361: PTR_ADDIU a0, 1 /* fill bytewise */
137 bne t1, a0, 1b
138 sb a1, -1(a0)
139
1402: jr ra /* done */
141 move a2, zero
142 END(memset)
143
144first_fixup:
145 jr ra
146 nop
147
148fwd_fixup:
149 PTR_L t0, TI_TASK($28)
150 LONG_L t0, THREAD_BUADDR(t0)
151 andi a2, 0x3f
152 LONG_ADDU a2, t1
153 jr ra
154 LONG_SUBU a2, t0
155
156partial_fixup:
157 PTR_L t0, TI_TASK($28)
158 LONG_L t0, THREAD_BUADDR(t0)
159 andi a2, LONGMASK
160 LONG_ADDU a2, t1
161 jr ra
162 LONG_SUBU a2, t0
163
164last_fixup:
165 jr ra
166 andi v1, a2, LONGMASK
diff --git a/arch/mips/lib/uncached.c b/arch/mips/lib/uncached.c
index 98ce89f8068b..2388f7f3ffde 100644
--- a/arch/mips/lib/uncached.c
+++ b/arch/mips/lib/uncached.c
@@ -44,20 +44,24 @@ unsigned long __init run_uncached(void *func)
44 44
45 if (sp >= (long)CKSEG0 && sp < (long)CKSEG2) 45 if (sp >= (long)CKSEG0 && sp < (long)CKSEG2)
46 usp = CKSEG1ADDR(sp); 46 usp = CKSEG1ADDR(sp);
47#ifdef CONFIG_64BIT
47 else if ((long long)sp >= (long long)PHYS_TO_XKPHYS(0LL, 0) && 48 else if ((long long)sp >= (long long)PHYS_TO_XKPHYS(0LL, 0) &&
48 (long long)sp < (long long)PHYS_TO_XKPHYS(8LL, 0)) 49 (long long)sp < (long long)PHYS_TO_XKPHYS(8LL, 0))
49 usp = PHYS_TO_XKPHYS((long long)K_CALG_UNCACHED, 50 usp = PHYS_TO_XKPHYS((long long)K_CALG_UNCACHED,
50 XKPHYS_TO_PHYS((long long)sp)); 51 XKPHYS_TO_PHYS((long long)sp));
52#endif
51 else { 53 else {
52 BUG(); 54 BUG();
53 usp = sp; 55 usp = sp;
54 } 56 }
55 if (lfunc >= (long)CKSEG0 && lfunc < (long)CKSEG2) 57 if (lfunc >= (long)CKSEG0 && lfunc < (long)CKSEG2)
56 ufunc = CKSEG1ADDR(lfunc); 58 ufunc = CKSEG1ADDR(lfunc);
59#ifdef CONFIG_64BIT
57 else if ((long long)lfunc >= (long long)PHYS_TO_XKPHYS(0LL, 0) && 60 else if ((long long)lfunc >= (long long)PHYS_TO_XKPHYS(0LL, 0) &&
58 (long long)lfunc < (long long)PHYS_TO_XKPHYS(8LL, 0)) 61 (long long)lfunc < (long long)PHYS_TO_XKPHYS(8LL, 0))
59 ufunc = PHYS_TO_XKPHYS((long long)K_CALG_UNCACHED, 62 ufunc = PHYS_TO_XKPHYS((long long)K_CALG_UNCACHED,
60 XKPHYS_TO_PHYS((long long)lfunc)); 63 XKPHYS_TO_PHYS((long long)lfunc));
64#endif
61 else { 65 else {
62 BUG(); 66 BUG();
63 ufunc = lfunc; 67 ufunc = lfunc;
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-03 14:21:56 -0500