MIPS: Rewrite sysmips(MIPS_ATOMIC_SET, ...) in C with inline assembler

This way it doesn't have to use CONFIG_CPU_HAS_LLSC anymore. Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
author: Ralf Baechle <ralf@linux-mips.org> 2009-09-16 20:25:05 -0400
committer: Ralf Baechle <ralf@linux-mips.org> 2009-09-17 14:07:49 -0400
commit: f1e39a4a616cd9981a9decfd5332fd07a01abb8b (patch)
tree: 555e0c78611bb30cef2caca1ae1b5043a9999290 /arch
parent: f4c6b6bc5a4fc8d607f2d89369008c85a3a12a8b (diff)
5 files changed, 114 insertions, 150 deletions
diff --git a/arch/mips/include/asm/system.h b/arch/mips/include/asm/system.h
index a2e9239b45aa..23f68b40d4bb 100644
--- a/arch/mips/include/asm/system.h
+++ b/arch/mips/include/asm/system.h
@@ -32,6 +32,9 @@ extern asmlinkage void *resume(void *last, void *next, void *next_ti);
 struct task_struct;
+extern unsigned int ll_bit;
+extern struct task_struct *ll_task;
 #ifdef CONFIG_MIPS_MT_FPAFF
 /*
diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S
index b57082123536..7c2de4f091c4 100644
--- a/arch/mips/kernel/scall32-o32.S
+++ b/arch/mips/kernel/scall32-o32.S
@@ -187,78 +187,6 @@ illegal_syscall:
        j       o32_syscall_exit
        END(handle_sys)
-        LEAF(mips_atomic_set)
-        andi    v0, a1, 3                       # must be word aligned
-        bnez    v0, bad_alignment
-        lw      v1, TI_ADDR_LIMIT($28)          # in legal address range?
-        addiu   a0, a1, 4
-        or      a0, a0, a1
-        and     a0, a0, v1
-        bltz    a0, bad_address
-#ifdef CONFIG_CPU_HAS_LLSC
-        /* Ok, this is the ll/sc case.  World is sane :-)  */
-1:      ll      v0, (a1)
-        move    a0, a2
-2:      sc      a0, (a1)
-#if R10000_LLSC_WAR
-        beqzl   a0, 1b
-#else
-        beqz    a0, 1b
-#endif
-        .section __ex_table,"a"
-        PTR     1b, bad_stack
-        PTR     2b, bad_stack
-        .previous
-#else
-        sw      a1, 16(sp)
-        sw      a2, 20(sp)
-        move    a0, sp
-        move    a2, a1
-        li      a1, 1
-        jal     do_page_fault
-        lw      a1, 16(sp)
-        lw      a2, 20(sp)
-        /*
-         * At this point the page should be readable and writable unless
-         * there was no more memory available.
-         */
-1:      lw      v0, (a1)
-2:      sw      a2, (a1)
-        .section __ex_table,"a"
-        PTR     1b, no_mem
-        PTR     2b, no_mem
-        .previous
-#endif
-        sw      zero, PT_R7(sp)         # success
-        sw      v0, PT_R2(sp)           # result
-        j       o32_syscall_exit        # continue like a normal syscall
-no_mem: li      v0, -ENOMEM
-        jr      ra
-bad_address:
-        li      v0, -EFAULT
-        jr      ra
-bad_alignment:
-        li      v0, -EINVAL
-        jr      ra
-        END(mips_atomic_set)
-        LEAF(sys_sysmips)
-        beq     a0, MIPS_ATOMIC_SET, mips_atomic_set
-        j       _sys_sysmips
-        END(sys_sysmips)
        LEAF(sys_syscall)
        subu    t0, a0, __NR_O32_Linux  # check syscall number
        sltiu   v0, t0, __NR_O32_Linux_syscalls + 1
diff --git a/arch/mips/kernel/scall64-64.S b/arch/mips/kernel/scall64-64.S
index 3d866f24e064..b97b993846d6 100644
--- a/arch/mips/kernel/scall64-64.S
+++ b/arch/mips/kernel/scall64-64.S
@@ -124,78 +124,6 @@ illegal_syscall:
        j       n64_syscall_exit
        END(handle_sys64)
-        LEAF(mips_atomic_set)
-        andi    v0, a1, 3                       # must be word aligned
-        bnez    v0, bad_alignment
-        LONG_L  v1, TI_ADDR_LIMIT($28)          # in legal address range?
-        LONG_ADDIU      a0, a1, 4
-        or      a0, a0, a1
-        and     a0, a0, v1
-        bltz    a0, bad_address
-#ifdef CONFIG_CPU_HAS_LLSC
-        /* Ok, this is the ll/sc case.  World is sane :-)  */
-1:      ll      v0, (a1)
-        move    a0, a2
-2:      sc      a0, (a1)
-#if R10000_LLSC_WAR
-        beqzl   a0, 1b
-#else
-        beqz    a0, 1b
-#endif
-        .section __ex_table,"a"
-        PTR     1b, bad_stack
-        PTR     2b, bad_stack
-        .previous
-#else
-        sw      a1, 16(sp)
-        sw      a2, 20(sp)
-        move    a0, sp
-        move    a2, a1
-        li      a1, 1
-        jal     do_page_fault
-        lw      a1, 16(sp)
-        lw      a2, 20(sp)
-        /*
-         * At this point the page should be readable and writable unless
-         * there was no more memory available.
-         */
-1:      lw      v0, (a1)
-2:      sw      a2, (a1)
-        .section __ex_table,"a"
-        PTR     1b, no_mem
-        PTR     2b, no_mem
-        .previous
-#endif
-        sd      zero, PT_R7(sp)         # success
-        sd      v0, PT_R2(sp)           # result
-        j       n64_syscall_exit        # continue like a normal syscall
-no_mem: li      v0, -ENOMEM
-        jr      ra
-bad_address:
-        li      v0, -EFAULT
-        jr      ra
-bad_alignment:
-        li      v0, -EINVAL
-        jr      ra
-        END(mips_atomic_set)
-        LEAF(sys_sysmips)
-        beq     a0, MIPS_ATOMIC_SET, mips_atomic_set
-        j       _sys_sysmips
-        END(sys_sysmips)
        .align  3
 sys_call_table:
        PTR     sys_read                        /* 5000 */
diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c
index 8cf384644040..3fe1fcfa2e73 100644
--- a/arch/mips/kernel/syscall.c
+++ b/arch/mips/kernel/syscall.c
@@ -28,7 +28,9 @@
 #include <linux/compiler.h>
 #include <linux/module.h>
 #include <linux/ipc.h>
+#include <linux/uaccess.h>
+#include <asm/asm.h>
 #include <asm/branch.h>
 #include <asm/cachectl.h>
 #include <asm/cacheflush.h>
@@ -290,12 +292,116 @@ SYSCALL_DEFINE1(set_thread_area, unsigned long, addr)
        return 0;
 }
-asmlinkage int _sys_sysmips(long cmd, long arg1, long arg2, long arg3)
+static inline int mips_atomic_set(struct pt_regs *regs,
+        unsigned long addr, unsigned long new)
 {
+        unsigned long old, tmp;
+        unsigned int err;
+        if (unlikely(addr & 3))
+                return -EINVAL;
+        if (unlikely(!access_ok(VERIFY_WRITE, addr, 4)))
+                return -EINVAL;
+        if (cpu_has_llsc && R10000_LLSC_WAR) {
+                __asm__ __volatile__ (
+                "       li      %[err], 0                               \n"
+                "1:     ll      %[old], (%[addr])                       \n"
+                "       move    %[tmp], %[new]                          \n"
+                "2:     sc      %[tmp], (%[addr])                       \n"
+                "       beqzl   %[tmp], 1b                              \n"
+                "3:                                                     \n"
+                "       .section .fixup,\"ax\"                          \n"
+                "4:     li      %[err], %[efault]                       \n"
+                "       j       3b                                      \n"
+                "       .previous                                       \n"
+                "       .section __ex_table,\"a\"                       \n"
+                "       "STR(PTR)"      1b, 4b                          \n"
+                "       "STR(PTR)"      2b, 4b                          \n"
+                "       .previous                                       \n"
+                : [old] "=&r" (old),
+                  [err] "=&r" (err),
+                  [tmp] "=&r" (tmp)
+                : [addr] "r" (addr),
+                  [new] "r" (new),
+                  [efault] "i" (-EFAULT)
+                : "memory");
+        } else if (cpu_has_llsc) {
+                __asm__ __volatile__ (
+                "       li      %[err], 0                               \n"
+                "1:     ll      %[old], (%[addr])                       \n"
+                "       move    %[tmp], %[new]                          \n"
+                "2:     sc      %[tmp], (%[addr])                       \n"
+                "       bnez    %[tmp], 4f                              \n"
+                "3:                                                     \n"
+                "       .subsection 2                                   \n"
+                "4:     b       1b                                      \n"
+                "       .previous                                       \n"
+                "                                                       \n"
+                "       .section .fixup,\"ax\"                          \n"
+                "5:     li      %[err], %[efault]                       \n"
+                "       j       3b                                      \n"
+                "       .previous                                       \n"
+                "       .section __ex_table,\"a\"                       \n"
+                "       "STR(PTR)"      1b, 5b                          \n"
+                "       "STR(PTR)"      2b, 5b                          \n"
+                "       .previous                                       \n"
+                : [old] "=&r" (old),
+                  [err] "=&r" (err),
+                  [tmp] "=&r" (tmp)
+                : [addr] "r" (addr),
+                  [new] "r" (new),
+                  [efault] "i" (-EFAULT)
+                : "memory");
+        } else {
+                do {
+                        preempt_disable();
+                        ll_bit = 1;
+                        ll_task = current;
+                        preempt_enable();
+                        err = __get_user(old, (unsigned int *) addr);
+                        err |= __put_user(new, (unsigned int *) addr);
+                        if (err)
+                                break;
+                        rmb();
+                } while (!ll_bit);
+        }
+        if (unlikely(err))
+                return err;
+        regs->regs[2] = old;
+        regs->regs[7] = 0;      /* No error */
+        /*
+         * Don't let your children do this ...
+         */
+        __asm__ __volatile__(
+        "       move    $29, %0                                         \n"
+        "       j       syscall_exit                                    \n"
+        : /* no outputs */
+        : "r" (regs));
+        /* unreached.  Honestly.  */
+        while (1);
+}
+save_static_function(sys_sysmips);
+static int __used noinline
+_sys_sysmips(nabi_no_regargs struct pt_regs regs)
+{
+        long cmd, arg1, arg2, arg3;
+        cmd = regs.regs[4];
+        arg1 = regs.regs[5];
+        arg2 = regs.regs[6];
+        arg3 = regs.regs[7];
        switch (cmd) {
        case MIPS_ATOMIC_SET:
-                printk(KERN_CRIT "How did I get here?\n");
+                return mips_atomic_set(&regs, arg1, arg2);
-                return -EINVAL;
        case MIPS_FIXADE:
                if (arg1 & ~3)
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 08f1edf355e8..0a18b4c62afb 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -466,9 +466,8 @@ asmlinkage void do_be(struct pt_regs *regs)
 * The ll_bit is cleared by r*_switch.S
 */
-unsigned long ll_bit;
+unsigned int ll_bit;
+struct task_struct *ll_task;
-static struct task_struct *ll_task = NULL;
 static inline int simulate_ll(struct pt_regs *regs, unsigned int opcode)
 {
author	Ralf Baechle <ralf@linux-mips.org>	2009-09-16 20:25:05 -0400
committer	Ralf Baechle <ralf@linux-mips.org>	2009-09-17 14:07:49 -0400
commit	f1e39a4a616cd9981a9decfd5332fd07a01abb8b (patch)
tree	555e0c78611bb30cef2caca1ae1b5043a9999290 /arch
parent	f4c6b6bc5a4fc8d607f2d89369008c85a3a12a8b (diff)

diff --git a/arch/mips/include/asm/system.h b/arch/mips/include/asm/system.h index a2e9239b45aa..23f68b40d4bb 100644 --- a/arch/mips/include/asm/system.h +++ b/arch/mips/include/asm/system.h
@@ -32,6 +32,9 @@ extern asmlinkage void resume(void last, void next, void next_ti);
32		32
33	struct task_struct;	33	struct task_struct;
34		34
		35	extern unsigned int ll_bit;
		36	extern struct task_struct *ll_task;
		37
35	#ifdef CONFIG_MIPS_MT_FPAFF	38	#ifdef CONFIG_MIPS_MT_FPAFF
36		39
37	/*	40	/*


diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S index b57082123536..7c2de4f091c4 100644 --- a/arch/mips/kernel/scall32-o32.S +++ b/arch/mips/kernel/scall32-o32.S
@@ -187,78 +187,6 @@ illegal_syscall:
187	j o32_syscall_exit	187	j o32_syscall_exit
188	END(handle_sys)	188	END(handle_sys)
189		189
190	LEAF(mips_atomic_set)
191	andi v0, a1, 3 # must be word aligned
192	bnez v0, bad_alignment
193
194	lw v1, TI_ADDR_LIMIT($28) # in legal address range?
195	addiu a0, a1, 4
196	or a0, a0, a1
197	and a0, a0, v1
198	bltz a0, bad_address
199
200	#ifdef CONFIG_CPU_HAS_LLSC
201	/* Ok, this is the ll/sc case. World is sane :-) */
202	1: ll v0, (a1)
203	move a0, a2
204	2: sc a0, (a1)
205	#if R10000_LLSC_WAR
206	beqzl a0, 1b
207	#else
208	beqz a0, 1b
209	#endif
210
211	.section __ex_table,"a"
212	PTR 1b, bad_stack
213	PTR 2b, bad_stack
214	.previous
215	#else
216	sw a1, 16(sp)
217	sw a2, 20(sp)
218
219	move a0, sp
220	move a2, a1
221	li a1, 1
222	jal do_page_fault
223
224	lw a1, 16(sp)
225	lw a2, 20(sp)
226
227	/*
228	* At this point the page should be readable and writable unless
229	* there was no more memory available.
230	*/
231	1: lw v0, (a1)
232	2: sw a2, (a1)
233
234	.section __ex_table,"a"
235	PTR 1b, no_mem
236	PTR 2b, no_mem
237	.previous
238	#endif
239
240	sw zero, PT_R7(sp) # success
241	sw v0, PT_R2(sp) # result
242
243	j o32_syscall_exit # continue like a normal syscall
244
245	no_mem: li v0, -ENOMEM
246	jr ra
247
248	bad_address:
249	li v0, -EFAULT
250	jr ra
251
252	bad_alignment:
253	li v0, -EINVAL
254	jr ra
255	END(mips_atomic_set)
256
257	LEAF(sys_sysmips)
258	beq a0, MIPS_ATOMIC_SET, mips_atomic_set
259	j _sys_sysmips
260	END(sys_sysmips)
261
262	LEAF(sys_syscall)	190	LEAF(sys_syscall)
263	subu t0, a0, __NR_O32_Linux # check syscall number	191	subu t0, a0, __NR_O32_Linux # check syscall number
264	sltiu v0, t0, __NR_O32_Linux_syscalls + 1	192	sltiu v0, t0, __NR_O32_Linux_syscalls + 1


diff --git a/arch/mips/kernel/scall64-64.S b/arch/mips/kernel/scall64-64.S index 3d866f24e064..b97b993846d6 100644 --- a/arch/mips/kernel/scall64-64.S +++ b/arch/mips/kernel/scall64-64.S
@@ -124,78 +124,6 @@ illegal_syscall:
124	j n64_syscall_exit	124	j n64_syscall_exit
125	END(handle_sys64)	125	END(handle_sys64)
126		126
127	LEAF(mips_atomic_set)
128	andi v0, a1, 3 # must be word aligned
129	bnez v0, bad_alignment
130
131	LONG_L v1, TI_ADDR_LIMIT($28) # in legal address range?
132	LONG_ADDIU a0, a1, 4
133	or a0, a0, a1
134	and a0, a0, v1
135	bltz a0, bad_address
136
137	#ifdef CONFIG_CPU_HAS_LLSC
138	/* Ok, this is the ll/sc case. World is sane :-) */
139	1: ll v0, (a1)
140	move a0, a2
141	2: sc a0, (a1)
142	#if R10000_LLSC_WAR
143	beqzl a0, 1b
144	#else
145	beqz a0, 1b
146	#endif
147
148	.section __ex_table,"a"
149	PTR 1b, bad_stack
150	PTR 2b, bad_stack
151	.previous
152	#else
153	sw a1, 16(sp)
154	sw a2, 20(sp)
155
156	move a0, sp
157	move a2, a1
158	li a1, 1
159	jal do_page_fault
160
161	lw a1, 16(sp)
162	lw a2, 20(sp)
163
164	/*
165	* At this point the page should be readable and writable unless
166	* there was no more memory available.
167	*/
168	1: lw v0, (a1)
169	2: sw a2, (a1)
170
171	.section __ex_table,"a"
172	PTR 1b, no_mem
173	PTR 2b, no_mem
174	.previous
175	#endif
176
177	sd zero, PT_R7(sp) # success
178	sd v0, PT_R2(sp) # result
179
180	j n64_syscall_exit # continue like a normal syscall
181
182	no_mem: li v0, -ENOMEM
183	jr ra
184
185	bad_address:
186	li v0, -EFAULT
187	jr ra
188
189	bad_alignment:
190	li v0, -EINVAL
191	jr ra
192	END(mips_atomic_set)
193
194	LEAF(sys_sysmips)
195	beq a0, MIPS_ATOMIC_SET, mips_atomic_set
196	j _sys_sysmips
197	END(sys_sysmips)
198
199	.align 3	127	.align 3
200	sys_call_table:	128	sys_call_table:
201	PTR sys_read /* 5000 */	129	PTR sys_read /* 5000 */


diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c index 8cf384644040..3fe1fcfa2e73 100644 --- a/arch/mips/kernel/syscall.c +++ b/arch/mips/kernel/syscall.c
@@ -28,7 +28,9 @@
28	#include <linux/compiler.h>	28	#include <linux/compiler.h>
29	#include <linux/module.h>	29	#include <linux/module.h>
30	#include <linux/ipc.h>	30	#include <linux/ipc.h>
		31	#include <linux/uaccess.h>
31		32
		33	#include <asm/asm.h>
32	#include <asm/branch.h>	34	#include <asm/branch.h>
33	#include <asm/cachectl.h>	35	#include <asm/cachectl.h>
34	#include <asm/cacheflush.h>	36	#include <asm/cacheflush.h>
@@ -290,12 +292,116 @@ SYSCALL_DEFINE1(set_thread_area, unsigned long, addr)
290	return 0;	292	return 0;
291	}	293	}
292		294
293	asmlinkage int _sys_sysmips(long cmd, long arg1, long arg2, long arg3)	295	static inline int mips_atomic_set(struct pt_regs *regs,
		296	unsigned long addr, unsigned long new)
294	{	297	{
		298	unsigned long old, tmp;
		299	unsigned int err;
		300
		301	if (unlikely(addr & 3))
		302	return -EINVAL;
		303
		304	if (unlikely(!access_ok(VERIFY_WRITE, addr, 4)))
		305	return -EINVAL;
		306
		307	if (cpu_has_llsc && R10000_LLSC_WAR) {
		308	__asm__ __volatile__ (
		309	" li %[err], 0 \n"
		310	"1: ll %[old], (%[addr]) \n"
		311	" move %[tmp], %[new] \n"
		312	"2: sc %[tmp], (%[addr]) \n"
		313	" beqzl %[tmp], 1b \n"
		314	"3: \n"
		315	" .section .fixup,\"ax\" \n"
		316	"4: li %[err], %[efault] \n"
		317	" j 3b \n"
		318	" .previous \n"
		319	" .section __ex_table,\"a\" \n"
		320	" "STR(PTR)" 1b, 4b \n"
		321	" "STR(PTR)" 2b, 4b \n"
		322	" .previous \n"
		323	: [old] "=&r" (old),
		324	[err] "=&r" (err),
		325	[tmp] "=&r" (tmp)
		326	: [addr] "r" (addr),
		327	[new] "r" (new),
		328	[efault] "i" (-EFAULT)
		329	: "memory");
		330	} else if (cpu_has_llsc) {
		331	__asm__ __volatile__ (
		332	" li %[err], 0 \n"
		333	"1: ll %[old], (%[addr]) \n"
		334	" move %[tmp], %[new] \n"
		335	"2: sc %[tmp], (%[addr]) \n"
		336	" bnez %[tmp], 4f \n"
		337	"3: \n"
		338	" .subsection 2 \n"
		339	"4: b 1b \n"
		340	" .previous \n"
		341	" \n"
		342	" .section .fixup,\"ax\" \n"
		343	"5: li %[err], %[efault] \n"
		344	" j 3b \n"
		345	" .previous \n"
		346	" .section __ex_table,\"a\" \n"
		347	" "STR(PTR)" 1b, 5b \n"
		348	" "STR(PTR)" 2b, 5b \n"
		349	" .previous \n"
		350	: [old] "=&r" (old),
		351	[err] "=&r" (err),
		352	[tmp] "=&r" (tmp)
		353	: [addr] "r" (addr),
		354	[new] "r" (new),
		355	[efault] "i" (-EFAULT)
		356	: "memory");
		357	} else {
		358	do {
		359	preempt_disable();
		360	ll_bit = 1;
		361	ll_task = current;
		362	preempt_enable();
		363
		364	err = __get_user(old, (unsigned int *) addr);
		365	err \|= __put_user(new, (unsigned int *) addr);
		366	if (err)
		367	break;
		368	rmb();
		369	} while (!ll_bit);
		370	}
		371
		372	if (unlikely(err))
		373	return err;
		374
		375	regs->regs[2] = old;
		376	regs->regs[7] = 0; /* No error */
		377
		378	/*
		379	* Don't let your children do this ...
		380	*/
		381	__asm__ __volatile__(
		382	" move $29, %0 \n"
		383	" j syscall_exit \n"
		384	: /* no outputs */
		385	: "r" (regs));
		386
		387	/* unreached. Honestly. */
		388	while (1);
		389	}
		390
		391	save_static_function(sys_sysmips);
		392	static int __used noinline
		393	_sys_sysmips(nabi_no_regargs struct pt_regs regs)
		394	{
		395	long cmd, arg1, arg2, arg3;
		396
		397	cmd = regs.regs[4];
		398	arg1 = regs.regs[5];
		399	arg2 = regs.regs[6];
		400	arg3 = regs.regs[7];
		401
295	switch (cmd) {	402	switch (cmd) {
296	case MIPS_ATOMIC_SET:	403	case MIPS_ATOMIC_SET:
297	printk(KERN_CRIT "How did I get here?\n");	404	return mips_atomic_set(&regs, arg1, arg2);
298	return -EINVAL;
299		405
300	case MIPS_FIXADE:	406	case MIPS_FIXADE:
301	if (arg1 & ~3)	407	if (arg1 & ~3)


diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 08f1edf355e8..0a18b4c62afb 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c
@@ -466,9 +466,8 @@ asmlinkage void do_be(struct pt_regs *regs)
466	* The ll_bit is cleared by r*_switch.S	466	* The ll_bit is cleared by r*_switch.S
467	*/	467	*/
468		468
469	unsigned long ll_bit;	469	unsigned int ll_bit;
470		470	struct task_struct *ll_task;
471	static struct task_struct *ll_task = NULL;
472		471
473	static inline int simulate_ll(struct pt_regs *regs, unsigned int opcode)	472	static inline int simulate_ll(struct pt_regs *regs, unsigned int opcode)
474	{	473	{