aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2005-09-10 03:25:56 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2005-09-10 13:06:21 -0400
commitfb1c8f93d869b34cacb8b8932e2b83d96a19d720 (patch)
treea006d078aa02e421a7dc4793c335308204859d36 /arch
parent4327edf6b8a7ac7dce144313947995538842d8fd (diff)
[PATCH] spinlock consolidation
This patch (written by me and also containing many suggestions of Arjan van de Ven) does a major cleanup of the spinlock code. It does the following things: - consolidates and enhances the spinlock/rwlock debugging code - simplifies the asm/spinlock.h files - encapsulates the raw spinlock type and moves generic spinlock features (such as ->break_lock) into the generic code. - cleans up the spinlock code hierarchy to get rid of the spaghetti. Most notably there's now only a single variant of the debugging code, located in lib/spinlock_debug.c. (previously we had one SMP debugging variant per architecture, plus a separate generic one for UP builds) Also, i've enhanced the rwlock debugging facility, it will now track write-owners. There is new spinlock-owner/CPU-tracking on SMP builds too. All locks have lockup detection now, which will work for both soft and hard spin/rwlock lockups. The arch-level include files now only contain the minimally necessary subset of the spinlock code - all the rest that can be generalized now lives in the generic headers: include/asm-i386/spinlock_types.h | 16 include/asm-x86_64/spinlock_types.h | 16 I have also split up the various spinlock variants into separate files, making it easier to see which does what. The new layout is: SMP | UP ----------------------------|----------------------------------- asm/spinlock_types_smp.h | linux/spinlock_types_up.h linux/spinlock_types.h | linux/spinlock_types.h asm/spinlock_smp.h | linux/spinlock_up.h linux/spinlock_api_smp.h | linux/spinlock_api_up.h linux/spinlock.h | linux/spinlock.h /* * here's the role of the various spinlock/rwlock related include files: * * on SMP builds: * * asm/spinlock_types.h: contains the raw_spinlock_t/raw_rwlock_t and the * initializers * * linux/spinlock_types.h: * defines the generic type and initializers * * asm/spinlock.h: contains the __raw_spin_*()/etc. lowlevel * implementations, mostly inline assembly code * * (also included on UP-debug builds:) * * linux/spinlock_api_smp.h: * contains the prototypes for the _spin_*() APIs. * * linux/spinlock.h: builds the final spin_*() APIs. * * on UP builds: * * linux/spinlock_type_up.h: * contains the generic, simplified UP spinlock type. * (which is an empty structure on non-debug builds) * * linux/spinlock_types.h: * defines the generic type and initializers * * linux/spinlock_up.h: * contains the __raw_spin_*()/etc. version of UP * builds. (which are NOPs on non-debug, non-preempt * builds) * * (included on UP-non-debug builds:) * * linux/spinlock_api_up.h: * builds the _spin_*() APIs. * * linux/spinlock.h: builds the final spin_*() APIs. */ All SMP and UP architectures are converted by this patch. arm, i386, ia64, ppc, ppc64, s390/s390x, x64 was build-tested via crosscompilers. m32r, mips, sh, sparc, have not been tested yet, but should be mostly fine. From: Grant Grundler <grundler@parisc-linux.org> Booted and lightly tested on a500-44 (64-bit, SMP kernel, dual CPU). Builds 32-bit SMP kernel (not booted or tested). I did not try to build non-SMP kernels. That should be trivial to fix up later if necessary. I converted bit ops atomic_hash lock to raw_spinlock_t. Doing so avoids some ugly nesting of linux/*.h and asm/*.h files. Those particular locks are well tested and contained entirely inside arch specific code. I do NOT expect any new issues to arise with them. If someone does ever need to use debug/metrics with them, then they will need to unravel this hairball between spinlocks, atomic ops, and bit ops that exist only because parisc has exactly one atomic instruction: LDCW (load and clear word). From: "Luck, Tony" <tony.luck@intel.com> ia64 fix Signed-off-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Arjan van de Ven <arjanv@infradead.org> Signed-off-by: Grant Grundler <grundler@parisc-linux.org> Cc: Matthew Wilcox <willy@debian.org> Signed-off-by: Hirokazu Takata <takata@linux-m32r.org> Signed-off-by: Mikael Pettersson <mikpe@csd.uu.se> Signed-off-by: Benoit Boissinot <benoit.boissinot@ens-lyon.org> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/kernel/alpha_ksyms.c9
-rw-r--r--arch/alpha/kernel/smp.c172
-rw-r--r--arch/ia64/kernel/mca.c11
-rw-r--r--arch/m32r/kernel/smp.c48
-rw-r--r--arch/mips/lib/dec_and_lock.c8
-rw-r--r--arch/parisc/lib/Makefile2
-rw-r--r--arch/parisc/lib/bitops.c4
-rw-r--r--arch/parisc/lib/debuglocks.c277
-rw-r--r--arch/ppc/lib/Makefile1
-rw-r--r--arch/ppc/lib/dec_and_lock.c8
-rw-r--r--arch/ppc64/lib/dec_and_lock.c8
-rw-r--r--arch/ppc64/lib/locks.c14
-rw-r--r--arch/s390/lib/spinlock.c12
-rw-r--r--arch/sparc/kernel/sparc_ksyms.c10
-rw-r--r--arch/sparc/lib/Makefile2
-rw-r--r--arch/sparc/lib/debuglocks.c202
-rw-r--r--arch/sparc64/kernel/process.c5
-rw-r--r--arch/sparc64/kernel/sparc64_ksyms.c5
-rw-r--r--arch/sparc64/lib/Makefile1
-rw-r--r--arch/sparc64/lib/debuglocks.c366
20 files changed, 28 insertions, 1137 deletions
diff --git a/arch/alpha/kernel/alpha_ksyms.c b/arch/alpha/kernel/alpha_ksyms.c
index fc5ef90c4fc9..24ae9a366073 100644
--- a/arch/alpha/kernel/alpha_ksyms.c
+++ b/arch/alpha/kernel/alpha_ksyms.c
@@ -185,15 +185,6 @@ EXPORT_SYMBOL(smp_num_cpus);
185EXPORT_SYMBOL(smp_call_function); 185EXPORT_SYMBOL(smp_call_function);
186EXPORT_SYMBOL(smp_call_function_on_cpu); 186EXPORT_SYMBOL(smp_call_function_on_cpu);
187EXPORT_SYMBOL(_atomic_dec_and_lock); 187EXPORT_SYMBOL(_atomic_dec_and_lock);
188#ifdef CONFIG_DEBUG_SPINLOCK
189EXPORT_SYMBOL(_raw_spin_unlock);
190EXPORT_SYMBOL(debug_spin_lock);
191EXPORT_SYMBOL(debug_spin_trylock);
192#endif
193#ifdef CONFIG_DEBUG_RWLOCK
194EXPORT_SYMBOL(_raw_write_lock);
195EXPORT_SYMBOL(_raw_read_lock);
196#endif
197EXPORT_SYMBOL(cpu_present_mask); 188EXPORT_SYMBOL(cpu_present_mask);
198#endif /* CONFIG_SMP */ 189#endif /* CONFIG_SMP */
199 190
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index e211aa7404e6..da0be3465791 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -989,175 +989,3 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
989 989
990 preempt_enable(); 990 preempt_enable();
991} 991}
992
993#ifdef CONFIG_DEBUG_SPINLOCK
994void
995_raw_spin_unlock(spinlock_t * lock)
996{
997 mb();
998 lock->lock = 0;
999
1000 lock->on_cpu = -1;
1001 lock->previous = NULL;
1002 lock->task = NULL;
1003 lock->base_file = "none";
1004 lock->line_no = 0;
1005}
1006
1007void
1008debug_spin_lock(spinlock_t * lock, const char *base_file, int line_no)
1009{
1010 long tmp;
1011 long stuck;
1012 void *inline_pc = __builtin_return_address(0);
1013 unsigned long started = jiffies;
1014 int printed = 0;
1015 int cpu = smp_processor_id();
1016
1017 stuck = 1L << 30;
1018 try_again:
1019
1020 /* Use sub-sections to put the actual loop at the end
1021 of this object file's text section so as to perfect
1022 branch prediction. */
1023 __asm__ __volatile__(
1024 "1: ldl_l %0,%1\n"
1025 " subq %2,1,%2\n"
1026 " blbs %0,2f\n"
1027 " or %0,1,%0\n"
1028 " stl_c %0,%1\n"
1029 " beq %0,3f\n"
1030 "4: mb\n"
1031 ".subsection 2\n"
1032 "2: ldl %0,%1\n"
1033 " subq %2,1,%2\n"
1034 "3: blt %2,4b\n"
1035 " blbs %0,2b\n"
1036 " br 1b\n"
1037 ".previous"
1038 : "=r" (tmp), "=m" (lock->lock), "=r" (stuck)
1039 : "m" (lock->lock), "2" (stuck) : "memory");
1040
1041 if (stuck < 0) {
1042 printk(KERN_WARNING
1043 "%s:%d spinlock stuck in %s at %p(%d)"
1044 " owner %s at %p(%d) %s:%d\n",
1045 base_file, line_no,
1046 current->comm, inline_pc, cpu,
1047 lock->task->comm, lock->previous,
1048 lock->on_cpu, lock->base_file, lock->line_no);
1049 stuck = 1L << 36;
1050 printed = 1;
1051 goto try_again;
1052 }
1053
1054 /* Exiting. Got the lock. */
1055 lock->on_cpu = cpu;
1056 lock->previous = inline_pc;
1057 lock->task = current;
1058 lock->base_file = base_file;
1059 lock->line_no = line_no;
1060
1061 if (printed) {
1062 printk(KERN_WARNING
1063 "%s:%d spinlock grabbed in %s at %p(%d) %ld ticks\n",
1064 base_file, line_no, current->comm, inline_pc,
1065 cpu, jiffies - started);
1066 }
1067}
1068
1069int
1070debug_spin_trylock(spinlock_t * lock, const char *base_file, int line_no)
1071{
1072 int ret;
1073 if ((ret = !test_and_set_bit(0, lock))) {
1074 lock->on_cpu = smp_processor_id();
1075 lock->previous = __builtin_return_address(0);
1076 lock->task = current;
1077 } else {
1078 lock->base_file = base_file;
1079 lock->line_no = line_no;
1080 }
1081 return ret;
1082}
1083#endif /* CONFIG_DEBUG_SPINLOCK */
1084
1085#ifdef CONFIG_DEBUG_RWLOCK
1086void _raw_write_lock(rwlock_t * lock)
1087{
1088 long regx, regy;
1089 int stuck_lock, stuck_reader;
1090 void *inline_pc = __builtin_return_address(0);
1091
1092 try_again:
1093
1094 stuck_lock = 1<<30;
1095 stuck_reader = 1<<30;
1096
1097 __asm__ __volatile__(
1098 "1: ldl_l %1,%0\n"
1099 " blbs %1,6f\n"
1100 " blt %1,8f\n"
1101 " mov 1,%1\n"
1102 " stl_c %1,%0\n"
1103 " beq %1,6f\n"
1104 "4: mb\n"
1105 ".subsection 2\n"
1106 "6: blt %3,4b # debug\n"
1107 " subl %3,1,%3 # debug\n"
1108 " ldl %1,%0\n"
1109 " blbs %1,6b\n"
1110 "8: blt %4,4b # debug\n"
1111 " subl %4,1,%4 # debug\n"
1112 " ldl %1,%0\n"
1113 " blt %1,8b\n"
1114 " br 1b\n"
1115 ".previous"
1116 : "=m" (*(volatile int *)lock), "=&r" (regx), "=&r" (regy),
1117 "=&r" (stuck_lock), "=&r" (stuck_reader)
1118 : "m" (*(volatile int *)lock), "3" (stuck_lock), "4" (stuck_reader) : "memory");
1119
1120 if (stuck_lock < 0) {
1121 printk(KERN_WARNING "write_lock stuck at %p\n", inline_pc);
1122 goto try_again;
1123 }
1124 if (stuck_reader < 0) {
1125 printk(KERN_WARNING "write_lock stuck on readers at %p\n",
1126 inline_pc);
1127 goto try_again;
1128 }
1129}
1130
1131void _raw_read_lock(rwlock_t * lock)
1132{
1133 long regx;
1134 int stuck_lock;
1135 void *inline_pc = __builtin_return_address(0);
1136
1137 try_again:
1138
1139 stuck_lock = 1<<30;
1140
1141 __asm__ __volatile__(
1142 "1: ldl_l %1,%0;"
1143 " blbs %1,6f;"
1144 " subl %1,2,%1;"
1145 " stl_c %1,%0;"
1146 " beq %1,6f;"
1147 "4: mb\n"
1148 ".subsection 2\n"
1149 "6: ldl %1,%0;"
1150 " blt %2,4b # debug\n"
1151 " subl %2,1,%2 # debug\n"
1152 " blbs %1,6b;"
1153 " br 1b\n"
1154 ".previous"
1155 : "=m" (*(volatile int *)lock), "=&r" (regx), "=&r" (stuck_lock)
1156 : "m" (*(volatile int *)lock), "2" (stuck_lock) : "memory");
1157
1158 if (stuck_lock < 0) {
1159 printk(KERN_WARNING "read_lock stuck at %p\n", inline_pc);
1160 goto try_again;
1161 }
1162}
1163#endif /* CONFIG_DEBUG_RWLOCK */
diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c
index 4ebbf3974381..8d484204a3ff 100644
--- a/arch/ia64/kernel/mca.c
+++ b/arch/ia64/kernel/mca.c
@@ -491,12 +491,7 @@ init_handler_platform (pal_min_state_area_t *ms,
491 unw_init_from_interruption(&info, current, pt, sw); 491 unw_init_from_interruption(&info, current, pt, sw);
492 ia64_do_show_stack(&info, NULL); 492 ia64_do_show_stack(&info, NULL);
493 493
494#ifdef CONFIG_SMP 494 if (read_trylock(&tasklist_lock)) {
495 /* read_trylock() would be handy... */
496 if (!tasklist_lock.write_lock)
497 read_lock(&tasklist_lock);
498#endif
499 {
500 struct task_struct *g, *t; 495 struct task_struct *g, *t;
501 do_each_thread (g, t) { 496 do_each_thread (g, t) {
502 if (t == current) 497 if (t == current)
@@ -506,10 +501,6 @@ init_handler_platform (pal_min_state_area_t *ms,
506 show_stack(t, NULL); 501 show_stack(t, NULL);
507 } while_each_thread (g, t); 502 } while_each_thread (g, t);
508 } 503 }
509#ifdef CONFIG_SMP
510 if (!tasklist_lock.write_lock)
511 read_unlock(&tasklist_lock);
512#endif
513 504
514 printk("\nINIT dump complete. Please reboot now.\n"); 505 printk("\nINIT dump complete. Please reboot now.\n");
515 while (1); /* hang city if no debugger */ 506 while (1); /* hang city if no debugger */
diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c
index 48b187f2d2b3..a4576ac7e870 100644
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -892,7 +892,6 @@ unsigned long send_IPI_mask_phys(cpumask_t physid_mask, int ipi_num,
892 int try) 892 int try)
893{ 893{
894 spinlock_t *ipilock; 894 spinlock_t *ipilock;
895 unsigned long flags = 0;
896 volatile unsigned long *ipicr_addr; 895 volatile unsigned long *ipicr_addr;
897 unsigned long ipicr_val; 896 unsigned long ipicr_val;
898 unsigned long my_physid_mask; 897 unsigned long my_physid_mask;
@@ -916,50 +915,27 @@ unsigned long send_IPI_mask_phys(cpumask_t physid_mask, int ipi_num,
916 * write IPICRi (send IPIi) 915 * write IPICRi (send IPIi)
917 * unlock ipi_lock[i] 916 * unlock ipi_lock[i]
918 */ 917 */
918 spin_lock(ipilock);
919 __asm__ __volatile__ ( 919 __asm__ __volatile__ (
920 ";; LOCK ipi_lock[i] \n\t" 920 ";; CHECK IPICRi == 0 \n\t"
921 ".fillinsn \n" 921 ".fillinsn \n"
922 "1: \n\t" 922 "1: \n\t"
923 "mvfc %1, psw \n\t" 923 "ld %0, @%1 \n\t"
924 "clrpsw #0x40 -> nop \n\t" 924 "and %0, %4 \n\t"
925 DCACHE_CLEAR("r4", "r5", "%2") 925 "beqz %0, 2f \n\t"
926 "lock r4, @%2 \n\t" 926 "bnez %3, 3f \n\t"
927 "addi r4, #-1 \n\t"
928 "unlock r4, @%2 \n\t"
929 "mvtc %1, psw \n\t"
930 "bnez r4, 2f \n\t"
931 LOCK_SECTION_START(".balign 4 \n\t")
932 ".fillinsn \n"
933 "2: \n\t"
934 "ld r4, @%2 \n\t"
935 "blez r4, 2b \n\t"
936 "bra 1b \n\t" 927 "bra 1b \n\t"
937 LOCK_SECTION_END
938 ";; CHECK IPICRi == 0 \n\t"
939 ".fillinsn \n"
940 "3: \n\t"
941 "ld %0, @%3 \n\t"
942 "and %0, %6 \n\t"
943 "beqz %0, 4f \n\t"
944 "bnez %5, 5f \n\t"
945 "bra 3b \n\t"
946 ";; WRITE IPICRi (send IPIi) \n\t" 928 ";; WRITE IPICRi (send IPIi) \n\t"
947 ".fillinsn \n" 929 ".fillinsn \n"
948 "4: \n\t" 930 "2: \n\t"
949 "st %4, @%3 \n\t" 931 "st %2, @%1 \n\t"
950 ";; UNLOCK ipi_lock[i] \n\t"
951 ".fillinsn \n" 932 ".fillinsn \n"
952 "5: \n\t" 933 "3: \n\t"
953 "ldi r4, #1 \n\t"
954 "st r4, @%2 \n\t"
955 : "=&r"(ipicr_val) 934 : "=&r"(ipicr_val)
956 : "r"(flags), "r"(&ipilock->slock), "r"(ipicr_addr), 935 : "r"(ipicr_addr), "r"(mask), "r"(try), "r"(my_physid_mask)
957 "r"(mask), "r"(try), "r"(my_physid_mask) 936 : "memory"
958 : "memory", "r4"
959#ifdef CONFIG_CHIP_M32700_TS1
960 , "r5"
961#endif /* CONFIG_CHIP_M32700_TS1 */
962 ); 937 );
938 spin_unlock(ipilock);
963 939
964 return ipicr_val; 940 return ipicr_val;
965} 941}
diff --git a/arch/mips/lib/dec_and_lock.c b/arch/mips/lib/dec_and_lock.c
index e44e9579bd36..fd82c84a93b7 100644
--- a/arch/mips/lib/dec_and_lock.c
+++ b/arch/mips/lib/dec_and_lock.c
@@ -20,14 +20,7 @@
20 * has a cmpxchg, and where atomic->value is an int holding 20 * has a cmpxchg, and where atomic->value is an int holding
21 * the value of the atomic (i.e. the high bits aren't used 21 * the value of the atomic (i.e. the high bits aren't used
22 * for a lock or anything like that). 22 * for a lock or anything like that).
23 *
24 * N.B. ATOMIC_DEC_AND_LOCK gets defined in include/linux/spinlock.h
25 * if spinlocks are empty and thus atomic_dec_and_lock is defined
26 * to be atomic_dec_and_test - in that case we don't need it
27 * defined here as well.
28 */ 23 */
29
30#ifndef ATOMIC_DEC_AND_LOCK
31int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) 24int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
32{ 25{
33 int counter; 26 int counter;
@@ -52,4 +45,3 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
52} 45}
53 46
54EXPORT_SYMBOL(_atomic_dec_and_lock); 47EXPORT_SYMBOL(_atomic_dec_and_lock);
55#endif /* ATOMIC_DEC_AND_LOCK */
diff --git a/arch/parisc/lib/Makefile b/arch/parisc/lib/Makefile
index 7bf705676297..5f2e6904d14a 100644
--- a/arch/parisc/lib/Makefile
+++ b/arch/parisc/lib/Makefile
@@ -5,5 +5,3 @@
5lib-y := lusercopy.o bitops.o checksum.o io.o memset.o fixup.o memcpy.o 5lib-y := lusercopy.o bitops.o checksum.o io.o memset.o fixup.o memcpy.o
6 6
7obj-y := iomap.o 7obj-y := iomap.o
8
9lib-$(CONFIG_SMP) += debuglocks.o
diff --git a/arch/parisc/lib/bitops.c b/arch/parisc/lib/bitops.c
index 2de182f6fe8a..90f400b10282 100644
--- a/arch/parisc/lib/bitops.c
+++ b/arch/parisc/lib/bitops.c
@@ -13,8 +13,8 @@
13#include <asm/atomic.h> 13#include <asm/atomic.h>
14 14
15#ifdef CONFIG_SMP 15#ifdef CONFIG_SMP
16spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned = { 16raw_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned = {
17 [0 ... (ATOMIC_HASH_SIZE-1)] = SPIN_LOCK_UNLOCKED 17 [0 ... (ATOMIC_HASH_SIZE-1)] = __RAW_SPIN_LOCK_UNLOCKED
18}; 18};
19#endif 19#endif
20 20
diff --git a/arch/parisc/lib/debuglocks.c b/arch/parisc/lib/debuglocks.c
deleted file mode 100644
index 1b33fe6e5b7a..000000000000
--- a/arch/parisc/lib/debuglocks.c
+++ /dev/null
@@ -1,277 +0,0 @@
1/*
2 * Debugging versions of SMP locking primitives.
3 *
4 * Copyright (C) 2004 Thibaut VARENE <varenet@parisc-linux.org>
5 *
6 * Some code stollen from alpha & sparc64 ;)
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 *
22 * We use pdc_printf() throughout the file for all output messages, to avoid
23 * losing messages because of disabled interrupts. Since we're using these
24 * messages for debugging purposes, it makes sense not to send them to the
25 * linux console.
26 */
27
28
29#include <linux/config.h>
30#include <linux/kernel.h>
31#include <linux/sched.h>
32#include <linux/spinlock.h>
33#include <linux/hardirq.h> /* in_interrupt() */
34#include <asm/system.h>
35#include <asm/hardirq.h> /* in_interrupt() */
36#include <asm/pdc.h>
37
38#undef INIT_STUCK
39#define INIT_STUCK 1L << 30
40
41#ifdef CONFIG_DEBUG_SPINLOCK
42
43
44void _dbg_spin_lock(spinlock_t * lock, const char *base_file, int line_no)
45{
46 volatile unsigned int *a;
47 long stuck = INIT_STUCK;
48 void *inline_pc = __builtin_return_address(0);
49 unsigned long started = jiffies;
50 int printed = 0;
51 int cpu = smp_processor_id();
52
53try_again:
54
55 /* Do the actual locking */
56 /* <T-Bone> ggg: we can't get stuck on the outter loop?
57 * <ggg> T-Bone: We can hit the outer loop
58 * alot if multiple CPUs are constantly racing for a lock
59 * and the backplane is NOT fair about which CPU sees
60 * the update first. But it won't hang since every failed
61 * attempt will drop us back into the inner loop and
62 * decrement `stuck'.
63 * <ggg> K-class and some of the others are NOT fair in the HW
64 * implementation so we could see false positives.
65 * But fixing the lock contention is easier than
66 * fixing the HW to be fair.
67 * <tausq> __ldcw() returns 1 if we get the lock; otherwise we
68 * spin until the value of the lock changes, or we time out.
69 */
70 mb();
71 a = __ldcw_align(lock);
72 while (stuck && (__ldcw(a) == 0))
73 while ((*a == 0) && --stuck);
74 mb();
75
76 if (unlikely(stuck <= 0)) {
77 pdc_printf(
78 "%s:%d: spin_lock(%s/%p) stuck in %s at %p(%d)"
79 " owned by %s:%d in %s at %p(%d)\n",
80 base_file, line_no, lock->module, lock,
81 current->comm, inline_pc, cpu,
82 lock->bfile, lock->bline, lock->task->comm,
83 lock->previous, lock->oncpu);
84 stuck = INIT_STUCK;
85 printed = 1;
86 goto try_again;
87 }
88
89 /* Exiting. Got the lock. */
90 lock->oncpu = cpu;
91 lock->previous = inline_pc;
92 lock->task = current;
93 lock->bfile = (char *)base_file;
94 lock->bline = line_no;
95
96 if (unlikely(printed)) {
97 pdc_printf(
98 "%s:%d: spin_lock grabbed in %s at %p(%d) %ld ticks\n",
99 base_file, line_no, current->comm, inline_pc,
100 cpu, jiffies - started);
101 }
102}
103
104void _dbg_spin_unlock(spinlock_t * lock, const char *base_file, int line_no)
105{
106 CHECK_LOCK(lock);
107 volatile unsigned int *a;
108 mb();
109 a = __ldcw_align(lock);
110 if (unlikely((*a != 0) && lock->babble)) {
111 lock->babble--;
112 pdc_printf(
113 "%s:%d: spin_unlock(%s:%p) not locked\n",
114 base_file, line_no, lock->module, lock);
115 }
116 *a = 1;
117 mb();
118}
119
120int _dbg_spin_trylock(spinlock_t * lock, const char *base_file, int line_no)
121{
122 int ret;
123 volatile unsigned int *a;
124 mb();
125 a = __ldcw_align(lock);
126 ret = (__ldcw(a) != 0);
127 mb();
128 if (ret) {
129 lock->oncpu = smp_processor_id();
130 lock->previous = __builtin_return_address(0);
131 lock->task = current;
132 } else {
133 lock->bfile = (char *)base_file;
134 lock->bline = line_no;
135 }
136 return ret;
137}
138
139#endif /* CONFIG_DEBUG_SPINLOCK */
140
141#ifdef CONFIG_DEBUG_RWLOCK
142
143/* Interrupts trouble detailed explanation, thx Grant:
144 *
145 * o writer (wants to modify data) attempts to acquire the rwlock
146 * o He gets the write lock.
147 * o Interupts are still enabled, we take an interrupt with the
148 * write still holding the lock.
149 * o interrupt handler tries to acquire the rwlock for read.
150 * o deadlock since the writer can't release it at this point.
151 *
152 * In general, any use of spinlocks that competes between "base"
153 * level and interrupt level code will risk deadlock. Interrupts
154 * need to be disabled in the base level routines to avoid it.
155 * Or more precisely, only the IRQ the base level routine
156 * is competing with for the lock. But it's more efficient/faster
157 * to just disable all interrupts on that CPU to guarantee
158 * once it gets the lock it can release it quickly too.
159 */
160
161void _dbg_write_lock(rwlock_t *rw, const char *bfile, int bline)
162{
163 void *inline_pc = __builtin_return_address(0);
164 unsigned long started = jiffies;
165 long stuck = INIT_STUCK;
166 int printed = 0;
167 int cpu = smp_processor_id();
168
169 if(unlikely(in_interrupt())) { /* acquiring write lock in interrupt context, bad idea */
170 pdc_printf("write_lock caller: %s:%d, IRQs enabled,\n", bfile, bline);
171 BUG();
172 }
173
174 /* Note: if interrupts are disabled (which is most likely), the printk
175 will never show on the console. We might need a polling method to flush
176 the dmesg buffer anyhow. */
177
178retry:
179 _raw_spin_lock(&rw->lock);
180
181 if(rw->counter != 0) {
182 /* this basically never happens */
183 _raw_spin_unlock(&rw->lock);
184
185 stuck--;
186 if ((unlikely(stuck <= 0)) && (rw->counter < 0)) {
187 pdc_printf(
188 "%s:%d: write_lock stuck on writer"
189 " in %s at %p(%d) %ld ticks\n",
190 bfile, bline, current->comm, inline_pc,
191 cpu, jiffies - started);
192 stuck = INIT_STUCK;
193 printed = 1;
194 }
195 else if (unlikely(stuck <= 0)) {
196 pdc_printf(
197 "%s:%d: write_lock stuck on reader"
198 " in %s at %p(%d) %ld ticks\n",
199 bfile, bline, current->comm, inline_pc,
200 cpu, jiffies - started);
201 stuck = INIT_STUCK;
202 printed = 1;
203 }
204
205 while(rw->counter != 0);
206
207 goto retry;
208 }
209
210 /* got it. now leave without unlocking */
211 rw->counter = -1; /* remember we are locked */
212
213 if (unlikely(printed)) {
214 pdc_printf(
215 "%s:%d: write_lock grabbed in %s at %p(%d) %ld ticks\n",
216 bfile, bline, current->comm, inline_pc,
217 cpu, jiffies - started);
218 }
219}
220
221int _dbg_write_trylock(rwlock_t *rw, const char *bfile, int bline)
222{
223#if 0
224 void *inline_pc = __builtin_return_address(0);
225 int cpu = smp_processor_id();
226#endif
227
228 if(unlikely(in_interrupt())) { /* acquiring write lock in interrupt context, bad idea */
229 pdc_printf("write_lock caller: %s:%d, IRQs enabled,\n", bfile, bline);
230 BUG();
231 }
232
233 /* Note: if interrupts are disabled (which is most likely), the printk
234 will never show on the console. We might need a polling method to flush
235 the dmesg buffer anyhow. */
236
237 _raw_spin_lock(&rw->lock);
238
239 if(rw->counter != 0) {
240 /* this basically never happens */
241 _raw_spin_unlock(&rw->lock);
242 return 0;
243 }
244
245 /* got it. now leave without unlocking */
246 rw->counter = -1; /* remember we are locked */
247#if 0
248 pdc_printf("%s:%d: try write_lock grabbed in %s at %p(%d)\n",
249 bfile, bline, current->comm, inline_pc, cpu);
250#endif
251 return 1;
252}
253
254void _dbg_read_lock(rwlock_t * rw, const char *bfile, int bline)
255{
256#if 0
257 void *inline_pc = __builtin_return_address(0);
258 unsigned long started = jiffies;
259 int cpu = smp_processor_id();
260#endif
261 unsigned long flags;
262
263 local_irq_save(flags);
264 _raw_spin_lock(&rw->lock);
265
266 rw->counter++;
267#if 0
268 pdc_printf(
269 "%s:%d: read_lock grabbed in %s at %p(%d) %ld ticks\n",
270 bfile, bline, current->comm, inline_pc,
271 cpu, jiffies - started);
272#endif
273 _raw_spin_unlock(&rw->lock);
274 local_irq_restore(flags);
275}
276
277#endif /* CONFIG_DEBUG_RWLOCK */
diff --git a/arch/ppc/lib/Makefile b/arch/ppc/lib/Makefile
index 1c380e67d435..f1e1fb4144f0 100644
--- a/arch/ppc/lib/Makefile
+++ b/arch/ppc/lib/Makefile
@@ -4,6 +4,5 @@
4 4
5obj-y := checksum.o string.o strcase.o dec_and_lock.o div64.o 5obj-y := checksum.o string.o strcase.o dec_and_lock.o div64.o
6 6
7obj-$(CONFIG_SMP) += locks.o
8obj-$(CONFIG_8xx) += rheap.o 7obj-$(CONFIG_8xx) += rheap.o
9obj-$(CONFIG_CPM2) += rheap.o 8obj-$(CONFIG_CPM2) += rheap.o
diff --git a/arch/ppc/lib/dec_and_lock.c b/arch/ppc/lib/dec_and_lock.c
index 4ee888070d91..b18f0d9a00fc 100644
--- a/arch/ppc/lib/dec_and_lock.c
+++ b/arch/ppc/lib/dec_and_lock.c
@@ -11,14 +11,7 @@
11 * has a cmpxchg, and where atomic->value is an int holding 11 * has a cmpxchg, and where atomic->value is an int holding
12 * the value of the atomic (i.e. the high bits aren't used 12 * the value of the atomic (i.e. the high bits aren't used
13 * for a lock or anything like that). 13 * for a lock or anything like that).
14 *
15 * N.B. ATOMIC_DEC_AND_LOCK gets defined in include/linux/spinlock.h
16 * if spinlocks are empty and thus atomic_dec_and_lock is defined
17 * to be atomic_dec_and_test - in that case we don't need it
18 * defined here as well.
19 */ 14 */
20
21#ifndef ATOMIC_DEC_AND_LOCK
22int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) 15int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
23{ 16{
24 int counter; 17 int counter;
@@ -43,4 +36,3 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
43} 36}
44 37
45EXPORT_SYMBOL(_atomic_dec_and_lock); 38EXPORT_SYMBOL(_atomic_dec_and_lock);
46#endif /* ATOMIC_DEC_AND_LOCK */
diff --git a/arch/ppc64/lib/dec_and_lock.c b/arch/ppc64/lib/dec_and_lock.c
index 6e8d8591708c..7b9d4da5cf92 100644
--- a/arch/ppc64/lib/dec_and_lock.c
+++ b/arch/ppc64/lib/dec_and_lock.c
@@ -20,14 +20,7 @@
20 * has a cmpxchg, and where atomic->value is an int holding 20 * has a cmpxchg, and where atomic->value is an int holding
21 * the value of the atomic (i.e. the high bits aren't used 21 * the value of the atomic (i.e. the high bits aren't used
22 * for a lock or anything like that). 22 * for a lock or anything like that).
23 *
24 * N.B. ATOMIC_DEC_AND_LOCK gets defined in include/linux/spinlock.h
25 * if spinlocks are empty and thus atomic_dec_and_lock is defined
26 * to be atomic_dec_and_test - in that case we don't need it
27 * defined here as well.
28 */ 23 */
29
30#ifndef ATOMIC_DEC_AND_LOCK
31int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) 24int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
32{ 25{
33 int counter; 26 int counter;
@@ -52,4 +45,3 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
52} 45}
53 46
54EXPORT_SYMBOL(_atomic_dec_and_lock); 47EXPORT_SYMBOL(_atomic_dec_and_lock);
55#endif /* ATOMIC_DEC_AND_LOCK */
diff --git a/arch/ppc64/lib/locks.c b/arch/ppc64/lib/locks.c
index ef70ef91abe2..033643ab69e0 100644
--- a/arch/ppc64/lib/locks.c
+++ b/arch/ppc64/lib/locks.c
@@ -23,12 +23,12 @@
23/* waiting for a spinlock... */ 23/* waiting for a spinlock... */
24#if defined(CONFIG_PPC_SPLPAR) || defined(CONFIG_PPC_ISERIES) 24#if defined(CONFIG_PPC_SPLPAR) || defined(CONFIG_PPC_ISERIES)
25 25
26void __spin_yield(spinlock_t *lock) 26void __spin_yield(raw_spinlock_t *lock)
27{ 27{
28 unsigned int lock_value, holder_cpu, yield_count; 28 unsigned int lock_value, holder_cpu, yield_count;
29 struct paca_struct *holder_paca; 29 struct paca_struct *holder_paca;
30 30
31 lock_value = lock->lock; 31 lock_value = lock->slock;
32 if (lock_value == 0) 32 if (lock_value == 0)
33 return; 33 return;
34 holder_cpu = lock_value & 0xffff; 34 holder_cpu = lock_value & 0xffff;
@@ -38,7 +38,7 @@ void __spin_yield(spinlock_t *lock)
38 if ((yield_count & 1) == 0) 38 if ((yield_count & 1) == 0)
39 return; /* virtual cpu is currently running */ 39 return; /* virtual cpu is currently running */
40 rmb(); 40 rmb();
41 if (lock->lock != lock_value) 41 if (lock->slock != lock_value)
42 return; /* something has changed */ 42 return; /* something has changed */
43#ifdef CONFIG_PPC_ISERIES 43#ifdef CONFIG_PPC_ISERIES
44 HvCall2(HvCallBaseYieldProcessor, HvCall_YieldToProc, 44 HvCall2(HvCallBaseYieldProcessor, HvCall_YieldToProc,
@@ -54,7 +54,7 @@ void __spin_yield(spinlock_t *lock)
54 * This turns out to be the same for read and write locks, since 54 * This turns out to be the same for read and write locks, since
55 * we only know the holder if it is write-locked. 55 * we only know the holder if it is write-locked.
56 */ 56 */
57void __rw_yield(rwlock_t *rw) 57void __rw_yield(raw_rwlock_t *rw)
58{ 58{
59 int lock_value; 59 int lock_value;
60 unsigned int holder_cpu, yield_count; 60 unsigned int holder_cpu, yield_count;
@@ -82,9 +82,9 @@ void __rw_yield(rwlock_t *rw)
82} 82}
83#endif 83#endif
84 84
85void spin_unlock_wait(spinlock_t *lock) 85void __raw_spin_unlock_wait(raw_spinlock_t *lock)
86{ 86{
87 while (lock->lock) { 87 while (lock->slock) {
88 HMT_low(); 88 HMT_low();
89 if (SHARED_PROCESSOR) 89 if (SHARED_PROCESSOR)
90 __spin_yield(lock); 90 __spin_yield(lock);
@@ -92,4 +92,4 @@ void spin_unlock_wait(spinlock_t *lock)
92 HMT_medium(); 92 HMT_medium();
93} 93}
94 94
95EXPORT_SYMBOL(spin_unlock_wait); 95EXPORT_SYMBOL(__raw_spin_unlock_wait);
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 888b5596c195..2dc14e9c8327 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -36,7 +36,7 @@ _diag44(void)
36} 36}
37 37
38void 38void
39_raw_spin_lock_wait(spinlock_t *lp, unsigned int pc) 39_raw_spin_lock_wait(raw_spinlock_t *lp, unsigned int pc)
40{ 40{
41 int count = spin_retry; 41 int count = spin_retry;
42 42
@@ -53,7 +53,7 @@ _raw_spin_lock_wait(spinlock_t *lp, unsigned int pc)
53EXPORT_SYMBOL(_raw_spin_lock_wait); 53EXPORT_SYMBOL(_raw_spin_lock_wait);
54 54
55int 55int
56_raw_spin_trylock_retry(spinlock_t *lp, unsigned int pc) 56_raw_spin_trylock_retry(raw_spinlock_t *lp, unsigned int pc)
57{ 57{
58 int count = spin_retry; 58 int count = spin_retry;
59 59
@@ -67,7 +67,7 @@ _raw_spin_trylock_retry(spinlock_t *lp, unsigned int pc)
67EXPORT_SYMBOL(_raw_spin_trylock_retry); 67EXPORT_SYMBOL(_raw_spin_trylock_retry);
68 68
69void 69void
70_raw_read_lock_wait(rwlock_t *rw) 70_raw_read_lock_wait(raw_rwlock_t *rw)
71{ 71{
72 unsigned int old; 72 unsigned int old;
73 int count = spin_retry; 73 int count = spin_retry;
@@ -86,7 +86,7 @@ _raw_read_lock_wait(rwlock_t *rw)
86EXPORT_SYMBOL(_raw_read_lock_wait); 86EXPORT_SYMBOL(_raw_read_lock_wait);
87 87
88int 88int
89_raw_read_trylock_retry(rwlock_t *rw) 89_raw_read_trylock_retry(raw_rwlock_t *rw)
90{ 90{
91 unsigned int old; 91 unsigned int old;
92 int count = spin_retry; 92 int count = spin_retry;
@@ -102,7 +102,7 @@ _raw_read_trylock_retry(rwlock_t *rw)
102EXPORT_SYMBOL(_raw_read_trylock_retry); 102EXPORT_SYMBOL(_raw_read_trylock_retry);
103 103
104void 104void
105_raw_write_lock_wait(rwlock_t *rw) 105_raw_write_lock_wait(raw_rwlock_t *rw)
106{ 106{
107 int count = spin_retry; 107 int count = spin_retry;
108 108
@@ -119,7 +119,7 @@ _raw_write_lock_wait(rwlock_t *rw)
119EXPORT_SYMBOL(_raw_write_lock_wait); 119EXPORT_SYMBOL(_raw_write_lock_wait);
120 120
121int 121int
122_raw_write_trylock_retry(rwlock_t *rw) 122_raw_write_trylock_retry(raw_rwlock_t *rw)
123{ 123{
124 int count = spin_retry; 124 int count = spin_retry;
125 125
diff --git a/arch/sparc/kernel/sparc_ksyms.c b/arch/sparc/kernel/sparc_ksyms.c
index 5d974a2b735a..f84809333624 100644
--- a/arch/sparc/kernel/sparc_ksyms.c
+++ b/arch/sparc/kernel/sparc_ksyms.c
@@ -114,17 +114,7 @@ DOT_ALIAS2(unsigned, urem, unsigned, unsigned)
114/* used by various drivers */ 114/* used by various drivers */
115EXPORT_SYMBOL(sparc_cpu_model); 115EXPORT_SYMBOL(sparc_cpu_model);
116EXPORT_SYMBOL(kernel_thread); 116EXPORT_SYMBOL(kernel_thread);
117#ifdef CONFIG_DEBUG_SPINLOCK
118#ifdef CONFIG_SMP 117#ifdef CONFIG_SMP
119EXPORT_SYMBOL(_do_spin_lock);
120EXPORT_SYMBOL(_do_spin_unlock);
121EXPORT_SYMBOL(_spin_trylock);
122EXPORT_SYMBOL(_do_read_lock);
123EXPORT_SYMBOL(_do_read_unlock);
124EXPORT_SYMBOL(_do_write_lock);
125EXPORT_SYMBOL(_do_write_unlock);
126#endif
127#else
128// XXX find what uses (or used) these. 118// XXX find what uses (or used) these.
129EXPORT_SYMBOL(___rw_read_enter); 119EXPORT_SYMBOL(___rw_read_enter);
130EXPORT_SYMBOL(___rw_read_exit); 120EXPORT_SYMBOL(___rw_read_exit);
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 2296ff9dc47a..fa5006946062 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -9,5 +9,3 @@ lib-y := mul.o rem.o sdiv.o udiv.o umul.o urem.o ashrdi3.o memcpy.o memset.o \
9 strncpy_from_user.o divdi3.o udivdi3.o strlen_user.o \ 9 strncpy_from_user.o divdi3.o udivdi3.o strlen_user.o \
10 copy_user.o locks.o atomic.o atomic32.o bitops.o \ 10 copy_user.o locks.o atomic.o atomic32.o bitops.o \
11 lshrdi3.o ashldi3.o rwsem.o muldi3.o bitext.o 11 lshrdi3.o ashldi3.o rwsem.o muldi3.o bitext.o
12
13lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
diff --git a/arch/sparc/lib/debuglocks.c b/arch/sparc/lib/debuglocks.c
deleted file mode 100644
index fb182352782c..000000000000
--- a/arch/sparc/lib/debuglocks.c
+++ /dev/null
@@ -1,202 +0,0 @@
1/* $Id: debuglocks.c,v 1.11 2001/09/20 00:35:31 davem Exp $
2 * debuglocks.c: Debugging versions of SMP locking primitives.
3 *
4 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1998-99 Anton Blanchard (anton@progsoc.uts.edu.au)
6 */
7
8#include <linux/kernel.h>
9#include <linux/sched.h>
10#include <linux/threads.h> /* For NR_CPUS */
11#include <linux/spinlock.h>
12#include <asm/psr.h>
13#include <asm/system.h>
14
15#ifdef CONFIG_SMP
16
17/* Some notes on how these debugging routines work. When a lock is acquired
18 * an extra debugging member lock->owner_pc is set to the caller of the lock
19 * acquisition routine. Right before releasing a lock, the debugging program
20 * counter is cleared to zero.
21 *
22 * Furthermore, since PC's are 4 byte aligned on Sparc, we stuff the CPU
23 * number of the owner in the lowest two bits.
24 */
25
26#define STORE_CALLER(A) __asm__ __volatile__("mov %%i7, %0" : "=r" (A));
27
28static inline void show(char *str, spinlock_t *lock, unsigned long caller)
29{
30 int cpu = smp_processor_id();
31
32 printk("%s(%p) CPU#%d stuck at %08lx, owner PC(%08lx):CPU(%lx)\n",str,
33 lock, cpu, caller, lock->owner_pc & ~3, lock->owner_pc & 3);
34}
35
36static inline void show_read(char *str, rwlock_t *lock, unsigned long caller)
37{
38 int cpu = smp_processor_id();
39
40 printk("%s(%p) CPU#%d stuck at %08lx, owner PC(%08lx):CPU(%lx)\n", str,
41 lock, cpu, caller, lock->owner_pc & ~3, lock->owner_pc & 3);
42}
43
44static inline void show_write(char *str, rwlock_t *lock, unsigned long caller)
45{
46 int cpu = smp_processor_id();
47 int i;
48
49 printk("%s(%p) CPU#%d stuck at %08lx, owner PC(%08lx):CPU(%lx)", str,
50 lock, cpu, caller, lock->owner_pc & ~3, lock->owner_pc & 3);
51
52 for(i = 0; i < NR_CPUS; i++)
53 printk(" reader[%d]=%08lx", i, lock->reader_pc[i]);
54
55 printk("\n");
56}
57
58#undef INIT_STUCK
59#define INIT_STUCK 100000000
60
61void _do_spin_lock(spinlock_t *lock, char *str)
62{
63 unsigned long caller;
64 unsigned long val;
65 int cpu = smp_processor_id();
66 int stuck = INIT_STUCK;
67
68 STORE_CALLER(caller);
69
70again:
71 __asm__ __volatile__("ldstub [%1], %0" : "=r" (val) : "r" (&(lock->lock)));
72 if(val) {
73 while(lock->lock) {
74 if (!--stuck) {
75 show(str, lock, caller);
76 stuck = INIT_STUCK;
77 }
78 barrier();
79 }
80 goto again;
81 }
82 lock->owner_pc = (cpu & 3) | (caller & ~3);
83}
84
85int _spin_trylock(spinlock_t *lock)
86{
87 unsigned long val;
88 unsigned long caller;
89 int cpu = smp_processor_id();
90
91 STORE_CALLER(caller);
92
93 __asm__ __volatile__("ldstub [%1], %0" : "=r" (val) : "r" (&(lock->lock)));
94 if(!val) {
95 /* We got it, record our identity for debugging. */
96 lock->owner_pc = (cpu & 3) | (caller & ~3);
97 }
98 return val == 0;
99}
100
101void _do_spin_unlock(spinlock_t *lock)
102{
103 lock->owner_pc = 0;
104 barrier();
105 lock->lock = 0;
106}
107
108void _do_read_lock(rwlock_t *rw, char *str)
109{
110 unsigned long caller;
111 unsigned long val;
112 int cpu = smp_processor_id();
113 int stuck = INIT_STUCK;
114
115 STORE_CALLER(caller);
116
117wlock_again:
118 __asm__ __volatile__("ldstub [%1 + 3], %0" : "=r" (val) : "r" (&(rw->lock)));
119 if(val) {
120 while(rw->lock & 0xff) {
121 if (!--stuck) {
122 show_read(str, rw, caller);
123 stuck = INIT_STUCK;
124 }
125 barrier();
126 }
127 goto wlock_again;
128 }
129
130 rw->reader_pc[cpu] = caller;
131 barrier();
132 rw->lock++;
133}
134
135void _do_read_unlock(rwlock_t *rw, char *str)
136{
137 unsigned long caller;
138 unsigned long val;
139 int cpu = smp_processor_id();
140 int stuck = INIT_STUCK;
141
142 STORE_CALLER(caller);
143
144wlock_again:
145 __asm__ __volatile__("ldstub [%1 + 3], %0" : "=r" (val) : "r" (&(rw->lock)));
146 if(val) {
147 while(rw->lock & 0xff) {
148 if (!--stuck) {
149 show_read(str, rw, caller);
150 stuck = INIT_STUCK;
151 }
152 barrier();
153 }
154 goto wlock_again;
155 }
156
157 rw->reader_pc[cpu] = 0;
158 barrier();
159 rw->lock -= 0x1ff;
160}
161
162void _do_write_lock(rwlock_t *rw, char *str)
163{
164 unsigned long caller;
165 unsigned long val;
166 int cpu = smp_processor_id();
167 int stuck = INIT_STUCK;
168
169 STORE_CALLER(caller);
170
171wlock_again:
172 __asm__ __volatile__("ldstub [%1 + 3], %0" : "=r" (val) : "r" (&(rw->lock)));
173 if(val) {
174wlock_wait:
175 while(rw->lock) {
176 if (!--stuck) {
177 show_write(str, rw, caller);
178 stuck = INIT_STUCK;
179 }
180 barrier();
181 }
182 goto wlock_again;
183 }
184
185 if (rw->lock & ~0xff) {
186 *(((unsigned char *)&rw->lock)+3) = 0;
187 barrier();
188 goto wlock_wait;
189 }
190
191 barrier();
192 rw->owner_pc = (cpu & 3) | (caller & ~3);
193}
194
195void _do_write_unlock(rwlock_t *rw)
196{
197 rw->owner_pc = 0;
198 barrier();
199 rw->lock = 0;
200}
201
202#endif /* SMP */
diff --git a/arch/sparc64/kernel/process.c b/arch/sparc64/kernel/process.c
index 66255434128a..7d10b0397091 100644
--- a/arch/sparc64/kernel/process.c
+++ b/arch/sparc64/kernel/process.c
@@ -607,11 +607,6 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
607 struct thread_info *t = p->thread_info; 607 struct thread_info *t = p->thread_info;
608 char *child_trap_frame; 608 char *child_trap_frame;
609 609
610#ifdef CONFIG_DEBUG_SPINLOCK
611 p->thread.smp_lock_count = 0;
612 p->thread.smp_lock_pc = 0;
613#endif
614
615 /* Calculate offset to stack_frame & pt_regs */ 610 /* Calculate offset to stack_frame & pt_regs */
616 child_trap_frame = ((char *)t) + (THREAD_SIZE - (TRACEREG_SZ+STACKFRAME_SZ)); 611 child_trap_frame = ((char *)t) + (THREAD_SIZE - (TRACEREG_SZ+STACKFRAME_SZ));
617 memcpy(child_trap_frame, (((struct sparc_stackf *)regs)-1), (TRACEREG_SZ+STACKFRAME_SZ)); 612 memcpy(child_trap_frame, (((struct sparc_stackf *)regs)-1), (TRACEREG_SZ+STACKFRAME_SZ));
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index 7d9a0f6c437d..cbb5e59824e5 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -115,17 +115,12 @@ EXPORT_PER_CPU_SYMBOL(__cpu_data);
115 115
116/* used by various drivers */ 116/* used by various drivers */
117#ifdef CONFIG_SMP 117#ifdef CONFIG_SMP
118#ifndef CONFIG_DEBUG_SPINLOCK
119/* Out of line rw-locking implementation. */ 118/* Out of line rw-locking implementation. */
120EXPORT_SYMBOL(__read_lock); 119EXPORT_SYMBOL(__read_lock);
121EXPORT_SYMBOL(__read_unlock); 120EXPORT_SYMBOL(__read_unlock);
122EXPORT_SYMBOL(__write_lock); 121EXPORT_SYMBOL(__write_lock);
123EXPORT_SYMBOL(__write_unlock); 122EXPORT_SYMBOL(__write_unlock);
124EXPORT_SYMBOL(__write_trylock); 123EXPORT_SYMBOL(__write_trylock);
125/* Out of line spin-locking implementation. */
126EXPORT_SYMBOL(_raw_spin_lock);
127EXPORT_SYMBOL(_raw_spin_lock_flags);
128#endif
129 124
130/* Hard IRQ locking */ 125/* Hard IRQ locking */
131EXPORT_SYMBOL(synchronize_irq); 126EXPORT_SYMBOL(synchronize_irq);
diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index 40dbeec7e5d6..d968aebe83b2 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -14,7 +14,6 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
14 copy_in_user.o user_fixup.o memmove.o \ 14 copy_in_user.o user_fixup.o memmove.o \
15 mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o 15 mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
16 16
17lib-$(CONFIG_DEBUG_SPINLOCK) += debuglocks.o
18lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o 17lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
19 18
20obj-y += iomap.o 19obj-y += iomap.o
diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
deleted file mode 100644
index f5f0b5586f01..000000000000
--- a/arch/sparc64/lib/debuglocks.c
+++ /dev/null
@@ -1,366 +0,0 @@
1/* $Id: debuglocks.c,v 1.9 2001/11/17 00:10:48 davem Exp $
2 * debuglocks.c: Debugging versions of SMP locking primitives.
3 *
4 * Copyright (C) 1998 David S. Miller (davem@redhat.com)
5 */
6
7#include <linux/config.h>
8#include <linux/kernel.h>
9#include <linux/sched.h>
10#include <linux/spinlock.h>
11#include <asm/system.h>
12
13#ifdef CONFIG_SMP
14
15static inline void show (char *str, spinlock_t *lock, unsigned long caller)
16{
17 int cpu = smp_processor_id();
18
19 printk("%s(%p) CPU#%d stuck at %08x, owner PC(%08x):CPU(%x)\n",
20 str, lock, cpu, (unsigned int) caller,
21 lock->owner_pc, lock->owner_cpu);
22}
23
24static inline void show_read (char *str, rwlock_t *lock, unsigned long caller)
25{
26 int cpu = smp_processor_id();
27
28 printk("%s(%p) CPU#%d stuck at %08x, writer PC(%08x):CPU(%x)\n",
29 str, lock, cpu, (unsigned int) caller,
30 lock->writer_pc, lock->writer_cpu);
31}
32
33static inline void show_write (char *str, rwlock_t *lock, unsigned long caller)
34{
35 int cpu = smp_processor_id();
36 int i;
37
38 printk("%s(%p) CPU#%d stuck at %08x\n",
39 str, lock, cpu, (unsigned int) caller);
40 printk("Writer: PC(%08x):CPU(%x)\n",
41 lock->writer_pc, lock->writer_cpu);
42 printk("Readers:");
43 for (i = 0; i < NR_CPUS; i++)
44 if (lock->reader_pc[i])
45 printk(" %d[%08x]", i, lock->reader_pc[i]);
46 printk("\n");
47}
48
49#undef INIT_STUCK
50#define INIT_STUCK 100000000
51
52void _do_spin_lock(spinlock_t *lock, char *str, unsigned long caller)
53{
54 unsigned long val;
55 int stuck = INIT_STUCK;
56 int cpu = get_cpu();
57 int shown = 0;
58
59again:
60 __asm__ __volatile__("ldstub [%1], %0"
61 : "=r" (val)
62 : "r" (&(lock->lock))
63 : "memory");
64 membar_storeload_storestore();
65 if (val) {
66 while (lock->lock) {
67 if (!--stuck) {
68 if (shown++ <= 2)
69 show(str, lock, caller);
70 stuck = INIT_STUCK;
71 }
72 rmb();
73 }
74 goto again;
75 }
76 lock->owner_pc = ((unsigned int)caller);
77 lock->owner_cpu = cpu;
78 current->thread.smp_lock_count++;
79 current->thread.smp_lock_pc = ((unsigned int)caller);
80
81 put_cpu();
82}
83
84int _do_spin_trylock(spinlock_t *lock, unsigned long caller)
85{
86 unsigned long val;
87 int cpu = get_cpu();
88
89 __asm__ __volatile__("ldstub [%1], %0"
90 : "=r" (val)
91 : "r" (&(lock->lock))
92 : "memory");
93 membar_storeload_storestore();
94 if (!val) {
95 lock->owner_pc = ((unsigned int)caller);
96 lock->owner_cpu = cpu;
97 current->thread.smp_lock_count++;
98 current->thread.smp_lock_pc = ((unsigned int)caller);
99 }
100
101 put_cpu();
102
103 return val == 0;
104}
105
106void _do_spin_unlock(spinlock_t *lock)
107{
108 lock->owner_pc = 0;
109 lock->owner_cpu = NO_PROC_ID;
110 membar_storestore_loadstore();
111 lock->lock = 0;
112 current->thread.smp_lock_count--;
113}
114
115/* Keep INIT_STUCK the same... */
116
117void _do_read_lock(rwlock_t *rw, char *str, unsigned long caller)
118{
119 unsigned long val;
120 int stuck = INIT_STUCK;
121 int cpu = get_cpu();
122 int shown = 0;
123
124wlock_again:
125 /* Wait for any writer to go away. */
126 while (((long)(rw->lock)) < 0) {
127 if (!--stuck) {
128 if (shown++ <= 2)
129 show_read(str, rw, caller);
130 stuck = INIT_STUCK;
131 }
132 rmb();
133 }
134 /* Try once to increment the counter. */
135 __asm__ __volatile__(
136" ldx [%0], %%g1\n"
137" brlz,a,pn %%g1, 2f\n"
138" mov 1, %0\n"
139" add %%g1, 1, %%g7\n"
140" casx [%0], %%g1, %%g7\n"
141" sub %%g1, %%g7, %0\n"
142"2:" : "=r" (val)
143 : "0" (&(rw->lock))
144 : "g1", "g7", "memory");
145 membar_storeload_storestore();
146 if (val)
147 goto wlock_again;
148 rw->reader_pc[cpu] = ((unsigned int)caller);
149 current->thread.smp_lock_count++;
150 current->thread.smp_lock_pc = ((unsigned int)caller);
151
152 put_cpu();
153}
154
155void _do_read_unlock(rwlock_t *rw, char *str, unsigned long caller)
156{
157 unsigned long val;
158 int stuck = INIT_STUCK;
159 int cpu = get_cpu();
160 int shown = 0;
161
162 /* Drop our identity _first_. */
163 rw->reader_pc[cpu] = 0;
164 current->thread.smp_lock_count--;
165runlock_again:
166 /* Spin trying to decrement the counter using casx. */
167 __asm__ __volatile__(
168" membar #StoreLoad | #LoadLoad\n"
169" ldx [%0], %%g1\n"
170" sub %%g1, 1, %%g7\n"
171" casx [%0], %%g1, %%g7\n"
172" membar #StoreLoad | #StoreStore\n"
173" sub %%g1, %%g7, %0\n"
174 : "=r" (val)
175 : "0" (&(rw->lock))
176 : "g1", "g7", "memory");
177 if (val) {
178 if (!--stuck) {
179 if (shown++ <= 2)
180 show_read(str, rw, caller);
181 stuck = INIT_STUCK;
182 }
183 goto runlock_again;
184 }
185
186 put_cpu();
187}
188
189void _do_write_lock(rwlock_t *rw, char *str, unsigned long caller)
190{
191 unsigned long val;
192 int stuck = INIT_STUCK;
193 int cpu = get_cpu();
194 int shown = 0;
195
196wlock_again:
197 /* Spin while there is another writer. */
198 while (((long)rw->lock) < 0) {
199 if (!--stuck) {
200 if (shown++ <= 2)
201 show_write(str, rw, caller);
202 stuck = INIT_STUCK;
203 }
204 rmb();
205 }
206
207 /* Try to acuire the write bit. */
208 __asm__ __volatile__(
209" mov 1, %%g3\n"
210" sllx %%g3, 63, %%g3\n"
211" ldx [%0], %%g1\n"
212" brlz,pn %%g1, 1f\n"
213" or %%g1, %%g3, %%g7\n"
214" casx [%0], %%g1, %%g7\n"
215" membar #StoreLoad | #StoreStore\n"
216" ba,pt %%xcc, 2f\n"
217" sub %%g1, %%g7, %0\n"
218"1: mov 1, %0\n"
219"2:" : "=r" (val)
220 : "0" (&(rw->lock))
221 : "g3", "g1", "g7", "memory");
222 if (val) {
223 /* We couldn't get the write bit. */
224 if (!--stuck) {
225 if (shown++ <= 2)
226 show_write(str, rw, caller);
227 stuck = INIT_STUCK;
228 }
229 goto wlock_again;
230 }
231 if ((rw->lock & ((1UL<<63)-1UL)) != 0UL) {
232 /* Readers still around, drop the write
233 * lock, spin, and try again.
234 */
235 if (!--stuck) {
236 if (shown++ <= 2)
237 show_write(str, rw, caller);
238 stuck = INIT_STUCK;
239 }
240 __asm__ __volatile__(
241" mov 1, %%g3\n"
242" sllx %%g3, 63, %%g3\n"
243"1: ldx [%0], %%g1\n"
244" andn %%g1, %%g3, %%g7\n"
245" casx [%0], %%g1, %%g7\n"
246" cmp %%g1, %%g7\n"
247" membar #StoreLoad | #StoreStore\n"
248" bne,pn %%xcc, 1b\n"
249" nop"
250 : /* no outputs */
251 : "r" (&(rw->lock))
252 : "g3", "g1", "g7", "cc", "memory");
253 while(rw->lock != 0) {
254 if (!--stuck) {
255 if (shown++ <= 2)
256 show_write(str, rw, caller);
257 stuck = INIT_STUCK;
258 }
259 rmb();
260 }
261 goto wlock_again;
262 }
263
264 /* We have it, say who we are. */
265 rw->writer_pc = ((unsigned int)caller);
266 rw->writer_cpu = cpu;
267 current->thread.smp_lock_count++;
268 current->thread.smp_lock_pc = ((unsigned int)caller);
269
270 put_cpu();
271}
272
273void _do_write_unlock(rwlock_t *rw, unsigned long caller)
274{
275 unsigned long val;
276 int stuck = INIT_STUCK;
277 int shown = 0;
278
279 /* Drop our identity _first_ */
280 rw->writer_pc = 0;
281 rw->writer_cpu = NO_PROC_ID;
282 current->thread.smp_lock_count--;
283wlock_again:
284 __asm__ __volatile__(
285" membar #StoreLoad | #LoadLoad\n"
286" mov 1, %%g3\n"
287" sllx %%g3, 63, %%g3\n"
288" ldx [%0], %%g1\n"
289" andn %%g1, %%g3, %%g7\n"
290" casx [%0], %%g1, %%g7\n"
291" membar #StoreLoad | #StoreStore\n"
292" sub %%g1, %%g7, %0\n"
293 : "=r" (val)
294 : "0" (&(rw->lock))
295 : "g3", "g1", "g7", "memory");
296 if (val) {
297 if (!--stuck) {
298 if (shown++ <= 2)
299 show_write("write_unlock", rw, caller);
300 stuck = INIT_STUCK;
301 }
302 goto wlock_again;
303 }
304}
305
306int _do_write_trylock(rwlock_t *rw, char *str, unsigned long caller)
307{
308 unsigned long val;
309 int cpu = get_cpu();
310
311 /* Try to acuire the write bit. */
312 __asm__ __volatile__(
313" mov 1, %%g3\n"
314" sllx %%g3, 63, %%g3\n"
315" ldx [%0], %%g1\n"
316" brlz,pn %%g1, 1f\n"
317" or %%g1, %%g3, %%g7\n"
318" casx [%0], %%g1, %%g7\n"
319" membar #StoreLoad | #StoreStore\n"
320" ba,pt %%xcc, 2f\n"
321" sub %%g1, %%g7, %0\n"
322"1: mov 1, %0\n"
323"2:" : "=r" (val)
324 : "0" (&(rw->lock))
325 : "g3", "g1", "g7", "memory");
326
327 if (val) {
328 put_cpu();
329 return 0;
330 }
331
332 if ((rw->lock & ((1UL<<63)-1UL)) != 0UL) {
333 /* Readers still around, drop the write
334 * lock, return failure.
335 */
336 __asm__ __volatile__(
337" mov 1, %%g3\n"
338" sllx %%g3, 63, %%g3\n"
339"1: ldx [%0], %%g1\n"
340" andn %%g1, %%g3, %%g7\n"
341" casx [%0], %%g1, %%g7\n"
342" cmp %%g1, %%g7\n"
343" membar #StoreLoad | #StoreStore\n"
344" bne,pn %%xcc, 1b\n"
345" nop"
346 : /* no outputs */
347 : "r" (&(rw->lock))
348 : "g3", "g1", "g7", "cc", "memory");
349
350 put_cpu();
351
352 return 0;
353 }
354
355 /* We have it, say who we are. */
356 rw->writer_pc = ((unsigned int)caller);
357 rw->writer_cpu = cpu;
358 current->thread.smp_lock_count++;
359 current->thread.smp_lock_pc = ((unsigned int)caller);
360
361 put_cpu();
362
363 return 1;
364}
365
366#endif /* CONFIG_SMP */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-07 05:48:29 -0500