aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc64
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc64')
-rw-r--r--arch/sparc64/Kconfig18
-rw-r--r--arch/sparc64/kernel/auxio.c2
-rw-r--r--arch/sparc64/kernel/entry.S137
-rw-r--r--arch/sparc64/kernel/irq.c752
-rw-r--r--arch/sparc64/kernel/kprobes.c5
-rw-r--r--arch/sparc64/kernel/pci_psycho.c3
-rw-r--r--arch/sparc64/kernel/pci_sabre.c46
-rw-r--r--arch/sparc64/kernel/pci_schizo.c78
-rw-r--r--arch/sparc64/kernel/semaphore.c12
-rw-r--r--arch/sparc64/kernel/sparc64_ksyms.c1
-rw-r--r--arch/sparc64/kernel/time.c2
-rw-r--r--arch/sparc64/kernel/trampoline.S3
-rw-r--r--arch/sparc64/lib/U1memcpy.S103
-rw-r--r--arch/sparc64/lib/VISsave.S15
-rw-r--r--arch/sparc64/lib/atomic.S42
-rw-r--r--arch/sparc64/lib/bitops.S31
-rw-r--r--arch/sparc64/lib/debuglocks.c6
-rw-r--r--arch/sparc64/lib/dec_and_lock.S6
-rw-r--r--arch/sparc64/lib/rwsem.S15
-rw-r--r--arch/sparc64/mm/init.c6
-rw-r--r--arch/sparc64/mm/ultra.S8
21 files changed, 515 insertions, 776 deletions
diff --git a/arch/sparc64/Kconfig b/arch/sparc64/Kconfig
index e2b050eb3b96..d78bc13ebbb9 100644
--- a/arch/sparc64/Kconfig
+++ b/arch/sparc64/Kconfig
@@ -444,6 +444,24 @@ config PRINTER
444 If you have more than 8 printers, you need to increase the LP_NO 444 If you have more than 8 printers, you need to increase the LP_NO
445 macro in lp.c and the PARPORT_MAX macro in parport.h. 445 macro in lp.c and the PARPORT_MAX macro in parport.h.
446 446
447config PPDEV
448 tristate "Support for user-space parallel port device drivers"
449 depends on PARPORT
450 ---help---
451 Saying Y to this adds support for /dev/parport device nodes. This
452 is needed for programs that want portable access to the parallel
453 port, for instance deviceid (which displays Plug-and-Play device
454 IDs).
455
456 This is the parallel port equivalent of SCSI generic support (sg).
457 It is safe to say N to this -- it is not needed for normal printing
458 or parallel port CD-ROM/disk support.
459
460 To compile this driver as a module, choose M here: the
461 module will be called ppdev.
462
463 If unsure, say N.
464
447config ENVCTRL 465config ENVCTRL
448 tristate "SUNW, envctrl support" 466 tristate "SUNW, envctrl support"
449 depends on PCI 467 depends on PCI
diff --git a/arch/sparc64/kernel/auxio.c b/arch/sparc64/kernel/auxio.c
index a0716ccc2f4a..8852c20c8d99 100644
--- a/arch/sparc64/kernel/auxio.c
+++ b/arch/sparc64/kernel/auxio.c
@@ -16,7 +16,7 @@
16#include <asm/ebus.h> 16#include <asm/ebus.h>
17#include <asm/auxio.h> 17#include <asm/auxio.h>
18 18
19/* This cannot be static, as it is referenced in entry.S */ 19/* This cannot be static, as it is referenced in irq.c */
20void __iomem *auxio_register = NULL; 20void __iomem *auxio_register = NULL;
21 21
22enum auxio_type { 22enum auxio_type {
diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index a47f2d0b1a29..d3973d8a7195 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -271,8 +271,9 @@ cplus_fptrap_insn_1:
271 fmuld %f0, %f2, %f26 271 fmuld %f0, %f2, %f26
272 faddd %f0, %f2, %f28 272 faddd %f0, %f2, %f28
273 fmuld %f0, %f2, %f30 273 fmuld %f0, %f2, %f30
274 membar #Sync
274 b,pt %xcc, fpdis_exit 275 b,pt %xcc, fpdis_exit
275 membar #Sync 276 nop
2762: andcc %g5, FPRS_DU, %g0 2772: andcc %g5, FPRS_DU, %g0
277 bne,pt %icc, 3f 278 bne,pt %icc, 3f
278 fzero %f32 279 fzero %f32
@@ -301,8 +302,9 @@ cplus_fptrap_insn_2:
301 fmuld %f32, %f34, %f58 302 fmuld %f32, %f34, %f58
302 faddd %f32, %f34, %f60 303 faddd %f32, %f34, %f60
303 fmuld %f32, %f34, %f62 304 fmuld %f32, %f34, %f62
305 membar #Sync
304 ba,pt %xcc, fpdis_exit 306 ba,pt %xcc, fpdis_exit
305 membar #Sync 307 nop
3063: mov SECONDARY_CONTEXT, %g3 3083: mov SECONDARY_CONTEXT, %g3
307 add %g6, TI_FPREGS, %g1 309 add %g6, TI_FPREGS, %g1
308 ldxa [%g3] ASI_DMMU, %g5 310 ldxa [%g3] ASI_DMMU, %g5
@@ -551,13 +553,11 @@ do_ivec:
551 sllx %g3, 5, %g3 553 sllx %g3, 5, %g3
552 or %g2, %lo(ivector_table), %g2 554 or %g2, %lo(ivector_table), %g2
553 add %g2, %g3, %g3 555 add %g2, %g3, %g3
554 ldx [%g3 + 0x08], %g2 /* irq_info */
555 ldub [%g3 + 0x04], %g4 /* pil */ 556 ldub [%g3 + 0x04], %g4 /* pil */
556 brz,pn %g2, do_ivec_spurious 557 mov 1, %g2
557 mov 1, %g2
558
559 sllx %g2, %g4, %g2 558 sllx %g2, %g4, %g2
560 sllx %g4, 2, %g4 559 sllx %g4, 2, %g4
560
561 lduw [%g6 + %g4], %g5 /* g5 = irq_work(cpu, pil) */ 561 lduw [%g6 + %g4], %g5 /* g5 = irq_work(cpu, pil) */
562 stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */ 562 stw %g5, [%g3 + 0x00] /* bucket->irq_chain = g5 */
563 stw %g3, [%g6 + %g4] /* irq_work(cpu, pil) = bucket */ 563 stw %g3, [%g6 + %g4] /* irq_work(cpu, pil) = bucket */
@@ -565,9 +565,9 @@ do_ivec:
565 retry 565 retry
566do_ivec_xcall: 566do_ivec_xcall:
567 mov 0x50, %g1 567 mov 0x50, %g1
568
569 ldxa [%g1 + %g0] ASI_INTR_R, %g1 568 ldxa [%g1 + %g0] ASI_INTR_R, %g1
570 srl %g3, 0, %g3 569 srl %g3, 0, %g3
570
571 mov 0x60, %g7 571 mov 0x60, %g7
572 ldxa [%g7 + %g0] ASI_INTR_R, %g7 572 ldxa [%g7 + %g0] ASI_INTR_R, %g7
573 stxa %g0, [%g0] ASI_INTR_RECEIVE 573 stxa %g0, [%g0] ASI_INTR_RECEIVE
@@ -579,19 +579,6 @@ do_ivec_xcall:
5791: jmpl %g3, %g0 5791: jmpl %g3, %g0
580 nop 580 nop
581 581
582do_ivec_spurious:
583 stw %g3, [%g6 + 0x00] /* irq_work(cpu, 0) = bucket */
584 rdpr %pstate, %g5
585
586 wrpr %g5, PSTATE_IG | PSTATE_AG, %pstate
587 sethi %hi(109f), %g7
588 ba,pt %xcc, etrap
589109: or %g7, %lo(109b), %g7
590 call catch_disabled_ivec
591 add %sp, PTREGS_OFF, %o0
592 ba,pt %xcc, rtrap
593 clr %l6
594
595 .globl save_alternate_globals 582 .globl save_alternate_globals
596save_alternate_globals: /* %o0 = save_area */ 583save_alternate_globals: /* %o0 = save_area */
597 rdpr %pstate, %o5 584 rdpr %pstate, %o5
@@ -699,116 +686,6 @@ utrap_ill:
699 ba,pt %xcc, rtrap 686 ba,pt %xcc, rtrap
700 clr %l6 687 clr %l6
701 688
702#ifdef CONFIG_BLK_DEV_FD
703 .globl floppy_hardint
704floppy_hardint:
705 wr %g0, (1 << 11), %clear_softint
706 sethi %hi(doing_pdma), %g1
707 ld [%g1 + %lo(doing_pdma)], %g2
708 brz,pn %g2, floppy_dosoftint
709 sethi %hi(fdc_status), %g3
710 ldx [%g3 + %lo(fdc_status)], %g3
711 sethi %hi(pdma_vaddr), %g5
712 ldx [%g5 + %lo(pdma_vaddr)], %g4
713 sethi %hi(pdma_size), %g5
714 ldx [%g5 + %lo(pdma_size)], %g5
715
716next_byte:
717 lduba [%g3] ASI_PHYS_BYPASS_EC_E, %g7
718 andcc %g7, 0x80, %g0
719 be,pn %icc, floppy_fifo_emptied
720 andcc %g7, 0x20, %g0
721 be,pn %icc, floppy_overrun
722 andcc %g7, 0x40, %g0
723 be,pn %icc, floppy_write
724 sub %g5, 1, %g5
725
726 inc %g3
727 lduba [%g3] ASI_PHYS_BYPASS_EC_E, %g7
728 dec %g3
729 orcc %g0, %g5, %g0
730 stb %g7, [%g4]
731 bne,pn %xcc, next_byte
732 add %g4, 1, %g4
733
734 b,pt %xcc, floppy_tdone
735 nop
736
737floppy_write:
738 ldub [%g4], %g7
739 orcc %g0, %g5, %g0
740 inc %g3
741 stba %g7, [%g3] ASI_PHYS_BYPASS_EC_E
742 dec %g3
743 bne,pn %xcc, next_byte
744 add %g4, 1, %g4
745
746floppy_tdone:
747 sethi %hi(pdma_vaddr), %g1
748 stx %g4, [%g1 + %lo(pdma_vaddr)]
749 sethi %hi(pdma_size), %g1
750 stx %g5, [%g1 + %lo(pdma_size)]
751 sethi %hi(auxio_register), %g1
752 ldx [%g1 + %lo(auxio_register)], %g7
753 lduba [%g7] ASI_PHYS_BYPASS_EC_E, %g5
754 or %g5, AUXIO_AUX1_FTCNT, %g5
755/* andn %g5, AUXIO_AUX1_MASK, %g5 */
756 stba %g5, [%g7] ASI_PHYS_BYPASS_EC_E
757 andn %g5, AUXIO_AUX1_FTCNT, %g5
758/* andn %g5, AUXIO_AUX1_MASK, %g5 */
759
760 nop; nop; nop; nop; nop; nop;
761 nop; nop; nop; nop; nop; nop;
762
763 stba %g5, [%g7] ASI_PHYS_BYPASS_EC_E
764 sethi %hi(doing_pdma), %g1
765 b,pt %xcc, floppy_dosoftint
766 st %g0, [%g1 + %lo(doing_pdma)]
767
768floppy_fifo_emptied:
769 sethi %hi(pdma_vaddr), %g1
770 stx %g4, [%g1 + %lo(pdma_vaddr)]
771 sethi %hi(pdma_size), %g1
772 stx %g5, [%g1 + %lo(pdma_size)]
773 sethi %hi(irq_action), %g1
774 or %g1, %lo(irq_action), %g1
775 ldx [%g1 + (11 << 3)], %g3 ! irqaction[floppy_irq]
776 ldx [%g3 + 0x08], %g4 ! action->flags>>48==ino
777 sethi %hi(ivector_table), %g3
778 srlx %g4, 48, %g4
779 or %g3, %lo(ivector_table), %g3
780 sllx %g4, 5, %g4
781 ldx [%g3 + %g4], %g4 ! &ivector_table[ino]
782 ldx [%g4 + 0x10], %g4 ! bucket->iclr
783 stwa %g0, [%g4] ASI_PHYS_BYPASS_EC_E ! ICLR_IDLE
784 membar #Sync ! probably not needed...
785 retry
786
787floppy_overrun:
788 sethi %hi(pdma_vaddr), %g1
789 stx %g4, [%g1 + %lo(pdma_vaddr)]
790 sethi %hi(pdma_size), %g1
791 stx %g5, [%g1 + %lo(pdma_size)]
792 sethi %hi(doing_pdma), %g1
793 st %g0, [%g1 + %lo(doing_pdma)]
794
795floppy_dosoftint:
796 rdpr %pil, %g2
797 wrpr %g0, 15, %pil
798 sethi %hi(109f), %g7
799 b,pt %xcc, etrap_irq
800109: or %g7, %lo(109b), %g7
801
802 mov 11, %o0
803 mov 0, %o1
804 call sparc_floppy_irq
805 add %sp, PTREGS_OFF, %o2
806
807 b,pt %xcc, rtrap_irq
808 nop
809
810#endif /* CONFIG_BLK_DEV_FD */
811
812 /* XXX Here is stuff we still need to write... -DaveM XXX */ 689 /* XXX Here is stuff we still need to write... -DaveM XXX */
813 .globl netbsd_syscall 690 .globl netbsd_syscall
814netbsd_syscall: 691netbsd_syscall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 4dcb8af94090..74a2e0808cbc 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -37,6 +37,7 @@
37#include <asm/uaccess.h> 37#include <asm/uaccess.h>
38#include <asm/cache.h> 38#include <asm/cache.h>
39#include <asm/cpudata.h> 39#include <asm/cpudata.h>
40#include <asm/auxio.h>
40 41
41#ifdef CONFIG_SMP 42#ifdef CONFIG_SMP
42static void distribute_irqs(void); 43static void distribute_irqs(void);
@@ -70,31 +71,7 @@ struct irq_work_struct {
70struct irq_work_struct __irq_work[NR_CPUS]; 71struct irq_work_struct __irq_work[NR_CPUS];
71#define irq_work(__cpu, __pil) &(__irq_work[(__cpu)].irq_worklists[(__pil)]) 72#define irq_work(__cpu, __pil) &(__irq_work[(__cpu)].irq_worklists[(__pil)])
72 73
73#ifdef CONFIG_PCI 74static struct irqaction *irq_action[NR_IRQS+1];
74/* This is a table of physical addresses used to deal with IBF_DMA_SYNC.
75 * It is used for PCI only to synchronize DMA transfers with IRQ delivery
76 * for devices behind busses other than APB on Sabre systems.
77 *
78 * Currently these physical addresses are just config space accesses
79 * to the command register for that device.
80 */
81unsigned long pci_dma_wsync;
82unsigned long dma_sync_reg_table[256];
83unsigned char dma_sync_reg_table_entry = 0;
84#endif
85
86/* This is based upon code in the 32-bit Sparc kernel written mostly by
87 * David Redman (djhr@tadpole.co.uk).
88 */
89#define MAX_STATIC_ALLOC 4
90static struct irqaction static_irqaction[MAX_STATIC_ALLOC];
91static int static_irq_count;
92
93/* This is exported so that fast IRQ handlers can get at it... -DaveM */
94struct irqaction *irq_action[NR_IRQS+1] = {
95 NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL,
96 NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL
97};
98 75
99/* This only synchronizes entities which modify IRQ handler 76/* This only synchronizes entities which modify IRQ handler
100 * state and some selected user-level spots that want to 77 * state and some selected user-level spots that want to
@@ -240,17 +217,22 @@ void disable_irq(unsigned int irq)
240 * the CPU %tick register and not by some normal vectored interrupt 217 * the CPU %tick register and not by some normal vectored interrupt
241 * source. To handle this special case, we use this dummy INO bucket. 218 * source. To handle this special case, we use this dummy INO bucket.
242 */ 219 */
220static struct irq_desc pil0_dummy_desc;
243static struct ino_bucket pil0_dummy_bucket = { 221static struct ino_bucket pil0_dummy_bucket = {
244 0, /* irq_chain */ 222 .irq_info = &pil0_dummy_desc,
245 0, /* pil */
246 0, /* pending */
247 0, /* flags */
248 0, /* __unused */
249 NULL, /* irq_info */
250 0UL, /* iclr */
251 0UL, /* imap */
252}; 223};
253 224
225static void build_irq_error(const char *msg, unsigned int ino, int pil, int inofixup,
226 unsigned long iclr, unsigned long imap,
227 struct ino_bucket *bucket)
228{
229 prom_printf("IRQ: INO %04x (%d:%016lx:%016lx) --> "
230 "(%d:%d:%016lx:%016lx), halting...\n",
231 ino, bucket->pil, bucket->iclr, bucket->imap,
232 pil, inofixup, iclr, imap);
233 prom_halt();
234}
235
254unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap) 236unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap)
255{ 237{
256 struct ino_bucket *bucket; 238 struct ino_bucket *bucket;
@@ -279,28 +261,35 @@ unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long
279 prom_halt(); 261 prom_halt();
280 } 262 }
281 263
282 /* Ok, looks good, set it up. Don't touch the irq_chain or
283 * the pending flag.
284 */
285 bucket = &ivector_table[ino]; 264 bucket = &ivector_table[ino];
286 if ((bucket->flags & IBF_ACTIVE) || 265 if (bucket->flags & IBF_ACTIVE)
287 (bucket->irq_info != NULL)) { 266 build_irq_error("IRQ: Trying to build active INO bucket.\n",
288 /* This is a gross fatal error if it happens here. */ 267 ino, pil, inofixup, iclr, imap, bucket);
289 prom_printf("IRQ: Trying to reinit INO bucket, fatal error.\n"); 268
290 prom_printf("IRQ: Request INO %04x (%d:%d:%016lx:%016lx)\n", 269 if (bucket->irq_info) {
291 ino, pil, inofixup, iclr, imap); 270 if (bucket->imap != imap || bucket->iclr != iclr)
292 prom_printf("IRQ: Existing (%d:%016lx:%016lx)\n", 271 build_irq_error("IRQ: Trying to reinit INO bucket.\n",
293 bucket->pil, bucket->iclr, bucket->imap); 272 ino, pil, inofixup, iclr, imap, bucket);
294 prom_printf("IRQ: Cannot continue, halting...\n"); 273
274 goto out;
275 }
276
277 bucket->irq_info = kmalloc(sizeof(struct irq_desc), GFP_ATOMIC);
278 if (!bucket->irq_info) {
279 prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
295 prom_halt(); 280 prom_halt();
296 } 281 }
282 memset(bucket->irq_info, 0, sizeof(struct irq_desc));
283
284 /* Ok, looks good, set it up. Don't touch the irq_chain or
285 * the pending flag.
286 */
297 bucket->imap = imap; 287 bucket->imap = imap;
298 bucket->iclr = iclr; 288 bucket->iclr = iclr;
299 bucket->pil = pil; 289 bucket->pil = pil;
300 bucket->flags = 0; 290 bucket->flags = 0;
301 291
302 bucket->irq_info = NULL; 292out:
303
304 return __irq(bucket); 293 return __irq(bucket);
305} 294}
306 295
@@ -318,26 +307,65 @@ static void atomic_bucket_insert(struct ino_bucket *bucket)
318 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); 307 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
319} 308}
320 309
310static int check_irq_sharing(int pil, unsigned long irqflags)
311{
312 struct irqaction *action, *tmp;
313
314 action = *(irq_action + pil);
315 if (action) {
316 if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) {
317 for (tmp = action; tmp->next; tmp = tmp->next)
318 ;
319 } else {
320 return -EBUSY;
321 }
322 }
323 return 0;
324}
325
326static void append_irq_action(int pil, struct irqaction *action)
327{
328 struct irqaction **pp = irq_action + pil;
329
330 while (*pp)
331 pp = &((*pp)->next);
332 *pp = action;
333}
334
335static struct irqaction *get_action_slot(struct ino_bucket *bucket)
336{
337 struct irq_desc *desc = bucket->irq_info;
338 int max_irq, i;
339
340 max_irq = 1;
341 if (bucket->flags & IBF_PCI)
342 max_irq = MAX_IRQ_DESC_ACTION;
343 for (i = 0; i < max_irq; i++) {
344 struct irqaction *p = &desc->action[i];
345 u32 mask = (1 << i);
346
347 if (desc->action_active_mask & mask)
348 continue;
349
350 desc->action_active_mask |= mask;
351 return p;
352 }
353 return NULL;
354}
355
321int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *), 356int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *),
322 unsigned long irqflags, const char *name, void *dev_id) 357 unsigned long irqflags, const char *name, void *dev_id)
323{ 358{
324 struct irqaction *action, *tmp = NULL; 359 struct irqaction *action;
325 struct ino_bucket *bucket = __bucket(irq); 360 struct ino_bucket *bucket = __bucket(irq);
326 unsigned long flags; 361 unsigned long flags;
327 int pending = 0; 362 int pending = 0;
328 363
329 if ((bucket != &pil0_dummy_bucket) && 364 if (unlikely(!handler))
330 (bucket < &ivector_table[0] ||
331 bucket >= &ivector_table[NUM_IVECS])) {
332 unsigned int *caller;
333
334 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
335 printk(KERN_CRIT "request_irq: Old style IRQ registry attempt "
336 "from %p, irq %08x.\n", caller, irq);
337 return -EINVAL; 365 return -EINVAL;
338 } 366
339 if (!handler) 367 if (unlikely(!bucket->irq_info))
340 return -EINVAL; 368 return -ENODEV;
341 369
342 if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) { 370 if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) {
343 /* 371 /*
@@ -355,93 +383,20 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
355 383
356 spin_lock_irqsave(&irq_action_lock, flags); 384 spin_lock_irqsave(&irq_action_lock, flags);
357 385
358 action = *(bucket->pil + irq_action); 386 if (check_irq_sharing(bucket->pil, irqflags)) {
359 if (action) { 387 spin_unlock_irqrestore(&irq_action_lock, flags);
360 if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) 388 return -EBUSY;
361 for (tmp = action; tmp->next; tmp = tmp->next)
362 ;
363 else {
364 spin_unlock_irqrestore(&irq_action_lock, flags);
365 return -EBUSY;
366 }
367 action = NULL; /* Or else! */
368 } 389 }
369 390
370 /* If this is flagged as statically allocated then we use our 391 action = get_action_slot(bucket);
371 * private struct which is never freed.
372 */
373 if (irqflags & SA_STATIC_ALLOC) {
374 if (static_irq_count < MAX_STATIC_ALLOC)
375 action = &static_irqaction[static_irq_count++];
376 else
377 printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
378 "using kmalloc\n", irq, name);
379 }
380 if (action == NULL)
381 action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
382 GFP_ATOMIC);
383
384 if (!action) { 392 if (!action) {
385 spin_unlock_irqrestore(&irq_action_lock, flags); 393 spin_unlock_irqrestore(&irq_action_lock, flags);
386 return -ENOMEM; 394 return -ENOMEM;
387 } 395 }
388 396
389 if (bucket == &pil0_dummy_bucket) { 397 bucket->flags |= IBF_ACTIVE;
390 bucket->irq_info = action; 398 pending = 0;
391 bucket->flags |= IBF_ACTIVE; 399 if (bucket != &pil0_dummy_bucket) {
392 } else {
393 if ((bucket->flags & IBF_ACTIVE) != 0) {
394 void *orig = bucket->irq_info;
395 void **vector = NULL;
396
397 if ((bucket->flags & IBF_PCI) == 0) {
398 printk("IRQ: Trying to share non-PCI bucket.\n");
399 goto free_and_ebusy;
400 }
401 if ((bucket->flags & IBF_MULTI) == 0) {
402 vector = kmalloc(sizeof(void *) * 4, GFP_ATOMIC);
403 if (vector == NULL)
404 goto free_and_enomem;
405
406 /* We might have slept. */
407 if ((bucket->flags & IBF_MULTI) != 0) {
408 int ent;
409
410 kfree(vector);
411 vector = (void **)bucket->irq_info;
412 for(ent = 0; ent < 4; ent++) {
413 if (vector[ent] == NULL) {
414 vector[ent] = action;
415 break;
416 }
417 }
418 if (ent == 4)
419 goto free_and_ebusy;
420 } else {
421 vector[0] = orig;
422 vector[1] = action;
423 vector[2] = NULL;
424 vector[3] = NULL;
425 bucket->irq_info = vector;
426 bucket->flags |= IBF_MULTI;
427 }
428 } else {
429 int ent;
430
431 vector = (void **)orig;
432 for (ent = 0; ent < 4; ent++) {
433 if (vector[ent] == NULL) {
434 vector[ent] = action;
435 break;
436 }
437 }
438 if (ent == 4)
439 goto free_and_ebusy;
440 }
441 } else {
442 bucket->irq_info = action;
443 bucket->flags |= IBF_ACTIVE;
444 }
445 pending = bucket->pending; 400 pending = bucket->pending;
446 if (pending) 401 if (pending)
447 bucket->pending = 0; 402 bucket->pending = 0;
@@ -455,10 +410,7 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
455 put_ino_in_irqaction(action, irq); 410 put_ino_in_irqaction(action, irq);
456 put_smpaff_in_irqaction(action, CPU_MASK_NONE); 411 put_smpaff_in_irqaction(action, CPU_MASK_NONE);
457 412
458 if (tmp) 413 append_irq_action(bucket->pil, action);
459 tmp->next = action;
460 else
461 *(bucket->pil + irq_action) = action;
462 414
463 enable_irq(irq); 415 enable_irq(irq);
464 416
@@ -467,147 +419,103 @@ int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_
467 atomic_bucket_insert(bucket); 419 atomic_bucket_insert(bucket);
468 set_softint(1 << bucket->pil); 420 set_softint(1 << bucket->pil);
469 } 421 }
422
470 spin_unlock_irqrestore(&irq_action_lock, flags); 423 spin_unlock_irqrestore(&irq_action_lock, flags);
471 if ((bucket != &pil0_dummy_bucket) && (!(irqflags & SA_STATIC_ALLOC))) 424
425 if (bucket != &pil0_dummy_bucket)
472 register_irq_proc(__irq_ino(irq)); 426 register_irq_proc(__irq_ino(irq));
473 427
474#ifdef CONFIG_SMP 428#ifdef CONFIG_SMP
475 distribute_irqs(); 429 distribute_irqs();
476#endif 430#endif
477 return 0; 431 return 0;
478
479free_and_ebusy:
480 kfree(action);
481 spin_unlock_irqrestore(&irq_action_lock, flags);
482 return -EBUSY;
483
484free_and_enomem:
485 kfree(action);
486 spin_unlock_irqrestore(&irq_action_lock, flags);
487 return -ENOMEM;
488} 432}
489 433
490EXPORT_SYMBOL(request_irq); 434EXPORT_SYMBOL(request_irq);
491 435
492void free_irq(unsigned int irq, void *dev_id) 436static struct irqaction *unlink_irq_action(unsigned int irq, void *dev_id)
493{ 437{
494 struct irqaction *action; 438 struct ino_bucket *bucket = __bucket(irq);
495 struct irqaction *tmp = NULL; 439 struct irqaction *action, **pp;
496 unsigned long flags;
497 struct ino_bucket *bucket = __bucket(irq), *bp;
498 440
499 if ((bucket != &pil0_dummy_bucket) && 441 pp = irq_action + bucket->pil;
500 (bucket < &ivector_table[0] || 442 action = *pp;
501 bucket >= &ivector_table[NUM_IVECS])) { 443 if (unlikely(!action))
502 unsigned int *caller; 444 return NULL;
503 445
504 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller)); 446 if (unlikely(!action->handler)) {
505 printk(KERN_CRIT "free_irq: Old style IRQ removal attempt "
506 "from %p, irq %08x.\n", caller, irq);
507 return;
508 }
509
510 spin_lock_irqsave(&irq_action_lock, flags);
511
512 action = *(bucket->pil + irq_action);
513 if (!action->handler) {
514 printk("Freeing free IRQ %d\n", bucket->pil); 447 printk("Freeing free IRQ %d\n", bucket->pil);
515 return; 448 return NULL;
516 }
517 if (dev_id) {
518 for ( ; action; action = action->next) {
519 if (action->dev_id == dev_id)
520 break;
521 tmp = action;
522 }
523 if (!action) {
524 printk("Trying to free free shared IRQ %d\n", bucket->pil);
525 spin_unlock_irqrestore(&irq_action_lock, flags);
526 return;
527 }
528 } else if (action->flags & SA_SHIRQ) {
529 printk("Trying to free shared IRQ %d with NULL device ID\n", bucket->pil);
530 spin_unlock_irqrestore(&irq_action_lock, flags);
531 return;
532 } 449 }
533 450
534 if (action->flags & SA_STATIC_ALLOC) { 451 while (action && action->dev_id != dev_id) {
535 printk("Attempt to free statically allocated IRQ %d (%s)\n", 452 pp = &action->next;
536 bucket->pil, action->name); 453 action = *pp;
537 spin_unlock_irqrestore(&irq_action_lock, flags);
538 return;
539 } 454 }
540 455
541 if (action && tmp) 456 if (likely(action))
542 tmp->next = action->next; 457 *pp = action->next;
543 else 458
544 *(bucket->pil + irq_action) = action->next; 459 return action;
460}
461
462void free_irq(unsigned int irq, void *dev_id)
463{
464 struct irqaction *action;
465 struct ino_bucket *bucket;
466 unsigned long flags;
467
468 spin_lock_irqsave(&irq_action_lock, flags);
469
470 action = unlink_irq_action(irq, dev_id);
545 471
546 spin_unlock_irqrestore(&irq_action_lock, flags); 472 spin_unlock_irqrestore(&irq_action_lock, flags);
547 473
474 if (unlikely(!action))
475 return;
476
548 synchronize_irq(irq); 477 synchronize_irq(irq);
549 478
550 spin_lock_irqsave(&irq_action_lock, flags); 479 spin_lock_irqsave(&irq_action_lock, flags);
551 480
481 bucket = __bucket(irq);
552 if (bucket != &pil0_dummy_bucket) { 482 if (bucket != &pil0_dummy_bucket) {
483 struct irq_desc *desc = bucket->irq_info;
553 unsigned long imap = bucket->imap; 484 unsigned long imap = bucket->imap;
554 void **vector, *orig; 485 int ent, i;
555 int ent;
556
557 orig = bucket->irq_info;
558 vector = (void **)orig;
559
560 if ((bucket->flags & IBF_MULTI) != 0) {
561 int other = 0;
562 void *orphan = NULL;
563 for (ent = 0; ent < 4; ent++) {
564 if (vector[ent] == action)
565 vector[ent] = NULL;
566 else if (vector[ent] != NULL) {
567 orphan = vector[ent];
568 other++;
569 }
570 }
571 486
572 /* Only free when no other shared irq 487 for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
573 * uses this bucket. 488 struct irqaction *p = &desc->action[i];
574 */ 489
575 if (other) { 490 if (p == action) {
576 if (other == 1) { 491 desc->action_active_mask &= ~(1 << i);
577 /* Convert back to non-shared bucket. */ 492 break;
578 bucket->irq_info = orphan;
579 bucket->flags &= ~(IBF_MULTI);
580 kfree(vector);
581 }
582 goto out;
583 } 493 }
584 } else {
585 bucket->irq_info = NULL;
586 } 494 }
587 495
588 /* This unique interrupt source is now inactive. */ 496 if (!desc->action_active_mask) {
589 bucket->flags &= ~IBF_ACTIVE; 497 /* This unique interrupt source is now inactive. */
498 bucket->flags &= ~IBF_ACTIVE;
590 499
591 /* See if any other buckets share this bucket's IMAP 500 /* See if any other buckets share this bucket's IMAP
592 * and are still active. 501 * and are still active.
593 */ 502 */
594 for (ent = 0; ent < NUM_IVECS; ent++) { 503 for (ent = 0; ent < NUM_IVECS; ent++) {
595 bp = &ivector_table[ent]; 504 struct ino_bucket *bp = &ivector_table[ent];
596 if (bp != bucket && 505 if (bp != bucket &&
597 bp->imap == imap && 506 bp->imap == imap &&
598 (bp->flags & IBF_ACTIVE) != 0) 507 (bp->flags & IBF_ACTIVE) != 0)
599 break; 508 break;
600 } 509 }
601 510
602 /* Only disable when no other sub-irq levels of 511 /* Only disable when no other sub-irq levels of
603 * the same IMAP are active. 512 * the same IMAP are active.
604 */ 513 */
605 if (ent == NUM_IVECS) 514 if (ent == NUM_IVECS)
606 disable_irq(irq); 515 disable_irq(irq);
516 }
607 } 517 }
608 518
609out:
610 kfree(action);
611 spin_unlock_irqrestore(&irq_action_lock, flags); 519 spin_unlock_irqrestore(&irq_action_lock, flags);
612} 520}
613 521
@@ -646,99 +554,55 @@ void synchronize_irq(unsigned int irq)
646} 554}
647#endif /* CONFIG_SMP */ 555#endif /* CONFIG_SMP */
648 556
649void catch_disabled_ivec(struct pt_regs *regs) 557static void process_bucket(int irq, struct ino_bucket *bp, struct pt_regs *regs)
650{
651 int cpu = smp_processor_id();
652 struct ino_bucket *bucket = __bucket(*irq_work(cpu, 0));
653
654 /* We can actually see this on Ultra/PCI PCI cards, which are bridges
655 * to other devices. Here a single IMAP enabled potentially multiple
656 * unique interrupt sources (which each do have a unique ICLR register.
657 *
658 * So what we do is just register that the IVEC arrived, when registered
659 * for real the request_irq() code will check the bit and signal
660 * a local CPU interrupt for it.
661 */
662#if 0
663 printk("IVEC: Spurious interrupt vector (%x) received at (%016lx)\n",
664 bucket - &ivector_table[0], regs->tpc);
665#endif
666 *irq_work(cpu, 0) = 0;
667 bucket->pending = 1;
668}
669
670/* Tune this... */
671#define FORWARD_VOLUME 12
672
673#ifdef CONFIG_SMP
674
675static inline void redirect_intr(int cpu, struct ino_bucket *bp)
676{ 558{
677 /* Ok, here is what is going on: 559 struct irq_desc *desc = bp->irq_info;
678 * 1) Retargeting IRQs on Starfire is very 560 unsigned char flags = bp->flags;
679 * expensive so just forget about it on them. 561 u32 action_mask, i;
680 * 2) Moving around very high priority interrupts 562 int random;
681 * is a losing game.
682 * 3) If the current cpu is idle, interrupts are
683 * useful work, so keep them here. But do not
684 * pass to our neighbour if he is not very idle.
685 * 4) If sysadmin explicitly asks for directed intrs,
686 * Just Do It.
687 */
688 struct irqaction *ap = bp->irq_info;
689 cpumask_t cpu_mask;
690 unsigned int buddy, ticks;
691 563
692 cpu_mask = get_smpaff_in_irqaction(ap); 564 bp->flags |= IBF_INPROGRESS;
693 cpus_and(cpu_mask, cpu_mask, cpu_online_map);
694 if (cpus_empty(cpu_mask))
695 cpu_mask = cpu_online_map;
696 565
697 if (this_is_starfire != 0 || 566 if (unlikely(!(flags & IBF_ACTIVE))) {
698 bp->pil >= 10 || current->pid == 0) 567 bp->pending = 1;
699 goto out; 568 goto out;
700
701 /* 'cpu' is the MID (ie. UPAID), calculate the MID
702 * of our buddy.
703 */
704 buddy = cpu + 1;
705 if (buddy >= NR_CPUS)
706 buddy = 0;
707
708 ticks = 0;
709 while (!cpu_isset(buddy, cpu_mask)) {
710 if (++buddy >= NR_CPUS)
711 buddy = 0;
712 if (++ticks > NR_CPUS) {
713 put_smpaff_in_irqaction(ap, CPU_MASK_NONE);
714 goto out;
715 }
716 } 569 }
717 570
718 if (buddy == cpu) 571 if (desc->pre_handler)
719 goto out; 572 desc->pre_handler(bp,
573 desc->pre_handler_arg1,
574 desc->pre_handler_arg2);
720 575
721 /* Voo-doo programming. */ 576 action_mask = desc->action_active_mask;
722 if (cpu_data(buddy).idle_volume < FORWARD_VOLUME) 577 random = 0;
723 goto out; 578 for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
579 struct irqaction *p = &desc->action[i];
580 u32 mask = (1 << i);
724 581
725 /* This just so happens to be correct on Cheetah 582 if (!(action_mask & mask))
726 * at the moment. 583 continue;
727 */
728 buddy <<= 26;
729 584
730 /* Push it to our buddy. */ 585 action_mask &= ~mask;
731 upa_writel(buddy | IMAP_VALID, bp->imap);
732 586
587 if (p->handler(__irq(bp), p->dev_id, regs) == IRQ_HANDLED)
588 random |= p->flags;
589
590 if (!action_mask)
591 break;
592 }
593 if (bp->pil != 0) {
594 upa_writel(ICLR_IDLE, bp->iclr);
595 /* Test and add entropy */
596 if (random & SA_SAMPLE_RANDOM)
597 add_interrupt_randomness(irq);
598 }
733out: 599out:
734 return; 600 bp->flags &= ~IBF_INPROGRESS;
735} 601}
736 602
737#endif
738
739void handler_irq(int irq, struct pt_regs *regs) 603void handler_irq(int irq, struct pt_regs *regs)
740{ 604{
741 struct ino_bucket *bp, *nbp; 605 struct ino_bucket *bp;
742 int cpu = smp_processor_id(); 606 int cpu = smp_processor_id();
743 607
744#ifndef CONFIG_SMP 608#ifndef CONFIG_SMP
@@ -756,8 +620,6 @@ void handler_irq(int irq, struct pt_regs *regs)
756 clear_softint(clr_mask); 620 clear_softint(clr_mask);
757 } 621 }
758#else 622#else
759 int should_forward = 0;
760
761 clear_softint(1 << irq); 623 clear_softint(1 << irq);
762#endif 624#endif
763 625
@@ -772,199 +634,76 @@ void handler_irq(int irq, struct pt_regs *regs)
772#else 634#else
773 bp = __bucket(xchg32(irq_work(cpu, irq), 0)); 635 bp = __bucket(xchg32(irq_work(cpu, irq), 0));
774#endif 636#endif
775 for ( ; bp != NULL; bp = nbp) { 637 while (bp) {
776 unsigned char flags = bp->flags; 638 struct ino_bucket *nbp = __bucket(bp->irq_chain);
777 unsigned char random = 0;
778 639
779 nbp = __bucket(bp->irq_chain);
780 bp->irq_chain = 0; 640 bp->irq_chain = 0;
781 641 process_bucket(irq, bp, regs);
782 bp->flags |= IBF_INPROGRESS; 642 bp = nbp;
783
784 if ((flags & IBF_ACTIVE) != 0) {
785#ifdef CONFIG_PCI
786 if ((flags & IBF_DMA_SYNC) != 0) {
787 upa_readl(dma_sync_reg_table[bp->synctab_ent]);
788 upa_readq(pci_dma_wsync);
789 }
790#endif
791 if ((flags & IBF_MULTI) == 0) {
792 struct irqaction *ap = bp->irq_info;
793 int ret;
794
795 ret = ap->handler(__irq(bp), ap->dev_id, regs);
796 if (ret == IRQ_HANDLED)
797 random |= ap->flags;
798 } else {
799 void **vector = (void **)bp->irq_info;
800 int ent;
801 for (ent = 0; ent < 4; ent++) {
802 struct irqaction *ap = vector[ent];
803 if (ap != NULL) {
804 int ret;
805
806 ret = ap->handler(__irq(bp),
807 ap->dev_id,
808 regs);
809 if (ret == IRQ_HANDLED)
810 random |= ap->flags;
811 }
812 }
813 }
814 /* Only the dummy bucket lacks IMAP/ICLR. */
815 if (bp->pil != 0) {
816#ifdef CONFIG_SMP
817 if (should_forward) {
818 redirect_intr(cpu, bp);
819 should_forward = 0;
820 }
821#endif
822 upa_writel(ICLR_IDLE, bp->iclr);
823
824 /* Test and add entropy */
825 if (random & SA_SAMPLE_RANDOM)
826 add_interrupt_randomness(irq);
827 }
828 } else
829 bp->pending = 1;
830
831 bp->flags &= ~IBF_INPROGRESS;
832 } 643 }
833 irq_exit(); 644 irq_exit();
834} 645}
835 646
836#ifdef CONFIG_BLK_DEV_FD 647#ifdef CONFIG_BLK_DEV_FD
837extern void floppy_interrupt(int irq, void *dev_cookie, struct pt_regs *regs); 648extern irqreturn_t floppy_interrupt(int, void *, struct pt_regs *);;
838
839void sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
840{
841 struct irqaction *action = *(irq + irq_action);
842 struct ino_bucket *bucket;
843 int cpu = smp_processor_id();
844
845 irq_enter();
846 kstat_this_cpu.irqs[irq]++;
847
848 *(irq_work(cpu, irq)) = 0;
849 bucket = get_ino_in_irqaction(action) + ivector_table;
850
851 bucket->flags |= IBF_INPROGRESS;
852
853 floppy_interrupt(irq, dev_cookie, regs);
854 upa_writel(ICLR_IDLE, bucket->iclr);
855 649
856 bucket->flags &= ~IBF_INPROGRESS; 650/* XXX No easy way to include asm/floppy.h XXX */
651extern unsigned char *pdma_vaddr;
652extern unsigned long pdma_size;
653extern volatile int doing_pdma;
654extern unsigned long fdc_status;
857 655
858 irq_exit(); 656irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
859}
860#endif
861
862/* The following assumes that the branch lies before the place we
863 * are branching to. This is the case for a trap vector...
864 * You have been warned.
865 */
866#define SPARC_BRANCH(dest_addr, inst_addr) \
867 (0x10800000 | ((((dest_addr)-(inst_addr))>>2)&0x3fffff))
868
869#define SPARC_NOP (0x01000000)
870
871static void install_fast_irq(unsigned int cpu_irq,
872 irqreturn_t (*handler)(int, void *, struct pt_regs *))
873{ 657{
874 extern unsigned long sparc64_ttable_tl0; 658 if (likely(doing_pdma)) {
875 unsigned long ttent = (unsigned long) &sparc64_ttable_tl0; 659 void __iomem *stat = (void __iomem *) fdc_status;
876 unsigned int *insns; 660 unsigned char *vaddr = pdma_vaddr;
877 661 unsigned long size = pdma_size;
878 ttent += 0x820; 662 u8 val;
879 ttent += (cpu_irq - 1) << 5; 663
880 insns = (unsigned int *) ttent; 664 while (size) {
881 insns[0] = SPARC_BRANCH(((unsigned long) handler), 665 val = readb(stat);
882 ((unsigned long)&insns[0])); 666 if (unlikely(!(val & 0x80))) {
883 insns[1] = SPARC_NOP; 667 pdma_vaddr = vaddr;
884 __asm__ __volatile__("membar #StoreStore; flush %0" : : "r" (ttent)); 668 pdma_size = size;
885} 669 return IRQ_HANDLED;
886 670 }
887int request_fast_irq(unsigned int irq, 671 if (unlikely(!(val & 0x20))) {
888 irqreturn_t (*handler)(int, void *, struct pt_regs *), 672 pdma_vaddr = vaddr;
889 unsigned long irqflags, const char *name, void *dev_id) 673 pdma_size = size;
890{ 674 doing_pdma = 0;
891 struct irqaction *action; 675 goto main_interrupt;
892 struct ino_bucket *bucket = __bucket(irq); 676 }
893 unsigned long flags; 677 if (val & 0x40) {
894 678 /* read */
895 /* No pil0 dummy buckets allowed here. */ 679 *vaddr++ = readb(stat + 1);
896 if (bucket < &ivector_table[0] || 680 } else {
897 bucket >= &ivector_table[NUM_IVECS]) { 681 unsigned char data = *vaddr++;
898 unsigned int *caller;
899
900 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
901 printk(KERN_CRIT "request_fast_irq: Old style IRQ registry attempt "
902 "from %p, irq %08x.\n", caller, irq);
903 return -EINVAL;
904 }
905
906 if (!handler)
907 return -EINVAL;
908 682
909 if ((bucket->pil == 0) || (bucket->pil == 14)) { 683 /* write */
910 printk("request_fast_irq: Trying to register shared IRQ 0 or 14.\n"); 684 writeb(data, stat + 1);
911 return -EBUSY; 685 }
912 } 686 size--;
687 }
913 688
914 spin_lock_irqsave(&irq_action_lock, flags); 689 pdma_vaddr = vaddr;
690 pdma_size = size;
915 691
916 action = *(bucket->pil + irq_action); 692 /* Send Terminal Count pulse to floppy controller. */
917 if (action) { 693 val = readb(auxio_register);
918 if (action->flags & SA_SHIRQ) 694 val |= AUXIO_AUX1_FTCNT;
919 panic("Trying to register fast irq when already shared.\n"); 695 writeb(val, auxio_register);
920 if (irqflags & SA_SHIRQ) 696 val &= AUXIO_AUX1_FTCNT;
921 panic("Trying to register fast irq as shared.\n"); 697 writeb(val, auxio_register);
922 printk("request_fast_irq: Trying to register yet already owned.\n");
923 spin_unlock_irqrestore(&irq_action_lock, flags);
924 return -EBUSY;
925 }
926 698
927 /* 699 doing_pdma = 0;
928 * We do not check for SA_SAMPLE_RANDOM in this path. Neither do we
929 * support smp intr affinity in this path.
930 */
931 if (irqflags & SA_STATIC_ALLOC) {
932 if (static_irq_count < MAX_STATIC_ALLOC)
933 action = &static_irqaction[static_irq_count++];
934 else
935 printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
936 "using kmalloc\n", bucket->pil, name);
937 }
938 if (action == NULL)
939 action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
940 GFP_ATOMIC);
941 if (!action) {
942 spin_unlock_irqrestore(&irq_action_lock, flags);
943 return -ENOMEM;
944 } 700 }
945 install_fast_irq(bucket->pil, handler);
946
947 bucket->irq_info = action;
948 bucket->flags |= IBF_ACTIVE;
949
950 action->handler = handler;
951 action->flags = irqflags;
952 action->dev_id = NULL;
953 action->name = name;
954 action->next = NULL;
955 put_ino_in_irqaction(action, irq);
956 put_smpaff_in_irqaction(action, CPU_MASK_NONE);
957
958 *(bucket->pil + irq_action) = action;
959 enable_irq(irq);
960 701
961 spin_unlock_irqrestore(&irq_action_lock, flags); 702main_interrupt:
962 703 return floppy_interrupt(irq, dev_cookie, regs);
963#ifdef CONFIG_SMP
964 distribute_irqs();
965#endif
966 return 0;
967} 704}
705EXPORT_SYMBOL(sparc_floppy_irq);
706#endif
968 707
969/* We really don't need these at all on the Sparc. We only have 708/* We really don't need these at all on the Sparc. We only have
970 * stubs here because they are exported to modules. 709 * stubs here because they are exported to modules.
@@ -1030,7 +769,10 @@ static void distribute_irqs(void)
1030 */ 769 */
1031 for (level = 1; level < NR_IRQS; level++) { 770 for (level = 1; level < NR_IRQS; level++) {
1032 struct irqaction *p = irq_action[level]; 771 struct irqaction *p = irq_action[level];
1033 if (level == 12) continue; 772
773 if (level == 12)
774 continue;
775
1034 while(p) { 776 while(p) {
1035 cpu = retarget_one_irq(p, cpu); 777 cpu = retarget_one_irq(p, cpu);
1036 p = p->next; 778 p = p->next;
diff --git a/arch/sparc64/kernel/kprobes.c b/arch/sparc64/kernel/kprobes.c
index bdac631cf011..bbf11f85dab1 100644
--- a/arch/sparc64/kernel/kprobes.c
+++ b/arch/sparc64/kernel/kprobes.c
@@ -433,3 +433,8 @@ int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
433 return 0; 433 return 0;
434} 434}
435 435
436/* architecture specific initialization */
437int arch_init_kprobes(void)
438{
439 return 0;
440}
diff --git a/arch/sparc64/kernel/pci_psycho.c b/arch/sparc64/kernel/pci_psycho.c
index 534320ef0db2..91ab466d6c66 100644
--- a/arch/sparc64/kernel/pci_psycho.c
+++ b/arch/sparc64/kernel/pci_psycho.c
@@ -1303,8 +1303,7 @@ static void psycho_controller_hwinit(struct pci_controller_info *p)
1303{ 1303{
1304 u64 tmp; 1304 u64 tmp;
1305 1305
1306 /* PROM sets the IRQ retry value too low, increase it. */ 1306 psycho_write(p->pbm_A.controller_regs + PSYCHO_IRQ_RETRY, 5);
1307 psycho_write(p->pbm_A.controller_regs + PSYCHO_IRQ_RETRY, 0xff);
1308 1307
1309 /* Enable arbiter for all PCI slots. */ 1308 /* Enable arbiter for all PCI slots. */
1310 tmp = psycho_read(p->pbm_A.controller_regs + PSYCHO_PCIA_CTRL); 1309 tmp = psycho_read(p->pbm_A.controller_regs + PSYCHO_PCIA_CTRL);
diff --git a/arch/sparc64/kernel/pci_sabre.c b/arch/sparc64/kernel/pci_sabre.c
index 53d333b4a4e8..52bf3431a422 100644
--- a/arch/sparc64/kernel/pci_sabre.c
+++ b/arch/sparc64/kernel/pci_sabre.c
@@ -595,6 +595,23 @@ static int __init sabre_ino_to_pil(struct pci_dev *pdev, unsigned int ino)
595 return ret; 595 return ret;
596} 596}
597 597
598/* When a device lives behind a bridge deeper in the PCI bus topology
599 * than APB, a special sequence must run to make sure all pending DMA
600 * transfers at the time of IRQ delivery are visible in the coherency
601 * domain by the cpu. This sequence is to perform a read on the far
602 * side of the non-APB bridge, then perform a read of Sabre's DMA
603 * write-sync register.
604 */
605static void sabre_wsync_handler(struct ino_bucket *bucket, void *_arg1, void *_arg2)
606{
607 struct pci_dev *pdev = _arg1;
608 unsigned long sync_reg = (unsigned long) _arg2;
609 u16 _unused;
610
611 pci_read_config_word(pdev, PCI_VENDOR_ID, &_unused);
612 sabre_read(sync_reg);
613}
614
598static unsigned int __init sabre_irq_build(struct pci_pbm_info *pbm, 615static unsigned int __init sabre_irq_build(struct pci_pbm_info *pbm,
599 struct pci_dev *pdev, 616 struct pci_dev *pdev,
600 unsigned int ino) 617 unsigned int ino)
@@ -639,24 +656,14 @@ static unsigned int __init sabre_irq_build(struct pci_pbm_info *pbm,
639 if (pdev) { 656 if (pdev) {
640 struct pcidev_cookie *pcp = pdev->sysdata; 657 struct pcidev_cookie *pcp = pdev->sysdata;
641 658
642 /* When a device lives behind a bridge deeper in the
643 * PCI bus topology than APB, a special sequence must
644 * run to make sure all pending DMA transfers at the
645 * time of IRQ delivery are visible in the coherency
646 * domain by the cpu. This sequence is to perform
647 * a read on the far side of the non-APB bridge, then
648 * perform a read of Sabre's DMA write-sync register.
649 *
650 * Currently, the PCI_CONFIG register for the device
651 * is used for this read from the far side of the bridge.
652 */
653 if (pdev->bus->number != pcp->pbm->pci_first_busno) { 659 if (pdev->bus->number != pcp->pbm->pci_first_busno) {
654 bucket->flags |= IBF_DMA_SYNC; 660 struct pci_controller_info *p = pcp->pbm->parent;
655 bucket->synctab_ent = dma_sync_reg_table_entry++; 661 struct irq_desc *d = bucket->irq_info;
656 dma_sync_reg_table[bucket->synctab_ent] = 662
657 (unsigned long) sabre_pci_config_mkaddr( 663 d->pre_handler = sabre_wsync_handler;
658 pcp->pbm, 664 d->pre_handler_arg1 = pdev;
659 pdev->bus->number, pdev->devfn, PCI_COMMAND); 665 d->pre_handler_arg2 = (void *)
666 p->pbm_A.controller_regs + SABRE_WRSYNC;
660 } 667 }
661 } 668 }
662 return __irq(bucket); 669 return __irq(bucket);
@@ -1626,10 +1633,9 @@ void __init sabre_init(int pnode, char *model_name)
1626 */ 1633 */
1627 p->pbm_A.controller_regs = pr_regs[0].phys_addr; 1634 p->pbm_A.controller_regs = pr_regs[0].phys_addr;
1628 p->pbm_B.controller_regs = pr_regs[0].phys_addr; 1635 p->pbm_B.controller_regs = pr_regs[0].phys_addr;
1629 pci_dma_wsync = p->pbm_A.controller_regs + SABRE_WRSYNC;
1630 1636
1631 printk("PCI: Found SABRE, main regs at %016lx, wsync at %016lx\n", 1637 printk("PCI: Found SABRE, main regs at %016lx\n",
1632 p->pbm_A.controller_regs, pci_dma_wsync); 1638 p->pbm_A.controller_regs);
1633 1639
1634 /* Clear interrupts */ 1640 /* Clear interrupts */
1635 1641
diff --git a/arch/sparc64/kernel/pci_schizo.c b/arch/sparc64/kernel/pci_schizo.c
index 5753175b94e6..6a182bb66281 100644
--- a/arch/sparc64/kernel/pci_schizo.c
+++ b/arch/sparc64/kernel/pci_schizo.c
@@ -15,6 +15,7 @@
15#include <asm/iommu.h> 15#include <asm/iommu.h>
16#include <asm/irq.h> 16#include <asm/irq.h>
17#include <asm/upa.h> 17#include <asm/upa.h>
18#include <asm/pstate.h>
18 19
19#include "pci_impl.h" 20#include "pci_impl.h"
20#include "iommu_common.h" 21#include "iommu_common.h"
@@ -326,6 +327,44 @@ static int __init schizo_ino_to_pil(struct pci_dev *pdev, unsigned int ino)
326 return ret; 327 return ret;
327} 328}
328 329
330static void tomatillo_wsync_handler(struct ino_bucket *bucket, void *_arg1, void *_arg2)
331{
332 unsigned long sync_reg = (unsigned long) _arg2;
333 u64 mask = 1 << (__irq_ino(__irq(bucket)) & IMAP_INO);
334 u64 val;
335 int limit;
336
337 schizo_write(sync_reg, mask);
338
339 limit = 100000;
340 val = 0;
341 while (--limit) {
342 val = schizo_read(sync_reg);
343 if (!(val & mask))
344 break;
345 }
346 if (limit <= 0) {
347 printk("tomatillo_wsync_handler: DMA won't sync [%lx:%lx]\n",
348 val, mask);
349 }
350
351 if (_arg1) {
352 static unsigned char cacheline[64]
353 __attribute__ ((aligned (64)));
354
355 __asm__ __volatile__("rd %%fprs, %0\n\t"
356 "or %0, %4, %1\n\t"
357 "wr %1, 0x0, %%fprs\n\t"
358 "stda %%f0, [%5] %6\n\t"
359 "wr %0, 0x0, %%fprs\n\t"
360 "membar #Sync"
361 : "=&r" (mask), "=&r" (val)
362 : "0" (mask), "1" (val),
363 "i" (FPRS_FEF), "r" (&cacheline[0]),
364 "i" (ASI_BLK_COMMIT_P));
365 }
366}
367
329static unsigned int schizo_irq_build(struct pci_pbm_info *pbm, 368static unsigned int schizo_irq_build(struct pci_pbm_info *pbm,
330 struct pci_dev *pdev, 369 struct pci_dev *pdev,
331 unsigned int ino) 370 unsigned int ino)
@@ -369,6 +408,15 @@ static unsigned int schizo_irq_build(struct pci_pbm_info *pbm,
369 bucket = __bucket(build_irq(pil, ign_fixup, iclr, imap)); 408 bucket = __bucket(build_irq(pil, ign_fixup, iclr, imap));
370 bucket->flags |= IBF_PCI; 409 bucket->flags |= IBF_PCI;
371 410
411 if (pdev && pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) {
412 struct irq_desc *p = bucket->irq_info;
413
414 p->pre_handler = tomatillo_wsync_handler;
415 p->pre_handler_arg1 = ((pbm->chip_version <= 4) ?
416 (void *) 1 : (void *) 0);
417 p->pre_handler_arg2 = (void *) pbm->sync_reg;
418 }
419
372 return __irq(bucket); 420 return __irq(bucket);
373} 421}
374 422
@@ -885,6 +933,7 @@ static irqreturn_t schizo_ce_intr(int irq, void *dev_id, struct pt_regs *regs)
885 933
886#define SCHIZO_PCI_CTRL (0x2000UL) 934#define SCHIZO_PCI_CTRL (0x2000UL)
887#define SCHIZO_PCICTRL_BUS_UNUS (1UL << 63UL) /* Safari */ 935#define SCHIZO_PCICTRL_BUS_UNUS (1UL << 63UL) /* Safari */
936#define SCHIZO_PCICTRL_DTO_INT (1UL << 61UL) /* Tomatillo */
888#define SCHIZO_PCICTRL_ARB_PRIO (0x1ff << 52UL) /* Tomatillo */ 937#define SCHIZO_PCICTRL_ARB_PRIO (0x1ff << 52UL) /* Tomatillo */
889#define SCHIZO_PCICTRL_ESLCK (1UL << 51UL) /* Safari */ 938#define SCHIZO_PCICTRL_ESLCK (1UL << 51UL) /* Safari */
890#define SCHIZO_PCICTRL_ERRSLOT (7UL << 48UL) /* Safari */ 939#define SCHIZO_PCICTRL_ERRSLOT (7UL << 48UL) /* Safari */
@@ -1887,37 +1936,27 @@ static void __init schizo_pbm_hw_init(struct pci_pbm_info *pbm)
1887{ 1936{
1888 u64 tmp; 1937 u64 tmp;
1889 1938
1890 /* Set IRQ retry to infinity. */ 1939 schizo_write(pbm->pbm_regs + SCHIZO_PCI_IRQ_RETRY, 5);
1891 schizo_write(pbm->pbm_regs + SCHIZO_PCI_IRQ_RETRY,
1892 SCHIZO_IRQ_RETRY_INF);
1893 1940
1894 /* Enable arbiter for all PCI slots. Also, disable PCI interval
1895 * timer so that DTO (Discard TimeOuts) are not reported because
1896 * some Schizo revisions report them erroneously.
1897 */
1898 tmp = schizo_read(pbm->pbm_regs + SCHIZO_PCI_CTRL); 1941 tmp = schizo_read(pbm->pbm_regs + SCHIZO_PCI_CTRL);
1899 if (pbm->chip_type == PBM_CHIP_TYPE_SCHIZO_PLUS &&
1900 pbm->chip_version == 0x5 &&
1901 pbm->chip_revision == 0x1)
1902 tmp |= 0x0f;
1903 else
1904 tmp |= 0xff;
1905 1942
1906 tmp &= ~SCHIZO_PCICTRL_PTO; 1943 /* Enable arbiter for all PCI slots. */
1944 tmp |= 0xff;
1945
1907 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO && 1946 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO &&
1908 pbm->chip_version >= 0x2) 1947 pbm->chip_version >= 0x2)
1909 tmp |= 0x3UL << SCHIZO_PCICTRL_PTO_SHIFT; 1948 tmp |= 0x3UL << SCHIZO_PCICTRL_PTO_SHIFT;
1910 else
1911 tmp |= 0x1UL << SCHIZO_PCICTRL_PTO_SHIFT;
1912 1949
1913 if (!prom_getbool(pbm->prom_node, "no-bus-parking")) 1950 if (!prom_getbool(pbm->prom_node, "no-bus-parking"))
1914 tmp |= SCHIZO_PCICTRL_PARK; 1951 tmp |= SCHIZO_PCICTRL_PARK;
1952 else
1953 tmp &= ~SCHIZO_PCICTRL_PARK;
1915 1954
1916 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO && 1955 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO &&
1917 pbm->chip_version <= 0x1) 1956 pbm->chip_version <= 0x1)
1918 tmp |= (1UL << 61); 1957 tmp |= SCHIZO_PCICTRL_DTO_INT;
1919 else 1958 else
1920 tmp &= ~(1UL << 61); 1959 tmp &= ~SCHIZO_PCICTRL_DTO_INT;
1921 1960
1922 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) 1961 if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO)
1923 tmp |= (SCHIZO_PCICTRL_MRM_PREF | 1962 tmp |= (SCHIZO_PCICTRL_MRM_PREF |
@@ -2015,6 +2054,9 @@ static void __init schizo_pbm_init(struct pci_controller_info *p,
2015 pbm->pbm_regs = pr_regs[0].phys_addr; 2054 pbm->pbm_regs = pr_regs[0].phys_addr;
2016 pbm->controller_regs = pr_regs[1].phys_addr - 0x10000UL; 2055 pbm->controller_regs = pr_regs[1].phys_addr - 0x10000UL;
2017 2056
2057 if (chip_type == PBM_CHIP_TYPE_TOMATILLO)
2058 pbm->sync_reg = pr_regs[3].phys_addr + 0x1a18UL;
2059
2018 sprintf(pbm->name, 2060 sprintf(pbm->name,
2019 (chip_type == PBM_CHIP_TYPE_TOMATILLO ? 2061 (chip_type == PBM_CHIP_TYPE_TOMATILLO ?
2020 "TOMATILLO%d PBM%c" : 2062 "TOMATILLO%d PBM%c" :
diff --git a/arch/sparc64/kernel/semaphore.c b/arch/sparc64/kernel/semaphore.c
index 63496c43fe17..a809e63f03ef 100644
--- a/arch/sparc64/kernel/semaphore.c
+++ b/arch/sparc64/kernel/semaphore.c
@@ -32,8 +32,9 @@ static __inline__ int __sem_update_count(struct semaphore *sem, int incr)
32" add %1, %4, %1\n" 32" add %1, %4, %1\n"
33" cas [%3], %0, %1\n" 33" cas [%3], %0, %1\n"
34" cmp %0, %1\n" 34" cmp %0, %1\n"
35" membar #StoreLoad | #StoreStore\n"
35" bne,pn %%icc, 1b\n" 36" bne,pn %%icc, 1b\n"
36" membar #StoreLoad | #StoreStore\n" 37" nop\n"
37 : "=&r" (old_count), "=&r" (tmp), "=m" (sem->count) 38 : "=&r" (old_count), "=&r" (tmp), "=m" (sem->count)
38 : "r" (&sem->count), "r" (incr), "m" (sem->count) 39 : "r" (&sem->count), "r" (incr), "m" (sem->count)
39 : "cc"); 40 : "cc");
@@ -71,8 +72,9 @@ void up(struct semaphore *sem)
71" cmp %%g1, %%g7\n" 72" cmp %%g1, %%g7\n"
72" bne,pn %%icc, 1b\n" 73" bne,pn %%icc, 1b\n"
73" addcc %%g7, 1, %%g0\n" 74" addcc %%g7, 1, %%g0\n"
75" membar #StoreLoad | #StoreStore\n"
74" ble,pn %%icc, 3f\n" 76" ble,pn %%icc, 3f\n"
75" membar #StoreLoad | #StoreStore\n" 77" nop\n"
76"2:\n" 78"2:\n"
77" .subsection 2\n" 79" .subsection 2\n"
78"3: mov %0, %%g1\n" 80"3: mov %0, %%g1\n"
@@ -128,8 +130,9 @@ void __sched down(struct semaphore *sem)
128" cmp %%g1, %%g7\n" 130" cmp %%g1, %%g7\n"
129" bne,pn %%icc, 1b\n" 131" bne,pn %%icc, 1b\n"
130" cmp %%g7, 1\n" 132" cmp %%g7, 1\n"
133" membar #StoreLoad | #StoreStore\n"
131" bl,pn %%icc, 3f\n" 134" bl,pn %%icc, 3f\n"
132" membar #StoreLoad | #StoreStore\n" 135" nop\n"
133"2:\n" 136"2:\n"
134" .subsection 2\n" 137" .subsection 2\n"
135"3: mov %0, %%g1\n" 138"3: mov %0, %%g1\n"
@@ -233,8 +236,9 @@ int __sched down_interruptible(struct semaphore *sem)
233" cmp %%g1, %%g7\n" 236" cmp %%g1, %%g7\n"
234" bne,pn %%icc, 1b\n" 237" bne,pn %%icc, 1b\n"
235" cmp %%g7, 1\n" 238" cmp %%g7, 1\n"
239" membar #StoreLoad | #StoreStore\n"
236" bl,pn %%icc, 3f\n" 240" bl,pn %%icc, 3f\n"
237" membar #StoreLoad | #StoreStore\n" 241" nop\n"
238"2:\n" 242"2:\n"
239" .subsection 2\n" 243" .subsection 2\n"
240"3: mov %2, %%g1\n" 244"3: mov %2, %%g1\n"
diff --git a/arch/sparc64/kernel/sparc64_ksyms.c b/arch/sparc64/kernel/sparc64_ksyms.c
index e78cc53594fa..56cd96f4a5cd 100644
--- a/arch/sparc64/kernel/sparc64_ksyms.c
+++ b/arch/sparc64/kernel/sparc64_ksyms.c
@@ -227,7 +227,6 @@ EXPORT_SYMBOL(__flush_dcache_range);
227 227
228EXPORT_SYMBOL(mostek_lock); 228EXPORT_SYMBOL(mostek_lock);
229EXPORT_SYMBOL(mstk48t02_regs); 229EXPORT_SYMBOL(mstk48t02_regs);
230EXPORT_SYMBOL(request_fast_irq);
231#ifdef CONFIG_SUN_AUXIO 230#ifdef CONFIG_SUN_AUXIO
232EXPORT_SYMBOL(auxio_set_led); 231EXPORT_SYMBOL(auxio_set_led);
233EXPORT_SYMBOL(auxio_set_lte); 232EXPORT_SYMBOL(auxio_set_lte);
diff --git a/arch/sparc64/kernel/time.c b/arch/sparc64/kernel/time.c
index 71b4e3807694..b40db389f90b 100644
--- a/arch/sparc64/kernel/time.c
+++ b/arch/sparc64/kernel/time.c
@@ -973,7 +973,7 @@ static void sparc64_start_timers(irqreturn_t (*cfunc)(int, void *, struct pt_reg
973 int err; 973 int err;
974 974
975 /* Register IRQ handler. */ 975 /* Register IRQ handler. */
976 err = request_irq(build_irq(0, 0, 0UL, 0UL), cfunc, SA_STATIC_ALLOC, 976 err = request_irq(build_irq(0, 0, 0UL, 0UL), cfunc, 0,
977 "timer", NULL); 977 "timer", NULL);
978 978
979 if (err) { 979 if (err) {
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 2c8f9344b4ee..3a145fc39cf2 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -98,8 +98,9 @@ startup_continue:
98 98
99 sethi %hi(prom_entry_lock), %g2 99 sethi %hi(prom_entry_lock), %g2
1001: ldstub [%g2 + %lo(prom_entry_lock)], %g1 1001: ldstub [%g2 + %lo(prom_entry_lock)], %g1
101 membar #StoreLoad | #StoreStore
101 brnz,pn %g1, 1b 102 brnz,pn %g1, 1b
102 membar #StoreLoad | #StoreStore 103 nop
103 104
104 sethi %hi(p1275buf), %g2 105 sethi %hi(p1275buf), %g2
105 or %g2, %lo(p1275buf), %g2 106 or %g2, %lo(p1275buf), %g2
diff --git a/arch/sparc64/lib/U1memcpy.S b/arch/sparc64/lib/U1memcpy.S
index da9b520c7189..bafd2fc07acb 100644
--- a/arch/sparc64/lib/U1memcpy.S
+++ b/arch/sparc64/lib/U1memcpy.S
@@ -87,14 +87,17 @@
87#define LOOP_CHUNK3(src, dest, len, branch_dest) \ 87#define LOOP_CHUNK3(src, dest, len, branch_dest) \
88 MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) 88 MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
89 89
90#define DO_SYNC membar #Sync;
90#define STORE_SYNC(dest, fsrc) \ 91#define STORE_SYNC(dest, fsrc) \
91 EX_ST(STORE_BLK(%fsrc, %dest)); \ 92 EX_ST(STORE_BLK(%fsrc, %dest)); \
92 add %dest, 0x40, %dest; 93 add %dest, 0x40, %dest; \
94 DO_SYNC
93 95
94#define STORE_JUMP(dest, fsrc, target) \ 96#define STORE_JUMP(dest, fsrc, target) \
95 EX_ST(STORE_BLK(%fsrc, %dest)); \ 97 EX_ST(STORE_BLK(%fsrc, %dest)); \
96 add %dest, 0x40, %dest; \ 98 add %dest, 0x40, %dest; \
97 ba,pt %xcc, target; 99 ba,pt %xcc, target; \
100 nop;
98 101
99#define FINISH_VISCHUNK(dest, f0, f1, left) \ 102#define FINISH_VISCHUNK(dest, f0, f1, left) \
100 subcc %left, 8, %left;\ 103 subcc %left, 8, %left;\
@@ -239,17 +242,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
239 ba,pt %xcc, 1b+4 242 ba,pt %xcc, 1b+4
240 faligndata %f0, %f2, %f48 243 faligndata %f0, %f2, %f48
2411: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) 2441: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
242 STORE_SYNC(o0, f48) membar #Sync 245 STORE_SYNC(o0, f48)
243 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) 246 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
244 STORE_JUMP(o0, f48, 40f) membar #Sync 247 STORE_JUMP(o0, f48, 40f)
2452: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) 2482: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
246 STORE_SYNC(o0, f48) membar #Sync 249 STORE_SYNC(o0, f48)
247 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) 250 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
248 STORE_JUMP(o0, f48, 48f) membar #Sync 251 STORE_JUMP(o0, f48, 48f)
2493: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) 2523: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
250 STORE_SYNC(o0, f48) membar #Sync 253 STORE_SYNC(o0, f48)
251 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) 254 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
252 STORE_JUMP(o0, f48, 56f) membar #Sync 255 STORE_JUMP(o0, f48, 56f)
253 256
2541: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) 2571: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
255 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 258 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -260,17 +263,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
260 ba,pt %xcc, 1b+4 263 ba,pt %xcc, 1b+4
261 faligndata %f2, %f4, %f48 264 faligndata %f2, %f4, %f48
2621: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) 2651: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
263 STORE_SYNC(o0, f48) membar #Sync 266 STORE_SYNC(o0, f48)
264 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) 267 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
265 STORE_JUMP(o0, f48, 41f) membar #Sync 268 STORE_JUMP(o0, f48, 41f)
2662: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) 2692: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
267 STORE_SYNC(o0, f48) membar #Sync 270 STORE_SYNC(o0, f48)
268 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) 271 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
269 STORE_JUMP(o0, f48, 49f) membar #Sync 272 STORE_JUMP(o0, f48, 49f)
2703: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) 2733: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
271 STORE_SYNC(o0, f48) membar #Sync 274 STORE_SYNC(o0, f48)
272 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) 275 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
273 STORE_JUMP(o0, f48, 57f) membar #Sync 276 STORE_JUMP(o0, f48, 57f)
274 277
2751: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) 2781: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
276 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 279 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -281,17 +284,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
281 ba,pt %xcc, 1b+4 284 ba,pt %xcc, 1b+4
282 faligndata %f4, %f6, %f48 285 faligndata %f4, %f6, %f48
2831: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) 2861: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
284 STORE_SYNC(o0, f48) membar #Sync 287 STORE_SYNC(o0, f48)
285 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) 288 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
286 STORE_JUMP(o0, f48, 42f) membar #Sync 289 STORE_JUMP(o0, f48, 42f)
2872: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) 2902: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
288 STORE_SYNC(o0, f48) membar #Sync 291 STORE_SYNC(o0, f48)
289 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) 292 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
290 STORE_JUMP(o0, f48, 50f) membar #Sync 293 STORE_JUMP(o0, f48, 50f)
2913: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) 2943: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
292 STORE_SYNC(o0, f48) membar #Sync 295 STORE_SYNC(o0, f48)
293 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) 296 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
294 STORE_JUMP(o0, f48, 58f) membar #Sync 297 STORE_JUMP(o0, f48, 58f)
295 298
2961: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) 2991: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
297 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 300 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -302,17 +305,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
302 ba,pt %xcc, 1b+4 305 ba,pt %xcc, 1b+4
303 faligndata %f6, %f8, %f48 306 faligndata %f6, %f8, %f48
3041: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) 3071: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
305 STORE_SYNC(o0, f48) membar #Sync 308 STORE_SYNC(o0, f48)
306 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 309 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
307 STORE_JUMP(o0, f48, 43f) membar #Sync 310 STORE_JUMP(o0, f48, 43f)
3082: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 3112: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
309 STORE_SYNC(o0, f48) membar #Sync 312 STORE_SYNC(o0, f48)
310 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) 313 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
311 STORE_JUMP(o0, f48, 51f) membar #Sync 314 STORE_JUMP(o0, f48, 51f)
3123: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) 3153: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
313 STORE_SYNC(o0, f48) membar #Sync 316 STORE_SYNC(o0, f48)
314 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) 317 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
315 STORE_JUMP(o0, f48, 59f) membar #Sync 318 STORE_JUMP(o0, f48, 59f)
316 319
3171: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) 3201: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
318 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 321 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -323,17 +326,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
323 ba,pt %xcc, 1b+4 326 ba,pt %xcc, 1b+4
324 faligndata %f8, %f10, %f48 327 faligndata %f8, %f10, %f48
3251: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) 3281: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
326 STORE_SYNC(o0, f48) membar #Sync 329 STORE_SYNC(o0, f48)
327 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) 330 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
328 STORE_JUMP(o0, f48, 44f) membar #Sync 331 STORE_JUMP(o0, f48, 44f)
3292: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) 3322: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
330 STORE_SYNC(o0, f48) membar #Sync 333 STORE_SYNC(o0, f48)
331 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) 334 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
332 STORE_JUMP(o0, f48, 52f) membar #Sync 335 STORE_JUMP(o0, f48, 52f)
3333: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) 3363: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
334 STORE_SYNC(o0, f48) membar #Sync 337 STORE_SYNC(o0, f48)
335 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) 338 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
336 STORE_JUMP(o0, f48, 60f) membar #Sync 339 STORE_JUMP(o0, f48, 60f)
337 340
3381: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) 3411: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
339 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 342 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -344,17 +347,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
344 ba,pt %xcc, 1b+4 347 ba,pt %xcc, 1b+4
345 faligndata %f10, %f12, %f48 348 faligndata %f10, %f12, %f48
3461: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) 3491: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
347 STORE_SYNC(o0, f48) membar #Sync 350 STORE_SYNC(o0, f48)
348 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) 351 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
349 STORE_JUMP(o0, f48, 45f) membar #Sync 352 STORE_JUMP(o0, f48, 45f)
3502: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) 3532: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
351 STORE_SYNC(o0, f48) membar #Sync 354 STORE_SYNC(o0, f48)
352 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) 355 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
353 STORE_JUMP(o0, f48, 53f) membar #Sync 356 STORE_JUMP(o0, f48, 53f)
3543: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) 3573: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
355 STORE_SYNC(o0, f48) membar #Sync 358 STORE_SYNC(o0, f48)
356 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) 359 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
357 STORE_JUMP(o0, f48, 61f) membar #Sync 360 STORE_JUMP(o0, f48, 61f)
358 361
3591: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) 3621: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
360 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 363 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -365,17 +368,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
365 ba,pt %xcc, 1b+4 368 ba,pt %xcc, 1b+4
366 faligndata %f12, %f14, %f48 369 faligndata %f12, %f14, %f48
3671: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) 3701: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
368 STORE_SYNC(o0, f48) membar #Sync 371 STORE_SYNC(o0, f48)
369 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) 372 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
370 STORE_JUMP(o0, f48, 46f) membar #Sync 373 STORE_JUMP(o0, f48, 46f)
3712: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) 3742: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
372 STORE_SYNC(o0, f48) membar #Sync 375 STORE_SYNC(o0, f48)
373 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) 376 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
374 STORE_JUMP(o0, f48, 54f) membar #Sync 377 STORE_JUMP(o0, f48, 54f)
3753: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) 3783: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
376 STORE_SYNC(o0, f48) membar #Sync 379 STORE_SYNC(o0, f48)
377 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) 380 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
378 STORE_JUMP(o0, f48, 62f) membar #Sync 381 STORE_JUMP(o0, f48, 62f)
379 382
3801: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) 3831: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
381 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) 384 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
@@ -386,17 +389,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
386 ba,pt %xcc, 1b+4 389 ba,pt %xcc, 1b+4
387 faligndata %f14, %f16, %f48 390 faligndata %f14, %f16, %f48
3881: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) 3911: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
389 STORE_SYNC(o0, f48) membar #Sync 392 STORE_SYNC(o0, f48)
390 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) 393 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
391 STORE_JUMP(o0, f48, 47f) membar #Sync 394 STORE_JUMP(o0, f48, 47f)
3922: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) 3952: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
393 STORE_SYNC(o0, f48) membar #Sync 396 STORE_SYNC(o0, f48)
394 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) 397 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
395 STORE_JUMP(o0, f48, 55f) membar #Sync 398 STORE_JUMP(o0, f48, 55f)
3963: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) 3993: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
397 STORE_SYNC(o0, f48) membar #Sync 400 STORE_SYNC(o0, f48)
398 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) 401 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
399 STORE_JUMP(o0, f48, 63f) membar #Sync 402 STORE_JUMP(o0, f48, 63f)
400 403
40140: FINISH_VISCHUNK(o0, f0, f2, g3) 40440: FINISH_VISCHUNK(o0, f0, f2, g3)
40241: FINISH_VISCHUNK(o0, f2, f4, g3) 40541: FINISH_VISCHUNK(o0, f2, f4, g3)
diff --git a/arch/sparc64/lib/VISsave.S b/arch/sparc64/lib/VISsave.S
index 65e328d600a8..4e18989bd602 100644
--- a/arch/sparc64/lib/VISsave.S
+++ b/arch/sparc64/lib/VISsave.S
@@ -72,7 +72,11 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3
72 72
73 stda %f48, [%g3 + %g1] ASI_BLK_P 73 stda %f48, [%g3 + %g1] ASI_BLK_P
745: membar #Sync 745: membar #Sync
75 jmpl %g7 + %g0, %g0 75 ba,pt %xcc, 80f
76 nop
77
78 .align 32
7980: jmpl %g7 + %g0, %g0
76 nop 80 nop
77 81
786: ldub [%g3 + TI_FPSAVED], %o5 826: ldub [%g3 + TI_FPSAVED], %o5
@@ -87,8 +91,11 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3
87 stda %f32, [%g2 + %g1] ASI_BLK_P 91 stda %f32, [%g2 + %g1] ASI_BLK_P
88 stda %f48, [%g3 + %g1] ASI_BLK_P 92 stda %f48, [%g3 + %g1] ASI_BLK_P
89 membar #Sync 93 membar #Sync
90 jmpl %g7 + %g0, %g0 94 ba,pt %xcc, 80f
95 nop
91 96
97 .align 32
9880: jmpl %g7 + %g0, %g0
92 nop 99 nop
93 100
94 .align 32 101 .align 32
@@ -126,6 +133,10 @@ VISenterhalf:
126 stda %f0, [%g2 + %g1] ASI_BLK_P 133 stda %f0, [%g2 + %g1] ASI_BLK_P
127 stda %f16, [%g3 + %g1] ASI_BLK_P 134 stda %f16, [%g3 + %g1] ASI_BLK_P
128 membar #Sync 135 membar #Sync
136 ba,pt %xcc, 4f
137 nop
138
139 .align 32
1294: and %o5, FPRS_DU, %o5 1404: and %o5, FPRS_DU, %o5
130 jmpl %g7 + %g0, %g0 141 jmpl %g7 + %g0, %g0
131 wr %o5, FPRS_FEF, %fprs 142 wr %o5, FPRS_FEF, %fprs
diff --git a/arch/sparc64/lib/atomic.S b/arch/sparc64/lib/atomic.S
index e528b8d1a3e6..faf87c31598b 100644
--- a/arch/sparc64/lib/atomic.S
+++ b/arch/sparc64/lib/atomic.S
@@ -7,18 +7,6 @@
7#include <linux/config.h> 7#include <linux/config.h>
8#include <asm/asi.h> 8#include <asm/asi.h>
9 9
10 /* On SMP we need to use memory barriers to ensure
11 * correct memory operation ordering, nop these out
12 * for uniprocessor.
13 */
14#ifdef CONFIG_SMP
15#define ATOMIC_PRE_BARRIER membar #StoreLoad | #LoadLoad
16#define ATOMIC_POST_BARRIER membar #StoreLoad | #StoreStore
17#else
18#define ATOMIC_PRE_BARRIER nop
19#define ATOMIC_POST_BARRIER nop
20#endif
21
22 .text 10 .text
23 11
24 /* Two versions of the atomic routines, one that 12 /* Two versions of the atomic routines, one that
@@ -52,6 +40,24 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
52 nop 40 nop
53 .size atomic_sub, .-atomic_sub 41 .size atomic_sub, .-atomic_sub
54 42
43 /* On SMP we need to use memory barriers to ensure
44 * correct memory operation ordering, nop these out
45 * for uniprocessor.
46 */
47#ifdef CONFIG_SMP
48
49#define ATOMIC_PRE_BARRIER membar #StoreLoad | #LoadLoad;
50#define ATOMIC_POST_BARRIER \
51 ba,pt %xcc, 80b; \
52 membar #StoreLoad | #StoreStore
53
5480: retl
55 nop
56#else
57#define ATOMIC_PRE_BARRIER
58#define ATOMIC_POST_BARRIER
59#endif
60
55 .globl atomic_add_ret 61 .globl atomic_add_ret
56 .type atomic_add_ret,#function 62 .type atomic_add_ret,#function
57atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ 63atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
@@ -62,9 +68,10 @@ atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
62 cmp %g1, %g7 68 cmp %g1, %g7
63 bne,pn %icc, 1b 69 bne,pn %icc, 1b
64 add %g7, %o0, %g7 70 add %g7, %o0, %g7
71 sra %g7, 0, %o0
65 ATOMIC_POST_BARRIER 72 ATOMIC_POST_BARRIER
66 retl 73 retl
67 sra %g7, 0, %o0 74 nop
68 .size atomic_add_ret, .-atomic_add_ret 75 .size atomic_add_ret, .-atomic_add_ret
69 76
70 .globl atomic_sub_ret 77 .globl atomic_sub_ret
@@ -77,9 +84,10 @@ atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
77 cmp %g1, %g7 84 cmp %g1, %g7
78 bne,pn %icc, 1b 85 bne,pn %icc, 1b
79 sub %g7, %o0, %g7 86 sub %g7, %o0, %g7
87 sra %g7, 0, %o0
80 ATOMIC_POST_BARRIER 88 ATOMIC_POST_BARRIER
81 retl 89 retl
82 sra %g7, 0, %o0 90 nop
83 .size atomic_sub_ret, .-atomic_sub_ret 91 .size atomic_sub_ret, .-atomic_sub_ret
84 92
85 .globl atomic64_add 93 .globl atomic64_add
@@ -118,9 +126,10 @@ atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
118 cmp %g1, %g7 126 cmp %g1, %g7
119 bne,pn %xcc, 1b 127 bne,pn %xcc, 1b
120 add %g7, %o0, %g7 128 add %g7, %o0, %g7
129 mov %g7, %o0
121 ATOMIC_POST_BARRIER 130 ATOMIC_POST_BARRIER
122 retl 131 retl
123 mov %g7, %o0 132 nop
124 .size atomic64_add_ret, .-atomic64_add_ret 133 .size atomic64_add_ret, .-atomic64_add_ret
125 134
126 .globl atomic64_sub_ret 135 .globl atomic64_sub_ret
@@ -133,7 +142,8 @@ atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
133 cmp %g1, %g7 142 cmp %g1, %g7
134 bne,pn %xcc, 1b 143 bne,pn %xcc, 1b
135 sub %g7, %o0, %g7 144 sub %g7, %o0, %g7
145 mov %g7, %o0
136 ATOMIC_POST_BARRIER 146 ATOMIC_POST_BARRIER
137 retl 147 retl
138 mov %g7, %o0 148 nop
139 .size atomic64_sub_ret, .-atomic64_sub_ret 149 .size atomic64_sub_ret, .-atomic64_sub_ret
diff --git a/arch/sparc64/lib/bitops.S b/arch/sparc64/lib/bitops.S
index 886dcd2b376a..31afbfe6c1e8 100644
--- a/arch/sparc64/lib/bitops.S
+++ b/arch/sparc64/lib/bitops.S
@@ -7,20 +7,26 @@
7#include <linux/config.h> 7#include <linux/config.h>
8#include <asm/asi.h> 8#include <asm/asi.h>
9 9
10 .text
11
10 /* On SMP we need to use memory barriers to ensure 12 /* On SMP we need to use memory barriers to ensure
11 * correct memory operation ordering, nop these out 13 * correct memory operation ordering, nop these out
12 * for uniprocessor. 14 * for uniprocessor.
13 */ 15 */
16
14#ifdef CONFIG_SMP 17#ifdef CONFIG_SMP
15#define BITOP_PRE_BARRIER membar #StoreLoad | #LoadLoad 18#define BITOP_PRE_BARRIER membar #StoreLoad | #LoadLoad
16#define BITOP_POST_BARRIER membar #StoreLoad | #StoreStore 19#define BITOP_POST_BARRIER \
20 ba,pt %xcc, 80b; \
21 membar #StoreLoad | #StoreStore
22
2380: retl
24 nop
17#else 25#else
18#define BITOP_PRE_BARRIER nop 26#define BITOP_PRE_BARRIER
19#define BITOP_POST_BARRIER nop 27#define BITOP_POST_BARRIER
20#endif 28#endif
21 29
22 .text
23
24 .globl test_and_set_bit 30 .globl test_and_set_bit
25 .type test_and_set_bit,#function 31 .type test_and_set_bit,#function
26test_and_set_bit: /* %o0=nr, %o1=addr */ 32test_and_set_bit: /* %o0=nr, %o1=addr */
@@ -37,10 +43,11 @@ test_and_set_bit: /* %o0=nr, %o1=addr */
37 cmp %g7, %g1 43 cmp %g7, %g1
38 bne,pn %xcc, 1b 44 bne,pn %xcc, 1b
39 and %g7, %o2, %g2 45 and %g7, %o2, %g2
40 BITOP_POST_BARRIER
41 clr %o0 46 clr %o0
47 movrne %g2, 1, %o0
48 BITOP_POST_BARRIER
42 retl 49 retl
43 movrne %g2, 1, %o0 50 nop
44 .size test_and_set_bit, .-test_and_set_bit 51 .size test_and_set_bit, .-test_and_set_bit
45 52
46 .globl test_and_clear_bit 53 .globl test_and_clear_bit
@@ -59,10 +66,11 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */
59 cmp %g7, %g1 66 cmp %g7, %g1
60 bne,pn %xcc, 1b 67 bne,pn %xcc, 1b
61 and %g7, %o2, %g2 68 and %g7, %o2, %g2
62 BITOP_POST_BARRIER
63 clr %o0 69 clr %o0
70 movrne %g2, 1, %o0
71 BITOP_POST_BARRIER
64 retl 72 retl
65 movrne %g2, 1, %o0 73 nop
66 .size test_and_clear_bit, .-test_and_clear_bit 74 .size test_and_clear_bit, .-test_and_clear_bit
67 75
68 .globl test_and_change_bit 76 .globl test_and_change_bit
@@ -81,10 +89,11 @@ test_and_change_bit: /* %o0=nr, %o1=addr */
81 cmp %g7, %g1 89 cmp %g7, %g1
82 bne,pn %xcc, 1b 90 bne,pn %xcc, 1b
83 and %g7, %o2, %g2 91 and %g7, %o2, %g2
84 BITOP_POST_BARRIER
85 clr %o0 92 clr %o0
93 movrne %g2, 1, %o0
94 BITOP_POST_BARRIER
86 retl 95 retl
87 movrne %g2, 1, %o0 96 nop
88 .size test_and_change_bit, .-test_and_change_bit 97 .size test_and_change_bit, .-test_and_change_bit
89 98
90 .globl set_bit 99 .globl set_bit
diff --git a/arch/sparc64/lib/debuglocks.c b/arch/sparc64/lib/debuglocks.c
index c421e0c65325..f03344cf784e 100644
--- a/arch/sparc64/lib/debuglocks.c
+++ b/arch/sparc64/lib/debuglocks.c
@@ -252,8 +252,9 @@ wlock_again:
252" andn %%g1, %%g3, %%g7\n" 252" andn %%g1, %%g3, %%g7\n"
253" casx [%0], %%g1, %%g7\n" 253" casx [%0], %%g1, %%g7\n"
254" cmp %%g1, %%g7\n" 254" cmp %%g1, %%g7\n"
255" membar #StoreLoad | #StoreStore\n"
255" bne,pn %%xcc, 1b\n" 256" bne,pn %%xcc, 1b\n"
256" membar #StoreLoad | #StoreStore" 257" nop"
257 : /* no outputs */ 258 : /* no outputs */
258 : "r" (&(rw->lock)) 259 : "r" (&(rw->lock))
259 : "g3", "g1", "g7", "cc", "memory"); 260 : "g3", "g1", "g7", "cc", "memory");
@@ -351,8 +352,9 @@ int _do_write_trylock (rwlock_t *rw, char *str)
351" andn %%g1, %%g3, %%g7\n" 352" andn %%g1, %%g3, %%g7\n"
352" casx [%0], %%g1, %%g7\n" 353" casx [%0], %%g1, %%g7\n"
353" cmp %%g1, %%g7\n" 354" cmp %%g1, %%g7\n"
355" membar #StoreLoad | #StoreStore\n"
354" bne,pn %%xcc, 1b\n" 356" bne,pn %%xcc, 1b\n"
355" membar #StoreLoad | #StoreStore" 357" nop"
356 : /* no outputs */ 358 : /* no outputs */
357 : "r" (&(rw->lock)) 359 : "r" (&(rw->lock))
358 : "g3", "g1", "g7", "cc", "memory"); 360 : "g3", "g1", "g7", "cc", "memory");
diff --git a/arch/sparc64/lib/dec_and_lock.S b/arch/sparc64/lib/dec_and_lock.S
index 7e6fdaebedba..8ee288dd0afc 100644
--- a/arch/sparc64/lib/dec_and_lock.S
+++ b/arch/sparc64/lib/dec_and_lock.S
@@ -48,8 +48,9 @@ start_to_zero:
48#endif 48#endif
49to_zero: 49to_zero:
50 ldstub [%o1], %g3 50 ldstub [%o1], %g3
51 membar #StoreLoad | #StoreStore
51 brnz,pn %g3, spin_on_lock 52 brnz,pn %g3, spin_on_lock
52 membar #StoreLoad | #StoreStore 53 nop
53loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */ 54loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
54 cmp %g2, %g7 55 cmp %g2, %g7
55 56
@@ -71,8 +72,9 @@ loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
71 nop 72 nop
72spin_on_lock: 73spin_on_lock:
73 ldub [%o1], %g3 74 ldub [%o1], %g3
75 membar #LoadLoad
74 brnz,pt %g3, spin_on_lock 76 brnz,pt %g3, spin_on_lock
75 membar #LoadLoad 77 nop
76 ba,pt %xcc, to_zero 78 ba,pt %xcc, to_zero
77 nop 79 nop
78 nop 80 nop
diff --git a/arch/sparc64/lib/rwsem.S b/arch/sparc64/lib/rwsem.S
index 174ff7b9164c..75f0e6b951d6 100644
--- a/arch/sparc64/lib/rwsem.S
+++ b/arch/sparc64/lib/rwsem.S
@@ -17,8 +17,9 @@ __down_read:
17 bne,pn %icc, 1b 17 bne,pn %icc, 1b
18 add %g7, 1, %g7 18 add %g7, 1, %g7
19 cmp %g7, 0 19 cmp %g7, 0
20 membar #StoreLoad | #StoreStore
20 bl,pn %icc, 3f 21 bl,pn %icc, 3f
21 membar #StoreLoad | #StoreStore 22 nop
222: 232:
23 retl 24 retl
24 nop 25 nop
@@ -57,8 +58,9 @@ __down_write:
57 cmp %g3, %g7 58 cmp %g3, %g7
58 bne,pn %icc, 1b 59 bne,pn %icc, 1b
59 cmp %g7, 0 60 cmp %g7, 0
61 membar #StoreLoad | #StoreStore
60 bne,pn %icc, 3f 62 bne,pn %icc, 3f
61 membar #StoreLoad | #StoreStore 63 nop
622: retl 642: retl
63 nop 65 nop
643: 663:
@@ -97,8 +99,9 @@ __up_read:
97 cmp %g1, %g7 99 cmp %g1, %g7
98 bne,pn %icc, 1b 100 bne,pn %icc, 1b
99 cmp %g7, 0 101 cmp %g7, 0
102 membar #StoreLoad | #StoreStore
100 bl,pn %icc, 3f 103 bl,pn %icc, 3f
101 membar #StoreLoad | #StoreStore 104 nop
1022: retl 1052: retl
103 nop 106 nop
1043: sethi %hi(RWSEM_ACTIVE_MASK), %g1 1073: sethi %hi(RWSEM_ACTIVE_MASK), %g1
@@ -126,8 +129,9 @@ __up_write:
126 bne,pn %icc, 1b 129 bne,pn %icc, 1b
127 sub %g7, %g1, %g7 130 sub %g7, %g1, %g7
128 cmp %g7, 0 131 cmp %g7, 0
132 membar #StoreLoad | #StoreStore
129 bl,pn %icc, 3f 133 bl,pn %icc, 3f
130 membar #StoreLoad | #StoreStore 134 nop
1312: 1352:
132 retl 136 retl
133 nop 137 nop
@@ -151,8 +155,9 @@ __downgrade_write:
151 bne,pn %icc, 1b 155 bne,pn %icc, 1b
152 sub %g7, %g1, %g7 156 sub %g7, %g1, %g7
153 cmp %g7, 0 157 cmp %g7, 0
158 membar #StoreLoad | #StoreStore
154 bl,pn %icc, 3f 159 bl,pn %icc, 3f
155 membar #StoreLoad | #StoreStore 160 nop
1562: 1612:
157 retl 162 retl
158 nop 163 nop
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index 9c5222075da9..8fc413cb6acd 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -136,8 +136,9 @@ static __inline__ void set_dcache_dirty(struct page *page, int this_cpu)
136 "or %%g1, %0, %%g1\n\t" 136 "or %%g1, %0, %%g1\n\t"
137 "casx [%2], %%g7, %%g1\n\t" 137 "casx [%2], %%g7, %%g1\n\t"
138 "cmp %%g7, %%g1\n\t" 138 "cmp %%g7, %%g1\n\t"
139 "membar #StoreLoad | #StoreStore\n\t"
139 "bne,pn %%xcc, 1b\n\t" 140 "bne,pn %%xcc, 1b\n\t"
140 " membar #StoreLoad | #StoreStore" 141 " nop"
141 : /* no outputs */ 142 : /* no outputs */
142 : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags) 143 : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
143 : "g1", "g7"); 144 : "g1", "g7");
@@ -157,8 +158,9 @@ static __inline__ void clear_dcache_dirty_cpu(struct page *page, unsigned long c
157 " andn %%g7, %1, %%g1\n\t" 158 " andn %%g7, %1, %%g1\n\t"
158 "casx [%2], %%g7, %%g1\n\t" 159 "casx [%2], %%g7, %%g1\n\t"
159 "cmp %%g7, %%g1\n\t" 160 "cmp %%g7, %%g1\n\t"
161 "membar #StoreLoad | #StoreStore\n\t"
160 "bne,pn %%xcc, 1b\n\t" 162 "bne,pn %%xcc, 1b\n\t"
161 " membar #StoreLoad | #StoreStore\n" 163 " nop\n"
162 "2:" 164 "2:"
163 : /* no outputs */ 165 : /* no outputs */
164 : "r" (cpu), "r" (mask), "r" (&page->flags), 166 : "r" (cpu), "r" (mask), "r" (&page->flags),
diff --git a/arch/sparc64/mm/ultra.S b/arch/sparc64/mm/ultra.S
index 7a0934321010..363770893797 100644
--- a/arch/sparc64/mm/ultra.S
+++ b/arch/sparc64/mm/ultra.S
@@ -72,6 +72,7 @@ __flush_tlb_pending:
72 flush %g6 72 flush %g6
73 retl 73 retl
74 wrpr %g7, 0x0, %pstate 74 wrpr %g7, 0x0, %pstate
75 nop
75 76
76 .align 32 77 .align 32
77 .globl __flush_tlb_kernel_range 78 .globl __flush_tlb_kernel_range
@@ -249,7 +250,7 @@ __cheetah_flush_tlb_mm: /* 15 insns */
249 retl 250 retl
250 wrpr %g7, 0x0, %pstate 251 wrpr %g7, 0x0, %pstate
251 252
252__cheetah_flush_tlb_pending: /* 22 insns */ 253__cheetah_flush_tlb_pending: /* 23 insns */
253 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ 254 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
254 rdpr %pstate, %g7 255 rdpr %pstate, %g7
255 sllx %o1, 3, %o1 256 sllx %o1, 3, %o1
@@ -266,8 +267,9 @@ __cheetah_flush_tlb_pending: /* 22 insns */
266 andn %o3, 1, %o3 267 andn %o3, 1, %o3
267 stxa %g0, [%o3] ASI_IMMU_DEMAP 268 stxa %g0, [%o3] ASI_IMMU_DEMAP
2682: stxa %g0, [%o3] ASI_DMMU_DEMAP 2692: stxa %g0, [%o3] ASI_DMMU_DEMAP
270 membar #Sync
269 brnz,pt %o1, 1b 271 brnz,pt %o1, 1b
270 membar #Sync 272 nop
271 stxa %g2, [%o4] ASI_DMMU 273 stxa %g2, [%o4] ASI_DMMU
272 flush %g6 274 flush %g6
273 wrpr %g0, 0, %tl 275 wrpr %g0, 0, %tl
@@ -316,7 +318,7 @@ cheetah_patch_cachetlbops:
316 sethi %hi(__cheetah_flush_tlb_pending), %o1 318 sethi %hi(__cheetah_flush_tlb_pending), %o1
317 or %o1, %lo(__cheetah_flush_tlb_pending), %o1 319 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
318 call cheetah_patch_one 320 call cheetah_patch_one
319 mov 22, %o2 321 mov 23, %o2
320 322
321#ifdef DCACHE_ALIASING_POSSIBLE 323#ifdef DCACHE_ALIASING_POSSIBLE
322 sethi %hi(__flush_dcache_page), %o0 324 sethi %hi(__flush_dcache_page), %o0
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-11 19:29:57 -0500