5 files changed, 38 insertions, 32 deletions

diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index 8059531bf0ac..9a785293203f 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -430,15 +430,15 @@ do_ivec:
         membar          #Sync
 
         sethi           %hi(ivector_table), %g2
-        sllx            %g3, 3, %g3
+        sllx            %g3, 4, %g3
         or              %g2, %lo(ivector_table), %g2
         add             %g2, %g3, %g3
 
         TRAP_LOAD_IRQ_WORK(%g6, %g1)
 
-        lduw            [%g6], %g5              /* g5 = irq_work(cpu) */
-        stw             %g5, [%g3 + 0x00]       /* bucket->irq_chain = g5 */
-        stw             %g3, [%g6]              /* irq_work(cpu) = bucket */
+        ldx             [%g6], %g5              /* g5 = irq_work(cpu) */
+        stx             %g5, [%g3 + 0x00]       /* bucket->irq_chain = g5 */
+        stx             %g3, [%g6]              /* irq_work(cpu) = bucket */
         wr              %g0, 1 << PIL_DEVICE_IRQ, %set_softint
         retry
 do_ivec_xcall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 045ab27d4271..4db4dd576210 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -59,23 +59,20 @@
  * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
  */
 struct ino_bucket {
-        /* Next handler in per-CPU IRQ worklist.  We know that
-         * bucket pointers have the high 32-bits clear, so to
-         * save space we only store the bits we need.
-         */
-/*0x00*/unsigned int irq_chain;
+/*0x00*/unsigned long irq_chain;
 
         /* Virtual interrupt number assigned to this INO.  */
-/*0x04*/unsigned int virt_irq;
+/*0x08*/unsigned int virt_irq;
+/*0x0c*/unsigned int __pad;
 };
 
 #define NUM_IVECS       (IMAP_INR + 1)
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
 #define __irq_ino(irq) \
-        (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
-#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
-#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+        (((struct ino_bucket *)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(irq))
+#define __irq(bucket) ((unsigned long)(bucket))
 
 /* This has to be in the main kernel image, it cannot be
  * turned into per-cpu data.  The reason is that the main
@@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
 #define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist)
 
 static struct {
-        unsigned int irq;
+        unsigned long irq;
         unsigned int dev_handle;
         unsigned int dev_ino;
 } virt_to_real_irq_table[NR_IRQS];
 static DEFINE_SPINLOCK(virt_irq_alloc_lock);
 
-unsigned char virt_irq_alloc(unsigned int real_irq)
+unsigned char virt_irq_alloc(unsigned long real_irq)
 {
         unsigned long flags;
         unsigned char ent;
@@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq)
 }
 #endif
 
-static unsigned int virt_to_real_irq(unsigned char virt_irq)
+static unsigned long virt_to_real_irq(unsigned char virt_irq)
 {
         return virt_to_real_irq_table[virt_irq].irq;
 }
@@ -227,7 +224,7 @@ struct irq_handler_data {
 
 static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
 {
-        unsigned int real_irq = virt_to_real_irq(virt_irq);
+        unsigned long real_irq = virt_to_real_irq(virt_irq);
         struct ino_bucket *bucket = NULL;
 
         if (likely(real_irq))
@@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs)
 {
         struct ino_bucket *bucket;
         struct pt_regs *old_regs;
+        unsigned long pstate;
 
         clear_softint(1 << irq);
 
         old_regs = set_irq_regs(regs);
         irq_enter();
 
-        /* Sliiiick... */
-        bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+        /* Grab an atomic snapshot of the pending IVECs.  */
+        __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
+                             "wrpr      %0, %3, %%pstate\n\t"
+                             "ldx       [%2], %1\n\t"
+                             "stx       %%g0, [%2]\n\t"
+                             "wrpr      %0, 0x0, %%pstate\n\t"
+                             : "=&r" (pstate), "=&r" (bucket)
+                             : "r" (irq_work(smp_processor_id())),
+                               "i" (PSTATE_IE)
+                             : "memory");
+
         while (bucket) {
                 struct ino_bucket *next = __bucket(bucket->irq_chain);
 
-                bucket->irq_chain = 0;
+                bucket->irq_chain = 0UL;
                 __do_IRQ(bucket->virt_irq);
 
                 bucket = next;
@@ -808,7 +815,7 @@ void init_irqwork_curcpu(void)
 {
         int cpu = hard_smp_processor_id();
 
-        trap_block[cpu].irq_worklist = 0;
+        trap_block[cpu].irq_worklist = 0UL;
 }
 
 /* Please be very careful with register_one_mondo() and
diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S
index 574bc248bca6..e3e9d4c1574b 100644
--- a/arch/sparc64/kernel/sun4v_ivec.S
+++ b/arch/sparc64/kernel/sun4v_ivec.S
@@ -101,14 +101,14 @@ sun4v_dev_mondo:
 
         /* Get &ivector_table[IVEC] into %g4.  */
         sethi   %hi(ivector_table), %g4
-        sllx    %g3, 3, %g3
+        sllx    %g3, 4, %g3
         or      %g4, %lo(ivector_table), %g4
         add     %g4, %g3, %g4
 
         /* Insert ivector_table[] entry into __irq_work[] queue.  */
-        lduw    [%g1], %g2              /* g2 = irq_work(cpu) */
-        stw     %g2, [%g4 + 0x00]       /* bucket->irq_chain = g2 */
-        stw     %g4, [%g1]              /* irq_work(cpu) = bucket */
+        ldx     [%g1], %g2              /* g2 = irq_work(cpu) */
+        stx     %g2, [%g4 + 0x00]       /* bucket->irq_chain = g2 */
+        stx     %g4, [%g1]              /* irq_work(cpu) = bucket */
 
         /* Signal the interrupt by setting (1 << pil) in %softint.  */
         wr      %g0, 1 << PIL_DEVICE_IRQ, %set_softint
diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
index 98a6e609163e..379c219c3b7a 100644
--- a/include/asm-sparc64/cpudata.h
+++ b/include/asm-sparc64/cpudata.h
@@ -75,12 +75,11 @@ struct trap_per_cpu {
         unsigned long           tsb_huge_temp;
 
 /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size.  */
-        unsigned int            irq_worklist;
+        unsigned long           irq_worklist;
         unsigned int            cpu_mondo_qmask;
         unsigned int            dev_mondo_qmask;
         unsigned int            resum_qmask;
         unsigned int            nonresum_qmask;
-        unsigned int            __pad2[1];
         void                    *hdesc;
 } __attribute__((aligned(64)));
 extern struct trap_per_cpu trap_block[NR_CPUS];
@@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
 #define TRAP_PER_CPU_TSB_HUGE           0xd0
 #define TRAP_PER_CPU_TSB_HUGE_TEMP      0xd8
 #define TRAP_PER_CPU_IRQ_WORKLIST       0xe0
-#define TRAP_PER_CPU_CPU_MONDO_QMASK    0xe4
-#define TRAP_PER_CPU_DEV_MONDO_QMASK    0xe8
-#define TRAP_PER_CPU_RESUM_QMASK        0xec
-#define TRAP_PER_CPU_NONRESUM_QMASK     0xf0
+#define TRAP_PER_CPU_CPU_MONDO_QMASK    0xe8
+#define TRAP_PER_CPU_DEV_MONDO_QMASK    0xec
+#define TRAP_PER_CPU_RESUM_QMASK        0xf0
+#define TRAP_PER_CPU_NONRESUM_QMASK     0xf4
 
 #define TRAP_BLOCK_SZ_SHIFT             8
 
diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h
index bad3c28cdee8..24841c22e81b 100644
--- a/include/asm-sparc64/irq.h
+++ b/include/asm-sparc64/irq.h
@@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p,
 extern void sun4u_destroy_msi(unsigned int virt_irq);
 extern unsigned int sbus_build_irq(void *sbus, unsigned int ino);
 
-extern unsigned char virt_irq_alloc(unsigned int real_irq);
+extern unsigned char virt_irq_alloc(unsigned long real_irq);
 #ifdef CONFIG_PCI_MSI
 extern void virt_irq_free(unsigned int virt_irq);
 #endif