[SPARC64]: Make IVEC pointers 64-bit.

Currently we chain IVEC entries using 32-bit "pointers"
because we know that the ivector_table is in the main
kernel image, thus below 4GB.

This uses proper 64-bit pointers instead.

Whilst this bloats up the kernel image size, this sets
the infrastructure necessary to significantly shrink the
kernel size by using physical addresses and dynamically
allocating the ivector table.

Signed-off-by: David S. Miller <davem@davemloft.net>

5 files changed, 38 insertions, 32 deletions

diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index 8059531bf0ac..9a785293203f 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -430,15 +430,15 @@ do_ivec:
         membar          #Sync
 
         sethi           %hi(ivector_table), %g2
-        sllx            %g3, 3, %g3
+        sllx            %g3, 4, %g3
         or              %g2, %lo(ivector_table), %g2
         add             %g2, %g3, %g3
 
         TRAP_LOAD_IRQ_WORK(%g6, %g1)
 
-        lduw            [%g6], %g5              /* g5 = irq_work(cpu) */
-        stw             %g5, [%g3 + 0x00]       /* bucket->irq_chain = g5 */
-        stw             %g3, [%g6]              /* irq_work(cpu) = bucket */
+        ldx             [%g6], %g5              /* g5 = irq_work(cpu) */
+        stx             %g5, [%g3 + 0x00]       /* bucket->irq_chain = g5 */
+        stx             %g3, [%g6]              /* irq_work(cpu) = bucket */
         wr              %g0, 1 << PIL_DEVICE_IRQ, %set_softint
         retry
 do_ivec_xcall:
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 045ab27d4271..4db4dd576210 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -59,23 +59,20 @@
  * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
  */
 struct ino_bucket {
-        /* Next handler in per-CPU IRQ worklist.  We know that
-         * bucket pointers have the high 32-bits clear, so to
-         * save space we only store the bits we need.
-         */
-/*0x00*/unsigned int irq_chain;
+/*0x00*/unsigned long irq_chain;
 
         /* Virtual interrupt number assigned to this INO.  */
-/*0x04*/unsigned int virt_irq;
+/*0x08*/unsigned int virt_irq;
+/*0x0c*/unsigned int __pad;
 };
 
 #define NUM_IVECS       (IMAP_INR + 1)
 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
 
 #define __irq_ino(irq) \
-        (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
-#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
-#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
+        (((struct ino_bucket *)(irq)) - &ivector_table[0])
+#define __bucket(irq) ((struct ino_bucket *)(irq))
+#define __irq(bucket) ((unsigned long)(bucket))
 
 /* This has to be in the main kernel image, it cannot be
  * turned into per-cpu data.  The reason is that the main
@@ -87,13 +84,13 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
 #define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist)
 
 static struct {
-        unsigned int irq;
+        unsigned long irq;
         unsigned int dev_handle;
         unsigned int dev_ino;
 } virt_to_real_irq_table[NR_IRQS];
 static DEFINE_SPINLOCK(virt_irq_alloc_lock);
 
-unsigned char virt_irq_alloc(unsigned int real_irq)
+unsigned char virt_irq_alloc(unsigned long real_irq)
 {
         unsigned long flags;
         unsigned char ent;
@@ -134,7 +131,7 @@ void virt_irq_free(unsigned int virt_irq)
 }
 #endif
 
-static unsigned int virt_to_real_irq(unsigned char virt_irq)
+static unsigned long virt_to_real_irq(unsigned char virt_irq)
 {
         return virt_to_real_irq_table[virt_irq].irq;
 }
@@ -227,7 +224,7 @@ struct irq_handler_data {
 
 static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
 {
-        unsigned int real_irq = virt_to_real_irq(virt_irq);
+        unsigned long real_irq = virt_to_real_irq(virt_irq);
         struct ino_bucket *bucket = NULL;
 
         if (likely(real_irq))
@@ -694,18 +691,28 @@ void handler_irq(int irq, struct pt_regs *regs)
 {
         struct ino_bucket *bucket;
         struct pt_regs *old_regs;
+        unsigned long pstate;
 
         clear_softint(1 << irq);
 
         old_regs = set_irq_regs(regs);
         irq_enter();
 
-        /* Sliiiick... */
-        bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
+        /* Grab an atomic snapshot of the pending IVECs.  */
+        __asm__ __volatile__("rdpr      %%pstate, %0\n\t"
+                             "wrpr      %0, %3, %%pstate\n\t"
+                             "ldx       [%2], %1\n\t"
+                             "stx       %%g0, [%2]\n\t"
+                             "wrpr      %0, 0x0, %%pstate\n\t"
+                             : "=&r" (pstate), "=&r" (bucket)
+                             : "r" (irq_work(smp_processor_id())),
+                               "i" (PSTATE_IE)
+                             : "memory");
+
         while (bucket) {
                 struct ino_bucket *next = __bucket(bucket->irq_chain);
 
-                bucket->irq_chain = 0;
+                bucket->irq_chain = 0UL;
                 __do_IRQ(bucket->virt_irq);
 
                 bucket = next;
@@ -808,7 +815,7 @@ void init_irqwork_curcpu(void)
 {
         int cpu = hard_smp_processor_id();
 
-        trap_block[cpu].irq_worklist = 0;
+        trap_block[cpu].irq_worklist = 0UL;
 }
 
 /* Please be very careful with register_one_mondo() and
diff --git a/arch/sparc64/kernel/sun4v_ivec.S b/arch/sparc64/kernel/sun4v_ivec.S
index 574bc248bca6..e3e9d4c1574b 100644
--- a/arch/sparc64/kernel/sun4v_ivec.S
+++ b/arch/sparc64/kernel/sun4v_ivec.S
@@ -101,14 +101,14 @@ sun4v_dev_mondo:
 
         /* Get &ivector_table[IVEC] into %g4.  */
         sethi   %hi(ivector_table), %g4
-        sllx    %g3, 3, %g3
+        sllx    %g3, 4, %g3
         or      %g4, %lo(ivector_table), %g4
         add     %g4, %g3, %g4
 
         /* Insert ivector_table[] entry into __irq_work[] queue.  */
-        lduw    [%g1], %g2              /* g2 = irq_work(cpu) */
-        stw     %g2, [%g4 + 0x00]       /* bucket->irq_chain = g2 */
-        stw     %g4, [%g1]              /* irq_work(cpu) = bucket */
+        ldx     [%g1], %g2              /* g2 = irq_work(cpu) */
+        stx     %g2, [%g4 + 0x00]       /* bucket->irq_chain = g2 */
+        stx     %g4, [%g1]              /* irq_work(cpu) = bucket */
 
         /* Signal the interrupt by setting (1 << pil) in %softint.  */
         wr      %g0, 1 << PIL_DEVICE_IRQ, %set_softint
diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
index 98a6e609163e..379c219c3b7a 100644
--- a/include/asm-sparc64/cpudata.h
+++ b/include/asm-sparc64/cpudata.h
@@ -75,12 +75,11 @@ struct trap_per_cpu {
         unsigned long           tsb_huge_temp;
 
 /* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size.  */
-        unsigned int            irq_worklist;
+        unsigned long           irq_worklist;
         unsigned int            cpu_mondo_qmask;
         unsigned int            dev_mondo_qmask;
         unsigned int            resum_qmask;
         unsigned int            nonresum_qmask;
-        unsigned int            __pad2[1];
         void                    *hdesc;
 } __attribute__((aligned(64)));
 extern struct trap_per_cpu trap_block[NR_CPUS];
@@ -129,10 +128,10 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
 #define TRAP_PER_CPU_TSB_HUGE           0xd0
 #define TRAP_PER_CPU_TSB_HUGE_TEMP      0xd8
 #define TRAP_PER_CPU_IRQ_WORKLIST       0xe0
-#define TRAP_PER_CPU_CPU_MONDO_QMASK    0xe4
-#define TRAP_PER_CPU_DEV_MONDO_QMASK    0xe8
-#define TRAP_PER_CPU_RESUM_QMASK        0xec
-#define TRAP_PER_CPU_NONRESUM_QMASK     0xf0
+#define TRAP_PER_CPU_CPU_MONDO_QMASK    0xe8
+#define TRAP_PER_CPU_DEV_MONDO_QMASK    0xec
+#define TRAP_PER_CPU_RESUM_QMASK        0xf0
+#define TRAP_PER_CPU_NONRESUM_QMASK     0xf4
 
 #define TRAP_BLOCK_SZ_SHIFT             8
 
diff --git a/include/asm-sparc64/irq.h b/include/asm-sparc64/irq.h
index bad3c28cdee8..24841c22e81b 100644
--- a/include/asm-sparc64/irq.h
+++ b/include/asm-sparc64/irq.h
@@ -59,7 +59,7 @@ extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p,
 extern void sun4u_destroy_msi(unsigned int virt_irq);
 extern unsigned int sbus_build_irq(void *sbus, unsigned int ino);
 
-extern unsigned char virt_irq_alloc(unsigned int real_irq);
+extern unsigned char virt_irq_alloc(unsigned long real_irq);
 #ifdef CONFIG_PCI_MSI
 extern void virt_irq_free(unsigned int virt_irq);
 #endif