21 files changed, 114 insertions, 137 deletions

diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 3c93f08ce18..2c2e38821f6 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -16,3 +16,8 @@ header-y += traps.h
 header-y += uctx.h
 header-y += utrap.h
 header-y += watchdog.h
+
+generic-y += div64.h
+generic-y += local64.h
+generic-y += irq_regs.h
+generic-y += local.h
diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h
index 7ae128b19d3..5c3c8b69884 100644
--- a/arch/sparc/include/asm/atomic_32.h
+++ b/arch/sparc/include/asm/atomic_32.h
@@ -22,7 +22,7 @@
 extern int __atomic_add_return(int, atomic_t *);
 extern int atomic_cmpxchg(atomic_t *, int, int);
 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
-extern int atomic_add_unless(atomic_t *, int, int);
+extern int __atomic_add_unless(atomic_t *, int, int);
 extern void atomic_set(atomic_t *, int);
 
 #define atomic_read(v)          (*(volatile int *)&(v)->counter)
@@ -52,7 +52,6 @@ extern void atomic_set(atomic_t *, int);
 #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
 #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
 
-#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
 /* This is the old 24-bit implementation.  It's still used internally
  * by some sparc-specific code, notably the semaphore implementation.
@@ -161,5 +160,4 @@ static inline int __atomic24_sub(int i, atomic24_t *v)
 
 #endif /* !(__KERNEL__) */
 
-#include <asm-generic/atomic-long.h>
 #endif /* !(__ARCH_SPARC_ATOMIC__) */
diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h
index bdb2ff880bd..9f421df46ae 100644
--- a/arch/sparc/include/asm/atomic_64.h
+++ b/arch/sparc/include/asm/atomic_64.h
@@ -70,7 +70,7 @@ extern long atomic64_sub_ret(long, atomic64_t *);
 #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
 
-static inline int atomic_add_unless(atomic_t *v, int a, int u)
+static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 {
         int c, old;
         c = atomic_read(v);
@@ -82,10 +82,9 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
                         break;
                 c = old;
         }
-        return c != (u);
+        return c;
 }
 
-#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
 #define atomic64_cmpxchg(v, o, n) \
         ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
@@ -114,5 +113,4 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u)
 #define smp_mb__before_atomic_inc()     barrier()
 #define smp_mb__after_atomic_inc()      barrier()
 
-#include <asm-generic/atomic-long.h>
 #endif /* !(__ARCH_SPARC64_ATOMIC__) */
diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h
index 38e9aa1b2ce..29011cc0e4b 100644
--- a/arch/sparc/include/asm/bitops_64.h
+++ b/arch/sparc/include/asm/bitops_64.h
@@ -26,61 +26,28 @@ extern void change_bit(unsigned long nr, volatile unsigned long *addr);
 #define smp_mb__before_clear_bit()      barrier()
 #define smp_mb__after_clear_bit()       barrier()
 
-#include <asm-generic/bitops/ffz.h>
-#include <asm-generic/bitops/__ffs.h>
 #include <asm-generic/bitops/fls.h>
 #include <asm-generic/bitops/__fls.h>
 #include <asm-generic/bitops/fls64.h>
 
 #ifdef __KERNEL__
 
+extern int ffs(int x);
+extern unsigned long __ffs(unsigned long);
+
+#include <asm-generic/bitops/ffz.h>
 #include <asm-generic/bitops/sched.h>
-#include <asm-generic/bitops/ffs.h>
 
 /*
  * hweightN: returns the hamming weight (i.e. the number
  * of bits set) of a N-bit word
  */
 
-#ifdef ULTRA_HAS_POPULATION_COUNT
-
-static inline unsigned int __arch_hweight64(unsigned long w)
-{
-        unsigned int res;
-
-        __asm__ ("popc %1,%0" : "=r" (res) : "r" (w));
-        return res;
-}
-
-static inline unsigned int __arch_hweight32(unsigned int w)
-{
-        unsigned int res;
-
-        __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xffffffff));
-        return res;
-}
+extern unsigned long __arch_hweight64(__u64 w);
+extern unsigned int __arch_hweight32(unsigned int w);
+extern unsigned int __arch_hweight16(unsigned int w);
+extern unsigned int __arch_hweight8(unsigned int w);
 
-static inline unsigned int __arch_hweight16(unsigned int w)
-{
-        unsigned int res;
-
-        __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xffff));
-        return res;
-}
-
-static inline unsigned int __arch_hweight8(unsigned int w)
-{
-        unsigned int res;
-
-        __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xff));
-        return res;
-}
-
-#else
-
-#include <asm-generic/bitops/arch_hweight.h>
-
-#endif
 #include <asm-generic/bitops/const_hweight.h>
 #include <asm-generic/bitops/lock.h>
 #endif /* __KERNEL__ */
@@ -91,10 +58,7 @@ static inline unsigned int __arch_hweight8(unsigned int w)
 
 #include <asm-generic/bitops/le.h>
 
-#define ext2_set_bit_atomic(lock,nr,addr) \
-        test_and_set_bit((nr) ^ 0x38,(unsigned long *)(addr))
-#define ext2_clear_bit_atomic(lock,nr,addr) \
-        test_and_clear_bit((nr) ^ 0x38,(unsigned long *)(addr))
+#include <asm-generic/bitops/ext2-atomic-setbit.h>
 
 #endif /* __KERNEL__ */
 
diff --git a/arch/sparc/include/asm/div64.h b/arch/sparc/include/asm/div64.h
deleted file mode 100644
index 6cd978cefb2..00000000000
--- a/arch/sparc/include/asm/div64.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/div64.h>
diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h
index cfa9cd2e551..7df8b7f544d 100644
--- a/arch/sparc/include/asm/elf_64.h
+++ b/arch/sparc/include/asm/elf_64.h
@@ -59,15 +59,33 @@
 #define R_SPARC_6               45
 
 /* Bits present in AT_HWCAP, primarily for Sparc32.  */
-
-#define HWCAP_SPARC_FLUSH       1    /* CPU supports flush instruction. */
-#define HWCAP_SPARC_STBAR       2
-#define HWCAP_SPARC_SWAP        4
-#define HWCAP_SPARC_MULDIV      8
-#define HWCAP_SPARC_V9          16
-#define HWCAP_SPARC_ULTRA3      32
-#define HWCAP_SPARC_BLKINIT     64
-#define HWCAP_SPARC_N2          128
+#define HWCAP_SPARC_FLUSH       0x00000001
+#define HWCAP_SPARC_STBAR       0x00000002
+#define HWCAP_SPARC_SWAP        0x00000004
+#define HWCAP_SPARC_MULDIV      0x00000008
+#define HWCAP_SPARC_V9          0x00000010
+#define HWCAP_SPARC_ULTRA3      0x00000020
+#define HWCAP_SPARC_BLKINIT     0x00000040
+#define HWCAP_SPARC_N2          0x00000080
+
+/* Solaris compatible AT_HWCAP bits. */
+#define AV_SPARC_MUL32          0x00000100 /* 32x32 multiply is efficient */
+#define AV_SPARC_DIV32          0x00000200 /* 32x32 divide is efficient */
+#define AV_SPARC_FSMULD         0x00000400 /* 'fsmuld' is efficient */
+#define AV_SPARC_V8PLUS         0x00000800 /* v9 insn available to 32bit */
+#define AV_SPARC_POPC           0x00001000 /* 'popc' is efficient */
+#define AV_SPARC_VIS            0x00002000 /* VIS insns available */
+#define AV_SPARC_VIS2           0x00004000 /* VIS2 insns available */
+#define AV_SPARC_ASI_BLK_INIT   0x00008000 /* block init ASIs available */
+#define AV_SPARC_FMAF           0x00010000 /* fused multiply-add */
+#define AV_SPARC_VIS3           0x00020000 /* VIS3 insns available */
+#define AV_SPARC_HPC            0x00040000 /* HPC insns available */
+#define AV_SPARC_RANDOM         0x00080000 /* 'random' insn available */
+#define AV_SPARC_TRANS          0x00100000 /* transaction insns available */
+#define AV_SPARC_FJFMAU         0x00200000 /* unfused multiply-add */
+#define AV_SPARC_IMA            0x00400000 /* integer multiply-add */
+#define AV_SPARC_ASI_CACHE_SPARING \
+                                0x00800000 /* cache sparing ASIs available */
 
 #define CORE_DUMP_USE_REGSET
 
@@ -162,31 +180,8 @@ typedef struct {
 #define ELF_ET_DYN_BASE         0x0000010000000000UL
 #define COMPAT_ELF_ET_DYN_BASE  0x0000000070000000UL
 
-
-/* This yields a mask that user programs can use to figure out what
-   instruction set this cpu supports.  */
-
-/* On Ultra, we support all of the v8 capabilities. */
-static inline unsigned int sparc64_elf_hwcap(void)
-{
-        unsigned int cap = (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR |
-                            HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV |
-                            HWCAP_SPARC_V9);
-
-        if (tlb_type == cheetah || tlb_type == cheetah_plus)
-                cap |= HWCAP_SPARC_ULTRA3;
-        else if (tlb_type == hypervisor) {
-                if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
-                    sun4v_chip_type == SUN4V_CHIP_NIAGARA2)
-                        cap |= HWCAP_SPARC_BLKINIT;
-                if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2)
-                        cap |= HWCAP_SPARC_N2;
-        }
-
-        return cap;
-}
-
-#define ELF_HWCAP       sparc64_elf_hwcap()
+extern unsigned long sparc64_elf_hwcap;
+#define ELF_HWCAP       sparc64_elf_hwcap
 
 /* This yields a string that ld.so will use to load implementation
    specific libraries for optimization.  This is more specific in
diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index 75686409be2..015a761eaa3 100644
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -2927,6 +2927,13 @@ extern unsigned long sun4v_ncs_request(unsigned long request,
 #define HV_FAST_FIRE_GET_PERFREG        0x120
 #define HV_FAST_FIRE_SET_PERFREG        0x121
 
+#define HV_FAST_REBOOT_DATA_SET         0x172
+
+#ifndef __ASSEMBLY__
+extern unsigned long sun4v_reboot_data_set(unsigned long ra,
+                                           unsigned long len);
+#endif
+
 /* Function numbers for HV_CORE_TRAP.  */
 #define HV_CORE_SET_VER                 0x00
 #define HV_CORE_PUTCHAR                 0x01
@@ -2940,16 +2947,23 @@ extern unsigned long sun4v_ncs_request(unsigned long request,
 #define HV_GRP_CORE                     0x0001
 #define HV_GRP_INTR                     0x0002
 #define HV_GRP_SOFT_STATE               0x0003
+#define HV_GRP_TM                       0x0080
 #define HV_GRP_PCI                      0x0100
 #define HV_GRP_LDOM                     0x0101
 #define HV_GRP_SVC_CHAN                 0x0102
 #define HV_GRP_NCS                      0x0103
 #define HV_GRP_RNG                      0x0104
+#define HV_GRP_PBOOT                    0x0105
+#define HV_GRP_TPM                      0x0107
+#define HV_GRP_SDIO                     0x0108
+#define HV_GRP_SDIO_ERR                 0x0109
+#define HV_GRP_REBOOT_DATA              0x0110
 #define HV_GRP_NIAG_PERF                0x0200
 #define HV_GRP_FIRE_PERF                0x0201
 #define HV_GRP_N2_CPU                   0x0202
 #define HV_GRP_NIU                      0x0204
 #define HV_GRP_VF_CPU                   0x0205
+#define HV_GRP_KT_CPU                   0x0209
 #define HV_GRP_DIAG                     0x0300
 
 #ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/irq_regs.h b/arch/sparc/include/asm/irq_regs.h
deleted file mode 100644
index 3dd9c0b7027..00000000000
--- a/arch/sparc/include/asm/irq_regs.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/irq_regs.h>
diff --git a/arch/sparc/include/asm/leon_pci.h b/arch/sparc/include/asm/leon_pci.h
index 42b4b31a82f..f48527ebdd8 100644
--- a/arch/sparc/include/asm/leon_pci.h
+++ b/arch/sparc/include/asm/leon_pci.h
@@ -12,7 +12,7 @@ struct leon_pci_info {
         struct pci_ops *ops;
         struct resource io_space;
         struct resource mem_space;
-        int (*map_irq)(struct pci_dev *dev, u8 slot, u8 pin);
+        int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin);
 };
 
 extern void leon_pci_init(struct platform_device *ofdev,
diff --git a/arch/sparc/include/asm/local.h b/arch/sparc/include/asm/local.h
deleted file mode 100644
index bc80815a435..00000000000
--- a/arch/sparc/include/asm/local.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef _SPARC_LOCAL_H
-#define _SPARC_LOCAL_H
-
-#include <asm-generic/local.h>
-
-#endif
diff --git a/arch/sparc/include/asm/local64.h b/arch/sparc/include/asm/local64.h
deleted file mode 100644
index 36c93b5cc23..00000000000
--- a/arch/sparc/include/asm/local64.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/local64.h>
diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h
index 56bbaadef64..edd3d3cde46 100644
--- a/arch/sparc/include/asm/prom.h
+++ b/arch/sparc/include/asm/prom.h
@@ -21,7 +21,7 @@
 #include <linux/of_pdt.h>
 #include <linux/proc_fs.h>
 #include <linux/mutex.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 #define OF_ROOT_NODE_ADDR_CELLS_DEFAULT 2
 #define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1
diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h
index b928b31424b..a0e1bcf843a 100644
--- a/arch/sparc/include/asm/ptrace.h
+++ b/arch/sparc/include/asm/ptrace.h
@@ -213,7 +213,6 @@ extern unsigned long profile_pc(struct pt_regs *);
 #else
 #define profile_pc(regs) instruction_pointer(regs)
 #endif
-extern void show_regs(struct pt_regs *);
 #endif /* (__KERNEL__) */
 
 #else /* __ASSEMBLY__ */
@@ -257,7 +256,6 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs)
 #define instruction_pointer(regs) ((regs)->pc)
 #define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP])
 unsigned long profile_pc(struct pt_regs *);
-extern void show_regs(struct pt_regs *);
 #endif /* (__KERNEL__) */
 
 #else /* (!__ASSEMBLY__) */
diff --git a/arch/sparc/include/asm/sigcontext.h b/arch/sparc/include/asm/sigcontext.h
index a1607d18035..69914d74813 100644
--- a/arch/sparc/include/asm/sigcontext.h
+++ b/arch/sparc/include/asm/sigcontext.h
@@ -45,6 +45,19 @@ typedef struct {
         int                     si_mask;
 } __siginfo32_t;
 
+#define __SIGC_MAXWIN   7
+
+typedef struct {
+        unsigned long locals[8];
+        unsigned long ins[8];
+} __siginfo_reg_window;
+
+typedef struct {
+        int                     wsaved;
+        __siginfo_reg_window    reg_window[__SIGC_MAXWIN];
+        unsigned long           rwbuf_stkptrs[__SIGC_MAXWIN];
+} __siginfo_rwin_t;
+
 #ifdef CONFIG_SPARC64
 typedef struct {
         unsigned   int si_float_regs [64];
@@ -73,6 +86,7 @@ struct sigcontext {
                 unsigned long   ss_size;
         }                       sigc_stack;
         unsigned long           sigc_mask;
+        __siginfo_rwin_t *      sigc_rwin_save;
 };
 
 #else
diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h
index 093f10843ff..01c51c70434 100644
--- a/arch/sparc/include/asm/smp_32.h
+++ b/arch/sparc/include/asm/smp_32.h
@@ -22,7 +22,7 @@
 
 #include <asm/ptrace.h>
 #include <asm/asi.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 
 /*
  *      Private routines/data
diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h
index 20bca895071..29862a9e906 100644
--- a/arch/sparc/include/asm/smp_64.h
+++ b/arch/sparc/include/asm/smp_64.h
@@ -27,7 +27,7 @@
  */
 
 #include <linux/bitops.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <asm/percpu.h>
 
 DECLARE_PER_CPU(cpumask_t, cpu_sibling_map);
diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h
index 5f5b8bf3f50..bcc98fc3528 100644
--- a/arch/sparc/include/asm/spinlock_32.h
+++ b/arch/sparc/include/asm/spinlock_32.h
@@ -131,6 +131,15 @@ static inline void arch_write_lock(arch_rwlock_t *rw)
         *(volatile __u32 *)&lp->lock = ~0U;
 }
 
+static void inline arch_write_unlock(arch_rwlock_t *lock)
+{
+        __asm__ __volatile__(
+"       st              %%g0, [%0]"
+        : /* no outputs */
+        : "r" (lock)
+        : "memory");
+}
+
 static inline int arch_write_trylock(arch_rwlock_t *rw)
 {
         unsigned int val;
@@ -175,8 +184,6 @@ static inline int __arch_read_trylock(arch_rwlock_t *rw)
         res; \
 })
 
-#define arch_write_unlock(rw)   do { (rw)->lock = 0; } while(0)
-
 #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
 #define arch_read_lock_flags(rw, flags)   arch_read_lock(rw)
 #define arch_write_lock_flags(rw, flags)  arch_write_lock(rw)
diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h
index 073936a8b27..96891769497 100644
--- a/arch/sparc/include/asm/spinlock_64.h
+++ b/arch/sparc/include/asm/spinlock_64.h
@@ -210,14 +210,8 @@ static int inline arch_write_trylock(arch_rwlock_t *lock)
         return result;
 }
 
-#define arch_read_lock(p)       arch_read_lock(p)
 #define arch_read_lock_flags(p, f) arch_read_lock(p)
-#define arch_read_trylock(p)    arch_read_trylock(p)
-#define arch_read_unlock(p)     arch_read_unlock(p)
-#define arch_write_lock(p)      arch_write_lock(p)
 #define arch_write_lock_flags(p, f) arch_write_lock(p)
-#define arch_write_unlock(p)    arch_write_unlock(p)
-#define arch_write_trylock(p)   arch_write_trylock(p)
 
 #define arch_read_can_lock(rw)          (!((rw)->lock & 0x80000000UL))
 #define arch_write_can_lock(rw) (!(rw)->lock)
diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h
index f0d0c40c44d..55a17c6efeb 100644
--- a/arch/sparc/include/asm/spitfire.h
+++ b/arch/sparc/include/asm/spitfire.h
@@ -42,6 +42,7 @@
 #define SUN4V_CHIP_INVALID      0x00
 #define SUN4V_CHIP_NIAGARA1     0x01
 #define SUN4V_CHIP_NIAGARA2     0x02
+#define SUN4V_CHIP_NIAGARA3     0x03
 #define SUN4V_CHIP_UNKNOWN      0xff
 
 #ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index 83c571d8c8a..1a8afd1ad04 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -133,29 +133,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         sub     TSB, 0x8, TSB;   \
         TSB_STORE(TSB, TAG);
 
-#define KTSB_LOAD_QUAD(TSB, REG) \
-        ldda            [TSB] ASI_NUCLEUS_QUAD_LDD, REG;
-
-#define KTSB_STORE(ADDR, VAL) \
-        stxa            VAL, [ADDR] ASI_N;
-
-#define KTSB_LOCK_TAG(TSB, REG1, REG2)  \
-99:     lduwa   [TSB] ASI_N, REG1;      \
-        sethi   %hi(TSB_TAG_LOCK_HIGH), REG2;\
-        andcc   REG1, REG2, %g0;        \
-        bne,pn  %icc, 99b;              \
-         nop;                           \
-        casa    [TSB] ASI_N, REG1, REG2;\
-        cmp     REG1, REG2;             \
-        bne,pn  %icc, 99b;              \
-         nop;                           \
-
-#define KTSB_WRITE(TSB, TTE, TAG) \
-        add     TSB, 0x8, TSB;   \
-        stxa    TTE, [TSB] ASI_N;     \
-        sub     TSB, 0x8, TSB;   \
-        stxa    TAG, [TSB] ASI_N;
-
         /* Do a kernel page table walk.  Leaves physical PTE pointer in
          * REG1.  Jumps to FAIL_LABEL on early page table walk termination.
          * VADDR will not be clobbered, but REG2 will.
@@ -239,6 +216,8 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         (KERNEL_TSB_SIZE_BYTES / 16)
 #define KERNEL_TSB4M_NENTRIES   4096
 
+#define KTSB_PHYS_SHIFT         15
+
         /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
          * on TSB hit.  REG1, REG2, REG3, and REG4 are used as temporaries
          * and the found TTE will be left in REG1.  REG3 and REG4 must
@@ -247,13 +226,22 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
          * VADDR and TAG will be preserved and not clobbered by this macro.
          */
 #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
-        sethi           %hi(swapper_tsb), REG1; \
+661:    sethi           %hi(swapper_tsb), REG1;                 \
         or              REG1, %lo(swapper_tsb), REG1; \
+        .section        .swapper_tsb_phys_patch, "ax"; \
+        .word           661b; \
+        .previous; \
+661:    nop; \
+        .section        .tsb_ldquad_phys_patch, "ax"; \
+        .word           661b; \
+        sllx            REG1, KTSB_PHYS_SHIFT, REG1; \
+        sllx            REG1, KTSB_PHYS_SHIFT, REG1; \
+        .previous; \
         srlx            VADDR, PAGE_SHIFT, REG2; \
         and             REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
         sllx            REG2, 4, REG2; \
         add             REG1, REG2, REG2; \
-        KTSB_LOAD_QUAD(REG2, REG3); \
+        TSB_LOAD_QUAD(REG2, REG3); \
         cmp             REG3, TAG; \
         be,a,pt         %xcc, OK_LABEL; \
          mov            REG4, REG1;
@@ -263,12 +251,21 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
          * we can make use of that for the index computation.
          */
 #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
-        sethi           %hi(swapper_4m_tsb), REG1; \
+661:    sethi           %hi(swapper_4m_tsb), REG1;           \
         or              REG1, %lo(swapper_4m_tsb), REG1; \
+        .section        .swapper_4m_tsb_phys_patch, "ax"; \
+        .word           661b; \
+        .previous; \
+661:    nop; \
+        .section        .tsb_ldquad_phys_patch, "ax"; \
+        .word           661b; \
+        sllx            REG1, KTSB_PHYS_SHIFT, REG1; \
+        sllx            REG1, KTSB_PHYS_SHIFT, REG1; \
+        .previous; \
         and             TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
         sllx            REG2, 4, REG2; \
         add             REG1, REG2, REG2; \
-        KTSB_LOAD_QUAD(REG2, REG3); \
+        TSB_LOAD_QUAD(REG2, REG3); \
         cmp             REG3, TAG; \
         be,a,pt         %xcc, OK_LABEL; \
          mov            REG4, REG1;
diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h
index bee4bf4be3a..9ed6ff679ab 100644
--- a/arch/sparc/include/asm/xor_64.h
+++ b/arch/sparc/include/asm/xor_64.h
@@ -65,6 +65,7 @@ static struct xor_block_template xor_block_niagara = {
 #define XOR_SELECT_TEMPLATE(FASTEST) \
         ((tlb_type == hypervisor && \
           (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || \
-           sun4v_chip_type == SUN4V_CHIP_NIAGARA2)) ? \
+           sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || \
+           sun4v_chip_type == SUN4V_CHIP_NIAGARA3)) ? \
          &xor_block_niagara : \
          &xor_block_VIS)