8 files changed, 74 insertions, 36 deletions

diff --git a/include/asm-sparc64/atomic.h b/include/asm-sparc64/atomic.h
index 25256bdc8aae..468eb48d8142 100644
--- a/include/asm-sparc64/atomic.h
+++ b/include/asm-sparc64/atomic.h
@@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *);
 ({                                                              \
         int c, old;                                             \
         c = atomic_read(v);                                     \
-        while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \
+        for (;;) {                                              \
+                if (unlikely(c == (u)))                         \
+                        break;                                  \
+                old = atomic_cmpxchg((v), c, c + (a));          \
+                if (likely(old == c))                           \
+                        break;                                  \
                 c = old;                                        \
-        c != (u);                                               \
+        }                                                       \
+        likely(c != (u));                                       \
 })
 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
 
diff --git a/include/asm-sparc64/cache.h b/include/asm-sparc64/cache.h
index f7d35a2ae9b8..e9df17acedde 100644
--- a/include/asm-sparc64/cache.h
+++ b/include/asm-sparc64/cache.h
@@ -13,4 +13,6 @@
 #define        SMP_CACHE_BYTES_SHIFT    6
 #define        SMP_CACHE_BYTES          (1 << SMP_CACHE_BYTES_SHIFT) /* L2 cache line size. */
 
+#define __read_mostly __attribute__((__section__(".data.read_mostly")))
+
 #endif
diff --git a/include/asm-sparc64/cpudata.h b/include/asm-sparc64/cpudata.h
index c66a81bbc84d..9d6a6dbaf126 100644
--- a/include/asm-sparc64/cpudata.h
+++ b/include/asm-sparc64/cpudata.h
@@ -71,7 +71,8 @@ struct trap_per_cpu {
 /* Dcache line 7: Physical addresses of CPU send mondo block and CPU list.  */
         unsigned long           cpu_mondo_block_pa;
         unsigned long           cpu_list_pa;
-        unsigned long           __pad1[2];
+        unsigned long           tsb_huge;
+        unsigned long           tsb_huge_temp;
 
 /* Dcache line 8: Unused, needed to keep trap_block a power-of-2 in size.  */
         unsigned long           __pad2[4];
@@ -116,6 +117,8 @@ extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch,
 #define TRAP_PER_CPU_FAULT_INFO         0x40
 #define TRAP_PER_CPU_CPU_MONDO_BLOCK_PA 0xc0
 #define TRAP_PER_CPU_CPU_LIST_PA        0xc8
+#define TRAP_PER_CPU_TSB_HUGE           0xd0
+#define TRAP_PER_CPU_TSB_HUGE_TEMP      0xd8
 
 #define TRAP_BLOCK_SZ_SHIFT             8
 
diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h
index 230ba678d3b0..2d4f2ea9568a 100644
--- a/include/asm-sparc64/mmu.h
+++ b/include/asm-sparc64/mmu.h
@@ -90,18 +90,39 @@ extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte
 extern void tsb_flush(unsigned long ent, unsigned long tag);
 extern void tsb_init(struct tsb *tsb, unsigned long size);
 
-typedef struct {
-        spinlock_t              lock;
-        unsigned long           sparc64_ctx_val;
+struct tsb_config {
         struct tsb              *tsb;
         unsigned long           tsb_rss_limit;
         unsigned long           tsb_nentries;
         unsigned long           tsb_reg_val;
         unsigned long           tsb_map_vaddr;
         unsigned long           tsb_map_pte;
-        struct hv_tsb_descr     tsb_descr;
+};
+
+#define MM_TSB_BASE     0
+
+#ifdef CONFIG_HUGETLB_PAGE
+#define MM_TSB_HUGE     1
+#define MM_NUM_TSBS     2
+#else
+#define MM_NUM_TSBS     1
+#endif
+
+typedef struct {
+        spinlock_t              lock;
+        unsigned long           sparc64_ctx_val;
+        unsigned long           huge_pte_count;
+        struct tsb_config       tsb_block[MM_NUM_TSBS];
+        struct hv_tsb_descr     tsb_descr[MM_NUM_TSBS];
 } mm_context_t;
 
 #endif /* !__ASSEMBLY__ */
 
+#define TSB_CONFIG_TSB          0x00
+#define TSB_CONFIG_RSS_LIMIT    0x08
+#define TSB_CONFIG_NENTRIES     0x10
+#define TSB_CONFIG_REG_VAL      0x18
+#define TSB_CONFIG_MAP_VADDR    0x20
+#define TSB_CONFIG_MAP_PTE      0x28
+
 #endif /* __MMU_H */
diff --git a/include/asm-sparc64/mmu_context.h b/include/asm-sparc64/mmu_context.h
index e7974321d052..2337eb487719 100644
--- a/include/asm-sparc64/mmu_context.h
+++ b/include/asm-sparc64/mmu_context.h
@@ -29,20 +29,25 @@ extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
 extern void destroy_context(struct mm_struct *mm);
 
 extern void __tsb_context_switch(unsigned long pgd_pa,
-                                 unsigned long tsb_reg,
-                                 unsigned long tsb_vaddr,
-                                 unsigned long tsb_pte,
+                                 struct tsb_config *tsb_base,
+                                 struct tsb_config *tsb_huge,
                                  unsigned long tsb_descr_pa);
 
 static inline void tsb_context_switch(struct mm_struct *mm)
 {
-        __tsb_context_switch(__pa(mm->pgd), mm->context.tsb_reg_val,
-                             mm->context.tsb_map_vaddr,
-                             mm->context.tsb_map_pte,
-                             __pa(&mm->context.tsb_descr));
+        __tsb_context_switch(__pa(mm->pgd),
+                             &mm->context.tsb_block[0],
+#ifdef CONFIG_HUGETLB_PAGE
+                             (mm->context.tsb_block[1].tsb ?
+                              &mm->context.tsb_block[1] :
+                              NULL)
+#else
+                             NULL
+#endif
+                             , __pa(&mm->context.tsb_descr[0]));
 }
 
-extern void tsb_grow(struct mm_struct *mm, unsigned long mm_rss);
+extern void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long mm_rss);
 #ifdef CONFIG_SMP
 extern void smp_tsb_sync(struct mm_struct *mm);
 #else
diff --git a/include/asm-sparc64/page.h b/include/asm-sparc64/page.h
index fcb2812265f4..66fe4ac59fd6 100644
--- a/include/asm-sparc64/page.h
+++ b/include/asm-sparc64/page.h
@@ -30,6 +30,23 @@
 
 #ifdef __KERNEL__
 
+#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
+#define HPAGE_SHIFT             22
+#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
+#define HPAGE_SHIFT             19
+#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
+#define HPAGE_SHIFT             16
+#endif
+
+#ifdef CONFIG_HUGETLB_PAGE
+#define HPAGE_SIZE              (_AC(1,UL) << HPAGE_SHIFT)
+#define HPAGE_MASK              (~(HPAGE_SIZE - 1UL))
+#define HUGETLB_PAGE_ORDER      (HPAGE_SHIFT - PAGE_SHIFT)
+#define ARCH_HAS_SETCLEAR_HUGE_PTE
+#define ARCH_HAS_HUGETLB_PREFAULT_HOOK
+#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
+#endif
+
 #ifndef __ASSEMBLY__
 
 extern void _clear_page(void *page);
@@ -90,23 +107,6 @@ typedef unsigned long pgprot_t;
 
 #endif /* (STRICT_MM_TYPECHECKS) */
 
-#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
-#define HPAGE_SHIFT             22
-#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
-#define HPAGE_SHIFT             19
-#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
-#define HPAGE_SHIFT             16
-#endif
-
-#ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SIZE              (_AC(1,UL) << HPAGE_SHIFT)
-#define HPAGE_MASK              (~(HPAGE_SIZE - 1UL))
-#define HUGETLB_PAGE_ORDER      (HPAGE_SHIFT - PAGE_SHIFT)
-#define ARCH_HAS_SETCLEAR_HUGE_PTE
-#define ARCH_HAS_HUGETLB_PREFAULT_HOOK
-#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
-#endif
-
 #define TASK_UNMAPPED_BASE      (test_thread_flag(TIF_32BIT) ? \
                                  (_AC(0x0000000070000000,UL)) : \
                                  (_AC(0xfffff80000000000,UL) + (1UL << 32UL)))
diff --git a/include/asm-sparc64/percpu.h b/include/asm-sparc64/percpu.h
index aea4e51e7cd1..82032e159a76 100644
--- a/include/asm-sparc64/percpu.h
+++ b/include/asm-sparc64/percpu.h
@@ -26,10 +26,9 @@ register unsigned long __local_per_cpu_offset asm("g5");
 #define percpu_modcopy(pcpudst, src, size)                      \
 do {                                                            \
         unsigned int __i;                                       \
-        for (__i = 0; __i < NR_CPUS; __i++)                     \
-                if (cpu_possible(__i))                          \
-                        memcpy((pcpudst)+__per_cpu_offset(__i), \
-                               (src), (size));                  \
+        for_each_cpu(__i)                                       \
+                memcpy((pcpudst)+__per_cpu_offset(__i),         \
+                       (src), (size));                          \
 } while (0)
 #else /* ! SMP */
 
diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index ed4124edf837..c44e7466534e 100644
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -105,6 +105,7 @@
 #define _PAGE_RES1_4U     _AC(0x0002000000000000,UL) /* Reserved             */
 #define _PAGE_SZ32MB_4U   _AC(0x0001000000000000,UL) /* (Panther) 32MB page  */
 #define _PAGE_SZ256MB_4U  _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
+#define _PAGE_SZALL_4U    _AC(0x6001000000000000,UL) /* All pgsz bits        */
 #define _PAGE_SN_4U       _AC(0x0000800000000000,UL) /* (Cheetah) Snoop      */
 #define _PAGE_RES2_4U     _AC(0x0000780000000000,UL) /* Reserved             */
 #define _PAGE_PADDR_4U    _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13]  */
@@ -150,6 +151,7 @@
 #define _PAGE_SZ512K_4V   _AC(0x0000000000000002,UL) /* 512K Page            */
 #define _PAGE_SZ64K_4V    _AC(0x0000000000000001,UL) /* 64K Page             */
 #define _PAGE_SZ8K_4V     _AC(0x0000000000000000,UL) /* 8K Page              */
+#define _PAGE_SZALL_4V    _AC(0x0000000000000007,UL) /* All pgsz bits        */
 
 #if PAGE_SHIFT == 13
 #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U