aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-11-15 00:16:30 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2013-11-15 00:16:30 -0500
commit1b2722752fe7bf21e76c87080add6d6c40af339f (patch)
tree2b98fb5d283c20a7d879bd8b3ed9d9ec67592f10 /arch
parent91838e2dab460ba589fb90db0fe1f504f5c04f12 (diff)
parent812cb83a56a908729c453a7db3fb2c262119bc9d (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next
Pull sparc update from David Miller: 1) Implement support for up to 47-bit physical addresses on sparc64. 2) Support HAVE_CONTEXT_TRACKING on sparc64, from Kirill Tkhai. 3) Fix Simba bridge window calculations, from Kjetil Oftedal. * git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next: sparc64: Implement HAVE_CONTEXT_TRACKING sparc64: Add self-IPI support for smp_send_reschedule() sparc: PCI: Fix incorrect address calculation of PCI Bridge windows on Simba-bridges sparc64: Encode huge PMDs using PTE encoding. sparc64: Move to 64-bit PGDs and PMDs. sparc64: Move from 4MB to 8MB huge pages. sparc64: Make PAGE_OFFSET variable. sparc64: Fix inconsistent max-physical-address defines. sparc64: Document the shift counts used to validate linear kernel addresses. sparc64: Define PAGE_OFFSET in terms of physical address bits. sparc64: Use PAGE_OFFSET instead of a magic constant. sparc64: Clean up 64-bit mmap exclusion defines.
Diffstat (limited to 'arch')
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/sparc/include/asm/mmu_64.h1
-rw-r--r--arch/sparc/include/asm/page_64.h49
-rw-r--r--arch/sparc/include/asm/pgtable_64.h209
-rw-r--r--arch/sparc/include/asm/sparsemem.h6
-rw-r--r--arch/sparc/include/asm/thread_info_64.h3
-rw-r--r--arch/sparc/include/asm/tsb.h105
-rw-r--r--arch/sparc/kernel/entry.h1
-rw-r--r--arch/sparc/kernel/kgdb_64.c5
-rw-r--r--arch/sparc/kernel/kprobes.c7
-rw-r--r--arch/sparc/kernel/ktlb.S30
-rw-r--r--arch/sparc/kernel/pci.c4
-rw-r--r--arch/sparc/kernel/process_64.c2
-rw-r--r--arch/sparc/kernel/ptrace_64.c10
-rw-r--r--arch/sparc/kernel/rtrap_64.S8
-rw-r--r--arch/sparc/kernel/signal_64.c13
-rw-r--r--arch/sparc/kernel/smp_64.c9
-rw-r--r--arch/sparc/kernel/sun4v_tlb_miss.S2
-rw-r--r--arch/sparc/kernel/sys_sparc_64.c6
-rw-r--r--arch/sparc/kernel/syscalls.S8
-rw-r--r--arch/sparc/kernel/traps_64.c85
-rw-r--r--arch/sparc/kernel/tsb.S2
-rw-r--r--arch/sparc/kernel/unaligned_64.c16
-rw-r--r--arch/sparc/kernel/vmlinux.lds.S5
-rw-r--r--arch/sparc/lib/clear_page.S4
-rw-r--r--arch/sparc/lib/copy_page.S4
-rw-r--r--arch/sparc/mm/fault_64.c14
-rw-r--r--arch/sparc/mm/gup.c9
-rw-r--r--arch/sparc/mm/hugetlbpage.c2
-rw-r--r--arch/sparc/mm/init_64.c282
-rw-r--r--arch/sparc/mm/init_64.h4
-rw-r--r--arch/sparc/mm/tlb.c13
-rw-r--r--arch/sparc/mm/tsb.c13
-rw-r--r--arch/sparc/mm/ultra.S12
34 files changed, 510 insertions, 434 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 8591b201d9cc..05fcfc634fcb 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -63,6 +63,7 @@ config SPARC64
63 select HAVE_DYNAMIC_FTRACE 63 select HAVE_DYNAMIC_FTRACE
64 select HAVE_FTRACE_MCOUNT_RECORD 64 select HAVE_FTRACE_MCOUNT_RECORD
65 select HAVE_SYSCALL_TRACEPOINTS 65 select HAVE_SYSCALL_TRACEPOINTS
66 select HAVE_CONTEXT_TRACKING
66 select HAVE_DEBUG_KMEMLEAK 67 select HAVE_DEBUG_KMEMLEAK
67 select RTC_DRV_CMOS 68 select RTC_DRV_CMOS
68 select RTC_DRV_BQ4802 69 select RTC_DRV_BQ4802
diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h
index 76092c4dd277..f668797ae234 100644
--- a/arch/sparc/include/asm/mmu_64.h
+++ b/arch/sparc/include/asm/mmu_64.h
@@ -93,7 +93,6 @@ typedef struct {
93 spinlock_t lock; 93 spinlock_t lock;
94 unsigned long sparc64_ctx_val; 94 unsigned long sparc64_ctx_val;
95 unsigned long huge_pte_count; 95 unsigned long huge_pte_count;
96 struct page *pgtable_page;
97 struct tsb_config tsb_block[MM_NUM_TSBS]; 96 struct tsb_config tsb_block[MM_NUM_TSBS];
98 struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; 97 struct hv_tsb_descr tsb_descr[MM_NUM_TSBS];
99} mm_context_t; 98} mm_context_t;
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index e15538899f3d..aac53fcea807 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -15,7 +15,10 @@
15#define DCACHE_ALIASING_POSSIBLE 15#define DCACHE_ALIASING_POSSIBLE
16#endif 16#endif
17 17
18#define HPAGE_SHIFT 22 18#define HPAGE_SHIFT 23
19#define REAL_HPAGE_SHIFT 22
20
21#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT)
19 22
20#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 23#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
21#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) 24#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT)
@@ -53,8 +56,8 @@ extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct pag
53/* These are used to make use of C type-checking.. */ 56/* These are used to make use of C type-checking.. */
54typedef struct { unsigned long pte; } pte_t; 57typedef struct { unsigned long pte; } pte_t;
55typedef struct { unsigned long iopte; } iopte_t; 58typedef struct { unsigned long iopte; } iopte_t;
56typedef struct { unsigned int pmd; } pmd_t; 59typedef struct { unsigned long pmd; } pmd_t;
57typedef struct { unsigned int pgd; } pgd_t; 60typedef struct { unsigned long pgd; } pgd_t;
58typedef struct { unsigned long pgprot; } pgprot_t; 61typedef struct { unsigned long pgprot; } pgprot_t;
59 62
60#define pte_val(x) ((x).pte) 63#define pte_val(x) ((x).pte)
@@ -73,8 +76,8 @@ typedef struct { unsigned long pgprot; } pgprot_t;
73/* .. while these make it easier on the compiler */ 76/* .. while these make it easier on the compiler */
74typedef unsigned long pte_t; 77typedef unsigned long pte_t;
75typedef unsigned long iopte_t; 78typedef unsigned long iopte_t;
76typedef unsigned int pmd_t; 79typedef unsigned long pmd_t;
77typedef unsigned int pgd_t; 80typedef unsigned long pgd_t;
78typedef unsigned long pgprot_t; 81typedef unsigned long pgprot_t;
79 82
80#define pte_val(x) (x) 83#define pte_val(x) (x)
@@ -93,18 +96,44 @@ typedef unsigned long pgprot_t;
93 96
94typedef pte_t *pgtable_t; 97typedef pte_t *pgtable_t;
95 98
99/* These two values define the virtual address space range in which we
100 * must forbid 64-bit user processes from making mappings. It used to
101 * represent precisely the virtual address space hole present in most
102 * early sparc64 chips including UltraSPARC-I. But now it also is
103 * further constrained by the limits of our page tables, which is
104 * 43-bits of virtual address.
105 */
106#define SPARC64_VA_HOLE_TOP _AC(0xfffffc0000000000,UL)
107#define SPARC64_VA_HOLE_BOTTOM _AC(0x0000040000000000,UL)
108
109/* The next two defines specify the actual exclusion region we
110 * enforce, wherein we use a 4GB red zone on each side of the VA hole.
111 */
112#define VA_EXCLUDE_START (SPARC64_VA_HOLE_BOTTOM - (1UL << 32UL))
113#define VA_EXCLUDE_END (SPARC64_VA_HOLE_TOP + (1UL << 32UL))
114
96#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ 115#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \
97 (_AC(0x0000000070000000,UL)) : \ 116 _AC(0x0000000070000000,UL) : \
98 (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) 117 VA_EXCLUDE_END)
99 118
100#include <asm-generic/memory_model.h> 119#include <asm-generic/memory_model.h>
101 120
121#define PAGE_OFFSET_BY_BITS(X) (-(_AC(1,UL) << (X)))
122extern unsigned long PAGE_OFFSET;
123
102#endif /* !(__ASSEMBLY__) */ 124#endif /* !(__ASSEMBLY__) */
103 125
104/* We used to stick this into a hard-coded global register (%g4) 126/* The maximum number of physical memory address bits we support, this
105 * but that does not make sense anymore. 127 * is used to size various tables used to manage kernel TLB misses and
128 * also the sparsemem code.
129 */
130#define MAX_PHYS_ADDRESS_BITS 47
131
132/* These two shift counts are used when indexing sparc64_valid_addr_bitmap
133 * and kpte_linear_bitmap.
106 */ 134 */
107#define PAGE_OFFSET _AC(0xFFFFF80000000000,UL) 135#define ILOG2_4MB 22
136#define ILOG2_256MB 28
108 137
109#ifndef __ASSEMBLY__ 138#ifndef __ASSEMBLY__
110 139
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 36760317814f..8358dc144959 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -48,18 +48,18 @@
48/* PMD_SHIFT determines the size of the area a second-level page 48/* PMD_SHIFT determines the size of the area a second-level page
49 * table can map 49 * table can map
50 */ 50 */
51#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4)) 51#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3))
52#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) 52#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT)
53#define PMD_MASK (~(PMD_SIZE-1)) 53#define PMD_MASK (~(PMD_SIZE-1))
54#define PMD_BITS (PAGE_SHIFT - 2) 54#define PMD_BITS (PAGE_SHIFT - 3)
55 55
56/* PGDIR_SHIFT determines what a third-level page table entry can map */ 56/* PGDIR_SHIFT determines what a third-level page table entry can map */
57#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4) + PMD_BITS) 57#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)
58#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) 58#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT)
59#define PGDIR_MASK (~(PGDIR_SIZE-1)) 59#define PGDIR_MASK (~(PGDIR_SIZE-1))
60#define PGDIR_BITS (PAGE_SHIFT - 2) 60#define PGDIR_BITS (PAGE_SHIFT - 3)
61 61
62#if (PGDIR_SHIFT + PGDIR_BITS) != 44 62#if (PGDIR_SHIFT + PGDIR_BITS) != 43
63#error Page table parameters do not cover virtual address space properly. 63#error Page table parameters do not cover virtual address space properly.
64#endif 64#endif
65 65
@@ -67,35 +67,12 @@
67#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. 67#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages.
68#endif 68#endif
69 69
70/* PMDs point to PTE tables which are 4K aligned. */
71#define PMD_PADDR _AC(0xfffffffe,UL)
72#define PMD_PADDR_SHIFT _AC(11,UL)
73
74#define PMD_ISHUGE _AC(0x00000001,UL)
75
76/* This is the PMD layout when PMD_ISHUGE is set. With 4MB huge
77 * pages, this frees up a bunch of bits in the layout that we can
78 * use for the protection settings and software metadata.
79 */
80#define PMD_HUGE_PADDR _AC(0xfffff800,UL)
81#define PMD_HUGE_PROTBITS _AC(0x000007ff,UL)
82#define PMD_HUGE_PRESENT _AC(0x00000400,UL)
83#define PMD_HUGE_WRITE _AC(0x00000200,UL)
84#define PMD_HUGE_DIRTY _AC(0x00000100,UL)
85#define PMD_HUGE_ACCESSED _AC(0x00000080,UL)
86#define PMD_HUGE_EXEC _AC(0x00000040,UL)
87#define PMD_HUGE_SPLITTING _AC(0x00000020,UL)
88
89/* PGDs point to PMD tables which are 8K aligned. */
90#define PGD_PADDR _AC(0xfffffffc,UL)
91#define PGD_PADDR_SHIFT _AC(11,UL)
92
93#ifndef __ASSEMBLY__ 70#ifndef __ASSEMBLY__
94 71
95#include <linux/sched.h> 72#include <linux/sched.h>
96 73
97/* Entries per page directory level. */ 74/* Entries per page directory level. */
98#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-4)) 75#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
99#define PTRS_PER_PMD (1UL << PMD_BITS) 76#define PTRS_PER_PMD (1UL << PMD_BITS)
100#define PTRS_PER_PGD (1UL << PGDIR_BITS) 77#define PTRS_PER_PGD (1UL << PGDIR_BITS)
101 78
@@ -112,6 +89,7 @@
112#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ 89#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */
113#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ 90#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
114#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ 91#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
92#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */
115 93
116/* Advertise support for _PAGE_SPECIAL */ 94/* Advertise support for _PAGE_SPECIAL */
117#define __HAVE_ARCH_PTE_SPECIAL 95#define __HAVE_ARCH_PTE_SPECIAL
@@ -125,6 +103,7 @@
125#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ 103#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */
126#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ 104#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
127#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ 105#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */
106#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */
128#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ 107#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */
129#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ 108#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */
130#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ 109#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
@@ -155,6 +134,7 @@
155#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ 134#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */
156#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ 135#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */
157#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ 136#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */
137#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */
158#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ 138#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */
159#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ 139#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */
160#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ 140#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */
@@ -180,6 +160,10 @@
180#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U 160#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U
181#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V 161#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V
182 162
163#if REAL_HPAGE_SHIFT != 22
164#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up
165#endif
166
183#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U 167#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U
184#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V 168#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V
185 169
@@ -239,16 +223,13 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
239#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) 223#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
240 224
241#ifdef CONFIG_TRANSPARENT_HUGEPAGE 225#ifdef CONFIG_TRANSPARENT_HUGEPAGE
242extern pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot); 226static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
243#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
244
245extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot);
246
247static inline pmd_t pmd_mkhuge(pmd_t pmd)
248{ 227{
249 /* Do nothing, mk_pmd() does this part. */ 228 pte_t pte = pfn_pte(page_nr, pgprot);
250 return pmd; 229
230 return __pmd(pte_val(pte));
251} 231}
232#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot))
252#endif 233#endif
253 234
254/* This one can be done with two shifts. */ 235/* This one can be done with two shifts. */
@@ -309,14 +290,25 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
309 : "=r" (mask), "=r" (tmp) 290 : "=r" (mask), "=r" (tmp)
310 : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | 291 : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
311 _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | 292 _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U |
312 _PAGE_SPECIAL), 293 _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U),
313 "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | 294 "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
314 _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | 295 _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V |
315 _PAGE_SPECIAL)); 296 _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V));
316 297
317 return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); 298 return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
318} 299}
319 300
301#ifdef CONFIG_TRANSPARENT_HUGEPAGE
302static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
303{
304 pte_t pte = __pte(pmd_val(pmd));
305
306 pte = pte_modify(pte, newprot);
307
308 return __pmd(pte_val(pte));
309}
310#endif
311
320static inline pte_t pgoff_to_pte(unsigned long off) 312static inline pte_t pgoff_to_pte(unsigned long off)
321{ 313{
322 off <<= PAGE_SHIFT; 314 off <<= PAGE_SHIFT;
@@ -357,7 +349,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot)
357 */ 349 */
358#define pgprot_noncached pgprot_noncached 350#define pgprot_noncached pgprot_noncached
359 351
360#ifdef CONFIG_HUGETLB_PAGE 352#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
361static inline pte_t pte_mkhuge(pte_t pte) 353static inline pte_t pte_mkhuge(pte_t pte)
362{ 354{
363 unsigned long mask; 355 unsigned long mask;
@@ -375,6 +367,17 @@ static inline pte_t pte_mkhuge(pte_t pte)
375 367
376 return __pte(pte_val(pte) | mask); 368 return __pte(pte_val(pte) | mask);
377} 369}
370#ifdef CONFIG_TRANSPARENT_HUGEPAGE
371static inline pmd_t pmd_mkhuge(pmd_t pmd)
372{
373 pte_t pte = __pte(pmd_val(pmd));
374
375 pte = pte_mkhuge(pte);
376 pte_val(pte) |= _PAGE_PMD_HUGE;
377
378 return __pmd(pte_val(pte));
379}
380#endif
378#endif 381#endif
379 382
380static inline pte_t pte_mkdirty(pte_t pte) 383static inline pte_t pte_mkdirty(pte_t pte)
@@ -626,91 +629,130 @@ static inline unsigned long pte_special(pte_t pte)
626 return pte_val(pte) & _PAGE_SPECIAL; 629 return pte_val(pte) & _PAGE_SPECIAL;
627} 630}
628 631
629static inline int pmd_large(pmd_t pmd) 632static inline unsigned long pmd_large(pmd_t pmd)
630{ 633{
631 return (pmd_val(pmd) & (PMD_ISHUGE | PMD_HUGE_PRESENT)) == 634 pte_t pte = __pte(pmd_val(pmd));
632 (PMD_ISHUGE | PMD_HUGE_PRESENT); 635
636 return (pte_val(pte) & _PAGE_PMD_HUGE) && pte_present(pte);
633} 637}
634 638
635#ifdef CONFIG_TRANSPARENT_HUGEPAGE 639#ifdef CONFIG_TRANSPARENT_HUGEPAGE
636static inline int pmd_young(pmd_t pmd) 640static inline unsigned long pmd_young(pmd_t pmd)
637{ 641{
638 return pmd_val(pmd) & PMD_HUGE_ACCESSED; 642 pte_t pte = __pte(pmd_val(pmd));
643
644 return pte_young(pte);
639} 645}
640 646
641static inline int pmd_write(pmd_t pmd) 647static inline unsigned long pmd_write(pmd_t pmd)
642{ 648{
643 return pmd_val(pmd) & PMD_HUGE_WRITE; 649 pte_t pte = __pte(pmd_val(pmd));
650
651 return pte_write(pte);
644} 652}
645 653
646static inline unsigned long pmd_pfn(pmd_t pmd) 654static inline unsigned long pmd_pfn(pmd_t pmd)
647{ 655{
648 unsigned long val = pmd_val(pmd) & PMD_HUGE_PADDR; 656 pte_t pte = __pte(pmd_val(pmd));
649 657
650 return val >> (PAGE_SHIFT - PMD_PADDR_SHIFT); 658 return pte_pfn(pte);
651} 659}
652 660
653static inline int pmd_trans_splitting(pmd_t pmd) 661static inline unsigned long pmd_trans_huge(pmd_t pmd)
654{ 662{
655 return (pmd_val(pmd) & (PMD_ISHUGE|PMD_HUGE_SPLITTING)) == 663 pte_t pte = __pte(pmd_val(pmd));
656 (PMD_ISHUGE|PMD_HUGE_SPLITTING); 664
665 return pte_val(pte) & _PAGE_PMD_HUGE;
657} 666}
658 667
659static inline int pmd_trans_huge(pmd_t pmd) 668static inline unsigned long pmd_trans_splitting(pmd_t pmd)
660{ 669{
661 return pmd_val(pmd) & PMD_ISHUGE; 670 pte_t pte = __pte(pmd_val(pmd));
671
672 return pmd_trans_huge(pmd) && pte_special(pte);
662} 673}
663 674
664#define has_transparent_hugepage() 1 675#define has_transparent_hugepage() 1
665 676
666static inline pmd_t pmd_mkold(pmd_t pmd) 677static inline pmd_t pmd_mkold(pmd_t pmd)
667{ 678{
668 pmd_val(pmd) &= ~PMD_HUGE_ACCESSED; 679 pte_t pte = __pte(pmd_val(pmd));
669 return pmd; 680
681 pte = pte_mkold(pte);
682
683 return __pmd(pte_val(pte));
670} 684}
671 685
672static inline pmd_t pmd_wrprotect(pmd_t pmd) 686static inline pmd_t pmd_wrprotect(pmd_t pmd)
673{ 687{
674 pmd_val(pmd) &= ~PMD_HUGE_WRITE; 688 pte_t pte = __pte(pmd_val(pmd));
675 return pmd; 689
690 pte = pte_wrprotect(pte);
691
692 return __pmd(pte_val(pte));
676} 693}
677 694
678static inline pmd_t pmd_mkdirty(pmd_t pmd) 695static inline pmd_t pmd_mkdirty(pmd_t pmd)
679{ 696{
680 pmd_val(pmd) |= PMD_HUGE_DIRTY; 697 pte_t pte = __pte(pmd_val(pmd));
681 return pmd; 698
699 pte = pte_mkdirty(pte);
700
701 return __pmd(pte_val(pte));
682} 702}
683 703
684static inline pmd_t pmd_mkyoung(pmd_t pmd) 704static inline pmd_t pmd_mkyoung(pmd_t pmd)
685{ 705{
686 pmd_val(pmd) |= PMD_HUGE_ACCESSED; 706 pte_t pte = __pte(pmd_val(pmd));
687 return pmd; 707
708 pte = pte_mkyoung(pte);
709
710 return __pmd(pte_val(pte));
688} 711}
689 712
690static inline pmd_t pmd_mkwrite(pmd_t pmd) 713static inline pmd_t pmd_mkwrite(pmd_t pmd)
691{ 714{
692 pmd_val(pmd) |= PMD_HUGE_WRITE; 715 pte_t pte = __pte(pmd_val(pmd));
693 return pmd; 716
717 pte = pte_mkwrite(pte);
718
719 return __pmd(pte_val(pte));
694} 720}
695 721
696static inline pmd_t pmd_mknotpresent(pmd_t pmd) 722static inline pmd_t pmd_mknotpresent(pmd_t pmd)
697{ 723{
698 pmd_val(pmd) &= ~PMD_HUGE_PRESENT; 724 unsigned long mask;
725
726 if (tlb_type == hypervisor)
727 mask = _PAGE_PRESENT_4V;
728 else
729 mask = _PAGE_PRESENT_4U;
730
731 pmd_val(pmd) &= ~mask;
732
699 return pmd; 733 return pmd;
700} 734}
701 735
702static inline pmd_t pmd_mksplitting(pmd_t pmd) 736static inline pmd_t pmd_mksplitting(pmd_t pmd)
703{ 737{
704 pmd_val(pmd) |= PMD_HUGE_SPLITTING; 738 pte_t pte = __pte(pmd_val(pmd));
705 return pmd; 739
740 pte = pte_mkspecial(pte);
741
742 return __pmd(pte_val(pte));
706} 743}
707 744
708extern pgprot_t pmd_pgprot(pmd_t entry); 745static inline pgprot_t pmd_pgprot(pmd_t entry)
746{
747 unsigned long val = pmd_val(entry);
748
749 return __pgprot(val);
750}
709#endif 751#endif
710 752
711static inline int pmd_present(pmd_t pmd) 753static inline int pmd_present(pmd_t pmd)
712{ 754{
713 return pmd_val(pmd) != 0U; 755 return pmd_val(pmd) != 0UL;
714} 756}
715 757
716#define pmd_none(pmd) (!pmd_val(pmd)) 758#define pmd_none(pmd) (!pmd_val(pmd))
@@ -728,33 +770,32 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
728 770
729static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) 771static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
730{ 772{
731 unsigned long val = __pa((unsigned long) (ptep)) >> PMD_PADDR_SHIFT; 773 unsigned long val = __pa((unsigned long) (ptep));
732 774
733 pmd_val(*pmdp) = val; 775 pmd_val(*pmdp) = val;
734} 776}
735 777
736#define pud_set(pudp, pmdp) \ 778#define pud_set(pudp, pmdp) \
737 (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)) >> PGD_PADDR_SHIFT)) 779 (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
738static inline unsigned long __pmd_page(pmd_t pmd) 780static inline unsigned long __pmd_page(pmd_t pmd)
739{ 781{
740 unsigned long paddr = (unsigned long) pmd_val(pmd); 782 pte_t pte = __pte(pmd_val(pmd));
741#ifdef CONFIG_TRANSPARENT_HUGEPAGE 783 unsigned long pfn;
742 if (pmd_val(pmd) & PMD_ISHUGE) 784
743 paddr &= PMD_HUGE_PADDR; 785 pfn = pte_pfn(pte);
744#endif 786
745 paddr <<= PMD_PADDR_SHIFT; 787 return ((unsigned long) __va(pfn << PAGE_SHIFT));
746 return ((unsigned long) __va(paddr));
747} 788}
748#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) 789#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd))
749#define pud_page_vaddr(pud) \ 790#define pud_page_vaddr(pud) \
750 ((unsigned long) __va((((unsigned long)pud_val(pud))<<PGD_PADDR_SHIFT))) 791 ((unsigned long) __va(pud_val(pud)))
751#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) 792#define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud))
752#define pmd_bad(pmd) (0) 793#define pmd_bad(pmd) (0)
753#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0U) 794#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
754#define pud_none(pud) (!pud_val(pud)) 795#define pud_none(pud) (!pud_val(pud))
755#define pud_bad(pud) (0) 796#define pud_bad(pud) (0)
756#define pud_present(pud) (pud_val(pud) != 0U) 797#define pud_present(pud) (pud_val(pud) != 0U)
757#define pud_clear(pudp) (pud_val(*(pudp)) = 0U) 798#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
758 799
759/* Same in both SUN4V and SUN4U. */ 800/* Same in both SUN4V and SUN4U. */
760#define pte_none(pte) (!pte_val(pte)) 801#define pte_none(pte) (!pte_val(pte))
@@ -789,7 +830,7 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
789 pmd_t *pmdp) 830 pmd_t *pmdp)
790{ 831{
791 pmd_t pmd = *pmdp; 832 pmd_t pmd = *pmdp;
792 set_pmd_at(mm, addr, pmdp, __pmd(0U)); 833 set_pmd_at(mm, addr, pmdp, __pmd(0UL));
793 return pmd; 834 return pmd;
794} 835}
795 836
@@ -837,8 +878,8 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
837}) 878})
838#endif 879#endif
839 880
840extern pgd_t swapper_pg_dir[2048]; 881extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
841extern pmd_t swapper_low_pmd_dir[2048]; 882extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD];
842 883
843extern void paging_init(void); 884extern void paging_init(void);
844extern unsigned long find_ecache_flush_span(unsigned long size); 885extern unsigned long find_ecache_flush_span(unsigned long size);
diff --git a/arch/sparc/include/asm/sparsemem.h b/arch/sparc/include/asm/sparsemem.h
index b99d4e4b6d28..e5e1752d5d78 100644
--- a/arch/sparc/include/asm/sparsemem.h
+++ b/arch/sparc/include/asm/sparsemem.h
@@ -3,9 +3,11 @@
3 3
4#ifdef __KERNEL__ 4#ifdef __KERNEL__
5 5
6#include <asm/page.h>
7
6#define SECTION_SIZE_BITS 30 8#define SECTION_SIZE_BITS 30
7#define MAX_PHYSADDR_BITS 42 9#define MAX_PHYSADDR_BITS MAX_PHYS_ADDRESS_BITS
8#define MAX_PHYSMEM_BITS 42 10#define MAX_PHYSMEM_BITS MAX_PHYS_ADDRESS_BITS
9 11
10#endif /* !(__KERNEL__) */ 12#endif /* !(__KERNEL__) */
11 13
diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h
index d5e504251079..5d9292ab1077 100644
--- a/arch/sparc/include/asm/thread_info_64.h
+++ b/arch/sparc/include/asm/thread_info_64.h
@@ -192,7 +192,7 @@ register struct thread_info *current_thread_info_reg asm("g6");
192#define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ 192#define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */
193/* flag bit 6 is available */ 193/* flag bit 6 is available */
194#define TIF_32BIT 7 /* 32-bit binary */ 194#define TIF_32BIT 7 /* 32-bit binary */
195/* flag bit 8 is available */ 195#define TIF_NOHZ 8 /* in adaptive nohz mode */
196#define TIF_SECCOMP 9 /* secure computing */ 196#define TIF_SECCOMP 9 /* secure computing */
197#define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ 197#define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */
198#define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ 198#define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */
@@ -210,6 +210,7 @@ register struct thread_info *current_thread_info_reg asm("g6");
210#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) 210#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
211#define _TIF_UNALIGNED (1<<TIF_UNALIGNED) 211#define _TIF_UNALIGNED (1<<TIF_UNALIGNED)
212#define _TIF_32BIT (1<<TIF_32BIT) 212#define _TIF_32BIT (1<<TIF_32BIT)
213#define _TIF_NOHZ (1<<TIF_NOHZ)
213#define _TIF_SECCOMP (1<<TIF_SECCOMP) 214#define _TIF_SECCOMP (1<<TIF_SECCOMP)
214#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) 215#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT)
215#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) 216#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index e696432b950d..2230f80d9fe3 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -142,98 +142,39 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
142 or REG1, %lo(swapper_pg_dir), REG1; \ 142 or REG1, %lo(swapper_pg_dir), REG1; \
143 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ 143 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
144 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 144 srlx REG2, 64 - PAGE_SHIFT, REG2; \
145 andn REG2, 0x3, REG2; \ 145 andn REG2, 0x7, REG2; \
146 lduw [REG1 + REG2], REG1; \ 146 ldx [REG1 + REG2], REG1; \
147 brz,pn REG1, FAIL_LABEL; \ 147 brz,pn REG1, FAIL_LABEL; \
148 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 148 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
149 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 149 srlx REG2, 64 - PAGE_SHIFT, REG2; \
150 sllx REG1, PGD_PADDR_SHIFT, REG1; \ 150 andn REG2, 0x7, REG2; \
151 andn REG2, 0x3, REG2; \ 151 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
152 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
153 brz,pn REG1, FAIL_LABEL; \ 152 brz,pn REG1, FAIL_LABEL; \
154 sllx VADDR, 64 - PMD_SHIFT, REG2; \ 153 sllx VADDR, 64 - PMD_SHIFT, REG2; \
155 srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ 154 srlx REG2, 64 - PAGE_SHIFT, REG2; \
156 sllx REG1, PMD_PADDR_SHIFT, REG1; \
157 andn REG2, 0x7, REG2; \ 155 andn REG2, 0x7, REG2; \
158 add REG1, REG2, REG1; 156 add REG1, REG2, REG1;
159 157
160 /* These macros exists only to make the PMD translator below
161 * easier to read. It hides the ELF section switch for the
162 * sun4v code patching.
163 */
164#define OR_PTE_BIT_1INSN(REG, NAME) \
165661: or REG, _PAGE_##NAME##_4U, REG; \
166 .section .sun4v_1insn_patch, "ax"; \
167 .word 661b; \
168 or REG, _PAGE_##NAME##_4V, REG; \
169 .previous;
170
171#define OR_PTE_BIT_2INSN(REG, TMP, NAME) \
172661: sethi %hi(_PAGE_##NAME##_4U), TMP; \
173 or REG, TMP, REG; \
174 .section .sun4v_2insn_patch, "ax"; \
175 .word 661b; \
176 mov -1, TMP; \
177 or REG, _PAGE_##NAME##_4V, REG; \
178 .previous;
179
180 /* Load into REG the PTE value for VALID, CACHE, and SZHUGE. */
181#define BUILD_PTE_VALID_SZHUGE_CACHE(REG) \
182661: sethi %uhi(_PAGE_VALID|_PAGE_SZHUGE_4U), REG; \
183 .section .sun4v_1insn_patch, "ax"; \
184 .word 661b; \
185 sethi %uhi(_PAGE_VALID), REG; \
186 .previous; \
187 sllx REG, 32, REG; \
188661: or REG, _PAGE_CP_4U|_PAGE_CV_4U, REG; \
189 .section .sun4v_1insn_patch, "ax"; \
190 .word 661b; \
191 or REG, _PAGE_CP_4V|_PAGE_CV_4V|_PAGE_SZHUGE_4V, REG; \
192 .previous;
193
194 /* PMD has been loaded into REG1, interpret the value, seeing 158 /* PMD has been loaded into REG1, interpret the value, seeing
195 * if it is a HUGE PMD or a normal one. If it is not valid 159 * if it is a HUGE PMD or a normal one. If it is not valid
196 * then jump to FAIL_LABEL. If it is a HUGE PMD, and it 160 * then jump to FAIL_LABEL. If it is a HUGE PMD, and it
197 * translates to a valid PTE, branch to PTE_LABEL. 161 * translates to a valid PTE, branch to PTE_LABEL.
198 * 162 *
199 * We translate the PMD by hand, one bit at a time, 163 * We have to propagate the 4MB bit of the virtual address
200 * constructing the huge PTE. 164 * because we are fabricating 8MB pages using 4MB hw pages.
201 *
202 * So we construct the PTE in REG2 as follows:
203 *
204 * 1) Extract the PMD PFN from REG1 and place it into REG2.
205 *
206 * 2) Translate PMD protection bits in REG1 into REG2, one bit
207 * at a time using andcc tests on REG1 and OR's into REG2.
208 *
209 * Only two bits to be concerned with here, EXEC and WRITE.
210 * Now REG1 is freed up and we can use it as a temporary.
211 *
212 * 3) Construct the VALID, CACHE, and page size PTE bits in
213 * REG1, OR with REG2 to form final PTE.
214 */ 165 */
215#ifdef CONFIG_TRANSPARENT_HUGEPAGE 166#ifdef CONFIG_TRANSPARENT_HUGEPAGE
216#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ 167#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
217 brz,pn REG1, FAIL_LABEL; \ 168 brz,pn REG1, FAIL_LABEL; \
218 andcc REG1, PMD_ISHUGE, %g0; \ 169 sethi %uhi(_PAGE_PMD_HUGE), REG2; \
219 be,pt %xcc, 700f; \ 170 sllx REG2, 32, REG2; \
220 and REG1, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED, REG2; \ 171 andcc REG1, REG2, %g0; \
221 cmp REG2, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED; \ 172 be,pt %xcc, 700f; \
222 bne,pn %xcc, FAIL_LABEL; \ 173 sethi %hi(4 * 1024 * 1024), REG2; \
223 andn REG1, PMD_HUGE_PROTBITS, REG2; \ 174 andn REG1, REG2, REG1; \
224 sllx REG2, PMD_PADDR_SHIFT, REG2; \ 175 and VADDR, REG2, REG2; \
225 /* REG2 now holds PFN << PAGE_SHIFT */ \ 176 brlz,pt REG1, PTE_LABEL; \
226 andcc REG1, PMD_HUGE_WRITE, %g0; \ 177 or REG1, REG2, REG1; \
227 bne,a,pt %xcc, 1f; \
228 OR_PTE_BIT_1INSN(REG2, W); \
2291: andcc REG1, PMD_HUGE_EXEC, %g0; \
230 be,pt %xcc, 1f; \
231 nop; \
232 OR_PTE_BIT_2INSN(REG2, REG1, EXEC); \
233 /* REG1 can now be clobbered, build final PTE */ \
2341: BUILD_PTE_VALID_SZHUGE_CACHE(REG1); \
235 ba,pt %xcc, PTE_LABEL; \
236 or REG1, REG2, REG1; \
237700: 178700:
238#else 179#else
239#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ 180#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
@@ -253,18 +194,16 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
253#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ 194#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \
254 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ 195 sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
255 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 196 srlx REG2, 64 - PAGE_SHIFT, REG2; \
256 andn REG2, 0x3, REG2; \ 197 andn REG2, 0x7, REG2; \
257 lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ 198 ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
258 brz,pn REG1, FAIL_LABEL; \ 199 brz,pn REG1, FAIL_LABEL; \
259 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 200 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
260 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 201 srlx REG2, 64 - PAGE_SHIFT, REG2; \
261 sllx REG1, PGD_PADDR_SHIFT, REG1; \ 202 andn REG2, 0x7, REG2; \
262 andn REG2, 0x3, REG2; \ 203 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
263 lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
264 USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ 204 USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
265 sllx VADDR, 64 - PMD_SHIFT, REG2; \ 205 sllx VADDR, 64 - PMD_SHIFT, REG2; \
266 srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ 206 srlx REG2, 64 - PAGE_SHIFT, REG2; \
267 sllx REG1, PMD_PADDR_SHIFT, REG1; \
268 andn REG2, 0x7, REG2; \ 207 andn REG2, 0x7, REG2; \
269 add REG1, REG2, REG1; \ 208 add REG1, REG2, REG1; \
270 ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ 209 ldxa [REG1] ASI_PHYS_USE_EC, REG1; \
diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h
index 9c179fbfb219..140966fbd303 100644
--- a/arch/sparc/kernel/entry.h
+++ b/arch/sparc/kernel/entry.h
@@ -88,7 +88,6 @@ extern asmlinkage void syscall_trace_leave(struct pt_regs *regs);
88 88
89extern void bad_trap_tl1(struct pt_regs *regs, long lvl); 89extern void bad_trap_tl1(struct pt_regs *regs, long lvl);
90 90
91extern void do_fpe_common(struct pt_regs *regs);
92extern void do_fpieee(struct pt_regs *regs); 91extern void do_fpieee(struct pt_regs *regs);
93extern void do_fpother(struct pt_regs *regs); 92extern void do_fpother(struct pt_regs *regs);
94extern void do_tof(struct pt_regs *regs); 93extern void do_tof(struct pt_regs *regs);
diff --git a/arch/sparc/kernel/kgdb_64.c b/arch/sparc/kernel/kgdb_64.c
index 53c0a82e6030..60b19f50c80a 100644
--- a/arch/sparc/kernel/kgdb_64.c
+++ b/arch/sparc/kernel/kgdb_64.c
@@ -159,11 +159,12 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code,
159 159
160asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) 160asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs)
161{ 161{
162 enum ctx_state prev_state = exception_enter();
162 unsigned long flags; 163 unsigned long flags;
163 164
164 if (user_mode(regs)) { 165 if (user_mode(regs)) {
165 bad_trap(regs, trap_level); 166 bad_trap(regs, trap_level);
166 return; 167 goto out;
167 } 168 }
168 169
169 flushw_all(); 170 flushw_all();
@@ -171,6 +172,8 @@ asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs)
171 local_irq_save(flags); 172 local_irq_save(flags);
172 kgdb_handle_exception(0x172, SIGTRAP, 0, regs); 173 kgdb_handle_exception(0x172, SIGTRAP, 0, regs);
173 local_irq_restore(flags); 174 local_irq_restore(flags);
175out:
176 exception_exit(prev_state);
174} 177}
175 178
176int kgdb_arch_init(void) 179int kgdb_arch_init(void)
diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c
index e72212148d2a..5a09fd315e5f 100644
--- a/arch/sparc/kernel/kprobes.c
+++ b/arch/sparc/kernel/kprobes.c
@@ -8,6 +8,7 @@
8#include <linux/module.h> 8#include <linux/module.h>
9#include <linux/kdebug.h> 9#include <linux/kdebug.h>
10#include <linux/slab.h> 10#include <linux/slab.h>
11#include <linux/context_tracking.h>
11#include <asm/signal.h> 12#include <asm/signal.h>
12#include <asm/cacheflush.h> 13#include <asm/cacheflush.h>
13#include <asm/uaccess.h> 14#include <asm/uaccess.h>
@@ -418,12 +419,14 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
418asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, 419asmlinkage void __kprobes kprobe_trap(unsigned long trap_level,
419 struct pt_regs *regs) 420 struct pt_regs *regs)
420{ 421{
422 enum ctx_state prev_state = exception_enter();
423
421 BUG_ON(trap_level != 0x170 && trap_level != 0x171); 424 BUG_ON(trap_level != 0x170 && trap_level != 0x171);
422 425
423 if (user_mode(regs)) { 426 if (user_mode(regs)) {
424 local_irq_enable(); 427 local_irq_enable();
425 bad_trap(regs, trap_level); 428 bad_trap(regs, trap_level);
426 return; 429 goto out;
427 } 430 }
428 431
429 /* trap_level == 0x170 --> ta 0x70 432 /* trap_level == 0x170 --> ta 0x70
@@ -433,6 +436,8 @@ asmlinkage void __kprobes kprobe_trap(unsigned long trap_level,
433 (trap_level == 0x170) ? "debug" : "debug_2", 436 (trap_level == 0x170) ? "debug" : "debug_2",
434 regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) 437 regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP)
435 bad_trap(regs, trap_level); 438 bad_trap(regs, trap_level);
439out:
440 exception_exit(prev_state);
436} 441}
437 442
438/* Jprobes support. */ 443/* Jprobes support. */
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
index fde5a419cf27..542e96ac4d39 100644
--- a/arch/sparc/kernel/ktlb.S
+++ b/arch/sparc/kernel/ktlb.S
@@ -153,12 +153,19 @@ kvmap_dtlb_tsb4m_miss:
153 /* Clear the PAGE_OFFSET top virtual bits, shift 153 /* Clear the PAGE_OFFSET top virtual bits, shift
154 * down to get PFN, and make sure PFN is in range. 154 * down to get PFN, and make sure PFN is in range.
155 */ 155 */
156 sllx %g4, 21, %g5 156661: sllx %g4, 0, %g5
157 .section .page_offset_shift_patch, "ax"
158 .word 661b
159 .previous
157 160
158 /* Check to see if we know about valid memory at the 4MB 161 /* Check to see if we know about valid memory at the 4MB
159 * chunk this physical address will reside within. 162 * chunk this physical address will reside within.
160 */ 163 */
161 srlx %g5, 21 + 41, %g2 164661: srlx %g5, MAX_PHYS_ADDRESS_BITS, %g2
165 .section .page_offset_shift_patch, "ax"
166 .word 661b
167 .previous
168
162 brnz,pn %g2, kvmap_dtlb_longpath 169 brnz,pn %g2, kvmap_dtlb_longpath
163 nop 170 nop
164 171
@@ -176,7 +183,11 @@ valid_addr_bitmap_patch:
176 or %g7, %lo(sparc64_valid_addr_bitmap), %g7 183 or %g7, %lo(sparc64_valid_addr_bitmap), %g7
177 .previous 184 .previous
178 185
179 srlx %g5, 21 + 22, %g2 186661: srlx %g5, ILOG2_4MB, %g2
187 .section .page_offset_shift_patch, "ax"
188 .word 661b
189 .previous
190
180 srlx %g2, 6, %g5 191 srlx %g2, 6, %g5
181 and %g2, 63, %g2 192 and %g2, 63, %g2
182 sllx %g5, 3, %g5 193 sllx %g5, 3, %g5
@@ -189,9 +200,18 @@ valid_addr_bitmap_patch:
1892: sethi %hi(kpte_linear_bitmap), %g2 2002: sethi %hi(kpte_linear_bitmap), %g2
190 201
191 /* Get the 256MB physical address index. */ 202 /* Get the 256MB physical address index. */
192 sllx %g4, 21, %g5 203661: sllx %g4, 0, %g5
204 .section .page_offset_shift_patch, "ax"
205 .word 661b
206 .previous
207
193 or %g2, %lo(kpte_linear_bitmap), %g2 208 or %g2, %lo(kpte_linear_bitmap), %g2
194 srlx %g5, 21 + 28, %g5 209
210661: srlx %g5, ILOG2_256MB, %g5
211 .section .page_offset_shift_patch, "ax"
212 .word 661b
213 .previous
214
195 and %g5, (32 - 1), %g7 215 and %g5, (32 - 1), %g7
196 216
197 /* Divide by 32 to get the offset into the bitmask. */ 217 /* Divide by 32 to get the offset into the bitmask. */
diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c
index bc4d3f5d2e5d..cb021453de2a 100644
--- a/arch/sparc/kernel/pci.c
+++ b/arch/sparc/kernel/pci.c
@@ -398,8 +398,8 @@ static void apb_fake_ranges(struct pci_dev *dev,
398 apb_calc_first_last(map, &first, &last); 398 apb_calc_first_last(map, &first, &last);
399 res = bus->resource[1]; 399 res = bus->resource[1];
400 res->flags = IORESOURCE_MEM; 400 res->flags = IORESOURCE_MEM;
401 region.start = (first << 21); 401 region.start = (first << 29);
402 region.end = (last << 21) + ((1 << 21) - 1); 402 region.end = (last << 29) + ((1 << 29) - 1);
403 pcibios_bus_to_resource(dev, res, &region); 403 pcibios_bus_to_resource(dev, res, &region);
404} 404}
405 405
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index baebab215492..32a280ec38c1 100644
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -31,6 +31,7 @@
31#include <linux/elfcore.h> 31#include <linux/elfcore.h>
32#include <linux/sysrq.h> 32#include <linux/sysrq.h>
33#include <linux/nmi.h> 33#include <linux/nmi.h>
34#include <linux/context_tracking.h>
34 35
35#include <asm/uaccess.h> 36#include <asm/uaccess.h>
36#include <asm/page.h> 37#include <asm/page.h>
@@ -557,6 +558,7 @@ void fault_in_user_windows(void)
557 558
558barf: 559barf:
559 set_thread_wsaved(window + 1); 560 set_thread_wsaved(window + 1);
561 user_exit();
560 do_exit(SIGILL); 562 do_exit(SIGILL);
561} 563}
562 564
diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c
index 773c1f2983ce..c13c9f25d83a 100644
--- a/arch/sparc/kernel/ptrace_64.c
+++ b/arch/sparc/kernel/ptrace_64.c
@@ -27,6 +27,7 @@
27#include <trace/syscall.h> 27#include <trace/syscall.h>
28#include <linux/compat.h> 28#include <linux/compat.h>
29#include <linux/elf.h> 29#include <linux/elf.h>
30#include <linux/context_tracking.h>
30 31
31#include <asm/asi.h> 32#include <asm/asi.h>
32#include <asm/pgtable.h> 33#include <asm/pgtable.h>
@@ -1066,6 +1067,9 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs)
1066 /* do the secure computing check first */ 1067 /* do the secure computing check first */
1067 secure_computing_strict(regs->u_regs[UREG_G1]); 1068 secure_computing_strict(regs->u_regs[UREG_G1]);
1068 1069
1070 if (test_thread_flag(TIF_NOHZ))
1071 user_exit();
1072
1069 if (test_thread_flag(TIF_SYSCALL_TRACE)) 1073 if (test_thread_flag(TIF_SYSCALL_TRACE))
1070 ret = tracehook_report_syscall_entry(regs); 1074 ret = tracehook_report_syscall_entry(regs);
1071 1075
@@ -1086,6 +1090,9 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs)
1086 1090
1087asmlinkage void syscall_trace_leave(struct pt_regs *regs) 1091asmlinkage void syscall_trace_leave(struct pt_regs *regs)
1088{ 1092{
1093 if (test_thread_flag(TIF_NOHZ))
1094 user_exit();
1095
1089 audit_syscall_exit(regs); 1096 audit_syscall_exit(regs);
1090 1097
1091 if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) 1098 if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
@@ -1093,4 +1100,7 @@ asmlinkage void syscall_trace_leave(struct pt_regs *regs)
1093 1100
1094 if (test_thread_flag(TIF_SYSCALL_TRACE)) 1101 if (test_thread_flag(TIF_SYSCALL_TRACE))
1095 tracehook_report_syscall_exit(regs, 0); 1102 tracehook_report_syscall_exit(regs, 0);
1103
1104 if (test_thread_flag(TIF_NOHZ))
1105 user_enter();
1096} 1106}
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S
index afa2a9e3d0a0..a954eb81881b 100644
--- a/arch/sparc/kernel/rtrap_64.S
+++ b/arch/sparc/kernel/rtrap_64.S
@@ -18,10 +18,16 @@
18#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) 18#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV)
19#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) 19#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG)
20 20
21#ifdef CONFIG_CONTEXT_TRACKING
22# define SCHEDULE_USER schedule_user
23#else
24# define SCHEDULE_USER schedule
25#endif
26
21 .text 27 .text
22 .align 32 28 .align 32
23__handle_preemption: 29__handle_preemption:
24 call schedule 30 call SCHEDULE_USER
25 wrpr %g0, RTRAP_PSTATE, %pstate 31 wrpr %g0, RTRAP_PSTATE, %pstate
26 ba,pt %xcc, __handle_preemption_continue 32 ba,pt %xcc, __handle_preemption_continue
27 wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate 33 wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate
diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c
index 35923e8abd82..cd91d010e6d3 100644
--- a/arch/sparc/kernel/signal_64.c
+++ b/arch/sparc/kernel/signal_64.c
@@ -23,6 +23,7 @@
23#include <linux/tty.h> 23#include <linux/tty.h>
24#include <linux/binfmts.h> 24#include <linux/binfmts.h>
25#include <linux/bitops.h> 25#include <linux/bitops.h>
26#include <linux/context_tracking.h>
26 27
27#include <asm/uaccess.h> 28#include <asm/uaccess.h>
28#include <asm/ptrace.h> 29#include <asm/ptrace.h>
@@ -43,6 +44,7 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs)
43{ 44{
44 struct ucontext __user *ucp = (struct ucontext __user *) 45 struct ucontext __user *ucp = (struct ucontext __user *)
45 regs->u_regs[UREG_I0]; 46 regs->u_regs[UREG_I0];
47 enum ctx_state prev_state = exception_enter();
46 mc_gregset_t __user *grp; 48 mc_gregset_t __user *grp;
47 unsigned long pc, npc, tstate; 49 unsigned long pc, npc, tstate;
48 unsigned long fp, i7; 50 unsigned long fp, i7;
@@ -129,16 +131,19 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs)
129 } 131 }
130 if (err) 132 if (err)
131 goto do_sigsegv; 133 goto do_sigsegv;
132 134out:
135 exception_exit(prev_state);
133 return; 136 return;
134do_sigsegv: 137do_sigsegv:
135 force_sig(SIGSEGV, current); 138 force_sig(SIGSEGV, current);
139 goto out;
136} 140}
137 141
138asmlinkage void sparc64_get_context(struct pt_regs *regs) 142asmlinkage void sparc64_get_context(struct pt_regs *regs)
139{ 143{
140 struct ucontext __user *ucp = (struct ucontext __user *) 144 struct ucontext __user *ucp = (struct ucontext __user *)
141 regs->u_regs[UREG_I0]; 145 regs->u_regs[UREG_I0];
146 enum ctx_state prev_state = exception_enter();
142 mc_gregset_t __user *grp; 147 mc_gregset_t __user *grp;
143 mcontext_t __user *mcp; 148 mcontext_t __user *mcp;
144 unsigned long fp, i7; 149 unsigned long fp, i7;
@@ -220,10 +225,12 @@ asmlinkage void sparc64_get_context(struct pt_regs *regs)
220 } 225 }
221 if (err) 226 if (err)
222 goto do_sigsegv; 227 goto do_sigsegv;
223 228out:
229 exception_exit(prev_state);
224 return; 230 return;
225do_sigsegv: 231do_sigsegv:
226 force_sig(SIGSEGV, current); 232 force_sig(SIGSEGV, current);
233 goto out;
227} 234}
228 235
229struct rt_signal_frame { 236struct rt_signal_frame {
@@ -528,11 +535,13 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0)
528 535
529void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) 536void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags)
530{ 537{
538 user_exit();
531 if (thread_info_flags & _TIF_SIGPENDING) 539 if (thread_info_flags & _TIF_SIGPENDING)
532 do_signal(regs, orig_i0); 540 do_signal(regs, orig_i0);
533 if (thread_info_flags & _TIF_NOTIFY_RESUME) { 541 if (thread_info_flags & _TIF_NOTIFY_RESUME) {
534 clear_thread_flag(TIF_NOTIFY_RESUME); 542 clear_thread_flag(TIF_NOTIFY_RESUME);
535 tracehook_notify_resume(regs); 543 tracehook_notify_resume(regs);
536 } 544 }
545 user_enter();
537} 546}
538 547
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index e142545244f2..b66a5338231e 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1399,8 +1399,13 @@ void __init smp_cpus_done(unsigned int max_cpus)
1399 1399
1400void smp_send_reschedule(int cpu) 1400void smp_send_reschedule(int cpu)
1401{ 1401{
1402 xcall_deliver((u64) &xcall_receive_signal, 0, 0, 1402 if (cpu == smp_processor_id()) {
1403 cpumask_of(cpu)); 1403 WARN_ON_ONCE(preemptible());
1404 set_softint(1 << PIL_SMP_RECEIVE_SIGNAL);
1405 } else {
1406 xcall_deliver((u64) &xcall_receive_signal,
1407 0, 0, cpumask_of(cpu));
1408 }
1404} 1409}
1405 1410
1406void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) 1411void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs)
diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S
index bde867fd71e8..e0c09bf85610 100644
--- a/arch/sparc/kernel/sun4v_tlb_miss.S
+++ b/arch/sparc/kernel/sun4v_tlb_miss.S
@@ -182,7 +182,7 @@ sun4v_tsb_miss_common:
182 cmp %g5, -1 182 cmp %g5, -1
183 be,pt %xcc, 80f 183 be,pt %xcc, 80f
184 nop 184 nop
185 COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) 185 COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
186 186
187 /* That clobbered %g2, reload it. */ 187 /* That clobbered %g2, reload it. */
188 ldxa [%g0] ASI_SCRATCHPAD, %g2 188 ldxa [%g0] ASI_SCRATCHPAD, %g2
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c
index 51561b8b15ba..beb0b5a5f21f 100644
--- a/arch/sparc/kernel/sys_sparc_64.c
+++ b/arch/sparc/kernel/sys_sparc_64.c
@@ -24,6 +24,7 @@
24#include <linux/personality.h> 24#include <linux/personality.h>
25#include <linux/random.h> 25#include <linux/random.h>
26#include <linux/export.h> 26#include <linux/export.h>
27#include <linux/context_tracking.h>
27 28
28#include <asm/uaccess.h> 29#include <asm/uaccess.h>
29#include <asm/utrap.h> 30#include <asm/utrap.h>
@@ -39,9 +40,6 @@ asmlinkage unsigned long sys_getpagesize(void)
39 return PAGE_SIZE; 40 return PAGE_SIZE;
40} 41}
41 42
42#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL))
43#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL))
44
45/* Does addr --> addr+len fall within 4GB of the VA-space hole or 43/* Does addr --> addr+len fall within 4GB of the VA-space hole or
46 * overflow past the end of the 64-bit address space? 44 * overflow past the end of the 64-bit address space?
47 */ 45 */
@@ -499,6 +497,7 @@ asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs)
499 497
500asmlinkage void sparc_breakpoint(struct pt_regs *regs) 498asmlinkage void sparc_breakpoint(struct pt_regs *regs)
501{ 499{
500 enum ctx_state prev_state = exception_enter();
502 siginfo_t info; 501 siginfo_t info;
503 502
504 if (test_thread_flag(TIF_32BIT)) { 503 if (test_thread_flag(TIF_32BIT)) {
@@ -517,6 +516,7 @@ asmlinkage void sparc_breakpoint(struct pt_regs *regs)
517#ifdef DEBUG_SPARC_BREAKPOINT 516#ifdef DEBUG_SPARC_BREAKPOINT
518 printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); 517 printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc);
519#endif 518#endif
519 exception_exit(prev_state);
520} 520}
521 521
522extern void check_pending(int signum); 522extern void check_pending(int signum);
diff --git a/arch/sparc/kernel/syscalls.S b/arch/sparc/kernel/syscalls.S
index d950197a17e1..87729fff13b9 100644
--- a/arch/sparc/kernel/syscalls.S
+++ b/arch/sparc/kernel/syscalls.S
@@ -52,7 +52,7 @@ sys32_rt_sigreturn:
52#endif 52#endif
53 .align 32 53 .align 32
541: ldx [%g6 + TI_FLAGS], %l5 541: ldx [%g6 + TI_FLAGS], %l5
55 andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 55 andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
56 be,pt %icc, rtrap 56 be,pt %icc, rtrap
57 nop 57 nop
58 call syscall_trace_leave 58 call syscall_trace_leave
@@ -184,7 +184,7 @@ linux_sparc_syscall32:
184 184
185 srl %i3, 0, %o3 ! IEU0 185 srl %i3, 0, %o3 ! IEU0
186 srl %i2, 0, %o2 ! IEU0 Group 186 srl %i2, 0, %o2 ! IEU0 Group
187 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 187 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
188 bne,pn %icc, linux_syscall_trace32 ! CTI 188 bne,pn %icc, linux_syscall_trace32 ! CTI
189 mov %i0, %l5 ! IEU1 189 mov %i0, %l5 ! IEU1
1905: call %l7 ! CTI Group brk forced 1905: call %l7 ! CTI Group brk forced
@@ -207,7 +207,7 @@ linux_sparc_syscall:
207 207
208 mov %i3, %o3 ! IEU1 208 mov %i3, %o3 ! IEU1
209 mov %i4, %o4 ! IEU0 Group 209 mov %i4, %o4 ! IEU0 Group
210 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 210 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
211 bne,pn %icc, linux_syscall_trace ! CTI Group 211 bne,pn %icc, linux_syscall_trace ! CTI Group
212 mov %i0, %l5 ! IEU0 212 mov %i0, %l5 ! IEU0
2132: call %l7 ! CTI Group brk forced 2132: call %l7 ! CTI Group brk forced
@@ -223,7 +223,7 @@ ret_sys_call:
223 223
224 cmp %o0, -ERESTART_RESTARTBLOCK 224 cmp %o0, -ERESTART_RESTARTBLOCK
225 bgeu,pn %xcc, 1f 225 bgeu,pn %xcc, 1f
226 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 226 andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0
227 ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc 227 ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc
228 228
2292: 2292:
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index b3f833ab90eb..4ced92f05358 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -20,6 +20,7 @@
20#include <linux/ftrace.h> 20#include <linux/ftrace.h>
21#include <linux/reboot.h> 21#include <linux/reboot.h>
22#include <linux/gfp.h> 22#include <linux/gfp.h>
23#include <linux/context_tracking.h>
23 24
24#include <asm/smp.h> 25#include <asm/smp.h>
25#include <asm/delay.h> 26#include <asm/delay.h>
@@ -186,11 +187,12 @@ EXPORT_SYMBOL_GPL(unregister_dimm_printer);
186 187
187void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 188void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
188{ 189{
190 enum ctx_state prev_state = exception_enter();
189 siginfo_t info; 191 siginfo_t info;
190 192
191 if (notify_die(DIE_TRAP, "instruction access exception", regs, 193 if (notify_die(DIE_TRAP, "instruction access exception", regs,
192 0, 0x8, SIGTRAP) == NOTIFY_STOP) 194 0, 0x8, SIGTRAP) == NOTIFY_STOP)
193 return; 195 goto out;
194 196
195 if (regs->tstate & TSTATE_PRIV) { 197 if (regs->tstate & TSTATE_PRIV) {
196 printk("spitfire_insn_access_exception: SFSR[%016lx] " 198 printk("spitfire_insn_access_exception: SFSR[%016lx] "
@@ -207,6 +209,8 @@ void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, un
207 info.si_addr = (void __user *)regs->tpc; 209 info.si_addr = (void __user *)regs->tpc;
208 info.si_trapno = 0; 210 info.si_trapno = 0;
209 force_sig_info(SIGSEGV, &info, current); 211 force_sig_info(SIGSEGV, &info, current);
212out:
213 exception_exit(prev_state);
210} 214}
211 215
212void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 216void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
@@ -260,11 +264,12 @@ void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, u
260 264
261void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 265void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
262{ 266{
267 enum ctx_state prev_state = exception_enter();
263 siginfo_t info; 268 siginfo_t info;
264 269
265 if (notify_die(DIE_TRAP, "data access exception", regs, 270 if (notify_die(DIE_TRAP, "data access exception", regs,
266 0, 0x30, SIGTRAP) == NOTIFY_STOP) 271 0, 0x30, SIGTRAP) == NOTIFY_STOP)
267 return; 272 goto out;
268 273
269 if (regs->tstate & TSTATE_PRIV) { 274 if (regs->tstate & TSTATE_PRIV) {
270 /* Test if this comes from uaccess places. */ 275 /* Test if this comes from uaccess places. */
@@ -280,7 +285,7 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un
280#endif 285#endif
281 regs->tpc = entry->fixup; 286 regs->tpc = entry->fixup;
282 regs->tnpc = regs->tpc + 4; 287 regs->tnpc = regs->tpc + 4;
283 return; 288 goto out;
284 } 289 }
285 /* Shit... */ 290 /* Shit... */
286 printk("spitfire_data_access_exception: SFSR[%016lx] " 291 printk("spitfire_data_access_exception: SFSR[%016lx] "
@@ -294,6 +299,8 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un
294 info.si_addr = (void __user *)sfar; 299 info.si_addr = (void __user *)sfar;
295 info.si_trapno = 0; 300 info.si_trapno = 0;
296 force_sig_info(SIGSEGV, &info, current); 301 force_sig_info(SIGSEGV, &info, current);
302out:
303 exception_exit(prev_state);
297} 304}
298 305
299void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) 306void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
@@ -1994,6 +2001,7 @@ static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1994 */ 2001 */
1995void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) 2002void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
1996{ 2003{
2004 enum ctx_state prev_state = exception_enter();
1997 struct sun4v_error_entry *ent, local_copy; 2005 struct sun4v_error_entry *ent, local_copy;
1998 struct trap_per_cpu *tb; 2006 struct trap_per_cpu *tb;
1999 unsigned long paddr; 2007 unsigned long paddr;
@@ -2022,12 +2030,14 @@ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2022 pr_info("Shutdown request, %u seconds...\n", 2030 pr_info("Shutdown request, %u seconds...\n",
2023 local_copy.err_secs); 2031 local_copy.err_secs);
2024 orderly_poweroff(true); 2032 orderly_poweroff(true);
2025 return; 2033 goto out;
2026 } 2034 }
2027 2035
2028 sun4v_log_error(regs, &local_copy, cpu, 2036 sun4v_log_error(regs, &local_copy, cpu,
2029 KERN_ERR "RESUMABLE ERROR", 2037 KERN_ERR "RESUMABLE ERROR",
2030 &sun4v_resum_oflow_cnt); 2038 &sun4v_resum_oflow_cnt);
2039out:
2040 exception_exit(prev_state);
2031} 2041}
2032 2042
2033/* If we try to printk() we'll probably make matters worse, by trying 2043/* If we try to printk() we'll probably make matters worse, by trying
@@ -2152,7 +2162,7 @@ void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2152 err, op); 2162 err, op);
2153} 2163}
2154 2164
2155void do_fpe_common(struct pt_regs *regs) 2165static void do_fpe_common(struct pt_regs *regs)
2156{ 2166{
2157 if (regs->tstate & TSTATE_PRIV) { 2167 if (regs->tstate & TSTATE_PRIV) {
2158 regs->tpc = regs->tnpc; 2168 regs->tpc = regs->tnpc;
@@ -2188,23 +2198,28 @@ void do_fpe_common(struct pt_regs *regs)
2188 2198
2189void do_fpieee(struct pt_regs *regs) 2199void do_fpieee(struct pt_regs *regs)
2190{ 2200{
2201 enum ctx_state prev_state = exception_enter();
2202
2191 if (notify_die(DIE_TRAP, "fpu exception ieee", regs, 2203 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2192 0, 0x24, SIGFPE) == NOTIFY_STOP) 2204 0, 0x24, SIGFPE) == NOTIFY_STOP)
2193 return; 2205 goto out;
2194 2206
2195 do_fpe_common(regs); 2207 do_fpe_common(regs);
2208out:
2209 exception_exit(prev_state);
2196} 2210}
2197 2211
2198extern int do_mathemu(struct pt_regs *, struct fpustate *, bool); 2212extern int do_mathemu(struct pt_regs *, struct fpustate *, bool);
2199 2213
2200void do_fpother(struct pt_regs *regs) 2214void do_fpother(struct pt_regs *regs)
2201{ 2215{
2216 enum ctx_state prev_state = exception_enter();
2202 struct fpustate *f = FPUSTATE; 2217 struct fpustate *f = FPUSTATE;
2203 int ret = 0; 2218 int ret = 0;
2204 2219
2205 if (notify_die(DIE_TRAP, "fpu exception other", regs, 2220 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2206 0, 0x25, SIGFPE) == NOTIFY_STOP) 2221 0, 0x25, SIGFPE) == NOTIFY_STOP)
2207 return; 2222 goto out;
2208 2223
2209 switch ((current_thread_info()->xfsr[0] & 0x1c000)) { 2224 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2210 case (2 << 14): /* unfinished_FPop */ 2225 case (2 << 14): /* unfinished_FPop */
@@ -2213,17 +2228,20 @@ void do_fpother(struct pt_regs *regs)
2213 break; 2228 break;
2214 } 2229 }
2215 if (ret) 2230 if (ret)
2216 return; 2231 goto out;
2217 do_fpe_common(regs); 2232 do_fpe_common(regs);
2233out:
2234 exception_exit(prev_state);
2218} 2235}
2219 2236
2220void do_tof(struct pt_regs *regs) 2237void do_tof(struct pt_regs *regs)
2221{ 2238{
2239 enum ctx_state prev_state = exception_enter();
2222 siginfo_t info; 2240 siginfo_t info;
2223 2241
2224 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, 2242 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2225 0, 0x26, SIGEMT) == NOTIFY_STOP) 2243 0, 0x26, SIGEMT) == NOTIFY_STOP)
2226 return; 2244 goto out;
2227 2245
2228 if (regs->tstate & TSTATE_PRIV) 2246 if (regs->tstate & TSTATE_PRIV)
2229 die_if_kernel("Penguin overflow trap from kernel mode", regs); 2247 die_if_kernel("Penguin overflow trap from kernel mode", regs);
@@ -2237,15 +2255,18 @@ void do_tof(struct pt_regs *regs)
2237 info.si_addr = (void __user *)regs->tpc; 2255 info.si_addr = (void __user *)regs->tpc;
2238 info.si_trapno = 0; 2256 info.si_trapno = 0;
2239 force_sig_info(SIGEMT, &info, current); 2257 force_sig_info(SIGEMT, &info, current);
2258out:
2259 exception_exit(prev_state);
2240} 2260}
2241 2261
2242void do_div0(struct pt_regs *regs) 2262void do_div0(struct pt_regs *regs)
2243{ 2263{
2264 enum ctx_state prev_state = exception_enter();
2244 siginfo_t info; 2265 siginfo_t info;
2245 2266
2246 if (notify_die(DIE_TRAP, "integer division by zero", regs, 2267 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2247 0, 0x28, SIGFPE) == NOTIFY_STOP) 2268 0, 0x28, SIGFPE) == NOTIFY_STOP)
2248 return; 2269 goto out;
2249 2270
2250 if (regs->tstate & TSTATE_PRIV) 2271 if (regs->tstate & TSTATE_PRIV)
2251 die_if_kernel("TL0: Kernel divide by zero.", regs); 2272 die_if_kernel("TL0: Kernel divide by zero.", regs);
@@ -2259,6 +2280,8 @@ void do_div0(struct pt_regs *regs)
2259 info.si_addr = (void __user *)regs->tpc; 2280 info.si_addr = (void __user *)regs->tpc;
2260 info.si_trapno = 0; 2281 info.si_trapno = 0;
2261 force_sig_info(SIGFPE, &info, current); 2282 force_sig_info(SIGFPE, &info, current);
2283out:
2284 exception_exit(prev_state);
2262} 2285}
2263 2286
2264static void instruction_dump(unsigned int *pc) 2287static void instruction_dump(unsigned int *pc)
@@ -2415,6 +2438,7 @@ extern int handle_ldf_stq(u32 insn, struct pt_regs *regs);
2415 2438
2416void do_illegal_instruction(struct pt_regs *regs) 2439void do_illegal_instruction(struct pt_regs *regs)
2417{ 2440{
2441 enum ctx_state prev_state = exception_enter();
2418 unsigned long pc = regs->tpc; 2442 unsigned long pc = regs->tpc;
2419 unsigned long tstate = regs->tstate; 2443 unsigned long tstate = regs->tstate;
2420 u32 insn; 2444 u32 insn;
@@ -2422,7 +2446,7 @@ void do_illegal_instruction(struct pt_regs *regs)
2422 2446
2423 if (notify_die(DIE_TRAP, "illegal instruction", regs, 2447 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2424 0, 0x10, SIGILL) == NOTIFY_STOP) 2448 0, 0x10, SIGILL) == NOTIFY_STOP)
2425 return; 2449 goto out;
2426 2450
2427 if (tstate & TSTATE_PRIV) 2451 if (tstate & TSTATE_PRIV)
2428 die_if_kernel("Kernel illegal instruction", regs); 2452 die_if_kernel("Kernel illegal instruction", regs);
@@ -2431,14 +2455,14 @@ void do_illegal_instruction(struct pt_regs *regs)
2431 if (get_user(insn, (u32 __user *) pc) != -EFAULT) { 2455 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2432 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { 2456 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2433 if (handle_popc(insn, regs)) 2457 if (handle_popc(insn, regs))
2434 return; 2458 goto out;
2435 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { 2459 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2436 if (handle_ldf_stq(insn, regs)) 2460 if (handle_ldf_stq(insn, regs))
2437 return; 2461 goto out;
2438 } else if (tlb_type == hypervisor) { 2462 } else if (tlb_type == hypervisor) {
2439 if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { 2463 if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2440 if (!vis_emul(regs, insn)) 2464 if (!vis_emul(regs, insn))
2441 return; 2465 goto out;
2442 } else { 2466 } else {
2443 struct fpustate *f = FPUSTATE; 2467 struct fpustate *f = FPUSTATE;
2444 2468
@@ -2448,7 +2472,7 @@ void do_illegal_instruction(struct pt_regs *regs)
2448 * Trap in the %fsr to unimplemented_FPop. 2472 * Trap in the %fsr to unimplemented_FPop.
2449 */ 2473 */
2450 if (do_mathemu(regs, f, true)) 2474 if (do_mathemu(regs, f, true))
2451 return; 2475 goto out;
2452 } 2476 }
2453 } 2477 }
2454 } 2478 }
@@ -2458,21 +2482,24 @@ void do_illegal_instruction(struct pt_regs *regs)
2458 info.si_addr = (void __user *)pc; 2482 info.si_addr = (void __user *)pc;
2459 info.si_trapno = 0; 2483 info.si_trapno = 0;
2460 force_sig_info(SIGILL, &info, current); 2484 force_sig_info(SIGILL, &info, current);
2485out:
2486 exception_exit(prev_state);
2461} 2487}
2462 2488
2463extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); 2489extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn);
2464 2490
2465void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 2491void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2466{ 2492{
2493 enum ctx_state prev_state = exception_enter();
2467 siginfo_t info; 2494 siginfo_t info;
2468 2495
2469 if (notify_die(DIE_TRAP, "memory address unaligned", regs, 2496 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2470 0, 0x34, SIGSEGV) == NOTIFY_STOP) 2497 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2471 return; 2498 goto out;
2472 2499
2473 if (regs->tstate & TSTATE_PRIV) { 2500 if (regs->tstate & TSTATE_PRIV) {
2474 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); 2501 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2475 return; 2502 goto out;
2476 } 2503 }
2477 info.si_signo = SIGBUS; 2504 info.si_signo = SIGBUS;
2478 info.si_errno = 0; 2505 info.si_errno = 0;
@@ -2480,6 +2507,8 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo
2480 info.si_addr = (void __user *)sfar; 2507 info.si_addr = (void __user *)sfar;
2481 info.si_trapno = 0; 2508 info.si_trapno = 0;
2482 force_sig_info(SIGBUS, &info, current); 2509 force_sig_info(SIGBUS, &info, current);
2510out:
2511 exception_exit(prev_state);
2483} 2512}
2484 2513
2485void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) 2514void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
@@ -2504,11 +2533,12 @@ void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_c
2504 2533
2505void do_privop(struct pt_regs *regs) 2534void do_privop(struct pt_regs *regs)
2506{ 2535{
2536 enum ctx_state prev_state = exception_enter();
2507 siginfo_t info; 2537 siginfo_t info;
2508 2538
2509 if (notify_die(DIE_TRAP, "privileged operation", regs, 2539 if (notify_die(DIE_TRAP, "privileged operation", regs,
2510 0, 0x11, SIGILL) == NOTIFY_STOP) 2540 0, 0x11, SIGILL) == NOTIFY_STOP)
2511 return; 2541 goto out;
2512 2542
2513 if (test_thread_flag(TIF_32BIT)) { 2543 if (test_thread_flag(TIF_32BIT)) {
2514 regs->tpc &= 0xffffffff; 2544 regs->tpc &= 0xffffffff;
@@ -2520,6 +2550,8 @@ void do_privop(struct pt_regs *regs)
2520 info.si_addr = (void __user *)regs->tpc; 2550 info.si_addr = (void __user *)regs->tpc;
2521 info.si_trapno = 0; 2551 info.si_trapno = 0;
2522 force_sig_info(SIGILL, &info, current); 2552 force_sig_info(SIGILL, &info, current);
2553out:
2554 exception_exit(prev_state);
2523} 2555}
2524 2556
2525void do_privact(struct pt_regs *regs) 2557void do_privact(struct pt_regs *regs)
@@ -2530,99 +2562,116 @@ void do_privact(struct pt_regs *regs)
2530/* Trap level 1 stuff or other traps we should never see... */ 2562/* Trap level 1 stuff or other traps we should never see... */
2531void do_cee(struct pt_regs *regs) 2563void do_cee(struct pt_regs *regs)
2532{ 2564{
2565 exception_enter();
2533 die_if_kernel("TL0: Cache Error Exception", regs); 2566 die_if_kernel("TL0: Cache Error Exception", regs);
2534} 2567}
2535 2568
2536void do_cee_tl1(struct pt_regs *regs) 2569void do_cee_tl1(struct pt_regs *regs)
2537{ 2570{
2571 exception_enter();
2538 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2572 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2539 die_if_kernel("TL1: Cache Error Exception", regs); 2573 die_if_kernel("TL1: Cache Error Exception", regs);
2540} 2574}
2541 2575
2542void do_dae_tl1(struct pt_regs *regs) 2576void do_dae_tl1(struct pt_regs *regs)
2543{ 2577{
2578 exception_enter();
2544 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2579 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2545 die_if_kernel("TL1: Data Access Exception", regs); 2580 die_if_kernel("TL1: Data Access Exception", regs);
2546} 2581}
2547 2582
2548void do_iae_tl1(struct pt_regs *regs) 2583void do_iae_tl1(struct pt_regs *regs)
2549{ 2584{
2585 exception_enter();
2550 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2586 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2551 die_if_kernel("TL1: Instruction Access Exception", regs); 2587 die_if_kernel("TL1: Instruction Access Exception", regs);
2552} 2588}
2553 2589
2554void do_div0_tl1(struct pt_regs *regs) 2590void do_div0_tl1(struct pt_regs *regs)
2555{ 2591{
2592 exception_enter();
2556 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2593 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2557 die_if_kernel("TL1: DIV0 Exception", regs); 2594 die_if_kernel("TL1: DIV0 Exception", regs);
2558} 2595}
2559 2596
2560void do_fpdis_tl1(struct pt_regs *regs) 2597void do_fpdis_tl1(struct pt_regs *regs)
2561{ 2598{
2599 exception_enter();
2562 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2600 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2563 die_if_kernel("TL1: FPU Disabled", regs); 2601 die_if_kernel("TL1: FPU Disabled", regs);
2564} 2602}
2565 2603
2566void do_fpieee_tl1(struct pt_regs *regs) 2604void do_fpieee_tl1(struct pt_regs *regs)
2567{ 2605{
2606 exception_enter();
2568 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2607 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2569 die_if_kernel("TL1: FPU IEEE Exception", regs); 2608 die_if_kernel("TL1: FPU IEEE Exception", regs);
2570} 2609}
2571 2610
2572void do_fpother_tl1(struct pt_regs *regs) 2611void do_fpother_tl1(struct pt_regs *regs)
2573{ 2612{
2613 exception_enter();
2574 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2614 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2575 die_if_kernel("TL1: FPU Other Exception", regs); 2615 die_if_kernel("TL1: FPU Other Exception", regs);
2576} 2616}
2577 2617
2578void do_ill_tl1(struct pt_regs *regs) 2618void do_ill_tl1(struct pt_regs *regs)
2579{ 2619{
2620 exception_enter();
2580 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2621 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2581 die_if_kernel("TL1: Illegal Instruction Exception", regs); 2622 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2582} 2623}
2583 2624
2584void do_irq_tl1(struct pt_regs *regs) 2625void do_irq_tl1(struct pt_regs *regs)
2585{ 2626{
2627 exception_enter();
2586 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2628 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2587 die_if_kernel("TL1: IRQ Exception", regs); 2629 die_if_kernel("TL1: IRQ Exception", regs);
2588} 2630}
2589 2631
2590void do_lddfmna_tl1(struct pt_regs *regs) 2632void do_lddfmna_tl1(struct pt_regs *regs)
2591{ 2633{
2634 exception_enter();
2592 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2635 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2593 die_if_kernel("TL1: LDDF Exception", regs); 2636 die_if_kernel("TL1: LDDF Exception", regs);
2594} 2637}
2595 2638
2596void do_stdfmna_tl1(struct pt_regs *regs) 2639void do_stdfmna_tl1(struct pt_regs *regs)
2597{ 2640{
2641 exception_enter();
2598 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2642 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2599 die_if_kernel("TL1: STDF Exception", regs); 2643 die_if_kernel("TL1: STDF Exception", regs);
2600} 2644}
2601 2645
2602void do_paw(struct pt_regs *regs) 2646void do_paw(struct pt_regs *regs)
2603{ 2647{
2648 exception_enter();
2604 die_if_kernel("TL0: Phys Watchpoint Exception", regs); 2649 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2605} 2650}
2606 2651
2607void do_paw_tl1(struct pt_regs *regs) 2652void do_paw_tl1(struct pt_regs *regs)
2608{ 2653{
2654 exception_enter();
2609 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2655 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2610 die_if_kernel("TL1: Phys Watchpoint Exception", regs); 2656 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2611} 2657}
2612 2658
2613void do_vaw(struct pt_regs *regs) 2659void do_vaw(struct pt_regs *regs)
2614{ 2660{
2661 exception_enter();
2615 die_if_kernel("TL0: Virt Watchpoint Exception", regs); 2662 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2616} 2663}
2617 2664
2618void do_vaw_tl1(struct pt_regs *regs) 2665void do_vaw_tl1(struct pt_regs *regs)
2619{ 2666{
2667 exception_enter();
2620 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2668 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2621 die_if_kernel("TL1: Virt Watchpoint Exception", regs); 2669 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2622} 2670}
2623 2671
2624void do_tof_tl1(struct pt_regs *regs) 2672void do_tof_tl1(struct pt_regs *regs)
2625{ 2673{
2674 exception_enter();
2626 dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); 2675 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2627 die_if_kernel("TL1: Tag Overflow Exception", regs); 2676 die_if_kernel("TL1: Tag Overflow Exception", regs);
2628} 2677}
diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S
index a313e4a9399b..14158d40ba76 100644
--- a/arch/sparc/kernel/tsb.S
+++ b/arch/sparc/kernel/tsb.S
@@ -75,7 +75,7 @@ tsb_miss_page_table_walk:
75 mov 512, %g7 75 mov 512, %g7
76 andn %g5, 0x7, %g5 76 andn %g5, 0x7, %g5
77 sllx %g7, %g6, %g7 77 sllx %g7, %g6, %g7
78 srlx %g4, HPAGE_SHIFT, %g6 78 srlx %g4, REAL_HPAGE_SHIFT, %g6
79 sub %g7, 1, %g7 79 sub %g7, 1, %g7
80 and %g6, %g7, %g6 80 and %g6, %g7, %g6
81 sllx %g6, 4, %g6 81 sllx %g6, 4, %g6
diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c
index 8201c25e7669..3c1a7cb31579 100644
--- a/arch/sparc/kernel/unaligned_64.c
+++ b/arch/sparc/kernel/unaligned_64.c
@@ -21,9 +21,12 @@
21#include <linux/bitops.h> 21#include <linux/bitops.h>
22#include <linux/perf_event.h> 22#include <linux/perf_event.h>
23#include <linux/ratelimit.h> 23#include <linux/ratelimit.h>
24#include <linux/context_tracking.h>
24#include <asm/fpumacro.h> 25#include <asm/fpumacro.h>
25#include <asm/cacheflush.h> 26#include <asm/cacheflush.h>
26 27
28#include "entry.h"
29
27enum direction { 30enum direction {
28 load, /* ld, ldd, ldh, ldsh */ 31 load, /* ld, ldd, ldh, ldsh */
29 store, /* st, std, sth, stsh */ 32 store, /* st, std, sth, stsh */
@@ -418,9 +421,6 @@ int handle_popc(u32 insn, struct pt_regs *regs)
418 421
419extern void do_fpother(struct pt_regs *regs); 422extern void do_fpother(struct pt_regs *regs);
420extern void do_privact(struct pt_regs *regs); 423extern void do_privact(struct pt_regs *regs);
421extern void spitfire_data_access_exception(struct pt_regs *regs,
422 unsigned long sfsr,
423 unsigned long sfar);
424extern void sun4v_data_access_exception(struct pt_regs *regs, 424extern void sun4v_data_access_exception(struct pt_regs *regs,
425 unsigned long addr, 425 unsigned long addr,
426 unsigned long type_ctx); 426 unsigned long type_ctx);
@@ -578,6 +578,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs)
578 578
579void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 579void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
580{ 580{
581 enum ctx_state prev_state = exception_enter();
581 unsigned long pc = regs->tpc; 582 unsigned long pc = regs->tpc;
582 unsigned long tstate = regs->tstate; 583 unsigned long tstate = regs->tstate;
583 u32 insn; 584 u32 insn;
@@ -632,13 +633,16 @@ daex:
632 sun4v_data_access_exception(regs, sfar, sfsr); 633 sun4v_data_access_exception(regs, sfar, sfsr);
633 else 634 else
634 spitfire_data_access_exception(regs, sfsr, sfar); 635 spitfire_data_access_exception(regs, sfsr, sfar);
635 return; 636 goto out;
636 } 637 }
637 advance(regs); 638 advance(regs);
639out:
640 exception_exit(prev_state);
638} 641}
639 642
640void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) 643void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
641{ 644{
645 enum ctx_state prev_state = exception_enter();
642 unsigned long pc = regs->tpc; 646 unsigned long pc = regs->tpc;
643 unsigned long tstate = regs->tstate; 647 unsigned long tstate = regs->tstate;
644 u32 insn; 648 u32 insn;
@@ -680,7 +684,9 @@ daex:
680 sun4v_data_access_exception(regs, sfar, sfsr); 684 sun4v_data_access_exception(regs, sfar, sfsr);
681 else 685 else
682 spitfire_data_access_exception(regs, sfsr, sfar); 686 spitfire_data_access_exception(regs, sfsr, sfar);
683 return; 687 goto out;
684 } 688 }
685 advance(regs); 689 advance(regs);
690out:
691 exception_exit(prev_state);
686} 692}
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index 0bacceb19150..932ff90fd760 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -122,6 +122,11 @@ SECTIONS
122 *(.swapper_4m_tsb_phys_patch) 122 *(.swapper_4m_tsb_phys_patch)
123 __swapper_4m_tsb_phys_patch_end = .; 123 __swapper_4m_tsb_phys_patch_end = .;
124 } 124 }
125 .page_offset_shift_patch : {
126 __page_offset_shift_patch = .;
127 *(.page_offset_shift_patch)
128 __page_offset_shift_patch_end = .;
129 }
125 .popc_3insn_patch : { 130 .popc_3insn_patch : {
126 __popc_3insn_patch = .; 131 __popc_3insn_patch = .;
127 *(.popc_3insn_patch) 132 *(.popc_3insn_patch)
diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S
index 77e531f6c2a7..46272dfc26e8 100644
--- a/arch/sparc/lib/clear_page.S
+++ b/arch/sparc/lib/clear_page.S
@@ -37,10 +37,10 @@ _clear_page: /* %o0=dest */
37 .globl clear_user_page 37 .globl clear_user_page
38clear_user_page: /* %o0=dest, %o1=vaddr */ 38clear_user_page: /* %o0=dest, %o1=vaddr */
39 lduw [%g6 + TI_PRE_COUNT], %o2 39 lduw [%g6 + TI_PRE_COUNT], %o2
40 sethi %uhi(PAGE_OFFSET), %g2 40 sethi %hi(PAGE_OFFSET), %g2
41 sethi %hi(PAGE_SIZE), %o4 41 sethi %hi(PAGE_SIZE), %o4
42 42
43 sllx %g2, 32, %g2 43 ldx [%g2 + %lo(PAGE_OFFSET)], %g2
44 sethi %hi(PAGE_KERNEL_LOCKED), %g3 44 sethi %hi(PAGE_KERNEL_LOCKED), %g3
45 45
46 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 46 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S
index 4d2df328e514..dd16c61f3263 100644
--- a/arch/sparc/lib/copy_page.S
+++ b/arch/sparc/lib/copy_page.S
@@ -46,10 +46,10 @@
46 .type copy_user_page,#function 46 .type copy_user_page,#function
47copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ 47copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
48 lduw [%g6 + TI_PRE_COUNT], %o4 48 lduw [%g6 + TI_PRE_COUNT], %o4
49 sethi %uhi(PAGE_OFFSET), %g2 49 sethi %hi(PAGE_OFFSET), %g2
50 sethi %hi(PAGE_SIZE), %o3 50 sethi %hi(PAGE_SIZE), %o3
51 51
52 sllx %g2, 32, %g2 52 ldx [%g2 + %lo(PAGE_OFFSET)], %g2
53 sethi %hi(PAGE_KERNEL_LOCKED), %g3 53 sethi %hi(PAGE_KERNEL_LOCKED), %g3
54 54
55 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 55 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 2ebec263d685..69bb818fdd79 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -21,6 +21,7 @@
21#include <linux/kprobes.h> 21#include <linux/kprobes.h>
22#include <linux/kdebug.h> 22#include <linux/kdebug.h>
23#include <linux/percpu.h> 23#include <linux/percpu.h>
24#include <linux/context_tracking.h>
24 25
25#include <asm/page.h> 26#include <asm/page.h>
26#include <asm/pgtable.h> 27#include <asm/pgtable.h>
@@ -272,6 +273,7 @@ static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs,
272 273
273asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) 274asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
274{ 275{
276 enum ctx_state prev_state = exception_enter();
275 struct mm_struct *mm = current->mm; 277 struct mm_struct *mm = current->mm;
276 struct vm_area_struct *vma; 278 struct vm_area_struct *vma;
277 unsigned int insn = 0; 279 unsigned int insn = 0;
@@ -282,7 +284,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
282 fault_code = get_thread_fault_code(); 284 fault_code = get_thread_fault_code();
283 285
284 if (notify_page_fault(regs)) 286 if (notify_page_fault(regs))
285 return; 287 goto exit_exception;
286 288
287 si_code = SEGV_MAPERR; 289 si_code = SEGV_MAPERR;
288 address = current_thread_info()->fault_address; 290 address = current_thread_info()->fault_address;
@@ -313,7 +315,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
313 /* Valid, no problems... */ 315 /* Valid, no problems... */
314 } else { 316 } else {
315 bad_kernel_pc(regs, address); 317 bad_kernel_pc(regs, address);
316 return; 318 goto exit_exception;
317 } 319 }
318 } else 320 } else
319 flags |= FAULT_FLAG_USER; 321 flags |= FAULT_FLAG_USER;
@@ -430,7 +432,7 @@ good_area:
430 fault = handle_mm_fault(mm, vma, address, flags); 432 fault = handle_mm_fault(mm, vma, address, flags);
431 433
432 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) 434 if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
433 return; 435 goto exit_exception;
434 436
435 if (unlikely(fault & VM_FAULT_ERROR)) { 437 if (unlikely(fault & VM_FAULT_ERROR)) {
436 if (fault & VM_FAULT_OOM) 438 if (fault & VM_FAULT_OOM)
@@ -482,6 +484,8 @@ good_area:
482 484
483 } 485 }
484#endif 486#endif
487exit_exception:
488 exception_exit(prev_state);
485 return; 489 return;
486 490
487 /* 491 /*
@@ -494,7 +498,7 @@ bad_area:
494 498
495handle_kernel_fault: 499handle_kernel_fault:
496 do_kernel_fault(regs, si_code, fault_code, insn, address); 500 do_kernel_fault(regs, si_code, fault_code, insn, address);
497 return; 501 goto exit_exception;
498 502
499/* 503/*
500 * We ran out of memory, or some other thing happened to us that made 504 * We ran out of memory, or some other thing happened to us that made
@@ -505,7 +509,7 @@ out_of_memory:
505 up_read(&mm->mmap_sem); 509 up_read(&mm->mmap_sem);
506 if (!(regs->tstate & TSTATE_PRIV)) { 510 if (!(regs->tstate & TSTATE_PRIV)) {
507 pagefault_out_of_memory(); 511 pagefault_out_of_memory();
508 return; 512 goto exit_exception;
509 } 513 }
510 goto handle_kernel_fault; 514 goto handle_kernel_fault;
511 515
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index 01ee23dd724d..c4d3da68b800 100644
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -71,13 +71,12 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
71 int *nr) 71 int *nr)
72{ 72{
73 struct page *head, *page, *tail; 73 struct page *head, *page, *tail;
74 u32 mask;
75 int refs; 74 int refs;
76 75
77 mask = PMD_HUGE_PRESENT; 76 if (!pmd_large(pmd))
78 if (write) 77 return 0;
79 mask |= PMD_HUGE_WRITE; 78
80 if ((pmd_val(pmd) & mask) != mask) 79 if (write && !pmd_write(pmd))
81 return 0; 80 return 0;
82 81
83 refs = 0; 82 refs = 0;
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 96399646570a..30963178d7e9 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -21,8 +21,6 @@
21/* Slightly simplified from the non-hugepage variant because by 21/* Slightly simplified from the non-hugepage variant because by
22 * definition we don't have to worry about any page coloring stuff 22 * definition we don't have to worry about any page coloring stuff
23 */ 23 */
24#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL))
25#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL))
26 24
27static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, 25static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
28 unsigned long addr, 26 unsigned long addr,
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index d6de9353ee11..6b643790e4fe 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -354,7 +354,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
354 354
355#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 355#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
356 if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) 356 if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
357 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, 357 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
358 address, pte_val(pte)); 358 address, pte_val(pte));
359 else 359 else
360#endif 360#endif
@@ -1557,6 +1557,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size)
1557 return ~0UL; 1557 return ~0UL;
1558} 1558}
1559 1559
1560unsigned long PAGE_OFFSET;
1561EXPORT_SYMBOL(PAGE_OFFSET);
1562
1563static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)
1564{
1565 unsigned long final_shift;
1566 unsigned int val = *insn;
1567 unsigned int cnt;
1568
1569 /* We are patching in ilog2(max_supported_phys_address), and
1570 * we are doing so in a manner similar to a relocation addend.
1571 * That is, we are adding the shift value to whatever value
1572 * is in the shift instruction count field already.
1573 */
1574 cnt = (val & 0x3f);
1575 val &= ~0x3f;
1576
1577 /* If we are trying to shift >= 64 bits, clear the destination
1578 * register. This can happen when phys_bits ends up being equal
1579 * to MAX_PHYS_ADDRESS_BITS.
1580 */
1581 final_shift = (cnt + (64 - phys_bits));
1582 if (final_shift >= 64) {
1583 unsigned int rd = (val >> 25) & 0x1f;
1584
1585 val = 0x80100000 | (rd << 25);
1586 } else {
1587 val |= final_shift;
1588 }
1589 *insn = val;
1590
1591 __asm__ __volatile__("flush %0"
1592 : /* no outputs */
1593 : "r" (insn));
1594}
1595
1596static void __init page_offset_shift_patch(unsigned long phys_bits)
1597{
1598 extern unsigned int __page_offset_shift_patch;
1599 extern unsigned int __page_offset_shift_patch_end;
1600 unsigned int *p;
1601
1602 p = &__page_offset_shift_patch;
1603 while (p < &__page_offset_shift_patch_end) {
1604 unsigned int *insn = (unsigned int *)(unsigned long)*p;
1605
1606 page_offset_shift_patch_one(insn, phys_bits);
1607
1608 p++;
1609 }
1610}
1611
1612static void __init setup_page_offset(void)
1613{
1614 unsigned long max_phys_bits = 40;
1615
1616 if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1617 max_phys_bits = 42;
1618 } else if (tlb_type == hypervisor) {
1619 switch (sun4v_chip_type) {
1620 case SUN4V_CHIP_NIAGARA1:
1621 case SUN4V_CHIP_NIAGARA2:
1622 max_phys_bits = 39;
1623 break;
1624 case SUN4V_CHIP_NIAGARA3:
1625 max_phys_bits = 43;
1626 break;
1627 case SUN4V_CHIP_NIAGARA4:
1628 case SUN4V_CHIP_NIAGARA5:
1629 case SUN4V_CHIP_SPARC64X:
1630 default:
1631 max_phys_bits = 47;
1632 break;
1633 }
1634 }
1635
1636 if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
1637 prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
1638 max_phys_bits);
1639 prom_halt();
1640 }
1641
1642 PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);
1643
1644 pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
1645 PAGE_OFFSET, max_phys_bits);
1646
1647 page_offset_shift_patch(max_phys_bits);
1648}
1649
1560static void __init tsb_phys_patch(void) 1650static void __init tsb_phys_patch(void)
1561{ 1651{
1562 struct tsb_ldquad_phys_patch_entry *pquad; 1652 struct tsb_ldquad_phys_patch_entry *pquad;
@@ -1722,7 +1812,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1722#ifndef CONFIG_DEBUG_PAGEALLOC 1812#ifndef CONFIG_DEBUG_PAGEALLOC
1723 if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { 1813 if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
1724 kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ 1814 kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
1725 0xfffff80000000000UL; 1815 PAGE_OFFSET;
1726 kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | 1816 kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1727 _PAGE_P_4V | _PAGE_W_4V); 1817 _PAGE_P_4V | _PAGE_W_4V);
1728 } else { 1818 } else {
@@ -1731,7 +1821,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1731 1821
1732 if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { 1822 if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
1733 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ 1823 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
1734 0xfffff80000000000UL; 1824 PAGE_OFFSET;
1735 kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | 1825 kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1736 _PAGE_P_4V | _PAGE_W_4V); 1826 _PAGE_P_4V | _PAGE_W_4V);
1737 } else { 1827 } else {
@@ -1740,7 +1830,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1740 1830
1741 if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { 1831 if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
1742 kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ 1832 kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
1743 0xfffff80000000000UL; 1833 PAGE_OFFSET;
1744 kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | 1834 kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |
1745 _PAGE_P_4V | _PAGE_W_4V); 1835 _PAGE_P_4V | _PAGE_W_4V);
1746 } else { 1836 } else {
@@ -1752,7 +1842,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1752/* paging_init() sets up the page tables */ 1842/* paging_init() sets up the page tables */
1753 1843
1754static unsigned long last_valid_pfn; 1844static unsigned long last_valid_pfn;
1755pgd_t swapper_pg_dir[2048]; 1845pgd_t swapper_pg_dir[PTRS_PER_PGD];
1756 1846
1757static void sun4u_pgprot_init(void); 1847static void sun4u_pgprot_init(void);
1758static void sun4v_pgprot_init(void); 1848static void sun4v_pgprot_init(void);
@@ -1763,6 +1853,8 @@ void __init paging_init(void)
1763 unsigned long real_end, i; 1853 unsigned long real_end, i;
1764 int node; 1854 int node;
1765 1855
1856 setup_page_offset();
1857
1766 /* These build time checkes make sure that the dcache_dirty_cpu() 1858 /* These build time checkes make sure that the dcache_dirty_cpu()
1767 * page->flags usage will work. 1859 * page->flags usage will work.
1768 * 1860 *
@@ -2261,10 +2353,10 @@ static void __init sun4u_pgprot_init(void)
2261 __ACCESS_BITS_4U | _PAGE_E_4U); 2353 __ACCESS_BITS_4U | _PAGE_E_4U);
2262 2354
2263#ifdef CONFIG_DEBUG_PAGEALLOC 2355#ifdef CONFIG_DEBUG_PAGEALLOC
2264 kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; 2356 kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
2265#else 2357#else
2266 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ 2358 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
2267 0xfffff80000000000UL; 2359 PAGE_OFFSET;
2268#endif 2360#endif
2269 kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | 2361 kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
2270 _PAGE_P_4U | _PAGE_W_4U); 2362 _PAGE_P_4U | _PAGE_W_4U);
@@ -2308,10 +2400,10 @@ static void __init sun4v_pgprot_init(void)
2308 _PAGE_CACHE = _PAGE_CACHE_4V; 2400 _PAGE_CACHE = _PAGE_CACHE_4V;
2309 2401
2310#ifdef CONFIG_DEBUG_PAGEALLOC 2402#ifdef CONFIG_DEBUG_PAGEALLOC
2311 kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; 2403 kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
2312#else 2404#else
2313 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ 2405 kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
2314 0xfffff80000000000UL; 2406 PAGE_OFFSET;
2315#endif 2407#endif
2316 kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | 2408 kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
2317 _PAGE_P_4V | _PAGE_W_4V); 2409 _PAGE_P_4V | _PAGE_W_4V);
@@ -2455,53 +2547,13 @@ void __flush_tlb_all(void)
2455 : : "r" (pstate)); 2547 : : "r" (pstate));
2456} 2548}
2457 2549
2458static pte_t *get_from_cache(struct mm_struct *mm)
2459{
2460 struct page *page;
2461 pte_t *ret;
2462
2463 spin_lock(&mm->page_table_lock);
2464 page = mm->context.pgtable_page;
2465 ret = NULL;
2466 if (page) {
2467 void *p = page_address(page);
2468
2469 mm->context.pgtable_page = NULL;
2470
2471 ret = (pte_t *) (p + (PAGE_SIZE / 2));
2472 }
2473 spin_unlock(&mm->page_table_lock);
2474
2475 return ret;
2476}
2477
2478static struct page *__alloc_for_cache(struct mm_struct *mm)
2479{
2480 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
2481 __GFP_REPEAT | __GFP_ZERO);
2482
2483 if (page) {
2484 spin_lock(&mm->page_table_lock);
2485 if (!mm->context.pgtable_page) {
2486 atomic_set(&page->_count, 2);
2487 mm->context.pgtable_page = page;
2488 }
2489 spin_unlock(&mm->page_table_lock);
2490 }
2491 return page;
2492}
2493
2494pte_t *pte_alloc_one_kernel(struct mm_struct *mm, 2550pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
2495 unsigned long address) 2551 unsigned long address)
2496{ 2552{
2497 struct page *page; 2553 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
2498 pte_t *pte; 2554 __GFP_REPEAT | __GFP_ZERO);
2499 2555 pte_t *pte = NULL;
2500 pte = get_from_cache(mm);
2501 if (pte)
2502 return pte;
2503 2556
2504 page = __alloc_for_cache(mm);
2505 if (page) 2557 if (page)
2506 pte = (pte_t *) page_address(page); 2558 pte = (pte_t *) page_address(page);
2507 2559
@@ -2511,14 +2563,10 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
2511pgtable_t pte_alloc_one(struct mm_struct *mm, 2563pgtable_t pte_alloc_one(struct mm_struct *mm,
2512 unsigned long address) 2564 unsigned long address)
2513{ 2565{
2514 struct page *page; 2566 struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
2515 pte_t *pte; 2567 __GFP_REPEAT | __GFP_ZERO);
2516 2568 pte_t *pte = NULL;
2517 pte = get_from_cache(mm);
2518 if (pte)
2519 return pte;
2520 2569
2521 page = __alloc_for_cache(mm);
2522 if (!page) 2570 if (!page)
2523 return NULL; 2571 return NULL;
2524 if (!pgtable_page_ctor(page)) { 2572 if (!pgtable_page_ctor(page)) {
@@ -2530,18 +2578,15 @@ pgtable_t pte_alloc_one(struct mm_struct *mm,
2530 2578
2531void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 2579void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
2532{ 2580{
2533 struct page *page = virt_to_page(pte); 2581 free_page((unsigned long)pte);
2534 if (put_page_testzero(page))
2535 free_hot_cold_page(page, 0);
2536} 2582}
2537 2583
2538static void __pte_free(pgtable_t pte) 2584static void __pte_free(pgtable_t pte)
2539{ 2585{
2540 struct page *page = virt_to_page(pte); 2586 struct page *page = virt_to_page(pte);
2541 if (put_page_testzero(page)) { 2587
2542 pgtable_page_dtor(page); 2588 pgtable_page_dtor(page);
2543 free_hot_cold_page(page, 0); 2589 __free_page(page);
2544 }
2545} 2590}
2546 2591
2547void pte_free(struct mm_struct *mm, pgtable_t pte) 2592void pte_free(struct mm_struct *mm, pgtable_t pte)
@@ -2558,124 +2603,27 @@ void pgtable_free(void *table, bool is_page)
2558} 2603}
2559 2604
2560#ifdef CONFIG_TRANSPARENT_HUGEPAGE 2605#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2561static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify)
2562{
2563 if (pgprot_val(pgprot) & _PAGE_VALID)
2564 pmd_val(pmd) |= PMD_HUGE_PRESENT;
2565 if (tlb_type == hypervisor) {
2566 if (pgprot_val(pgprot) & _PAGE_WRITE_4V)
2567 pmd_val(pmd) |= PMD_HUGE_WRITE;
2568 if (pgprot_val(pgprot) & _PAGE_EXEC_4V)
2569 pmd_val(pmd) |= PMD_HUGE_EXEC;
2570
2571 if (!for_modify) {
2572 if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V)
2573 pmd_val(pmd) |= PMD_HUGE_ACCESSED;
2574 if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V)
2575 pmd_val(pmd) |= PMD_HUGE_DIRTY;
2576 }
2577 } else {
2578 if (pgprot_val(pgprot) & _PAGE_WRITE_4U)
2579 pmd_val(pmd) |= PMD_HUGE_WRITE;
2580 if (pgprot_val(pgprot) & _PAGE_EXEC_4U)
2581 pmd_val(pmd) |= PMD_HUGE_EXEC;
2582
2583 if (!for_modify) {
2584 if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U)
2585 pmd_val(pmd) |= PMD_HUGE_ACCESSED;
2586 if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U)
2587 pmd_val(pmd) |= PMD_HUGE_DIRTY;
2588 }
2589 }
2590
2591 return pmd;
2592}
2593
2594pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
2595{
2596 pmd_t pmd;
2597
2598 pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT)));
2599 pmd_val(pmd) |= PMD_ISHUGE;
2600 pmd = pmd_set_protbits(pmd, pgprot, false);
2601 return pmd;
2602}
2603
2604pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
2605{
2606 pmd_val(pmd) &= ~(PMD_HUGE_PRESENT |
2607 PMD_HUGE_WRITE |
2608 PMD_HUGE_EXEC);
2609 pmd = pmd_set_protbits(pmd, newprot, true);
2610 return pmd;
2611}
2612
2613pgprot_t pmd_pgprot(pmd_t entry)
2614{
2615 unsigned long pte = 0;
2616
2617 if (pmd_val(entry) & PMD_HUGE_PRESENT)
2618 pte |= _PAGE_VALID;
2619
2620 if (tlb_type == hypervisor) {
2621 if (pmd_val(entry) & PMD_HUGE_PRESENT)
2622 pte |= _PAGE_PRESENT_4V;
2623 if (pmd_val(entry) & PMD_HUGE_EXEC)
2624 pte |= _PAGE_EXEC_4V;
2625 if (pmd_val(entry) & PMD_HUGE_WRITE)
2626 pte |= _PAGE_W_4V;
2627 if (pmd_val(entry) & PMD_HUGE_ACCESSED)
2628 pte |= _PAGE_ACCESSED_4V;
2629 if (pmd_val(entry) & PMD_HUGE_DIRTY)
2630 pte |= _PAGE_MODIFIED_4V;
2631 pte |= _PAGE_CP_4V|_PAGE_CV_4V;
2632 } else {
2633 if (pmd_val(entry) & PMD_HUGE_PRESENT)
2634 pte |= _PAGE_PRESENT_4U;
2635 if (pmd_val(entry) & PMD_HUGE_EXEC)
2636 pte |= _PAGE_EXEC_4U;
2637 if (pmd_val(entry) & PMD_HUGE_WRITE)
2638 pte |= _PAGE_W_4U;
2639 if (pmd_val(entry) & PMD_HUGE_ACCESSED)
2640 pte |= _PAGE_ACCESSED_4U;
2641 if (pmd_val(entry) & PMD_HUGE_DIRTY)
2642 pte |= _PAGE_MODIFIED_4U;
2643 pte |= _PAGE_CP_4U|_PAGE_CV_4U;
2644 }
2645
2646 return __pgprot(pte);
2647}
2648
2649void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, 2606void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
2650 pmd_t *pmd) 2607 pmd_t *pmd)
2651{ 2608{
2652 unsigned long pte, flags; 2609 unsigned long pte, flags;
2653 struct mm_struct *mm; 2610 struct mm_struct *mm;
2654 pmd_t entry = *pmd; 2611 pmd_t entry = *pmd;
2655 pgprot_t prot;
2656 2612
2657 if (!pmd_large(entry) || !pmd_young(entry)) 2613 if (!pmd_large(entry) || !pmd_young(entry))
2658 return; 2614 return;
2659 2615
2660 pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS); 2616 pte = pmd_val(entry);
2661 pte <<= PMD_PADDR_SHIFT;
2662 pte |= _PAGE_VALID;
2663
2664 prot = pmd_pgprot(entry);
2665
2666 if (tlb_type == hypervisor)
2667 pgprot_val(prot) |= _PAGE_SZHUGE_4V;
2668 else
2669 pgprot_val(prot) |= _PAGE_SZHUGE_4U;
2670 2617
2671 pte |= pgprot_val(prot); 2618 /* We are fabricating 8MB pages using 4MB real hw pages. */
2619 pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
2672 2620
2673 mm = vma->vm_mm; 2621 mm = vma->vm_mm;
2674 2622
2675 spin_lock_irqsave(&mm->context.lock, flags); 2623 spin_lock_irqsave(&mm->context.lock, flags);
2676 2624
2677 if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) 2625 if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
2678 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, 2626 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
2679 addr, pte); 2627 addr, pte);
2680 2628
2681 spin_unlock_irqrestore(&mm->context.lock, flags); 2629 spin_unlock_irqrestore(&mm->context.lock, flags);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 0661aa606dec..5d3782deb403 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -1,11 +1,13 @@
1#ifndef _SPARC64_MM_INIT_H 1#ifndef _SPARC64_MM_INIT_H
2#define _SPARC64_MM_INIT_H 2#define _SPARC64_MM_INIT_H
3 3
4#include <asm/page.h>
5
4/* Most of the symbols in this file are defined in init.c and 6/* Most of the symbols in this file are defined in init.c and
5 * marked non-static so that assembler code can get at them. 7 * marked non-static so that assembler code can get at them.
6 */ 8 */
7 9
8#define MAX_PHYS_ADDRESS (1UL << 41UL) 10#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS)
9#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) 11#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
10#define KPTE_BITMAP_BYTES \ 12#define KPTE_BITMAP_BYTES \
11 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) 13 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4)
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 656cc46a81f5..ad3bf4b4324d 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -161,8 +161,8 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
161 if (mm == &init_mm) 161 if (mm == &init_mm)
162 return; 162 return;
163 163
164 if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) { 164 if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
165 if (pmd_val(pmd) & PMD_ISHUGE) 165 if (pmd_val(pmd) & _PAGE_PMD_HUGE)
166 mm->context.huge_pte_count++; 166 mm->context.huge_pte_count++;
167 else 167 else
168 mm->context.huge_pte_count--; 168 mm->context.huge_pte_count--;
@@ -178,13 +178,16 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
178 } 178 }
179 179
180 if (!pmd_none(orig)) { 180 if (!pmd_none(orig)) {
181 bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0); 181 pte_t orig_pte = __pte(pmd_val(orig));
182 bool exec = pte_exec(orig_pte);
182 183
183 addr &= HPAGE_MASK; 184 addr &= HPAGE_MASK;
184 if (pmd_val(orig) & PMD_ISHUGE) 185 if (pmd_trans_huge(orig)) {
185 tlb_batch_add_one(mm, addr, exec); 186 tlb_batch_add_one(mm, addr, exec);
186 else 187 tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
188 } else {
187 tlb_batch_pmd_scan(mm, addr, orig, exec); 189 tlb_batch_pmd_scan(mm, addr, orig, exec);
190 }
188 } 191 }
189} 192}
190 193
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 2cc3bce5ee91..3b3a360b429a 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -87,7 +87,7 @@ void flush_tsb_user(struct tlb_batch *tb)
87 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; 87 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
88 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 88 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
89 base = __pa(base); 89 base = __pa(base);
90 __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries); 90 __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries);
91 } 91 }
92#endif 92#endif
93 spin_unlock_irqrestore(&mm->context.lock, flags); 93 spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -111,7 +111,7 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
111 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; 111 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
112 if (tlb_type == cheetah_plus || tlb_type == hypervisor) 112 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
113 base = __pa(base); 113 base = __pa(base);
114 __flush_tsb_one_entry(base, vaddr, HPAGE_SHIFT, nentries); 114 __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries);
115 } 115 }
116#endif 116#endif
117 spin_unlock_irqrestore(&mm->context.lock, flags); 117 spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -472,8 +472,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
472 mm->context.huge_pte_count = 0; 472 mm->context.huge_pte_count = 0;
473#endif 473#endif
474 474
475 mm->context.pgtable_page = NULL;
476
477 /* copy_mm() copies over the parent's mm_struct before calling 475 /* copy_mm() copies over the parent's mm_struct before calling
478 * us, so we need to zero out the TSB pointer or else tsb_grow() 476 * us, so we need to zero out the TSB pointer or else tsb_grow()
479 * will be confused and think there is an older TSB to free up. 477 * will be confused and think there is an older TSB to free up.
@@ -512,17 +510,10 @@ static void tsb_destroy_one(struct tsb_config *tp)
512void destroy_context(struct mm_struct *mm) 510void destroy_context(struct mm_struct *mm)
513{ 511{
514 unsigned long flags, i; 512 unsigned long flags, i;
515 struct page *page;
516 513
517 for (i = 0; i < MM_NUM_TSBS; i++) 514 for (i = 0; i < MM_NUM_TSBS; i++)
518 tsb_destroy_one(&mm->context.tsb_block[i]); 515 tsb_destroy_one(&mm->context.tsb_block[i]);
519 516
520 page = mm->context.pgtable_page;
521 if (page && put_page_testzero(page)) {
522 pgtable_page_dtor(page);
523 free_hot_cold_page(page, 0);
524 }
525
526 spin_lock_irqsave(&ctx_alloc_lock, flags); 517 spin_lock_irqsave(&ctx_alloc_lock, flags);
527 518
528 if (CTX_VALID(mm->context)) { 519 if (CTX_VALID(mm->context)) {
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index 432aa0cb1b38..b4f4733abc6e 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -153,10 +153,10 @@ __spitfire_flush_tlb_mm_slow:
153 .globl __flush_icache_page 153 .globl __flush_icache_page
154__flush_icache_page: /* %o0 = phys_page */ 154__flush_icache_page: /* %o0 = phys_page */
155 srlx %o0, PAGE_SHIFT, %o0 155 srlx %o0, PAGE_SHIFT, %o0
156 sethi %uhi(PAGE_OFFSET), %g1 156 sethi %hi(PAGE_OFFSET), %g1
157 sllx %o0, PAGE_SHIFT, %o0 157 sllx %o0, PAGE_SHIFT, %o0
158 sethi %hi(PAGE_SIZE), %g2 158 sethi %hi(PAGE_SIZE), %g2
159 sllx %g1, 32, %g1 159 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
160 add %o0, %g1, %o0 160 add %o0, %g1, %o0
1611: subcc %g2, 32, %g2 1611: subcc %g2, 32, %g2
162 bne,pt %icc, 1b 162 bne,pt %icc, 1b
@@ -178,8 +178,8 @@ __flush_icache_page: /* %o0 = phys_page */
178 .align 64 178 .align 64
179 .globl __flush_dcache_page 179 .globl __flush_dcache_page
180__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ 180__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
181 sethi %uhi(PAGE_OFFSET), %g1 181 sethi %hi(PAGE_OFFSET), %g1
182 sllx %g1, 32, %g1 182 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
183 sub %o0, %g1, %o0 ! physical address 183 sub %o0, %g1, %o0 ! physical address
184 srlx %o0, 11, %o0 ! make D-cache TAG 184 srlx %o0, 11, %o0 ! make D-cache TAG
185 sethi %hi(1 << 14), %o2 ! D-cache size 185 sethi %hi(1 << 14), %o2 ! D-cache size
@@ -287,8 +287,8 @@ __cheetah_flush_tlb_pending: /* 27 insns */
287 287
288#ifdef DCACHE_ALIASING_POSSIBLE 288#ifdef DCACHE_ALIASING_POSSIBLE
289__cheetah_flush_dcache_page: /* 11 insns */ 289__cheetah_flush_dcache_page: /* 11 insns */
290 sethi %uhi(PAGE_OFFSET), %g1 290 sethi %hi(PAGE_OFFSET), %g1
291 sllx %g1, 32, %g1 291 ldx [%g1 + %lo(PAGE_OFFSET)], %g1
292 sub %o0, %g1, %o0 292 sub %o0, %g1, %o0
293 sethi %hi(PAGE_SIZE), %o4 293 sethi %hi(PAGE_SIZE), %o4
2941: subcc %o4, (1 << 5), %o4 2941: subcc %o4, (1 << 5), %o4