diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2013-11-15 00:16:30 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2013-11-15 00:16:30 -0500 |
commit | 1b2722752fe7bf21e76c87080add6d6c40af339f (patch) | |
tree | 2b98fb5d283c20a7d879bd8b3ed9d9ec67592f10 /arch | |
parent | 91838e2dab460ba589fb90db0fe1f504f5c04f12 (diff) | |
parent | 812cb83a56a908729c453a7db3fb2c262119bc9d (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next
Pull sparc update from David Miller:
1) Implement support for up to 47-bit physical addresses on sparc64.
2) Support HAVE_CONTEXT_TRACKING on sparc64, from Kirill Tkhai.
3) Fix Simba bridge window calculations, from Kjetil Oftedal.
* git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-next:
sparc64: Implement HAVE_CONTEXT_TRACKING
sparc64: Add self-IPI support for smp_send_reschedule()
sparc: PCI: Fix incorrect address calculation of PCI Bridge windows on Simba-bridges
sparc64: Encode huge PMDs using PTE encoding.
sparc64: Move to 64-bit PGDs and PMDs.
sparc64: Move from 4MB to 8MB huge pages.
sparc64: Make PAGE_OFFSET variable.
sparc64: Fix inconsistent max-physical-address defines.
sparc64: Document the shift counts used to validate linear kernel addresses.
sparc64: Define PAGE_OFFSET in terms of physical address bits.
sparc64: Use PAGE_OFFSET instead of a magic constant.
sparc64: Clean up 64-bit mmap exclusion defines.
Diffstat (limited to 'arch')
34 files changed, 510 insertions, 434 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 8591b201d9cc..05fcfc634fcb 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig | |||
@@ -63,6 +63,7 @@ config SPARC64 | |||
63 | select HAVE_DYNAMIC_FTRACE | 63 | select HAVE_DYNAMIC_FTRACE |
64 | select HAVE_FTRACE_MCOUNT_RECORD | 64 | select HAVE_FTRACE_MCOUNT_RECORD |
65 | select HAVE_SYSCALL_TRACEPOINTS | 65 | select HAVE_SYSCALL_TRACEPOINTS |
66 | select HAVE_CONTEXT_TRACKING | ||
66 | select HAVE_DEBUG_KMEMLEAK | 67 | select HAVE_DEBUG_KMEMLEAK |
67 | select RTC_DRV_CMOS | 68 | select RTC_DRV_CMOS |
68 | select RTC_DRV_BQ4802 | 69 | select RTC_DRV_BQ4802 |
diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index 76092c4dd277..f668797ae234 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h | |||
@@ -93,7 +93,6 @@ typedef struct { | |||
93 | spinlock_t lock; | 93 | spinlock_t lock; |
94 | unsigned long sparc64_ctx_val; | 94 | unsigned long sparc64_ctx_val; |
95 | unsigned long huge_pte_count; | 95 | unsigned long huge_pte_count; |
96 | struct page *pgtable_page; | ||
97 | struct tsb_config tsb_block[MM_NUM_TSBS]; | 96 | struct tsb_config tsb_block[MM_NUM_TSBS]; |
98 | struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; | 97 | struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; |
99 | } mm_context_t; | 98 | } mm_context_t; |
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index e15538899f3d..aac53fcea807 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h | |||
@@ -15,7 +15,10 @@ | |||
15 | #define DCACHE_ALIASING_POSSIBLE | 15 | #define DCACHE_ALIASING_POSSIBLE |
16 | #endif | 16 | #endif |
17 | 17 | ||
18 | #define HPAGE_SHIFT 22 | 18 | #define HPAGE_SHIFT 23 |
19 | #define REAL_HPAGE_SHIFT 22 | ||
20 | |||
21 | #define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) | ||
19 | 22 | ||
20 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) | 23 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) |
21 | #define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) | 24 | #define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) |
@@ -53,8 +56,8 @@ extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct pag | |||
53 | /* These are used to make use of C type-checking.. */ | 56 | /* These are used to make use of C type-checking.. */ |
54 | typedef struct { unsigned long pte; } pte_t; | 57 | typedef struct { unsigned long pte; } pte_t; |
55 | typedef struct { unsigned long iopte; } iopte_t; | 58 | typedef struct { unsigned long iopte; } iopte_t; |
56 | typedef struct { unsigned int pmd; } pmd_t; | 59 | typedef struct { unsigned long pmd; } pmd_t; |
57 | typedef struct { unsigned int pgd; } pgd_t; | 60 | typedef struct { unsigned long pgd; } pgd_t; |
58 | typedef struct { unsigned long pgprot; } pgprot_t; | 61 | typedef struct { unsigned long pgprot; } pgprot_t; |
59 | 62 | ||
60 | #define pte_val(x) ((x).pte) | 63 | #define pte_val(x) ((x).pte) |
@@ -73,8 +76,8 @@ typedef struct { unsigned long pgprot; } pgprot_t; | |||
73 | /* .. while these make it easier on the compiler */ | 76 | /* .. while these make it easier on the compiler */ |
74 | typedef unsigned long pte_t; | 77 | typedef unsigned long pte_t; |
75 | typedef unsigned long iopte_t; | 78 | typedef unsigned long iopte_t; |
76 | typedef unsigned int pmd_t; | 79 | typedef unsigned long pmd_t; |
77 | typedef unsigned int pgd_t; | 80 | typedef unsigned long pgd_t; |
78 | typedef unsigned long pgprot_t; | 81 | typedef unsigned long pgprot_t; |
79 | 82 | ||
80 | #define pte_val(x) (x) | 83 | #define pte_val(x) (x) |
@@ -93,18 +96,44 @@ typedef unsigned long pgprot_t; | |||
93 | 96 | ||
94 | typedef pte_t *pgtable_t; | 97 | typedef pte_t *pgtable_t; |
95 | 98 | ||
99 | /* These two values define the virtual address space range in which we | ||
100 | * must forbid 64-bit user processes from making mappings. It used to | ||
101 | * represent precisely the virtual address space hole present in most | ||
102 | * early sparc64 chips including UltraSPARC-I. But now it also is | ||
103 | * further constrained by the limits of our page tables, which is | ||
104 | * 43-bits of virtual address. | ||
105 | */ | ||
106 | #define SPARC64_VA_HOLE_TOP _AC(0xfffffc0000000000,UL) | ||
107 | #define SPARC64_VA_HOLE_BOTTOM _AC(0x0000040000000000,UL) | ||
108 | |||
109 | /* The next two defines specify the actual exclusion region we | ||
110 | * enforce, wherein we use a 4GB red zone on each side of the VA hole. | ||
111 | */ | ||
112 | #define VA_EXCLUDE_START (SPARC64_VA_HOLE_BOTTOM - (1UL << 32UL)) | ||
113 | #define VA_EXCLUDE_END (SPARC64_VA_HOLE_TOP + (1UL << 32UL)) | ||
114 | |||
96 | #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ | 115 | #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ |
97 | (_AC(0x0000000070000000,UL)) : \ | 116 | _AC(0x0000000070000000,UL) : \ |
98 | (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) | 117 | VA_EXCLUDE_END) |
99 | 118 | ||
100 | #include <asm-generic/memory_model.h> | 119 | #include <asm-generic/memory_model.h> |
101 | 120 | ||
121 | #define PAGE_OFFSET_BY_BITS(X) (-(_AC(1,UL) << (X))) | ||
122 | extern unsigned long PAGE_OFFSET; | ||
123 | |||
102 | #endif /* !(__ASSEMBLY__) */ | 124 | #endif /* !(__ASSEMBLY__) */ |
103 | 125 | ||
104 | /* We used to stick this into a hard-coded global register (%g4) | 126 | /* The maximum number of physical memory address bits we support, this |
105 | * but that does not make sense anymore. | 127 | * is used to size various tables used to manage kernel TLB misses and |
128 | * also the sparsemem code. | ||
129 | */ | ||
130 | #define MAX_PHYS_ADDRESS_BITS 47 | ||
131 | |||
132 | /* These two shift counts are used when indexing sparc64_valid_addr_bitmap | ||
133 | * and kpte_linear_bitmap. | ||
106 | */ | 134 | */ |
107 | #define PAGE_OFFSET _AC(0xFFFFF80000000000,UL) | 135 | #define ILOG2_4MB 22 |
136 | #define ILOG2_256MB 28 | ||
108 | 137 | ||
109 | #ifndef __ASSEMBLY__ | 138 | #ifndef __ASSEMBLY__ |
110 | 139 | ||
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 36760317814f..8358dc144959 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h | |||
@@ -48,18 +48,18 @@ | |||
48 | /* PMD_SHIFT determines the size of the area a second-level page | 48 | /* PMD_SHIFT determines the size of the area a second-level page |
49 | * table can map | 49 | * table can map |
50 | */ | 50 | */ |
51 | #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4)) | 51 | #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3)) |
52 | #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) | 52 | #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) |
53 | #define PMD_MASK (~(PMD_SIZE-1)) | 53 | #define PMD_MASK (~(PMD_SIZE-1)) |
54 | #define PMD_BITS (PAGE_SHIFT - 2) | 54 | #define PMD_BITS (PAGE_SHIFT - 3) |
55 | 55 | ||
56 | /* PGDIR_SHIFT determines what a third-level page table entry can map */ | 56 | /* PGDIR_SHIFT determines what a third-level page table entry can map */ |
57 | #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-4) + PMD_BITS) | 57 | #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS) |
58 | #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) | 58 | #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) |
59 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) | 59 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) |
60 | #define PGDIR_BITS (PAGE_SHIFT - 2) | 60 | #define PGDIR_BITS (PAGE_SHIFT - 3) |
61 | 61 | ||
62 | #if (PGDIR_SHIFT + PGDIR_BITS) != 44 | 62 | #if (PGDIR_SHIFT + PGDIR_BITS) != 43 |
63 | #error Page table parameters do not cover virtual address space properly. | 63 | #error Page table parameters do not cover virtual address space properly. |
64 | #endif | 64 | #endif |
65 | 65 | ||
@@ -67,35 +67,12 @@ | |||
67 | #error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. | 67 | #error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. |
68 | #endif | 68 | #endif |
69 | 69 | ||
70 | /* PMDs point to PTE tables which are 4K aligned. */ | ||
71 | #define PMD_PADDR _AC(0xfffffffe,UL) | ||
72 | #define PMD_PADDR_SHIFT _AC(11,UL) | ||
73 | |||
74 | #define PMD_ISHUGE _AC(0x00000001,UL) | ||
75 | |||
76 | /* This is the PMD layout when PMD_ISHUGE is set. With 4MB huge | ||
77 | * pages, this frees up a bunch of bits in the layout that we can | ||
78 | * use for the protection settings and software metadata. | ||
79 | */ | ||
80 | #define PMD_HUGE_PADDR _AC(0xfffff800,UL) | ||
81 | #define PMD_HUGE_PROTBITS _AC(0x000007ff,UL) | ||
82 | #define PMD_HUGE_PRESENT _AC(0x00000400,UL) | ||
83 | #define PMD_HUGE_WRITE _AC(0x00000200,UL) | ||
84 | #define PMD_HUGE_DIRTY _AC(0x00000100,UL) | ||
85 | #define PMD_HUGE_ACCESSED _AC(0x00000080,UL) | ||
86 | #define PMD_HUGE_EXEC _AC(0x00000040,UL) | ||
87 | #define PMD_HUGE_SPLITTING _AC(0x00000020,UL) | ||
88 | |||
89 | /* PGDs point to PMD tables which are 8K aligned. */ | ||
90 | #define PGD_PADDR _AC(0xfffffffc,UL) | ||
91 | #define PGD_PADDR_SHIFT _AC(11,UL) | ||
92 | |||
93 | #ifndef __ASSEMBLY__ | 70 | #ifndef __ASSEMBLY__ |
94 | 71 | ||
95 | #include <linux/sched.h> | 72 | #include <linux/sched.h> |
96 | 73 | ||
97 | /* Entries per page directory level. */ | 74 | /* Entries per page directory level. */ |
98 | #define PTRS_PER_PTE (1UL << (PAGE_SHIFT-4)) | 75 | #define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) |
99 | #define PTRS_PER_PMD (1UL << PMD_BITS) | 76 | #define PTRS_PER_PMD (1UL << PMD_BITS) |
100 | #define PTRS_PER_PGD (1UL << PGDIR_BITS) | 77 | #define PTRS_PER_PGD (1UL << PGDIR_BITS) |
101 | 78 | ||
@@ -112,6 +89,7 @@ | |||
112 | #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ | 89 | #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ |
113 | #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ | 90 | #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ |
114 | #define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ | 91 | #define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ |
92 | #define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ | ||
115 | 93 | ||
116 | /* Advertise support for _PAGE_SPECIAL */ | 94 | /* Advertise support for _PAGE_SPECIAL */ |
117 | #define __HAVE_ARCH_PTE_SPECIAL | 95 | #define __HAVE_ARCH_PTE_SPECIAL |
@@ -125,6 +103,7 @@ | |||
125 | #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ | 103 | #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ |
126 | #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ | 104 | #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ |
127 | #define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ | 105 | #define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ |
106 | #define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */ | ||
128 | #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ | 107 | #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ |
129 | #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ | 108 | #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ |
130 | #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ | 109 | #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ |
@@ -155,6 +134,7 @@ | |||
155 | #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ | 134 | #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ |
156 | #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ | 135 | #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ |
157 | #define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ | 136 | #define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ |
137 | #define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */ | ||
158 | #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ | 138 | #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ |
159 | #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ | 139 | #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ |
160 | #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ | 140 | #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ |
@@ -180,6 +160,10 @@ | |||
180 | #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U | 160 | #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U |
181 | #define _PAGE_SZBITS_4V _PAGE_SZ8K_4V | 161 | #define _PAGE_SZBITS_4V _PAGE_SZ8K_4V |
182 | 162 | ||
163 | #if REAL_HPAGE_SHIFT != 22 | ||
164 | #error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up | ||
165 | #endif | ||
166 | |||
183 | #define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U | 167 | #define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U |
184 | #define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V | 168 | #define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V |
185 | 169 | ||
@@ -239,16 +223,13 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) | |||
239 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | 223 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) |
240 | 224 | ||
241 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 225 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
242 | extern pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot); | 226 | static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) |
243 | #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) | ||
244 | |||
245 | extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); | ||
246 | |||
247 | static inline pmd_t pmd_mkhuge(pmd_t pmd) | ||
248 | { | 227 | { |
249 | /* Do nothing, mk_pmd() does this part. */ | 228 | pte_t pte = pfn_pte(page_nr, pgprot); |
250 | return pmd; | 229 | |
230 | return __pmd(pte_val(pte)); | ||
251 | } | 231 | } |
232 | #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) | ||
252 | #endif | 233 | #endif |
253 | 234 | ||
254 | /* This one can be done with two shifts. */ | 235 | /* This one can be done with two shifts. */ |
@@ -309,14 +290,25 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) | |||
309 | : "=r" (mask), "=r" (tmp) | 290 | : "=r" (mask), "=r" (tmp) |
310 | : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | | 291 | : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | |
311 | _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | | 292 | _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | |
312 | _PAGE_SPECIAL), | 293 | _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U), |
313 | "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | | 294 | "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | |
314 | _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | | 295 | _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | |
315 | _PAGE_SPECIAL)); | 296 | _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V)); |
316 | 297 | ||
317 | return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); | 298 | return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); |
318 | } | 299 | } |
319 | 300 | ||
301 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
302 | static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) | ||
303 | { | ||
304 | pte_t pte = __pte(pmd_val(pmd)); | ||
305 | |||
306 | pte = pte_modify(pte, newprot); | ||
307 | |||
308 | return __pmd(pte_val(pte)); | ||
309 | } | ||
310 | #endif | ||
311 | |||
320 | static inline pte_t pgoff_to_pte(unsigned long off) | 312 | static inline pte_t pgoff_to_pte(unsigned long off) |
321 | { | 313 | { |
322 | off <<= PAGE_SHIFT; | 314 | off <<= PAGE_SHIFT; |
@@ -357,7 +349,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) | |||
357 | */ | 349 | */ |
358 | #define pgprot_noncached pgprot_noncached | 350 | #define pgprot_noncached pgprot_noncached |
359 | 351 | ||
360 | #ifdef CONFIG_HUGETLB_PAGE | 352 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) |
361 | static inline pte_t pte_mkhuge(pte_t pte) | 353 | static inline pte_t pte_mkhuge(pte_t pte) |
362 | { | 354 | { |
363 | unsigned long mask; | 355 | unsigned long mask; |
@@ -375,6 +367,17 @@ static inline pte_t pte_mkhuge(pte_t pte) | |||
375 | 367 | ||
376 | return __pte(pte_val(pte) | mask); | 368 | return __pte(pte_val(pte) | mask); |
377 | } | 369 | } |
370 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | ||
371 | static inline pmd_t pmd_mkhuge(pmd_t pmd) | ||
372 | { | ||
373 | pte_t pte = __pte(pmd_val(pmd)); | ||
374 | |||
375 | pte = pte_mkhuge(pte); | ||
376 | pte_val(pte) |= _PAGE_PMD_HUGE; | ||
377 | |||
378 | return __pmd(pte_val(pte)); | ||
379 | } | ||
380 | #endif | ||
378 | #endif | 381 | #endif |
379 | 382 | ||
380 | static inline pte_t pte_mkdirty(pte_t pte) | 383 | static inline pte_t pte_mkdirty(pte_t pte) |
@@ -626,91 +629,130 @@ static inline unsigned long pte_special(pte_t pte) | |||
626 | return pte_val(pte) & _PAGE_SPECIAL; | 629 | return pte_val(pte) & _PAGE_SPECIAL; |
627 | } | 630 | } |
628 | 631 | ||
629 | static inline int pmd_large(pmd_t pmd) | 632 | static inline unsigned long pmd_large(pmd_t pmd) |
630 | { | 633 | { |
631 | return (pmd_val(pmd) & (PMD_ISHUGE | PMD_HUGE_PRESENT)) == | 634 | pte_t pte = __pte(pmd_val(pmd)); |
632 | (PMD_ISHUGE | PMD_HUGE_PRESENT); | 635 | |
636 | return (pte_val(pte) & _PAGE_PMD_HUGE) && pte_present(pte); | ||
633 | } | 637 | } |
634 | 638 | ||
635 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 639 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
636 | static inline int pmd_young(pmd_t pmd) | 640 | static inline unsigned long pmd_young(pmd_t pmd) |
637 | { | 641 | { |
638 | return pmd_val(pmd) & PMD_HUGE_ACCESSED; | 642 | pte_t pte = __pte(pmd_val(pmd)); |
643 | |||
644 | return pte_young(pte); | ||
639 | } | 645 | } |
640 | 646 | ||
641 | static inline int pmd_write(pmd_t pmd) | 647 | static inline unsigned long pmd_write(pmd_t pmd) |
642 | { | 648 | { |
643 | return pmd_val(pmd) & PMD_HUGE_WRITE; | 649 | pte_t pte = __pte(pmd_val(pmd)); |
650 | |||
651 | return pte_write(pte); | ||
644 | } | 652 | } |
645 | 653 | ||
646 | static inline unsigned long pmd_pfn(pmd_t pmd) | 654 | static inline unsigned long pmd_pfn(pmd_t pmd) |
647 | { | 655 | { |
648 | unsigned long val = pmd_val(pmd) & PMD_HUGE_PADDR; | 656 | pte_t pte = __pte(pmd_val(pmd)); |
649 | 657 | ||
650 | return val >> (PAGE_SHIFT - PMD_PADDR_SHIFT); | 658 | return pte_pfn(pte); |
651 | } | 659 | } |
652 | 660 | ||
653 | static inline int pmd_trans_splitting(pmd_t pmd) | 661 | static inline unsigned long pmd_trans_huge(pmd_t pmd) |
654 | { | 662 | { |
655 | return (pmd_val(pmd) & (PMD_ISHUGE|PMD_HUGE_SPLITTING)) == | 663 | pte_t pte = __pte(pmd_val(pmd)); |
656 | (PMD_ISHUGE|PMD_HUGE_SPLITTING); | 664 | |
665 | return pte_val(pte) & _PAGE_PMD_HUGE; | ||
657 | } | 666 | } |
658 | 667 | ||
659 | static inline int pmd_trans_huge(pmd_t pmd) | 668 | static inline unsigned long pmd_trans_splitting(pmd_t pmd) |
660 | { | 669 | { |
661 | return pmd_val(pmd) & PMD_ISHUGE; | 670 | pte_t pte = __pte(pmd_val(pmd)); |
671 | |||
672 | return pmd_trans_huge(pmd) && pte_special(pte); | ||
662 | } | 673 | } |
663 | 674 | ||
664 | #define has_transparent_hugepage() 1 | 675 | #define has_transparent_hugepage() 1 |
665 | 676 | ||
666 | static inline pmd_t pmd_mkold(pmd_t pmd) | 677 | static inline pmd_t pmd_mkold(pmd_t pmd) |
667 | { | 678 | { |
668 | pmd_val(pmd) &= ~PMD_HUGE_ACCESSED; | 679 | pte_t pte = __pte(pmd_val(pmd)); |
669 | return pmd; | 680 | |
681 | pte = pte_mkold(pte); | ||
682 | |||
683 | return __pmd(pte_val(pte)); | ||
670 | } | 684 | } |
671 | 685 | ||
672 | static inline pmd_t pmd_wrprotect(pmd_t pmd) | 686 | static inline pmd_t pmd_wrprotect(pmd_t pmd) |
673 | { | 687 | { |
674 | pmd_val(pmd) &= ~PMD_HUGE_WRITE; | 688 | pte_t pte = __pte(pmd_val(pmd)); |
675 | return pmd; | 689 | |
690 | pte = pte_wrprotect(pte); | ||
691 | |||
692 | return __pmd(pte_val(pte)); | ||
676 | } | 693 | } |
677 | 694 | ||
678 | static inline pmd_t pmd_mkdirty(pmd_t pmd) | 695 | static inline pmd_t pmd_mkdirty(pmd_t pmd) |
679 | { | 696 | { |
680 | pmd_val(pmd) |= PMD_HUGE_DIRTY; | 697 | pte_t pte = __pte(pmd_val(pmd)); |
681 | return pmd; | 698 | |
699 | pte = pte_mkdirty(pte); | ||
700 | |||
701 | return __pmd(pte_val(pte)); | ||
682 | } | 702 | } |
683 | 703 | ||
684 | static inline pmd_t pmd_mkyoung(pmd_t pmd) | 704 | static inline pmd_t pmd_mkyoung(pmd_t pmd) |
685 | { | 705 | { |
686 | pmd_val(pmd) |= PMD_HUGE_ACCESSED; | 706 | pte_t pte = __pte(pmd_val(pmd)); |
687 | return pmd; | 707 | |
708 | pte = pte_mkyoung(pte); | ||
709 | |||
710 | return __pmd(pte_val(pte)); | ||
688 | } | 711 | } |
689 | 712 | ||
690 | static inline pmd_t pmd_mkwrite(pmd_t pmd) | 713 | static inline pmd_t pmd_mkwrite(pmd_t pmd) |
691 | { | 714 | { |
692 | pmd_val(pmd) |= PMD_HUGE_WRITE; | 715 | pte_t pte = __pte(pmd_val(pmd)); |
693 | return pmd; | 716 | |
717 | pte = pte_mkwrite(pte); | ||
718 | |||
719 | return __pmd(pte_val(pte)); | ||
694 | } | 720 | } |
695 | 721 | ||
696 | static inline pmd_t pmd_mknotpresent(pmd_t pmd) | 722 | static inline pmd_t pmd_mknotpresent(pmd_t pmd) |
697 | { | 723 | { |
698 | pmd_val(pmd) &= ~PMD_HUGE_PRESENT; | 724 | unsigned long mask; |
725 | |||
726 | if (tlb_type == hypervisor) | ||
727 | mask = _PAGE_PRESENT_4V; | ||
728 | else | ||
729 | mask = _PAGE_PRESENT_4U; | ||
730 | |||
731 | pmd_val(pmd) &= ~mask; | ||
732 | |||
699 | return pmd; | 733 | return pmd; |
700 | } | 734 | } |
701 | 735 | ||
702 | static inline pmd_t pmd_mksplitting(pmd_t pmd) | 736 | static inline pmd_t pmd_mksplitting(pmd_t pmd) |
703 | { | 737 | { |
704 | pmd_val(pmd) |= PMD_HUGE_SPLITTING; | 738 | pte_t pte = __pte(pmd_val(pmd)); |
705 | return pmd; | 739 | |
740 | pte = pte_mkspecial(pte); | ||
741 | |||
742 | return __pmd(pte_val(pte)); | ||
706 | } | 743 | } |
707 | 744 | ||
708 | extern pgprot_t pmd_pgprot(pmd_t entry); | 745 | static inline pgprot_t pmd_pgprot(pmd_t entry) |
746 | { | ||
747 | unsigned long val = pmd_val(entry); | ||
748 | |||
749 | return __pgprot(val); | ||
750 | } | ||
709 | #endif | 751 | #endif |
710 | 752 | ||
711 | static inline int pmd_present(pmd_t pmd) | 753 | static inline int pmd_present(pmd_t pmd) |
712 | { | 754 | { |
713 | return pmd_val(pmd) != 0U; | 755 | return pmd_val(pmd) != 0UL; |
714 | } | 756 | } |
715 | 757 | ||
716 | #define pmd_none(pmd) (!pmd_val(pmd)) | 758 | #define pmd_none(pmd) (!pmd_val(pmd)) |
@@ -728,33 +770,32 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, | |||
728 | 770 | ||
729 | static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) | 771 | static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) |
730 | { | 772 | { |
731 | unsigned long val = __pa((unsigned long) (ptep)) >> PMD_PADDR_SHIFT; | 773 | unsigned long val = __pa((unsigned long) (ptep)); |
732 | 774 | ||
733 | pmd_val(*pmdp) = val; | 775 | pmd_val(*pmdp) = val; |
734 | } | 776 | } |
735 | 777 | ||
736 | #define pud_set(pudp, pmdp) \ | 778 | #define pud_set(pudp, pmdp) \ |
737 | (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)) >> PGD_PADDR_SHIFT)) | 779 | (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)))) |
738 | static inline unsigned long __pmd_page(pmd_t pmd) | 780 | static inline unsigned long __pmd_page(pmd_t pmd) |
739 | { | 781 | { |
740 | unsigned long paddr = (unsigned long) pmd_val(pmd); | 782 | pte_t pte = __pte(pmd_val(pmd)); |
741 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 783 | unsigned long pfn; |
742 | if (pmd_val(pmd) & PMD_ISHUGE) | 784 | |
743 | paddr &= PMD_HUGE_PADDR; | 785 | pfn = pte_pfn(pte); |
744 | #endif | 786 | |
745 | paddr <<= PMD_PADDR_SHIFT; | 787 | return ((unsigned long) __va(pfn << PAGE_SHIFT)); |
746 | return ((unsigned long) __va(paddr)); | ||
747 | } | 788 | } |
748 | #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) | 789 | #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) |
749 | #define pud_page_vaddr(pud) \ | 790 | #define pud_page_vaddr(pud) \ |
750 | ((unsigned long) __va((((unsigned long)pud_val(pud))<<PGD_PADDR_SHIFT))) | 791 | ((unsigned long) __va(pud_val(pud))) |
751 | #define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) | 792 | #define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) |
752 | #define pmd_bad(pmd) (0) | 793 | #define pmd_bad(pmd) (0) |
753 | #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0U) | 794 | #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) |
754 | #define pud_none(pud) (!pud_val(pud)) | 795 | #define pud_none(pud) (!pud_val(pud)) |
755 | #define pud_bad(pud) (0) | 796 | #define pud_bad(pud) (0) |
756 | #define pud_present(pud) (pud_val(pud) != 0U) | 797 | #define pud_present(pud) (pud_val(pud) != 0U) |
757 | #define pud_clear(pudp) (pud_val(*(pudp)) = 0U) | 798 | #define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) |
758 | 799 | ||
759 | /* Same in both SUN4V and SUN4U. */ | 800 | /* Same in both SUN4V and SUN4U. */ |
760 | #define pte_none(pte) (!pte_val(pte)) | 801 | #define pte_none(pte) (!pte_val(pte)) |
@@ -789,7 +830,7 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, | |||
789 | pmd_t *pmdp) | 830 | pmd_t *pmdp) |
790 | { | 831 | { |
791 | pmd_t pmd = *pmdp; | 832 | pmd_t pmd = *pmdp; |
792 | set_pmd_at(mm, addr, pmdp, __pmd(0U)); | 833 | set_pmd_at(mm, addr, pmdp, __pmd(0UL)); |
793 | return pmd; | 834 | return pmd; |
794 | } | 835 | } |
795 | 836 | ||
@@ -837,8 +878,8 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, | |||
837 | }) | 878 | }) |
838 | #endif | 879 | #endif |
839 | 880 | ||
840 | extern pgd_t swapper_pg_dir[2048]; | 881 | extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; |
841 | extern pmd_t swapper_low_pmd_dir[2048]; | 882 | extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD]; |
842 | 883 | ||
843 | extern void paging_init(void); | 884 | extern void paging_init(void); |
844 | extern unsigned long find_ecache_flush_span(unsigned long size); | 885 | extern unsigned long find_ecache_flush_span(unsigned long size); |
diff --git a/arch/sparc/include/asm/sparsemem.h b/arch/sparc/include/asm/sparsemem.h index b99d4e4b6d28..e5e1752d5d78 100644 --- a/arch/sparc/include/asm/sparsemem.h +++ b/arch/sparc/include/asm/sparsemem.h | |||
@@ -3,9 +3,11 @@ | |||
3 | 3 | ||
4 | #ifdef __KERNEL__ | 4 | #ifdef __KERNEL__ |
5 | 5 | ||
6 | #include <asm/page.h> | ||
7 | |||
6 | #define SECTION_SIZE_BITS 30 | 8 | #define SECTION_SIZE_BITS 30 |
7 | #define MAX_PHYSADDR_BITS 42 | 9 | #define MAX_PHYSADDR_BITS MAX_PHYS_ADDRESS_BITS |
8 | #define MAX_PHYSMEM_BITS 42 | 10 | #define MAX_PHYSMEM_BITS MAX_PHYS_ADDRESS_BITS |
9 | 11 | ||
10 | #endif /* !(__KERNEL__) */ | 12 | #endif /* !(__KERNEL__) */ |
11 | 13 | ||
diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index d5e504251079..5d9292ab1077 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h | |||
@@ -192,7 +192,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); | |||
192 | #define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ | 192 | #define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ |
193 | /* flag bit 6 is available */ | 193 | /* flag bit 6 is available */ |
194 | #define TIF_32BIT 7 /* 32-bit binary */ | 194 | #define TIF_32BIT 7 /* 32-bit binary */ |
195 | /* flag bit 8 is available */ | 195 | #define TIF_NOHZ 8 /* in adaptive nohz mode */ |
196 | #define TIF_SECCOMP 9 /* secure computing */ | 196 | #define TIF_SECCOMP 9 /* secure computing */ |
197 | #define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ | 197 | #define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ |
198 | #define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ | 198 | #define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ |
@@ -210,6 +210,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); | |||
210 | #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) | 210 | #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) |
211 | #define _TIF_UNALIGNED (1<<TIF_UNALIGNED) | 211 | #define _TIF_UNALIGNED (1<<TIF_UNALIGNED) |
212 | #define _TIF_32BIT (1<<TIF_32BIT) | 212 | #define _TIF_32BIT (1<<TIF_32BIT) |
213 | #define _TIF_NOHZ (1<<TIF_NOHZ) | ||
213 | #define _TIF_SECCOMP (1<<TIF_SECCOMP) | 214 | #define _TIF_SECCOMP (1<<TIF_SECCOMP) |
214 | #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) | 215 | #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) |
215 | #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) | 216 | #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) |
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index e696432b950d..2230f80d9fe3 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h | |||
@@ -142,98 +142,39 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
142 | or REG1, %lo(swapper_pg_dir), REG1; \ | 142 | or REG1, %lo(swapper_pg_dir), REG1; \ |
143 | sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ | 143 | sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ |
144 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ | 144 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
145 | andn REG2, 0x3, REG2; \ | 145 | andn REG2, 0x7, REG2; \ |
146 | lduw [REG1 + REG2], REG1; \ | 146 | ldx [REG1 + REG2], REG1; \ |
147 | brz,pn REG1, FAIL_LABEL; \ | 147 | brz,pn REG1, FAIL_LABEL; \ |
148 | sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ | 148 | sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ |
149 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ | 149 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
150 | sllx REG1, PGD_PADDR_SHIFT, REG1; \ | 150 | andn REG2, 0x7, REG2; \ |
151 | andn REG2, 0x3, REG2; \ | 151 | ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ |
152 | lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ | ||
153 | brz,pn REG1, FAIL_LABEL; \ | 152 | brz,pn REG1, FAIL_LABEL; \ |
154 | sllx VADDR, 64 - PMD_SHIFT, REG2; \ | 153 | sllx VADDR, 64 - PMD_SHIFT, REG2; \ |
155 | srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ | 154 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
156 | sllx REG1, PMD_PADDR_SHIFT, REG1; \ | ||
157 | andn REG2, 0x7, REG2; \ | 155 | andn REG2, 0x7, REG2; \ |
158 | add REG1, REG2, REG1; | 156 | add REG1, REG2, REG1; |
159 | 157 | ||
160 | /* These macros exists only to make the PMD translator below | ||
161 | * easier to read. It hides the ELF section switch for the | ||
162 | * sun4v code patching. | ||
163 | */ | ||
164 | #define OR_PTE_BIT_1INSN(REG, NAME) \ | ||
165 | 661: or REG, _PAGE_##NAME##_4U, REG; \ | ||
166 | .section .sun4v_1insn_patch, "ax"; \ | ||
167 | .word 661b; \ | ||
168 | or REG, _PAGE_##NAME##_4V, REG; \ | ||
169 | .previous; | ||
170 | |||
171 | #define OR_PTE_BIT_2INSN(REG, TMP, NAME) \ | ||
172 | 661: sethi %hi(_PAGE_##NAME##_4U), TMP; \ | ||
173 | or REG, TMP, REG; \ | ||
174 | .section .sun4v_2insn_patch, "ax"; \ | ||
175 | .word 661b; \ | ||
176 | mov -1, TMP; \ | ||
177 | or REG, _PAGE_##NAME##_4V, REG; \ | ||
178 | .previous; | ||
179 | |||
180 | /* Load into REG the PTE value for VALID, CACHE, and SZHUGE. */ | ||
181 | #define BUILD_PTE_VALID_SZHUGE_CACHE(REG) \ | ||
182 | 661: sethi %uhi(_PAGE_VALID|_PAGE_SZHUGE_4U), REG; \ | ||
183 | .section .sun4v_1insn_patch, "ax"; \ | ||
184 | .word 661b; \ | ||
185 | sethi %uhi(_PAGE_VALID), REG; \ | ||
186 | .previous; \ | ||
187 | sllx REG, 32, REG; \ | ||
188 | 661: or REG, _PAGE_CP_4U|_PAGE_CV_4U, REG; \ | ||
189 | .section .sun4v_1insn_patch, "ax"; \ | ||
190 | .word 661b; \ | ||
191 | or REG, _PAGE_CP_4V|_PAGE_CV_4V|_PAGE_SZHUGE_4V, REG; \ | ||
192 | .previous; | ||
193 | |||
194 | /* PMD has been loaded into REG1, interpret the value, seeing | 158 | /* PMD has been loaded into REG1, interpret the value, seeing |
195 | * if it is a HUGE PMD or a normal one. If it is not valid | 159 | * if it is a HUGE PMD or a normal one. If it is not valid |
196 | * then jump to FAIL_LABEL. If it is a HUGE PMD, and it | 160 | * then jump to FAIL_LABEL. If it is a HUGE PMD, and it |
197 | * translates to a valid PTE, branch to PTE_LABEL. | 161 | * translates to a valid PTE, branch to PTE_LABEL. |
198 | * | 162 | * |
199 | * We translate the PMD by hand, one bit at a time, | 163 | * We have to propagate the 4MB bit of the virtual address |
200 | * constructing the huge PTE. | 164 | * because we are fabricating 8MB pages using 4MB hw pages. |
201 | * | ||
202 | * So we construct the PTE in REG2 as follows: | ||
203 | * | ||
204 | * 1) Extract the PMD PFN from REG1 and place it into REG2. | ||
205 | * | ||
206 | * 2) Translate PMD protection bits in REG1 into REG2, one bit | ||
207 | * at a time using andcc tests on REG1 and OR's into REG2. | ||
208 | * | ||
209 | * Only two bits to be concerned with here, EXEC and WRITE. | ||
210 | * Now REG1 is freed up and we can use it as a temporary. | ||
211 | * | ||
212 | * 3) Construct the VALID, CACHE, and page size PTE bits in | ||
213 | * REG1, OR with REG2 to form final PTE. | ||
214 | */ | 165 | */ |
215 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 166 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
216 | #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ | 167 | #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ |
217 | brz,pn REG1, FAIL_LABEL; \ | 168 | brz,pn REG1, FAIL_LABEL; \ |
218 | andcc REG1, PMD_ISHUGE, %g0; \ | 169 | sethi %uhi(_PAGE_PMD_HUGE), REG2; \ |
219 | be,pt %xcc, 700f; \ | 170 | sllx REG2, 32, REG2; \ |
220 | and REG1, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED, REG2; \ | 171 | andcc REG1, REG2, %g0; \ |
221 | cmp REG2, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED; \ | 172 | be,pt %xcc, 700f; \ |
222 | bne,pn %xcc, FAIL_LABEL; \ | 173 | sethi %hi(4 * 1024 * 1024), REG2; \ |
223 | andn REG1, PMD_HUGE_PROTBITS, REG2; \ | 174 | andn REG1, REG2, REG1; \ |
224 | sllx REG2, PMD_PADDR_SHIFT, REG2; \ | 175 | and VADDR, REG2, REG2; \ |
225 | /* REG2 now holds PFN << PAGE_SHIFT */ \ | 176 | brlz,pt REG1, PTE_LABEL; \ |
226 | andcc REG1, PMD_HUGE_WRITE, %g0; \ | 177 | or REG1, REG2, REG1; \ |
227 | bne,a,pt %xcc, 1f; \ | ||
228 | OR_PTE_BIT_1INSN(REG2, W); \ | ||
229 | 1: andcc REG1, PMD_HUGE_EXEC, %g0; \ | ||
230 | be,pt %xcc, 1f; \ | ||
231 | nop; \ | ||
232 | OR_PTE_BIT_2INSN(REG2, REG1, EXEC); \ | ||
233 | /* REG1 can now be clobbered, build final PTE */ \ | ||
234 | 1: BUILD_PTE_VALID_SZHUGE_CACHE(REG1); \ | ||
235 | ba,pt %xcc, PTE_LABEL; \ | ||
236 | or REG1, REG2, REG1; \ | ||
237 | 700: | 178 | 700: |
238 | #else | 179 | #else |
239 | #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ | 180 | #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ |
@@ -253,18 +194,16 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; | |||
253 | #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ | 194 | #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ |
254 | sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ | 195 | sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ |
255 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ | 196 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
256 | andn REG2, 0x3, REG2; \ | 197 | andn REG2, 0x7, REG2; \ |
257 | lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ | 198 | ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ |
258 | brz,pn REG1, FAIL_LABEL; \ | 199 | brz,pn REG1, FAIL_LABEL; \ |
259 | sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ | 200 | sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ |
260 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ | 201 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
261 | sllx REG1, PGD_PADDR_SHIFT, REG1; \ | 202 | andn REG2, 0x7, REG2; \ |
262 | andn REG2, 0x3, REG2; \ | 203 | ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ |
263 | lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ | ||
264 | USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ | 204 | USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ |
265 | sllx VADDR, 64 - PMD_SHIFT, REG2; \ | 205 | sllx VADDR, 64 - PMD_SHIFT, REG2; \ |
266 | srlx REG2, 64 - (PAGE_SHIFT - 1), REG2; \ | 206 | srlx REG2, 64 - PAGE_SHIFT, REG2; \ |
267 | sllx REG1, PMD_PADDR_SHIFT, REG1; \ | ||
268 | andn REG2, 0x7, REG2; \ | 207 | andn REG2, 0x7, REG2; \ |
269 | add REG1, REG2, REG1; \ | 208 | add REG1, REG2, REG1; \ |
270 | ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ | 209 | ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ |
diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index 9c179fbfb219..140966fbd303 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h | |||
@@ -88,7 +88,6 @@ extern asmlinkage void syscall_trace_leave(struct pt_regs *regs); | |||
88 | 88 | ||
89 | extern void bad_trap_tl1(struct pt_regs *regs, long lvl); | 89 | extern void bad_trap_tl1(struct pt_regs *regs, long lvl); |
90 | 90 | ||
91 | extern void do_fpe_common(struct pt_regs *regs); | ||
92 | extern void do_fpieee(struct pt_regs *regs); | 91 | extern void do_fpieee(struct pt_regs *regs); |
93 | extern void do_fpother(struct pt_regs *regs); | 92 | extern void do_fpother(struct pt_regs *regs); |
94 | extern void do_tof(struct pt_regs *regs); | 93 | extern void do_tof(struct pt_regs *regs); |
diff --git a/arch/sparc/kernel/kgdb_64.c b/arch/sparc/kernel/kgdb_64.c index 53c0a82e6030..60b19f50c80a 100644 --- a/arch/sparc/kernel/kgdb_64.c +++ b/arch/sparc/kernel/kgdb_64.c | |||
@@ -159,11 +159,12 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, | |||
159 | 159 | ||
160 | asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) | 160 | asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) |
161 | { | 161 | { |
162 | enum ctx_state prev_state = exception_enter(); | ||
162 | unsigned long flags; | 163 | unsigned long flags; |
163 | 164 | ||
164 | if (user_mode(regs)) { | 165 | if (user_mode(regs)) { |
165 | bad_trap(regs, trap_level); | 166 | bad_trap(regs, trap_level); |
166 | return; | 167 | goto out; |
167 | } | 168 | } |
168 | 169 | ||
169 | flushw_all(); | 170 | flushw_all(); |
@@ -171,6 +172,8 @@ asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) | |||
171 | local_irq_save(flags); | 172 | local_irq_save(flags); |
172 | kgdb_handle_exception(0x172, SIGTRAP, 0, regs); | 173 | kgdb_handle_exception(0x172, SIGTRAP, 0, regs); |
173 | local_irq_restore(flags); | 174 | local_irq_restore(flags); |
175 | out: | ||
176 | exception_exit(prev_state); | ||
174 | } | 177 | } |
175 | 178 | ||
176 | int kgdb_arch_init(void) | 179 | int kgdb_arch_init(void) |
diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index e72212148d2a..5a09fd315e5f 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c | |||
@@ -8,6 +8,7 @@ | |||
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | #include <linux/kdebug.h> | 9 | #include <linux/kdebug.h> |
10 | #include <linux/slab.h> | 10 | #include <linux/slab.h> |
11 | #include <linux/context_tracking.h> | ||
11 | #include <asm/signal.h> | 12 | #include <asm/signal.h> |
12 | #include <asm/cacheflush.h> | 13 | #include <asm/cacheflush.h> |
13 | #include <asm/uaccess.h> | 14 | #include <asm/uaccess.h> |
@@ -418,12 +419,14 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, | |||
418 | asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, | 419 | asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, |
419 | struct pt_regs *regs) | 420 | struct pt_regs *regs) |
420 | { | 421 | { |
422 | enum ctx_state prev_state = exception_enter(); | ||
423 | |||
421 | BUG_ON(trap_level != 0x170 && trap_level != 0x171); | 424 | BUG_ON(trap_level != 0x170 && trap_level != 0x171); |
422 | 425 | ||
423 | if (user_mode(regs)) { | 426 | if (user_mode(regs)) { |
424 | local_irq_enable(); | 427 | local_irq_enable(); |
425 | bad_trap(regs, trap_level); | 428 | bad_trap(regs, trap_level); |
426 | return; | 429 | goto out; |
427 | } | 430 | } |
428 | 431 | ||
429 | /* trap_level == 0x170 --> ta 0x70 | 432 | /* trap_level == 0x170 --> ta 0x70 |
@@ -433,6 +436,8 @@ asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, | |||
433 | (trap_level == 0x170) ? "debug" : "debug_2", | 436 | (trap_level == 0x170) ? "debug" : "debug_2", |
434 | regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) | 437 | regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) |
435 | bad_trap(regs, trap_level); | 438 | bad_trap(regs, trap_level); |
439 | out: | ||
440 | exception_exit(prev_state); | ||
436 | } | 441 | } |
437 | 442 | ||
438 | /* Jprobes support. */ | 443 | /* Jprobes support. */ |
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S index fde5a419cf27..542e96ac4d39 100644 --- a/arch/sparc/kernel/ktlb.S +++ b/arch/sparc/kernel/ktlb.S | |||
@@ -153,12 +153,19 @@ kvmap_dtlb_tsb4m_miss: | |||
153 | /* Clear the PAGE_OFFSET top virtual bits, shift | 153 | /* Clear the PAGE_OFFSET top virtual bits, shift |
154 | * down to get PFN, and make sure PFN is in range. | 154 | * down to get PFN, and make sure PFN is in range. |
155 | */ | 155 | */ |
156 | sllx %g4, 21, %g5 | 156 | 661: sllx %g4, 0, %g5 |
157 | .section .page_offset_shift_patch, "ax" | ||
158 | .word 661b | ||
159 | .previous | ||
157 | 160 | ||
158 | /* Check to see if we know about valid memory at the 4MB | 161 | /* Check to see if we know about valid memory at the 4MB |
159 | * chunk this physical address will reside within. | 162 | * chunk this physical address will reside within. |
160 | */ | 163 | */ |
161 | srlx %g5, 21 + 41, %g2 | 164 | 661: srlx %g5, MAX_PHYS_ADDRESS_BITS, %g2 |
165 | .section .page_offset_shift_patch, "ax" | ||
166 | .word 661b | ||
167 | .previous | ||
168 | |||
162 | brnz,pn %g2, kvmap_dtlb_longpath | 169 | brnz,pn %g2, kvmap_dtlb_longpath |
163 | nop | 170 | nop |
164 | 171 | ||
@@ -176,7 +183,11 @@ valid_addr_bitmap_patch: | |||
176 | or %g7, %lo(sparc64_valid_addr_bitmap), %g7 | 183 | or %g7, %lo(sparc64_valid_addr_bitmap), %g7 |
177 | .previous | 184 | .previous |
178 | 185 | ||
179 | srlx %g5, 21 + 22, %g2 | 186 | 661: srlx %g5, ILOG2_4MB, %g2 |
187 | .section .page_offset_shift_patch, "ax" | ||
188 | .word 661b | ||
189 | .previous | ||
190 | |||
180 | srlx %g2, 6, %g5 | 191 | srlx %g2, 6, %g5 |
181 | and %g2, 63, %g2 | 192 | and %g2, 63, %g2 |
182 | sllx %g5, 3, %g5 | 193 | sllx %g5, 3, %g5 |
@@ -189,9 +200,18 @@ valid_addr_bitmap_patch: | |||
189 | 2: sethi %hi(kpte_linear_bitmap), %g2 | 200 | 2: sethi %hi(kpte_linear_bitmap), %g2 |
190 | 201 | ||
191 | /* Get the 256MB physical address index. */ | 202 | /* Get the 256MB physical address index. */ |
192 | sllx %g4, 21, %g5 | 203 | 661: sllx %g4, 0, %g5 |
204 | .section .page_offset_shift_patch, "ax" | ||
205 | .word 661b | ||
206 | .previous | ||
207 | |||
193 | or %g2, %lo(kpte_linear_bitmap), %g2 | 208 | or %g2, %lo(kpte_linear_bitmap), %g2 |
194 | srlx %g5, 21 + 28, %g5 | 209 | |
210 | 661: srlx %g5, ILOG2_256MB, %g5 | ||
211 | .section .page_offset_shift_patch, "ax" | ||
212 | .word 661b | ||
213 | .previous | ||
214 | |||
195 | and %g5, (32 - 1), %g7 | 215 | and %g5, (32 - 1), %g7 |
196 | 216 | ||
197 | /* Divide by 32 to get the offset into the bitmask. */ | 217 | /* Divide by 32 to get the offset into the bitmask. */ |
diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index bc4d3f5d2e5d..cb021453de2a 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c | |||
@@ -398,8 +398,8 @@ static void apb_fake_ranges(struct pci_dev *dev, | |||
398 | apb_calc_first_last(map, &first, &last); | 398 | apb_calc_first_last(map, &first, &last); |
399 | res = bus->resource[1]; | 399 | res = bus->resource[1]; |
400 | res->flags = IORESOURCE_MEM; | 400 | res->flags = IORESOURCE_MEM; |
401 | region.start = (first << 21); | 401 | region.start = (first << 29); |
402 | region.end = (last << 21) + ((1 << 21) - 1); | 402 | region.end = (last << 29) + ((1 << 29) - 1); |
403 | pcibios_bus_to_resource(dev, res, ®ion); | 403 | pcibios_bus_to_resource(dev, res, ®ion); |
404 | } | 404 | } |
405 | 405 | ||
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index baebab215492..32a280ec38c1 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c | |||
@@ -31,6 +31,7 @@ | |||
31 | #include <linux/elfcore.h> | 31 | #include <linux/elfcore.h> |
32 | #include <linux/sysrq.h> | 32 | #include <linux/sysrq.h> |
33 | #include <linux/nmi.h> | 33 | #include <linux/nmi.h> |
34 | #include <linux/context_tracking.h> | ||
34 | 35 | ||
35 | #include <asm/uaccess.h> | 36 | #include <asm/uaccess.h> |
36 | #include <asm/page.h> | 37 | #include <asm/page.h> |
@@ -557,6 +558,7 @@ void fault_in_user_windows(void) | |||
557 | 558 | ||
558 | barf: | 559 | barf: |
559 | set_thread_wsaved(window + 1); | 560 | set_thread_wsaved(window + 1); |
561 | user_exit(); | ||
560 | do_exit(SIGILL); | 562 | do_exit(SIGILL); |
561 | } | 563 | } |
562 | 564 | ||
diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c index 773c1f2983ce..c13c9f25d83a 100644 --- a/arch/sparc/kernel/ptrace_64.c +++ b/arch/sparc/kernel/ptrace_64.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <trace/syscall.h> | 27 | #include <trace/syscall.h> |
28 | #include <linux/compat.h> | 28 | #include <linux/compat.h> |
29 | #include <linux/elf.h> | 29 | #include <linux/elf.h> |
30 | #include <linux/context_tracking.h> | ||
30 | 31 | ||
31 | #include <asm/asi.h> | 32 | #include <asm/asi.h> |
32 | #include <asm/pgtable.h> | 33 | #include <asm/pgtable.h> |
@@ -1066,6 +1067,9 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) | |||
1066 | /* do the secure computing check first */ | 1067 | /* do the secure computing check first */ |
1067 | secure_computing_strict(regs->u_regs[UREG_G1]); | 1068 | secure_computing_strict(regs->u_regs[UREG_G1]); |
1068 | 1069 | ||
1070 | if (test_thread_flag(TIF_NOHZ)) | ||
1071 | user_exit(); | ||
1072 | |||
1069 | if (test_thread_flag(TIF_SYSCALL_TRACE)) | 1073 | if (test_thread_flag(TIF_SYSCALL_TRACE)) |
1070 | ret = tracehook_report_syscall_entry(regs); | 1074 | ret = tracehook_report_syscall_entry(regs); |
1071 | 1075 | ||
@@ -1086,6 +1090,9 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) | |||
1086 | 1090 | ||
1087 | asmlinkage void syscall_trace_leave(struct pt_regs *regs) | 1091 | asmlinkage void syscall_trace_leave(struct pt_regs *regs) |
1088 | { | 1092 | { |
1093 | if (test_thread_flag(TIF_NOHZ)) | ||
1094 | user_exit(); | ||
1095 | |||
1089 | audit_syscall_exit(regs); | 1096 | audit_syscall_exit(regs); |
1090 | 1097 | ||
1091 | if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) | 1098 | if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) |
@@ -1093,4 +1100,7 @@ asmlinkage void syscall_trace_leave(struct pt_regs *regs) | |||
1093 | 1100 | ||
1094 | if (test_thread_flag(TIF_SYSCALL_TRACE)) | 1101 | if (test_thread_flag(TIF_SYSCALL_TRACE)) |
1095 | tracehook_report_syscall_exit(regs, 0); | 1102 | tracehook_report_syscall_exit(regs, 0); |
1103 | |||
1104 | if (test_thread_flag(TIF_NOHZ)) | ||
1105 | user_enter(); | ||
1096 | } | 1106 | } |
diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index afa2a9e3d0a0..a954eb81881b 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S | |||
@@ -18,10 +18,16 @@ | |||
18 | #define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) | 18 | #define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) |
19 | #define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) | 19 | #define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) |
20 | 20 | ||
21 | #ifdef CONFIG_CONTEXT_TRACKING | ||
22 | # define SCHEDULE_USER schedule_user | ||
23 | #else | ||
24 | # define SCHEDULE_USER schedule | ||
25 | #endif | ||
26 | |||
21 | .text | 27 | .text |
22 | .align 32 | 28 | .align 32 |
23 | __handle_preemption: | 29 | __handle_preemption: |
24 | call schedule | 30 | call SCHEDULE_USER |
25 | wrpr %g0, RTRAP_PSTATE, %pstate | 31 | wrpr %g0, RTRAP_PSTATE, %pstate |
26 | ba,pt %xcc, __handle_preemption_continue | 32 | ba,pt %xcc, __handle_preemption_continue |
27 | wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate | 33 | wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate |
diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c index 35923e8abd82..cd91d010e6d3 100644 --- a/arch/sparc/kernel/signal_64.c +++ b/arch/sparc/kernel/signal_64.c | |||
@@ -23,6 +23,7 @@ | |||
23 | #include <linux/tty.h> | 23 | #include <linux/tty.h> |
24 | #include <linux/binfmts.h> | 24 | #include <linux/binfmts.h> |
25 | #include <linux/bitops.h> | 25 | #include <linux/bitops.h> |
26 | #include <linux/context_tracking.h> | ||
26 | 27 | ||
27 | #include <asm/uaccess.h> | 28 | #include <asm/uaccess.h> |
28 | #include <asm/ptrace.h> | 29 | #include <asm/ptrace.h> |
@@ -43,6 +44,7 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) | |||
43 | { | 44 | { |
44 | struct ucontext __user *ucp = (struct ucontext __user *) | 45 | struct ucontext __user *ucp = (struct ucontext __user *) |
45 | regs->u_regs[UREG_I0]; | 46 | regs->u_regs[UREG_I0]; |
47 | enum ctx_state prev_state = exception_enter(); | ||
46 | mc_gregset_t __user *grp; | 48 | mc_gregset_t __user *grp; |
47 | unsigned long pc, npc, tstate; | 49 | unsigned long pc, npc, tstate; |
48 | unsigned long fp, i7; | 50 | unsigned long fp, i7; |
@@ -129,16 +131,19 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) | |||
129 | } | 131 | } |
130 | if (err) | 132 | if (err) |
131 | goto do_sigsegv; | 133 | goto do_sigsegv; |
132 | 134 | out: | |
135 | exception_exit(prev_state); | ||
133 | return; | 136 | return; |
134 | do_sigsegv: | 137 | do_sigsegv: |
135 | force_sig(SIGSEGV, current); | 138 | force_sig(SIGSEGV, current); |
139 | goto out; | ||
136 | } | 140 | } |
137 | 141 | ||
138 | asmlinkage void sparc64_get_context(struct pt_regs *regs) | 142 | asmlinkage void sparc64_get_context(struct pt_regs *regs) |
139 | { | 143 | { |
140 | struct ucontext __user *ucp = (struct ucontext __user *) | 144 | struct ucontext __user *ucp = (struct ucontext __user *) |
141 | regs->u_regs[UREG_I0]; | 145 | regs->u_regs[UREG_I0]; |
146 | enum ctx_state prev_state = exception_enter(); | ||
142 | mc_gregset_t __user *grp; | 147 | mc_gregset_t __user *grp; |
143 | mcontext_t __user *mcp; | 148 | mcontext_t __user *mcp; |
144 | unsigned long fp, i7; | 149 | unsigned long fp, i7; |
@@ -220,10 +225,12 @@ asmlinkage void sparc64_get_context(struct pt_regs *regs) | |||
220 | } | 225 | } |
221 | if (err) | 226 | if (err) |
222 | goto do_sigsegv; | 227 | goto do_sigsegv; |
223 | 228 | out: | |
229 | exception_exit(prev_state); | ||
224 | return; | 230 | return; |
225 | do_sigsegv: | 231 | do_sigsegv: |
226 | force_sig(SIGSEGV, current); | 232 | force_sig(SIGSEGV, current); |
233 | goto out; | ||
227 | } | 234 | } |
228 | 235 | ||
229 | struct rt_signal_frame { | 236 | struct rt_signal_frame { |
@@ -528,11 +535,13 @@ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) | |||
528 | 535 | ||
529 | void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) | 536 | void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) |
530 | { | 537 | { |
538 | user_exit(); | ||
531 | if (thread_info_flags & _TIF_SIGPENDING) | 539 | if (thread_info_flags & _TIF_SIGPENDING) |
532 | do_signal(regs, orig_i0); | 540 | do_signal(regs, orig_i0); |
533 | if (thread_info_flags & _TIF_NOTIFY_RESUME) { | 541 | if (thread_info_flags & _TIF_NOTIFY_RESUME) { |
534 | clear_thread_flag(TIF_NOTIFY_RESUME); | 542 | clear_thread_flag(TIF_NOTIFY_RESUME); |
535 | tracehook_notify_resume(regs); | 543 | tracehook_notify_resume(regs); |
536 | } | 544 | } |
545 | user_enter(); | ||
537 | } | 546 | } |
538 | 547 | ||
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index e142545244f2..b66a5338231e 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c | |||
@@ -1399,8 +1399,13 @@ void __init smp_cpus_done(unsigned int max_cpus) | |||
1399 | 1399 | ||
1400 | void smp_send_reschedule(int cpu) | 1400 | void smp_send_reschedule(int cpu) |
1401 | { | 1401 | { |
1402 | xcall_deliver((u64) &xcall_receive_signal, 0, 0, | 1402 | if (cpu == smp_processor_id()) { |
1403 | cpumask_of(cpu)); | 1403 | WARN_ON_ONCE(preemptible()); |
1404 | set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); | ||
1405 | } else { | ||
1406 | xcall_deliver((u64) &xcall_receive_signal, | ||
1407 | 0, 0, cpumask_of(cpu)); | ||
1408 | } | ||
1404 | } | 1409 | } |
1405 | 1410 | ||
1406 | void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) | 1411 | void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) |
diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S index bde867fd71e8..e0c09bf85610 100644 --- a/arch/sparc/kernel/sun4v_tlb_miss.S +++ b/arch/sparc/kernel/sun4v_tlb_miss.S | |||
@@ -182,7 +182,7 @@ sun4v_tsb_miss_common: | |||
182 | cmp %g5, -1 | 182 | cmp %g5, -1 |
183 | be,pt %xcc, 80f | 183 | be,pt %xcc, 80f |
184 | nop | 184 | nop |
185 | COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) | 185 | COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) |
186 | 186 | ||
187 | /* That clobbered %g2, reload it. */ | 187 | /* That clobbered %g2, reload it. */ |
188 | ldxa [%g0] ASI_SCRATCHPAD, %g2 | 188 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 51561b8b15ba..beb0b5a5f21f 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c | |||
@@ -24,6 +24,7 @@ | |||
24 | #include <linux/personality.h> | 24 | #include <linux/personality.h> |
25 | #include <linux/random.h> | 25 | #include <linux/random.h> |
26 | #include <linux/export.h> | 26 | #include <linux/export.h> |
27 | #include <linux/context_tracking.h> | ||
27 | 28 | ||
28 | #include <asm/uaccess.h> | 29 | #include <asm/uaccess.h> |
29 | #include <asm/utrap.h> | 30 | #include <asm/utrap.h> |
@@ -39,9 +40,6 @@ asmlinkage unsigned long sys_getpagesize(void) | |||
39 | return PAGE_SIZE; | 40 | return PAGE_SIZE; |
40 | } | 41 | } |
41 | 42 | ||
42 | #define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) | ||
43 | #define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) | ||
44 | |||
45 | /* Does addr --> addr+len fall within 4GB of the VA-space hole or | 43 | /* Does addr --> addr+len fall within 4GB of the VA-space hole or |
46 | * overflow past the end of the 64-bit address space? | 44 | * overflow past the end of the 64-bit address space? |
47 | */ | 45 | */ |
@@ -499,6 +497,7 @@ asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs) | |||
499 | 497 | ||
500 | asmlinkage void sparc_breakpoint(struct pt_regs *regs) | 498 | asmlinkage void sparc_breakpoint(struct pt_regs *regs) |
501 | { | 499 | { |
500 | enum ctx_state prev_state = exception_enter(); | ||
502 | siginfo_t info; | 501 | siginfo_t info; |
503 | 502 | ||
504 | if (test_thread_flag(TIF_32BIT)) { | 503 | if (test_thread_flag(TIF_32BIT)) { |
@@ -517,6 +516,7 @@ asmlinkage void sparc_breakpoint(struct pt_regs *regs) | |||
517 | #ifdef DEBUG_SPARC_BREAKPOINT | 516 | #ifdef DEBUG_SPARC_BREAKPOINT |
518 | printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); | 517 | printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); |
519 | #endif | 518 | #endif |
519 | exception_exit(prev_state); | ||
520 | } | 520 | } |
521 | 521 | ||
522 | extern void check_pending(int signum); | 522 | extern void check_pending(int signum); |
diff --git a/arch/sparc/kernel/syscalls.S b/arch/sparc/kernel/syscalls.S index d950197a17e1..87729fff13b9 100644 --- a/arch/sparc/kernel/syscalls.S +++ b/arch/sparc/kernel/syscalls.S | |||
@@ -52,7 +52,7 @@ sys32_rt_sigreturn: | |||
52 | #endif | 52 | #endif |
53 | .align 32 | 53 | .align 32 |
54 | 1: ldx [%g6 + TI_FLAGS], %l5 | 54 | 1: ldx [%g6 + TI_FLAGS], %l5 |
55 | andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 | 55 | andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 |
56 | be,pt %icc, rtrap | 56 | be,pt %icc, rtrap |
57 | nop | 57 | nop |
58 | call syscall_trace_leave | 58 | call syscall_trace_leave |
@@ -184,7 +184,7 @@ linux_sparc_syscall32: | |||
184 | 184 | ||
185 | srl %i3, 0, %o3 ! IEU0 | 185 | srl %i3, 0, %o3 ! IEU0 |
186 | srl %i2, 0, %o2 ! IEU0 Group | 186 | srl %i2, 0, %o2 ! IEU0 Group |
187 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 | 187 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 |
188 | bne,pn %icc, linux_syscall_trace32 ! CTI | 188 | bne,pn %icc, linux_syscall_trace32 ! CTI |
189 | mov %i0, %l5 ! IEU1 | 189 | mov %i0, %l5 ! IEU1 |
190 | 5: call %l7 ! CTI Group brk forced | 190 | 5: call %l7 ! CTI Group brk forced |
@@ -207,7 +207,7 @@ linux_sparc_syscall: | |||
207 | 207 | ||
208 | mov %i3, %o3 ! IEU1 | 208 | mov %i3, %o3 ! IEU1 |
209 | mov %i4, %o4 ! IEU0 Group | 209 | mov %i4, %o4 ! IEU0 Group |
210 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 | 210 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 |
211 | bne,pn %icc, linux_syscall_trace ! CTI Group | 211 | bne,pn %icc, linux_syscall_trace ! CTI Group |
212 | mov %i0, %l5 ! IEU0 | 212 | mov %i0, %l5 ! IEU0 |
213 | 2: call %l7 ! CTI Group brk forced | 213 | 2: call %l7 ! CTI Group brk forced |
@@ -223,7 +223,7 @@ ret_sys_call: | |||
223 | 223 | ||
224 | cmp %o0, -ERESTART_RESTARTBLOCK | 224 | cmp %o0, -ERESTART_RESTARTBLOCK |
225 | bgeu,pn %xcc, 1f | 225 | bgeu,pn %xcc, 1f |
226 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 | 226 | andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 |
227 | ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc | 227 | ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc |
228 | 228 | ||
229 | 2: | 229 | 2: |
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index b3f833ab90eb..4ced92f05358 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c | |||
@@ -20,6 +20,7 @@ | |||
20 | #include <linux/ftrace.h> | 20 | #include <linux/ftrace.h> |
21 | #include <linux/reboot.h> | 21 | #include <linux/reboot.h> |
22 | #include <linux/gfp.h> | 22 | #include <linux/gfp.h> |
23 | #include <linux/context_tracking.h> | ||
23 | 24 | ||
24 | #include <asm/smp.h> | 25 | #include <asm/smp.h> |
25 | #include <asm/delay.h> | 26 | #include <asm/delay.h> |
@@ -186,11 +187,12 @@ EXPORT_SYMBOL_GPL(unregister_dimm_printer); | |||
186 | 187 | ||
187 | void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) | 188 | void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) |
188 | { | 189 | { |
190 | enum ctx_state prev_state = exception_enter(); | ||
189 | siginfo_t info; | 191 | siginfo_t info; |
190 | 192 | ||
191 | if (notify_die(DIE_TRAP, "instruction access exception", regs, | 193 | if (notify_die(DIE_TRAP, "instruction access exception", regs, |
192 | 0, 0x8, SIGTRAP) == NOTIFY_STOP) | 194 | 0, 0x8, SIGTRAP) == NOTIFY_STOP) |
193 | return; | 195 | goto out; |
194 | 196 | ||
195 | if (regs->tstate & TSTATE_PRIV) { | 197 | if (regs->tstate & TSTATE_PRIV) { |
196 | printk("spitfire_insn_access_exception: SFSR[%016lx] " | 198 | printk("spitfire_insn_access_exception: SFSR[%016lx] " |
@@ -207,6 +209,8 @@ void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, un | |||
207 | info.si_addr = (void __user *)regs->tpc; | 209 | info.si_addr = (void __user *)regs->tpc; |
208 | info.si_trapno = 0; | 210 | info.si_trapno = 0; |
209 | force_sig_info(SIGSEGV, &info, current); | 211 | force_sig_info(SIGSEGV, &info, current); |
212 | out: | ||
213 | exception_exit(prev_state); | ||
210 | } | 214 | } |
211 | 215 | ||
212 | void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) | 216 | void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) |
@@ -260,11 +264,12 @@ void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, u | |||
260 | 264 | ||
261 | void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) | 265 | void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) |
262 | { | 266 | { |
267 | enum ctx_state prev_state = exception_enter(); | ||
263 | siginfo_t info; | 268 | siginfo_t info; |
264 | 269 | ||
265 | if (notify_die(DIE_TRAP, "data access exception", regs, | 270 | if (notify_die(DIE_TRAP, "data access exception", regs, |
266 | 0, 0x30, SIGTRAP) == NOTIFY_STOP) | 271 | 0, 0x30, SIGTRAP) == NOTIFY_STOP) |
267 | return; | 272 | goto out; |
268 | 273 | ||
269 | if (regs->tstate & TSTATE_PRIV) { | 274 | if (regs->tstate & TSTATE_PRIV) { |
270 | /* Test if this comes from uaccess places. */ | 275 | /* Test if this comes from uaccess places. */ |
@@ -280,7 +285,7 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un | |||
280 | #endif | 285 | #endif |
281 | regs->tpc = entry->fixup; | 286 | regs->tpc = entry->fixup; |
282 | regs->tnpc = regs->tpc + 4; | 287 | regs->tnpc = regs->tpc + 4; |
283 | return; | 288 | goto out; |
284 | } | 289 | } |
285 | /* Shit... */ | 290 | /* Shit... */ |
286 | printk("spitfire_data_access_exception: SFSR[%016lx] " | 291 | printk("spitfire_data_access_exception: SFSR[%016lx] " |
@@ -294,6 +299,8 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un | |||
294 | info.si_addr = (void __user *)sfar; | 299 | info.si_addr = (void __user *)sfar; |
295 | info.si_trapno = 0; | 300 | info.si_trapno = 0; |
296 | force_sig_info(SIGSEGV, &info, current); | 301 | force_sig_info(SIGSEGV, &info, current); |
302 | out: | ||
303 | exception_exit(prev_state); | ||
297 | } | 304 | } |
298 | 305 | ||
299 | void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) | 306 | void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) |
@@ -1994,6 +2001,7 @@ static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, | |||
1994 | */ | 2001 | */ |
1995 | void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) | 2002 | void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) |
1996 | { | 2003 | { |
2004 | enum ctx_state prev_state = exception_enter(); | ||
1997 | struct sun4v_error_entry *ent, local_copy; | 2005 | struct sun4v_error_entry *ent, local_copy; |
1998 | struct trap_per_cpu *tb; | 2006 | struct trap_per_cpu *tb; |
1999 | unsigned long paddr; | 2007 | unsigned long paddr; |
@@ -2022,12 +2030,14 @@ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) | |||
2022 | pr_info("Shutdown request, %u seconds...\n", | 2030 | pr_info("Shutdown request, %u seconds...\n", |
2023 | local_copy.err_secs); | 2031 | local_copy.err_secs); |
2024 | orderly_poweroff(true); | 2032 | orderly_poweroff(true); |
2025 | return; | 2033 | goto out; |
2026 | } | 2034 | } |
2027 | 2035 | ||
2028 | sun4v_log_error(regs, &local_copy, cpu, | 2036 | sun4v_log_error(regs, &local_copy, cpu, |
2029 | KERN_ERR "RESUMABLE ERROR", | 2037 | KERN_ERR "RESUMABLE ERROR", |
2030 | &sun4v_resum_oflow_cnt); | 2038 | &sun4v_resum_oflow_cnt); |
2039 | out: | ||
2040 | exception_exit(prev_state); | ||
2031 | } | 2041 | } |
2032 | 2042 | ||
2033 | /* If we try to printk() we'll probably make matters worse, by trying | 2043 | /* If we try to printk() we'll probably make matters worse, by trying |
@@ -2152,7 +2162,7 @@ void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op) | |||
2152 | err, op); | 2162 | err, op); |
2153 | } | 2163 | } |
2154 | 2164 | ||
2155 | void do_fpe_common(struct pt_regs *regs) | 2165 | static void do_fpe_common(struct pt_regs *regs) |
2156 | { | 2166 | { |
2157 | if (regs->tstate & TSTATE_PRIV) { | 2167 | if (regs->tstate & TSTATE_PRIV) { |
2158 | regs->tpc = regs->tnpc; | 2168 | regs->tpc = regs->tnpc; |
@@ -2188,23 +2198,28 @@ void do_fpe_common(struct pt_regs *regs) | |||
2188 | 2198 | ||
2189 | void do_fpieee(struct pt_regs *regs) | 2199 | void do_fpieee(struct pt_regs *regs) |
2190 | { | 2200 | { |
2201 | enum ctx_state prev_state = exception_enter(); | ||
2202 | |||
2191 | if (notify_die(DIE_TRAP, "fpu exception ieee", regs, | 2203 | if (notify_die(DIE_TRAP, "fpu exception ieee", regs, |
2192 | 0, 0x24, SIGFPE) == NOTIFY_STOP) | 2204 | 0, 0x24, SIGFPE) == NOTIFY_STOP) |
2193 | return; | 2205 | goto out; |
2194 | 2206 | ||
2195 | do_fpe_common(regs); | 2207 | do_fpe_common(regs); |
2208 | out: | ||
2209 | exception_exit(prev_state); | ||
2196 | } | 2210 | } |
2197 | 2211 | ||
2198 | extern int do_mathemu(struct pt_regs *, struct fpustate *, bool); | 2212 | extern int do_mathemu(struct pt_regs *, struct fpustate *, bool); |
2199 | 2213 | ||
2200 | void do_fpother(struct pt_regs *regs) | 2214 | void do_fpother(struct pt_regs *regs) |
2201 | { | 2215 | { |
2216 | enum ctx_state prev_state = exception_enter(); | ||
2202 | struct fpustate *f = FPUSTATE; | 2217 | struct fpustate *f = FPUSTATE; |
2203 | int ret = 0; | 2218 | int ret = 0; |
2204 | 2219 | ||
2205 | if (notify_die(DIE_TRAP, "fpu exception other", regs, | 2220 | if (notify_die(DIE_TRAP, "fpu exception other", regs, |
2206 | 0, 0x25, SIGFPE) == NOTIFY_STOP) | 2221 | 0, 0x25, SIGFPE) == NOTIFY_STOP) |
2207 | return; | 2222 | goto out; |
2208 | 2223 | ||
2209 | switch ((current_thread_info()->xfsr[0] & 0x1c000)) { | 2224 | switch ((current_thread_info()->xfsr[0] & 0x1c000)) { |
2210 | case (2 << 14): /* unfinished_FPop */ | 2225 | case (2 << 14): /* unfinished_FPop */ |
@@ -2213,17 +2228,20 @@ void do_fpother(struct pt_regs *regs) | |||
2213 | break; | 2228 | break; |
2214 | } | 2229 | } |
2215 | if (ret) | 2230 | if (ret) |
2216 | return; | 2231 | goto out; |
2217 | do_fpe_common(regs); | 2232 | do_fpe_common(regs); |
2233 | out: | ||
2234 | exception_exit(prev_state); | ||
2218 | } | 2235 | } |
2219 | 2236 | ||
2220 | void do_tof(struct pt_regs *regs) | 2237 | void do_tof(struct pt_regs *regs) |
2221 | { | 2238 | { |
2239 | enum ctx_state prev_state = exception_enter(); | ||
2222 | siginfo_t info; | 2240 | siginfo_t info; |
2223 | 2241 | ||
2224 | if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, | 2242 | if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, |
2225 | 0, 0x26, SIGEMT) == NOTIFY_STOP) | 2243 | 0, 0x26, SIGEMT) == NOTIFY_STOP) |
2226 | return; | 2244 | goto out; |
2227 | 2245 | ||
2228 | if (regs->tstate & TSTATE_PRIV) | 2246 | if (regs->tstate & TSTATE_PRIV) |
2229 | die_if_kernel("Penguin overflow trap from kernel mode", regs); | 2247 | die_if_kernel("Penguin overflow trap from kernel mode", regs); |
@@ -2237,15 +2255,18 @@ void do_tof(struct pt_regs *regs) | |||
2237 | info.si_addr = (void __user *)regs->tpc; | 2255 | info.si_addr = (void __user *)regs->tpc; |
2238 | info.si_trapno = 0; | 2256 | info.si_trapno = 0; |
2239 | force_sig_info(SIGEMT, &info, current); | 2257 | force_sig_info(SIGEMT, &info, current); |
2258 | out: | ||
2259 | exception_exit(prev_state); | ||
2240 | } | 2260 | } |
2241 | 2261 | ||
2242 | void do_div0(struct pt_regs *regs) | 2262 | void do_div0(struct pt_regs *regs) |
2243 | { | 2263 | { |
2264 | enum ctx_state prev_state = exception_enter(); | ||
2244 | siginfo_t info; | 2265 | siginfo_t info; |
2245 | 2266 | ||
2246 | if (notify_die(DIE_TRAP, "integer division by zero", regs, | 2267 | if (notify_die(DIE_TRAP, "integer division by zero", regs, |
2247 | 0, 0x28, SIGFPE) == NOTIFY_STOP) | 2268 | 0, 0x28, SIGFPE) == NOTIFY_STOP) |
2248 | return; | 2269 | goto out; |
2249 | 2270 | ||
2250 | if (regs->tstate & TSTATE_PRIV) | 2271 | if (regs->tstate & TSTATE_PRIV) |
2251 | die_if_kernel("TL0: Kernel divide by zero.", regs); | 2272 | die_if_kernel("TL0: Kernel divide by zero.", regs); |
@@ -2259,6 +2280,8 @@ void do_div0(struct pt_regs *regs) | |||
2259 | info.si_addr = (void __user *)regs->tpc; | 2280 | info.si_addr = (void __user *)regs->tpc; |
2260 | info.si_trapno = 0; | 2281 | info.si_trapno = 0; |
2261 | force_sig_info(SIGFPE, &info, current); | 2282 | force_sig_info(SIGFPE, &info, current); |
2283 | out: | ||
2284 | exception_exit(prev_state); | ||
2262 | } | 2285 | } |
2263 | 2286 | ||
2264 | static void instruction_dump(unsigned int *pc) | 2287 | static void instruction_dump(unsigned int *pc) |
@@ -2415,6 +2438,7 @@ extern int handle_ldf_stq(u32 insn, struct pt_regs *regs); | |||
2415 | 2438 | ||
2416 | void do_illegal_instruction(struct pt_regs *regs) | 2439 | void do_illegal_instruction(struct pt_regs *regs) |
2417 | { | 2440 | { |
2441 | enum ctx_state prev_state = exception_enter(); | ||
2418 | unsigned long pc = regs->tpc; | 2442 | unsigned long pc = regs->tpc; |
2419 | unsigned long tstate = regs->tstate; | 2443 | unsigned long tstate = regs->tstate; |
2420 | u32 insn; | 2444 | u32 insn; |
@@ -2422,7 +2446,7 @@ void do_illegal_instruction(struct pt_regs *regs) | |||
2422 | 2446 | ||
2423 | if (notify_die(DIE_TRAP, "illegal instruction", regs, | 2447 | if (notify_die(DIE_TRAP, "illegal instruction", regs, |
2424 | 0, 0x10, SIGILL) == NOTIFY_STOP) | 2448 | 0, 0x10, SIGILL) == NOTIFY_STOP) |
2425 | return; | 2449 | goto out; |
2426 | 2450 | ||
2427 | if (tstate & TSTATE_PRIV) | 2451 | if (tstate & TSTATE_PRIV) |
2428 | die_if_kernel("Kernel illegal instruction", regs); | 2452 | die_if_kernel("Kernel illegal instruction", regs); |
@@ -2431,14 +2455,14 @@ void do_illegal_instruction(struct pt_regs *regs) | |||
2431 | if (get_user(insn, (u32 __user *) pc) != -EFAULT) { | 2455 | if (get_user(insn, (u32 __user *) pc) != -EFAULT) { |
2432 | if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { | 2456 | if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { |
2433 | if (handle_popc(insn, regs)) | 2457 | if (handle_popc(insn, regs)) |
2434 | return; | 2458 | goto out; |
2435 | } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { | 2459 | } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { |
2436 | if (handle_ldf_stq(insn, regs)) | 2460 | if (handle_ldf_stq(insn, regs)) |
2437 | return; | 2461 | goto out; |
2438 | } else if (tlb_type == hypervisor) { | 2462 | } else if (tlb_type == hypervisor) { |
2439 | if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { | 2463 | if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { |
2440 | if (!vis_emul(regs, insn)) | 2464 | if (!vis_emul(regs, insn)) |
2441 | return; | 2465 | goto out; |
2442 | } else { | 2466 | } else { |
2443 | struct fpustate *f = FPUSTATE; | 2467 | struct fpustate *f = FPUSTATE; |
2444 | 2468 | ||
@@ -2448,7 +2472,7 @@ void do_illegal_instruction(struct pt_regs *regs) | |||
2448 | * Trap in the %fsr to unimplemented_FPop. | 2472 | * Trap in the %fsr to unimplemented_FPop. |
2449 | */ | 2473 | */ |
2450 | if (do_mathemu(regs, f, true)) | 2474 | if (do_mathemu(regs, f, true)) |
2451 | return; | 2475 | goto out; |
2452 | } | 2476 | } |
2453 | } | 2477 | } |
2454 | } | 2478 | } |
@@ -2458,21 +2482,24 @@ void do_illegal_instruction(struct pt_regs *regs) | |||
2458 | info.si_addr = (void __user *)pc; | 2482 | info.si_addr = (void __user *)pc; |
2459 | info.si_trapno = 0; | 2483 | info.si_trapno = 0; |
2460 | force_sig_info(SIGILL, &info, current); | 2484 | force_sig_info(SIGILL, &info, current); |
2485 | out: | ||
2486 | exception_exit(prev_state); | ||
2461 | } | 2487 | } |
2462 | 2488 | ||
2463 | extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); | 2489 | extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); |
2464 | 2490 | ||
2465 | void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) | 2491 | void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) |
2466 | { | 2492 | { |
2493 | enum ctx_state prev_state = exception_enter(); | ||
2467 | siginfo_t info; | 2494 | siginfo_t info; |
2468 | 2495 | ||
2469 | if (notify_die(DIE_TRAP, "memory address unaligned", regs, | 2496 | if (notify_die(DIE_TRAP, "memory address unaligned", regs, |
2470 | 0, 0x34, SIGSEGV) == NOTIFY_STOP) | 2497 | 0, 0x34, SIGSEGV) == NOTIFY_STOP) |
2471 | return; | 2498 | goto out; |
2472 | 2499 | ||
2473 | if (regs->tstate & TSTATE_PRIV) { | 2500 | if (regs->tstate & TSTATE_PRIV) { |
2474 | kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); | 2501 | kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); |
2475 | return; | 2502 | goto out; |
2476 | } | 2503 | } |
2477 | info.si_signo = SIGBUS; | 2504 | info.si_signo = SIGBUS; |
2478 | info.si_errno = 0; | 2505 | info.si_errno = 0; |
@@ -2480,6 +2507,8 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo | |||
2480 | info.si_addr = (void __user *)sfar; | 2507 | info.si_addr = (void __user *)sfar; |
2481 | info.si_trapno = 0; | 2508 | info.si_trapno = 0; |
2482 | force_sig_info(SIGBUS, &info, current); | 2509 | force_sig_info(SIGBUS, &info, current); |
2510 | out: | ||
2511 | exception_exit(prev_state); | ||
2483 | } | 2512 | } |
2484 | 2513 | ||
2485 | void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) | 2514 | void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) |
@@ -2504,11 +2533,12 @@ void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_c | |||
2504 | 2533 | ||
2505 | void do_privop(struct pt_regs *regs) | 2534 | void do_privop(struct pt_regs *regs) |
2506 | { | 2535 | { |
2536 | enum ctx_state prev_state = exception_enter(); | ||
2507 | siginfo_t info; | 2537 | siginfo_t info; |
2508 | 2538 | ||
2509 | if (notify_die(DIE_TRAP, "privileged operation", regs, | 2539 | if (notify_die(DIE_TRAP, "privileged operation", regs, |
2510 | 0, 0x11, SIGILL) == NOTIFY_STOP) | 2540 | 0, 0x11, SIGILL) == NOTIFY_STOP) |
2511 | return; | 2541 | goto out; |
2512 | 2542 | ||
2513 | if (test_thread_flag(TIF_32BIT)) { | 2543 | if (test_thread_flag(TIF_32BIT)) { |
2514 | regs->tpc &= 0xffffffff; | 2544 | regs->tpc &= 0xffffffff; |
@@ -2520,6 +2550,8 @@ void do_privop(struct pt_regs *regs) | |||
2520 | info.si_addr = (void __user *)regs->tpc; | 2550 | info.si_addr = (void __user *)regs->tpc; |
2521 | info.si_trapno = 0; | 2551 | info.si_trapno = 0; |
2522 | force_sig_info(SIGILL, &info, current); | 2552 | force_sig_info(SIGILL, &info, current); |
2553 | out: | ||
2554 | exception_exit(prev_state); | ||
2523 | } | 2555 | } |
2524 | 2556 | ||
2525 | void do_privact(struct pt_regs *regs) | 2557 | void do_privact(struct pt_regs *regs) |
@@ -2530,99 +2562,116 @@ void do_privact(struct pt_regs *regs) | |||
2530 | /* Trap level 1 stuff or other traps we should never see... */ | 2562 | /* Trap level 1 stuff or other traps we should never see... */ |
2531 | void do_cee(struct pt_regs *regs) | 2563 | void do_cee(struct pt_regs *regs) |
2532 | { | 2564 | { |
2565 | exception_enter(); | ||
2533 | die_if_kernel("TL0: Cache Error Exception", regs); | 2566 | die_if_kernel("TL0: Cache Error Exception", regs); |
2534 | } | 2567 | } |
2535 | 2568 | ||
2536 | void do_cee_tl1(struct pt_regs *regs) | 2569 | void do_cee_tl1(struct pt_regs *regs) |
2537 | { | 2570 | { |
2571 | exception_enter(); | ||
2538 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2572 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2539 | die_if_kernel("TL1: Cache Error Exception", regs); | 2573 | die_if_kernel("TL1: Cache Error Exception", regs); |
2540 | } | 2574 | } |
2541 | 2575 | ||
2542 | void do_dae_tl1(struct pt_regs *regs) | 2576 | void do_dae_tl1(struct pt_regs *regs) |
2543 | { | 2577 | { |
2578 | exception_enter(); | ||
2544 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2579 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2545 | die_if_kernel("TL1: Data Access Exception", regs); | 2580 | die_if_kernel("TL1: Data Access Exception", regs); |
2546 | } | 2581 | } |
2547 | 2582 | ||
2548 | void do_iae_tl1(struct pt_regs *regs) | 2583 | void do_iae_tl1(struct pt_regs *regs) |
2549 | { | 2584 | { |
2585 | exception_enter(); | ||
2550 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2586 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2551 | die_if_kernel("TL1: Instruction Access Exception", regs); | 2587 | die_if_kernel("TL1: Instruction Access Exception", regs); |
2552 | } | 2588 | } |
2553 | 2589 | ||
2554 | void do_div0_tl1(struct pt_regs *regs) | 2590 | void do_div0_tl1(struct pt_regs *regs) |
2555 | { | 2591 | { |
2592 | exception_enter(); | ||
2556 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2593 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2557 | die_if_kernel("TL1: DIV0 Exception", regs); | 2594 | die_if_kernel("TL1: DIV0 Exception", regs); |
2558 | } | 2595 | } |
2559 | 2596 | ||
2560 | void do_fpdis_tl1(struct pt_regs *regs) | 2597 | void do_fpdis_tl1(struct pt_regs *regs) |
2561 | { | 2598 | { |
2599 | exception_enter(); | ||
2562 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2600 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2563 | die_if_kernel("TL1: FPU Disabled", regs); | 2601 | die_if_kernel("TL1: FPU Disabled", regs); |
2564 | } | 2602 | } |
2565 | 2603 | ||
2566 | void do_fpieee_tl1(struct pt_regs *regs) | 2604 | void do_fpieee_tl1(struct pt_regs *regs) |
2567 | { | 2605 | { |
2606 | exception_enter(); | ||
2568 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2607 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2569 | die_if_kernel("TL1: FPU IEEE Exception", regs); | 2608 | die_if_kernel("TL1: FPU IEEE Exception", regs); |
2570 | } | 2609 | } |
2571 | 2610 | ||
2572 | void do_fpother_tl1(struct pt_regs *regs) | 2611 | void do_fpother_tl1(struct pt_regs *regs) |
2573 | { | 2612 | { |
2613 | exception_enter(); | ||
2574 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2614 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2575 | die_if_kernel("TL1: FPU Other Exception", regs); | 2615 | die_if_kernel("TL1: FPU Other Exception", regs); |
2576 | } | 2616 | } |
2577 | 2617 | ||
2578 | void do_ill_tl1(struct pt_regs *regs) | 2618 | void do_ill_tl1(struct pt_regs *regs) |
2579 | { | 2619 | { |
2620 | exception_enter(); | ||
2580 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2621 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2581 | die_if_kernel("TL1: Illegal Instruction Exception", regs); | 2622 | die_if_kernel("TL1: Illegal Instruction Exception", regs); |
2582 | } | 2623 | } |
2583 | 2624 | ||
2584 | void do_irq_tl1(struct pt_regs *regs) | 2625 | void do_irq_tl1(struct pt_regs *regs) |
2585 | { | 2626 | { |
2627 | exception_enter(); | ||
2586 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2628 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2587 | die_if_kernel("TL1: IRQ Exception", regs); | 2629 | die_if_kernel("TL1: IRQ Exception", regs); |
2588 | } | 2630 | } |
2589 | 2631 | ||
2590 | void do_lddfmna_tl1(struct pt_regs *regs) | 2632 | void do_lddfmna_tl1(struct pt_regs *regs) |
2591 | { | 2633 | { |
2634 | exception_enter(); | ||
2592 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2635 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2593 | die_if_kernel("TL1: LDDF Exception", regs); | 2636 | die_if_kernel("TL1: LDDF Exception", regs); |
2594 | } | 2637 | } |
2595 | 2638 | ||
2596 | void do_stdfmna_tl1(struct pt_regs *regs) | 2639 | void do_stdfmna_tl1(struct pt_regs *regs) |
2597 | { | 2640 | { |
2641 | exception_enter(); | ||
2598 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2642 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2599 | die_if_kernel("TL1: STDF Exception", regs); | 2643 | die_if_kernel("TL1: STDF Exception", regs); |
2600 | } | 2644 | } |
2601 | 2645 | ||
2602 | void do_paw(struct pt_regs *regs) | 2646 | void do_paw(struct pt_regs *regs) |
2603 | { | 2647 | { |
2648 | exception_enter(); | ||
2604 | die_if_kernel("TL0: Phys Watchpoint Exception", regs); | 2649 | die_if_kernel("TL0: Phys Watchpoint Exception", regs); |
2605 | } | 2650 | } |
2606 | 2651 | ||
2607 | void do_paw_tl1(struct pt_regs *regs) | 2652 | void do_paw_tl1(struct pt_regs *regs) |
2608 | { | 2653 | { |
2654 | exception_enter(); | ||
2609 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2655 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2610 | die_if_kernel("TL1: Phys Watchpoint Exception", regs); | 2656 | die_if_kernel("TL1: Phys Watchpoint Exception", regs); |
2611 | } | 2657 | } |
2612 | 2658 | ||
2613 | void do_vaw(struct pt_regs *regs) | 2659 | void do_vaw(struct pt_regs *regs) |
2614 | { | 2660 | { |
2661 | exception_enter(); | ||
2615 | die_if_kernel("TL0: Virt Watchpoint Exception", regs); | 2662 | die_if_kernel("TL0: Virt Watchpoint Exception", regs); |
2616 | } | 2663 | } |
2617 | 2664 | ||
2618 | void do_vaw_tl1(struct pt_regs *regs) | 2665 | void do_vaw_tl1(struct pt_regs *regs) |
2619 | { | 2666 | { |
2667 | exception_enter(); | ||
2620 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2668 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2621 | die_if_kernel("TL1: Virt Watchpoint Exception", regs); | 2669 | die_if_kernel("TL1: Virt Watchpoint Exception", regs); |
2622 | } | 2670 | } |
2623 | 2671 | ||
2624 | void do_tof_tl1(struct pt_regs *regs) | 2672 | void do_tof_tl1(struct pt_regs *regs) |
2625 | { | 2673 | { |
2674 | exception_enter(); | ||
2626 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); | 2675 | dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); |
2627 | die_if_kernel("TL1: Tag Overflow Exception", regs); | 2676 | die_if_kernel("TL1: Tag Overflow Exception", regs); |
2628 | } | 2677 | } |
diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index a313e4a9399b..14158d40ba76 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S | |||
@@ -75,7 +75,7 @@ tsb_miss_page_table_walk: | |||
75 | mov 512, %g7 | 75 | mov 512, %g7 |
76 | andn %g5, 0x7, %g5 | 76 | andn %g5, 0x7, %g5 |
77 | sllx %g7, %g6, %g7 | 77 | sllx %g7, %g6, %g7 |
78 | srlx %g4, HPAGE_SHIFT, %g6 | 78 | srlx %g4, REAL_HPAGE_SHIFT, %g6 |
79 | sub %g7, 1, %g7 | 79 | sub %g7, 1, %g7 |
80 | and %g6, %g7, %g6 | 80 | and %g6, %g7, %g6 |
81 | sllx %g6, 4, %g6 | 81 | sllx %g6, 4, %g6 |
diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c index 8201c25e7669..3c1a7cb31579 100644 --- a/arch/sparc/kernel/unaligned_64.c +++ b/arch/sparc/kernel/unaligned_64.c | |||
@@ -21,9 +21,12 @@ | |||
21 | #include <linux/bitops.h> | 21 | #include <linux/bitops.h> |
22 | #include <linux/perf_event.h> | 22 | #include <linux/perf_event.h> |
23 | #include <linux/ratelimit.h> | 23 | #include <linux/ratelimit.h> |
24 | #include <linux/context_tracking.h> | ||
24 | #include <asm/fpumacro.h> | 25 | #include <asm/fpumacro.h> |
25 | #include <asm/cacheflush.h> | 26 | #include <asm/cacheflush.h> |
26 | 27 | ||
28 | #include "entry.h" | ||
29 | |||
27 | enum direction { | 30 | enum direction { |
28 | load, /* ld, ldd, ldh, ldsh */ | 31 | load, /* ld, ldd, ldh, ldsh */ |
29 | store, /* st, std, sth, stsh */ | 32 | store, /* st, std, sth, stsh */ |
@@ -418,9 +421,6 @@ int handle_popc(u32 insn, struct pt_regs *regs) | |||
418 | 421 | ||
419 | extern void do_fpother(struct pt_regs *regs); | 422 | extern void do_fpother(struct pt_regs *regs); |
420 | extern void do_privact(struct pt_regs *regs); | 423 | extern void do_privact(struct pt_regs *regs); |
421 | extern void spitfire_data_access_exception(struct pt_regs *regs, | ||
422 | unsigned long sfsr, | ||
423 | unsigned long sfar); | ||
424 | extern void sun4v_data_access_exception(struct pt_regs *regs, | 424 | extern void sun4v_data_access_exception(struct pt_regs *regs, |
425 | unsigned long addr, | 425 | unsigned long addr, |
426 | unsigned long type_ctx); | 426 | unsigned long type_ctx); |
@@ -578,6 +578,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) | |||
578 | 578 | ||
579 | void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) | 579 | void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) |
580 | { | 580 | { |
581 | enum ctx_state prev_state = exception_enter(); | ||
581 | unsigned long pc = regs->tpc; | 582 | unsigned long pc = regs->tpc; |
582 | unsigned long tstate = regs->tstate; | 583 | unsigned long tstate = regs->tstate; |
583 | u32 insn; | 584 | u32 insn; |
@@ -632,13 +633,16 @@ daex: | |||
632 | sun4v_data_access_exception(regs, sfar, sfsr); | 633 | sun4v_data_access_exception(regs, sfar, sfsr); |
633 | else | 634 | else |
634 | spitfire_data_access_exception(regs, sfsr, sfar); | 635 | spitfire_data_access_exception(regs, sfsr, sfar); |
635 | return; | 636 | goto out; |
636 | } | 637 | } |
637 | advance(regs); | 638 | advance(regs); |
639 | out: | ||
640 | exception_exit(prev_state); | ||
638 | } | 641 | } |
639 | 642 | ||
640 | void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) | 643 | void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) |
641 | { | 644 | { |
645 | enum ctx_state prev_state = exception_enter(); | ||
642 | unsigned long pc = regs->tpc; | 646 | unsigned long pc = regs->tpc; |
643 | unsigned long tstate = regs->tstate; | 647 | unsigned long tstate = regs->tstate; |
644 | u32 insn; | 648 | u32 insn; |
@@ -680,7 +684,9 @@ daex: | |||
680 | sun4v_data_access_exception(regs, sfar, sfsr); | 684 | sun4v_data_access_exception(regs, sfar, sfsr); |
681 | else | 685 | else |
682 | spitfire_data_access_exception(regs, sfsr, sfar); | 686 | spitfire_data_access_exception(regs, sfsr, sfar); |
683 | return; | 687 | goto out; |
684 | } | 688 | } |
685 | advance(regs); | 689 | advance(regs); |
690 | out: | ||
691 | exception_exit(prev_state); | ||
686 | } | 692 | } |
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 0bacceb19150..932ff90fd760 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S | |||
@@ -122,6 +122,11 @@ SECTIONS | |||
122 | *(.swapper_4m_tsb_phys_patch) | 122 | *(.swapper_4m_tsb_phys_patch) |
123 | __swapper_4m_tsb_phys_patch_end = .; | 123 | __swapper_4m_tsb_phys_patch_end = .; |
124 | } | 124 | } |
125 | .page_offset_shift_patch : { | ||
126 | __page_offset_shift_patch = .; | ||
127 | *(.page_offset_shift_patch) | ||
128 | __page_offset_shift_patch_end = .; | ||
129 | } | ||
125 | .popc_3insn_patch : { | 130 | .popc_3insn_patch : { |
126 | __popc_3insn_patch = .; | 131 | __popc_3insn_patch = .; |
127 | *(.popc_3insn_patch) | 132 | *(.popc_3insn_patch) |
diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S index 77e531f6c2a7..46272dfc26e8 100644 --- a/arch/sparc/lib/clear_page.S +++ b/arch/sparc/lib/clear_page.S | |||
@@ -37,10 +37,10 @@ _clear_page: /* %o0=dest */ | |||
37 | .globl clear_user_page | 37 | .globl clear_user_page |
38 | clear_user_page: /* %o0=dest, %o1=vaddr */ | 38 | clear_user_page: /* %o0=dest, %o1=vaddr */ |
39 | lduw [%g6 + TI_PRE_COUNT], %o2 | 39 | lduw [%g6 + TI_PRE_COUNT], %o2 |
40 | sethi %uhi(PAGE_OFFSET), %g2 | 40 | sethi %hi(PAGE_OFFSET), %g2 |
41 | sethi %hi(PAGE_SIZE), %o4 | 41 | sethi %hi(PAGE_SIZE), %o4 |
42 | 42 | ||
43 | sllx %g2, 32, %g2 | 43 | ldx [%g2 + %lo(PAGE_OFFSET)], %g2 |
44 | sethi %hi(PAGE_KERNEL_LOCKED), %g3 | 44 | sethi %hi(PAGE_KERNEL_LOCKED), %g3 |
45 | 45 | ||
46 | ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 | 46 | ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 |
diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S index 4d2df328e514..dd16c61f3263 100644 --- a/arch/sparc/lib/copy_page.S +++ b/arch/sparc/lib/copy_page.S | |||
@@ -46,10 +46,10 @@ | |||
46 | .type copy_user_page,#function | 46 | .type copy_user_page,#function |
47 | copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ | 47 | copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ |
48 | lduw [%g6 + TI_PRE_COUNT], %o4 | 48 | lduw [%g6 + TI_PRE_COUNT], %o4 |
49 | sethi %uhi(PAGE_OFFSET), %g2 | 49 | sethi %hi(PAGE_OFFSET), %g2 |
50 | sethi %hi(PAGE_SIZE), %o3 | 50 | sethi %hi(PAGE_SIZE), %o3 |
51 | 51 | ||
52 | sllx %g2, 32, %g2 | 52 | ldx [%g2 + %lo(PAGE_OFFSET)], %g2 |
53 | sethi %hi(PAGE_KERNEL_LOCKED), %g3 | 53 | sethi %hi(PAGE_KERNEL_LOCKED), %g3 |
54 | 54 | ||
55 | ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 | 55 | ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 |
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 2ebec263d685..69bb818fdd79 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c | |||
@@ -21,6 +21,7 @@ | |||
21 | #include <linux/kprobes.h> | 21 | #include <linux/kprobes.h> |
22 | #include <linux/kdebug.h> | 22 | #include <linux/kdebug.h> |
23 | #include <linux/percpu.h> | 23 | #include <linux/percpu.h> |
24 | #include <linux/context_tracking.h> | ||
24 | 25 | ||
25 | #include <asm/page.h> | 26 | #include <asm/page.h> |
26 | #include <asm/pgtable.h> | 27 | #include <asm/pgtable.h> |
@@ -272,6 +273,7 @@ static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs, | |||
272 | 273 | ||
273 | asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) | 274 | asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) |
274 | { | 275 | { |
276 | enum ctx_state prev_state = exception_enter(); | ||
275 | struct mm_struct *mm = current->mm; | 277 | struct mm_struct *mm = current->mm; |
276 | struct vm_area_struct *vma; | 278 | struct vm_area_struct *vma; |
277 | unsigned int insn = 0; | 279 | unsigned int insn = 0; |
@@ -282,7 +284,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) | |||
282 | fault_code = get_thread_fault_code(); | 284 | fault_code = get_thread_fault_code(); |
283 | 285 | ||
284 | if (notify_page_fault(regs)) | 286 | if (notify_page_fault(regs)) |
285 | return; | 287 | goto exit_exception; |
286 | 288 | ||
287 | si_code = SEGV_MAPERR; | 289 | si_code = SEGV_MAPERR; |
288 | address = current_thread_info()->fault_address; | 290 | address = current_thread_info()->fault_address; |
@@ -313,7 +315,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) | |||
313 | /* Valid, no problems... */ | 315 | /* Valid, no problems... */ |
314 | } else { | 316 | } else { |
315 | bad_kernel_pc(regs, address); | 317 | bad_kernel_pc(regs, address); |
316 | return; | 318 | goto exit_exception; |
317 | } | 319 | } |
318 | } else | 320 | } else |
319 | flags |= FAULT_FLAG_USER; | 321 | flags |= FAULT_FLAG_USER; |
@@ -430,7 +432,7 @@ good_area: | |||
430 | fault = handle_mm_fault(mm, vma, address, flags); | 432 | fault = handle_mm_fault(mm, vma, address, flags); |
431 | 433 | ||
432 | if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) | 434 | if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) |
433 | return; | 435 | goto exit_exception; |
434 | 436 | ||
435 | if (unlikely(fault & VM_FAULT_ERROR)) { | 437 | if (unlikely(fault & VM_FAULT_ERROR)) { |
436 | if (fault & VM_FAULT_OOM) | 438 | if (fault & VM_FAULT_OOM) |
@@ -482,6 +484,8 @@ good_area: | |||
482 | 484 | ||
483 | } | 485 | } |
484 | #endif | 486 | #endif |
487 | exit_exception: | ||
488 | exception_exit(prev_state); | ||
485 | return; | 489 | return; |
486 | 490 | ||
487 | /* | 491 | /* |
@@ -494,7 +498,7 @@ bad_area: | |||
494 | 498 | ||
495 | handle_kernel_fault: | 499 | handle_kernel_fault: |
496 | do_kernel_fault(regs, si_code, fault_code, insn, address); | 500 | do_kernel_fault(regs, si_code, fault_code, insn, address); |
497 | return; | 501 | goto exit_exception; |
498 | 502 | ||
499 | /* | 503 | /* |
500 | * We ran out of memory, or some other thing happened to us that made | 504 | * We ran out of memory, or some other thing happened to us that made |
@@ -505,7 +509,7 @@ out_of_memory: | |||
505 | up_read(&mm->mmap_sem); | 509 | up_read(&mm->mmap_sem); |
506 | if (!(regs->tstate & TSTATE_PRIV)) { | 510 | if (!(regs->tstate & TSTATE_PRIV)) { |
507 | pagefault_out_of_memory(); | 511 | pagefault_out_of_memory(); |
508 | return; | 512 | goto exit_exception; |
509 | } | 513 | } |
510 | goto handle_kernel_fault; | 514 | goto handle_kernel_fault; |
511 | 515 | ||
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c index 01ee23dd724d..c4d3da68b800 100644 --- a/arch/sparc/mm/gup.c +++ b/arch/sparc/mm/gup.c | |||
@@ -71,13 +71,12 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, | |||
71 | int *nr) | 71 | int *nr) |
72 | { | 72 | { |
73 | struct page *head, *page, *tail; | 73 | struct page *head, *page, *tail; |
74 | u32 mask; | ||
75 | int refs; | 74 | int refs; |
76 | 75 | ||
77 | mask = PMD_HUGE_PRESENT; | 76 | if (!pmd_large(pmd)) |
78 | if (write) | 77 | return 0; |
79 | mask |= PMD_HUGE_WRITE; | 78 | |
80 | if ((pmd_val(pmd) & mask) != mask) | 79 | if (write && !pmd_write(pmd)) |
81 | return 0; | 80 | return 0; |
82 | 81 | ||
83 | refs = 0; | 82 | refs = 0; |
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 96399646570a..30963178d7e9 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c | |||
@@ -21,8 +21,6 @@ | |||
21 | /* Slightly simplified from the non-hugepage variant because by | 21 | /* Slightly simplified from the non-hugepage variant because by |
22 | * definition we don't have to worry about any page coloring stuff | 22 | * definition we don't have to worry about any page coloring stuff |
23 | */ | 23 | */ |
24 | #define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) | ||
25 | #define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) | ||
26 | 24 | ||
27 | static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, | 25 | static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, |
28 | unsigned long addr, | 26 | unsigned long addr, |
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index d6de9353ee11..6b643790e4fe 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c | |||
@@ -354,7 +354,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * | |||
354 | 354 | ||
355 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) | 355 | #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) |
356 | if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) | 356 | if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) |
357 | __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, | 357 | __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, |
358 | address, pte_val(pte)); | 358 | address, pte_val(pte)); |
359 | else | 359 | else |
360 | #endif | 360 | #endif |
@@ -1557,6 +1557,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size) | |||
1557 | return ~0UL; | 1557 | return ~0UL; |
1558 | } | 1558 | } |
1559 | 1559 | ||
1560 | unsigned long PAGE_OFFSET; | ||
1561 | EXPORT_SYMBOL(PAGE_OFFSET); | ||
1562 | |||
1563 | static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits) | ||
1564 | { | ||
1565 | unsigned long final_shift; | ||
1566 | unsigned int val = *insn; | ||
1567 | unsigned int cnt; | ||
1568 | |||
1569 | /* We are patching in ilog2(max_supported_phys_address), and | ||
1570 | * we are doing so in a manner similar to a relocation addend. | ||
1571 | * That is, we are adding the shift value to whatever value | ||
1572 | * is in the shift instruction count field already. | ||
1573 | */ | ||
1574 | cnt = (val & 0x3f); | ||
1575 | val &= ~0x3f; | ||
1576 | |||
1577 | /* If we are trying to shift >= 64 bits, clear the destination | ||
1578 | * register. This can happen when phys_bits ends up being equal | ||
1579 | * to MAX_PHYS_ADDRESS_BITS. | ||
1580 | */ | ||
1581 | final_shift = (cnt + (64 - phys_bits)); | ||
1582 | if (final_shift >= 64) { | ||
1583 | unsigned int rd = (val >> 25) & 0x1f; | ||
1584 | |||
1585 | val = 0x80100000 | (rd << 25); | ||
1586 | } else { | ||
1587 | val |= final_shift; | ||
1588 | } | ||
1589 | *insn = val; | ||
1590 | |||
1591 | __asm__ __volatile__("flush %0" | ||
1592 | : /* no outputs */ | ||
1593 | : "r" (insn)); | ||
1594 | } | ||
1595 | |||
1596 | static void __init page_offset_shift_patch(unsigned long phys_bits) | ||
1597 | { | ||
1598 | extern unsigned int __page_offset_shift_patch; | ||
1599 | extern unsigned int __page_offset_shift_patch_end; | ||
1600 | unsigned int *p; | ||
1601 | |||
1602 | p = &__page_offset_shift_patch; | ||
1603 | while (p < &__page_offset_shift_patch_end) { | ||
1604 | unsigned int *insn = (unsigned int *)(unsigned long)*p; | ||
1605 | |||
1606 | page_offset_shift_patch_one(insn, phys_bits); | ||
1607 | |||
1608 | p++; | ||
1609 | } | ||
1610 | } | ||
1611 | |||
1612 | static void __init setup_page_offset(void) | ||
1613 | { | ||
1614 | unsigned long max_phys_bits = 40; | ||
1615 | |||
1616 | if (tlb_type == cheetah || tlb_type == cheetah_plus) { | ||
1617 | max_phys_bits = 42; | ||
1618 | } else if (tlb_type == hypervisor) { | ||
1619 | switch (sun4v_chip_type) { | ||
1620 | case SUN4V_CHIP_NIAGARA1: | ||
1621 | case SUN4V_CHIP_NIAGARA2: | ||
1622 | max_phys_bits = 39; | ||
1623 | break; | ||
1624 | case SUN4V_CHIP_NIAGARA3: | ||
1625 | max_phys_bits = 43; | ||
1626 | break; | ||
1627 | case SUN4V_CHIP_NIAGARA4: | ||
1628 | case SUN4V_CHIP_NIAGARA5: | ||
1629 | case SUN4V_CHIP_SPARC64X: | ||
1630 | default: | ||
1631 | max_phys_bits = 47; | ||
1632 | break; | ||
1633 | } | ||
1634 | } | ||
1635 | |||
1636 | if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) { | ||
1637 | prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n", | ||
1638 | max_phys_bits); | ||
1639 | prom_halt(); | ||
1640 | } | ||
1641 | |||
1642 | PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits); | ||
1643 | |||
1644 | pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", | ||
1645 | PAGE_OFFSET, max_phys_bits); | ||
1646 | |||
1647 | page_offset_shift_patch(max_phys_bits); | ||
1648 | } | ||
1649 | |||
1560 | static void __init tsb_phys_patch(void) | 1650 | static void __init tsb_phys_patch(void) |
1561 | { | 1651 | { |
1562 | struct tsb_ldquad_phys_patch_entry *pquad; | 1652 | struct tsb_ldquad_phys_patch_entry *pquad; |
@@ -1722,7 +1812,7 @@ static void __init sun4v_linear_pte_xor_finalize(void) | |||
1722 | #ifndef CONFIG_DEBUG_PAGEALLOC | 1812 | #ifndef CONFIG_DEBUG_PAGEALLOC |
1723 | if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { | 1813 | if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { |
1724 | kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ | 1814 | kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ |
1725 | 0xfffff80000000000UL; | 1815 | PAGE_OFFSET; |
1726 | kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | | 1816 | kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | |
1727 | _PAGE_P_4V | _PAGE_W_4V); | 1817 | _PAGE_P_4V | _PAGE_W_4V); |
1728 | } else { | 1818 | } else { |
@@ -1731,7 +1821,7 @@ static void __init sun4v_linear_pte_xor_finalize(void) | |||
1731 | 1821 | ||
1732 | if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { | 1822 | if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { |
1733 | kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ | 1823 | kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ |
1734 | 0xfffff80000000000UL; | 1824 | PAGE_OFFSET; |
1735 | kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | | 1825 | kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | |
1736 | _PAGE_P_4V | _PAGE_W_4V); | 1826 | _PAGE_P_4V | _PAGE_W_4V); |
1737 | } else { | 1827 | } else { |
@@ -1740,7 +1830,7 @@ static void __init sun4v_linear_pte_xor_finalize(void) | |||
1740 | 1830 | ||
1741 | if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { | 1831 | if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { |
1742 | kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ | 1832 | kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ |
1743 | 0xfffff80000000000UL; | 1833 | PAGE_OFFSET; |
1744 | kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | | 1834 | kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | |
1745 | _PAGE_P_4V | _PAGE_W_4V); | 1835 | _PAGE_P_4V | _PAGE_W_4V); |
1746 | } else { | 1836 | } else { |
@@ -1752,7 +1842,7 @@ static void __init sun4v_linear_pte_xor_finalize(void) | |||
1752 | /* paging_init() sets up the page tables */ | 1842 | /* paging_init() sets up the page tables */ |
1753 | 1843 | ||
1754 | static unsigned long last_valid_pfn; | 1844 | static unsigned long last_valid_pfn; |
1755 | pgd_t swapper_pg_dir[2048]; | 1845 | pgd_t swapper_pg_dir[PTRS_PER_PGD]; |
1756 | 1846 | ||
1757 | static void sun4u_pgprot_init(void); | 1847 | static void sun4u_pgprot_init(void); |
1758 | static void sun4v_pgprot_init(void); | 1848 | static void sun4v_pgprot_init(void); |
@@ -1763,6 +1853,8 @@ void __init paging_init(void) | |||
1763 | unsigned long real_end, i; | 1853 | unsigned long real_end, i; |
1764 | int node; | 1854 | int node; |
1765 | 1855 | ||
1856 | setup_page_offset(); | ||
1857 | |||
1766 | /* These build time checkes make sure that the dcache_dirty_cpu() | 1858 | /* These build time checkes make sure that the dcache_dirty_cpu() |
1767 | * page->flags usage will work. | 1859 | * page->flags usage will work. |
1768 | * | 1860 | * |
@@ -2261,10 +2353,10 @@ static void __init sun4u_pgprot_init(void) | |||
2261 | __ACCESS_BITS_4U | _PAGE_E_4U); | 2353 | __ACCESS_BITS_4U | _PAGE_E_4U); |
2262 | 2354 | ||
2263 | #ifdef CONFIG_DEBUG_PAGEALLOC | 2355 | #ifdef CONFIG_DEBUG_PAGEALLOC |
2264 | kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; | 2356 | kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; |
2265 | #else | 2357 | #else |
2266 | kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ | 2358 | kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ |
2267 | 0xfffff80000000000UL; | 2359 | PAGE_OFFSET; |
2268 | #endif | 2360 | #endif |
2269 | kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | | 2361 | kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | |
2270 | _PAGE_P_4U | _PAGE_W_4U); | 2362 | _PAGE_P_4U | _PAGE_W_4U); |
@@ -2308,10 +2400,10 @@ static void __init sun4v_pgprot_init(void) | |||
2308 | _PAGE_CACHE = _PAGE_CACHE_4V; | 2400 | _PAGE_CACHE = _PAGE_CACHE_4V; |
2309 | 2401 | ||
2310 | #ifdef CONFIG_DEBUG_PAGEALLOC | 2402 | #ifdef CONFIG_DEBUG_PAGEALLOC |
2311 | kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL; | 2403 | kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; |
2312 | #else | 2404 | #else |
2313 | kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ | 2405 | kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ |
2314 | 0xfffff80000000000UL; | 2406 | PAGE_OFFSET; |
2315 | #endif | 2407 | #endif |
2316 | kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | | 2408 | kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | |
2317 | _PAGE_P_4V | _PAGE_W_4V); | 2409 | _PAGE_P_4V | _PAGE_W_4V); |
@@ -2455,53 +2547,13 @@ void __flush_tlb_all(void) | |||
2455 | : : "r" (pstate)); | 2547 | : : "r" (pstate)); |
2456 | } | 2548 | } |
2457 | 2549 | ||
2458 | static pte_t *get_from_cache(struct mm_struct *mm) | ||
2459 | { | ||
2460 | struct page *page; | ||
2461 | pte_t *ret; | ||
2462 | |||
2463 | spin_lock(&mm->page_table_lock); | ||
2464 | page = mm->context.pgtable_page; | ||
2465 | ret = NULL; | ||
2466 | if (page) { | ||
2467 | void *p = page_address(page); | ||
2468 | |||
2469 | mm->context.pgtable_page = NULL; | ||
2470 | |||
2471 | ret = (pte_t *) (p + (PAGE_SIZE / 2)); | ||
2472 | } | ||
2473 | spin_unlock(&mm->page_table_lock); | ||
2474 | |||
2475 | return ret; | ||
2476 | } | ||
2477 | |||
2478 | static struct page *__alloc_for_cache(struct mm_struct *mm) | ||
2479 | { | ||
2480 | struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | | ||
2481 | __GFP_REPEAT | __GFP_ZERO); | ||
2482 | |||
2483 | if (page) { | ||
2484 | spin_lock(&mm->page_table_lock); | ||
2485 | if (!mm->context.pgtable_page) { | ||
2486 | atomic_set(&page->_count, 2); | ||
2487 | mm->context.pgtable_page = page; | ||
2488 | } | ||
2489 | spin_unlock(&mm->page_table_lock); | ||
2490 | } | ||
2491 | return page; | ||
2492 | } | ||
2493 | |||
2494 | pte_t *pte_alloc_one_kernel(struct mm_struct *mm, | 2550 | pte_t *pte_alloc_one_kernel(struct mm_struct *mm, |
2495 | unsigned long address) | 2551 | unsigned long address) |
2496 | { | 2552 | { |
2497 | struct page *page; | 2553 | struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | |
2498 | pte_t *pte; | 2554 | __GFP_REPEAT | __GFP_ZERO); |
2499 | 2555 | pte_t *pte = NULL; | |
2500 | pte = get_from_cache(mm); | ||
2501 | if (pte) | ||
2502 | return pte; | ||
2503 | 2556 | ||
2504 | page = __alloc_for_cache(mm); | ||
2505 | if (page) | 2557 | if (page) |
2506 | pte = (pte_t *) page_address(page); | 2558 | pte = (pte_t *) page_address(page); |
2507 | 2559 | ||
@@ -2511,14 +2563,10 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, | |||
2511 | pgtable_t pte_alloc_one(struct mm_struct *mm, | 2563 | pgtable_t pte_alloc_one(struct mm_struct *mm, |
2512 | unsigned long address) | 2564 | unsigned long address) |
2513 | { | 2565 | { |
2514 | struct page *page; | 2566 | struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | |
2515 | pte_t *pte; | 2567 | __GFP_REPEAT | __GFP_ZERO); |
2516 | 2568 | pte_t *pte = NULL; | |
2517 | pte = get_from_cache(mm); | ||
2518 | if (pte) | ||
2519 | return pte; | ||
2520 | 2569 | ||
2521 | page = __alloc_for_cache(mm); | ||
2522 | if (!page) | 2570 | if (!page) |
2523 | return NULL; | 2571 | return NULL; |
2524 | if (!pgtable_page_ctor(page)) { | 2572 | if (!pgtable_page_ctor(page)) { |
@@ -2530,18 +2578,15 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, | |||
2530 | 2578 | ||
2531 | void pte_free_kernel(struct mm_struct *mm, pte_t *pte) | 2579 | void pte_free_kernel(struct mm_struct *mm, pte_t *pte) |
2532 | { | 2580 | { |
2533 | struct page *page = virt_to_page(pte); | 2581 | free_page((unsigned long)pte); |
2534 | if (put_page_testzero(page)) | ||
2535 | free_hot_cold_page(page, 0); | ||
2536 | } | 2582 | } |
2537 | 2583 | ||
2538 | static void __pte_free(pgtable_t pte) | 2584 | static void __pte_free(pgtable_t pte) |
2539 | { | 2585 | { |
2540 | struct page *page = virt_to_page(pte); | 2586 | struct page *page = virt_to_page(pte); |
2541 | if (put_page_testzero(page)) { | 2587 | |
2542 | pgtable_page_dtor(page); | 2588 | pgtable_page_dtor(page); |
2543 | free_hot_cold_page(page, 0); | 2589 | __free_page(page); |
2544 | } | ||
2545 | } | 2590 | } |
2546 | 2591 | ||
2547 | void pte_free(struct mm_struct *mm, pgtable_t pte) | 2592 | void pte_free(struct mm_struct *mm, pgtable_t pte) |
@@ -2558,124 +2603,27 @@ void pgtable_free(void *table, bool is_page) | |||
2558 | } | 2603 | } |
2559 | 2604 | ||
2560 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE | 2605 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
2561 | static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify) | ||
2562 | { | ||
2563 | if (pgprot_val(pgprot) & _PAGE_VALID) | ||
2564 | pmd_val(pmd) |= PMD_HUGE_PRESENT; | ||
2565 | if (tlb_type == hypervisor) { | ||
2566 | if (pgprot_val(pgprot) & _PAGE_WRITE_4V) | ||
2567 | pmd_val(pmd) |= PMD_HUGE_WRITE; | ||
2568 | if (pgprot_val(pgprot) & _PAGE_EXEC_4V) | ||
2569 | pmd_val(pmd) |= PMD_HUGE_EXEC; | ||
2570 | |||
2571 | if (!for_modify) { | ||
2572 | if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V) | ||
2573 | pmd_val(pmd) |= PMD_HUGE_ACCESSED; | ||
2574 | if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V) | ||
2575 | pmd_val(pmd) |= PMD_HUGE_DIRTY; | ||
2576 | } | ||
2577 | } else { | ||
2578 | if (pgprot_val(pgprot) & _PAGE_WRITE_4U) | ||
2579 | pmd_val(pmd) |= PMD_HUGE_WRITE; | ||
2580 | if (pgprot_val(pgprot) & _PAGE_EXEC_4U) | ||
2581 | pmd_val(pmd) |= PMD_HUGE_EXEC; | ||
2582 | |||
2583 | if (!for_modify) { | ||
2584 | if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U) | ||
2585 | pmd_val(pmd) |= PMD_HUGE_ACCESSED; | ||
2586 | if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U) | ||
2587 | pmd_val(pmd) |= PMD_HUGE_DIRTY; | ||
2588 | } | ||
2589 | } | ||
2590 | |||
2591 | return pmd; | ||
2592 | } | ||
2593 | |||
2594 | pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) | ||
2595 | { | ||
2596 | pmd_t pmd; | ||
2597 | |||
2598 | pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT))); | ||
2599 | pmd_val(pmd) |= PMD_ISHUGE; | ||
2600 | pmd = pmd_set_protbits(pmd, pgprot, false); | ||
2601 | return pmd; | ||
2602 | } | ||
2603 | |||
2604 | pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) | ||
2605 | { | ||
2606 | pmd_val(pmd) &= ~(PMD_HUGE_PRESENT | | ||
2607 | PMD_HUGE_WRITE | | ||
2608 | PMD_HUGE_EXEC); | ||
2609 | pmd = pmd_set_protbits(pmd, newprot, true); | ||
2610 | return pmd; | ||
2611 | } | ||
2612 | |||
2613 | pgprot_t pmd_pgprot(pmd_t entry) | ||
2614 | { | ||
2615 | unsigned long pte = 0; | ||
2616 | |||
2617 | if (pmd_val(entry) & PMD_HUGE_PRESENT) | ||
2618 | pte |= _PAGE_VALID; | ||
2619 | |||
2620 | if (tlb_type == hypervisor) { | ||
2621 | if (pmd_val(entry) & PMD_HUGE_PRESENT) | ||
2622 | pte |= _PAGE_PRESENT_4V; | ||
2623 | if (pmd_val(entry) & PMD_HUGE_EXEC) | ||
2624 | pte |= _PAGE_EXEC_4V; | ||
2625 | if (pmd_val(entry) & PMD_HUGE_WRITE) | ||
2626 | pte |= _PAGE_W_4V; | ||
2627 | if (pmd_val(entry) & PMD_HUGE_ACCESSED) | ||
2628 | pte |= _PAGE_ACCESSED_4V; | ||
2629 | if (pmd_val(entry) & PMD_HUGE_DIRTY) | ||
2630 | pte |= _PAGE_MODIFIED_4V; | ||
2631 | pte |= _PAGE_CP_4V|_PAGE_CV_4V; | ||
2632 | } else { | ||
2633 | if (pmd_val(entry) & PMD_HUGE_PRESENT) | ||
2634 | pte |= _PAGE_PRESENT_4U; | ||
2635 | if (pmd_val(entry) & PMD_HUGE_EXEC) | ||
2636 | pte |= _PAGE_EXEC_4U; | ||
2637 | if (pmd_val(entry) & PMD_HUGE_WRITE) | ||
2638 | pte |= _PAGE_W_4U; | ||
2639 | if (pmd_val(entry) & PMD_HUGE_ACCESSED) | ||
2640 | pte |= _PAGE_ACCESSED_4U; | ||
2641 | if (pmd_val(entry) & PMD_HUGE_DIRTY) | ||
2642 | pte |= _PAGE_MODIFIED_4U; | ||
2643 | pte |= _PAGE_CP_4U|_PAGE_CV_4U; | ||
2644 | } | ||
2645 | |||
2646 | return __pgprot(pte); | ||
2647 | } | ||
2648 | |||
2649 | void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, | 2606 | void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, |
2650 | pmd_t *pmd) | 2607 | pmd_t *pmd) |
2651 | { | 2608 | { |
2652 | unsigned long pte, flags; | 2609 | unsigned long pte, flags; |
2653 | struct mm_struct *mm; | 2610 | struct mm_struct *mm; |
2654 | pmd_t entry = *pmd; | 2611 | pmd_t entry = *pmd; |
2655 | pgprot_t prot; | ||
2656 | 2612 | ||
2657 | if (!pmd_large(entry) || !pmd_young(entry)) | 2613 | if (!pmd_large(entry) || !pmd_young(entry)) |
2658 | return; | 2614 | return; |
2659 | 2615 | ||
2660 | pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS); | 2616 | pte = pmd_val(entry); |
2661 | pte <<= PMD_PADDR_SHIFT; | ||
2662 | pte |= _PAGE_VALID; | ||
2663 | |||
2664 | prot = pmd_pgprot(entry); | ||
2665 | |||
2666 | if (tlb_type == hypervisor) | ||
2667 | pgprot_val(prot) |= _PAGE_SZHUGE_4V; | ||
2668 | else | ||
2669 | pgprot_val(prot) |= _PAGE_SZHUGE_4U; | ||
2670 | 2617 | ||
2671 | pte |= pgprot_val(prot); | 2618 | /* We are fabricating 8MB pages using 4MB real hw pages. */ |
2619 | pte |= (addr & (1UL << REAL_HPAGE_SHIFT)); | ||
2672 | 2620 | ||
2673 | mm = vma->vm_mm; | 2621 | mm = vma->vm_mm; |
2674 | 2622 | ||
2675 | spin_lock_irqsave(&mm->context.lock, flags); | 2623 | spin_lock_irqsave(&mm->context.lock, flags); |
2676 | 2624 | ||
2677 | if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) | 2625 | if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) |
2678 | __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT, | 2626 | __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, |
2679 | addr, pte); | 2627 | addr, pte); |
2680 | 2628 | ||
2681 | spin_unlock_irqrestore(&mm->context.lock, flags); | 2629 | spin_unlock_irqrestore(&mm->context.lock, flags); |
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h index 0661aa606dec..5d3782deb403 100644 --- a/arch/sparc/mm/init_64.h +++ b/arch/sparc/mm/init_64.h | |||
@@ -1,11 +1,13 @@ | |||
1 | #ifndef _SPARC64_MM_INIT_H | 1 | #ifndef _SPARC64_MM_INIT_H |
2 | #define _SPARC64_MM_INIT_H | 2 | #define _SPARC64_MM_INIT_H |
3 | 3 | ||
4 | #include <asm/page.h> | ||
5 | |||
4 | /* Most of the symbols in this file are defined in init.c and | 6 | /* Most of the symbols in this file are defined in init.c and |
5 | * marked non-static so that assembler code can get at them. | 7 | * marked non-static so that assembler code can get at them. |
6 | */ | 8 | */ |
7 | 9 | ||
8 | #define MAX_PHYS_ADDRESS (1UL << 41UL) | 10 | #define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) |
9 | #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) | 11 | #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) |
10 | #define KPTE_BITMAP_BYTES \ | 12 | #define KPTE_BITMAP_BYTES \ |
11 | ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) | 13 | ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) |
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index 656cc46a81f5..ad3bf4b4324d 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c | |||
@@ -161,8 +161,8 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, | |||
161 | if (mm == &init_mm) | 161 | if (mm == &init_mm) |
162 | return; | 162 | return; |
163 | 163 | ||
164 | if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) { | 164 | if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { |
165 | if (pmd_val(pmd) & PMD_ISHUGE) | 165 | if (pmd_val(pmd) & _PAGE_PMD_HUGE) |
166 | mm->context.huge_pte_count++; | 166 | mm->context.huge_pte_count++; |
167 | else | 167 | else |
168 | mm->context.huge_pte_count--; | 168 | mm->context.huge_pte_count--; |
@@ -178,13 +178,16 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, | |||
178 | } | 178 | } |
179 | 179 | ||
180 | if (!pmd_none(orig)) { | 180 | if (!pmd_none(orig)) { |
181 | bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0); | 181 | pte_t orig_pte = __pte(pmd_val(orig)); |
182 | bool exec = pte_exec(orig_pte); | ||
182 | 183 | ||
183 | addr &= HPAGE_MASK; | 184 | addr &= HPAGE_MASK; |
184 | if (pmd_val(orig) & PMD_ISHUGE) | 185 | if (pmd_trans_huge(orig)) { |
185 | tlb_batch_add_one(mm, addr, exec); | 186 | tlb_batch_add_one(mm, addr, exec); |
186 | else | 187 | tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); |
188 | } else { | ||
187 | tlb_batch_pmd_scan(mm, addr, orig, exec); | 189 | tlb_batch_pmd_scan(mm, addr, orig, exec); |
190 | } | ||
188 | } | 191 | } |
189 | } | 192 | } |
190 | 193 | ||
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 2cc3bce5ee91..3b3a360b429a 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c | |||
@@ -87,7 +87,7 @@ void flush_tsb_user(struct tlb_batch *tb) | |||
87 | nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; | 87 | nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; |
88 | if (tlb_type == cheetah_plus || tlb_type == hypervisor) | 88 | if (tlb_type == cheetah_plus || tlb_type == hypervisor) |
89 | base = __pa(base); | 89 | base = __pa(base); |
90 | __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries); | 90 | __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); |
91 | } | 91 | } |
92 | #endif | 92 | #endif |
93 | spin_unlock_irqrestore(&mm->context.lock, flags); | 93 | spin_unlock_irqrestore(&mm->context.lock, flags); |
@@ -111,7 +111,7 @@ void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) | |||
111 | nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; | 111 | nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; |
112 | if (tlb_type == cheetah_plus || tlb_type == hypervisor) | 112 | if (tlb_type == cheetah_plus || tlb_type == hypervisor) |
113 | base = __pa(base); | 113 | base = __pa(base); |
114 | __flush_tsb_one_entry(base, vaddr, HPAGE_SHIFT, nentries); | 114 | __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); |
115 | } | 115 | } |
116 | #endif | 116 | #endif |
117 | spin_unlock_irqrestore(&mm->context.lock, flags); | 117 | spin_unlock_irqrestore(&mm->context.lock, flags); |
@@ -472,8 +472,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) | |||
472 | mm->context.huge_pte_count = 0; | 472 | mm->context.huge_pte_count = 0; |
473 | #endif | 473 | #endif |
474 | 474 | ||
475 | mm->context.pgtable_page = NULL; | ||
476 | |||
477 | /* copy_mm() copies over the parent's mm_struct before calling | 475 | /* copy_mm() copies over the parent's mm_struct before calling |
478 | * us, so we need to zero out the TSB pointer or else tsb_grow() | 476 | * us, so we need to zero out the TSB pointer or else tsb_grow() |
479 | * will be confused and think there is an older TSB to free up. | 477 | * will be confused and think there is an older TSB to free up. |
@@ -512,17 +510,10 @@ static void tsb_destroy_one(struct tsb_config *tp) | |||
512 | void destroy_context(struct mm_struct *mm) | 510 | void destroy_context(struct mm_struct *mm) |
513 | { | 511 | { |
514 | unsigned long flags, i; | 512 | unsigned long flags, i; |
515 | struct page *page; | ||
516 | 513 | ||
517 | for (i = 0; i < MM_NUM_TSBS; i++) | 514 | for (i = 0; i < MM_NUM_TSBS; i++) |
518 | tsb_destroy_one(&mm->context.tsb_block[i]); | 515 | tsb_destroy_one(&mm->context.tsb_block[i]); |
519 | 516 | ||
520 | page = mm->context.pgtable_page; | ||
521 | if (page && put_page_testzero(page)) { | ||
522 | pgtable_page_dtor(page); | ||
523 | free_hot_cold_page(page, 0); | ||
524 | } | ||
525 | |||
526 | spin_lock_irqsave(&ctx_alloc_lock, flags); | 517 | spin_lock_irqsave(&ctx_alloc_lock, flags); |
527 | 518 | ||
528 | if (CTX_VALID(mm->context)) { | 519 | if (CTX_VALID(mm->context)) { |
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S index 432aa0cb1b38..b4f4733abc6e 100644 --- a/arch/sparc/mm/ultra.S +++ b/arch/sparc/mm/ultra.S | |||
@@ -153,10 +153,10 @@ __spitfire_flush_tlb_mm_slow: | |||
153 | .globl __flush_icache_page | 153 | .globl __flush_icache_page |
154 | __flush_icache_page: /* %o0 = phys_page */ | 154 | __flush_icache_page: /* %o0 = phys_page */ |
155 | srlx %o0, PAGE_SHIFT, %o0 | 155 | srlx %o0, PAGE_SHIFT, %o0 |
156 | sethi %uhi(PAGE_OFFSET), %g1 | 156 | sethi %hi(PAGE_OFFSET), %g1 |
157 | sllx %o0, PAGE_SHIFT, %o0 | 157 | sllx %o0, PAGE_SHIFT, %o0 |
158 | sethi %hi(PAGE_SIZE), %g2 | 158 | sethi %hi(PAGE_SIZE), %g2 |
159 | sllx %g1, 32, %g1 | 159 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
160 | add %o0, %g1, %o0 | 160 | add %o0, %g1, %o0 |
161 | 1: subcc %g2, 32, %g2 | 161 | 1: subcc %g2, 32, %g2 |
162 | bne,pt %icc, 1b | 162 | bne,pt %icc, 1b |
@@ -178,8 +178,8 @@ __flush_icache_page: /* %o0 = phys_page */ | |||
178 | .align 64 | 178 | .align 64 |
179 | .globl __flush_dcache_page | 179 | .globl __flush_dcache_page |
180 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ | 180 | __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ |
181 | sethi %uhi(PAGE_OFFSET), %g1 | 181 | sethi %hi(PAGE_OFFSET), %g1 |
182 | sllx %g1, 32, %g1 | 182 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
183 | sub %o0, %g1, %o0 ! physical address | 183 | sub %o0, %g1, %o0 ! physical address |
184 | srlx %o0, 11, %o0 ! make D-cache TAG | 184 | srlx %o0, 11, %o0 ! make D-cache TAG |
185 | sethi %hi(1 << 14), %o2 ! D-cache size | 185 | sethi %hi(1 << 14), %o2 ! D-cache size |
@@ -287,8 +287,8 @@ __cheetah_flush_tlb_pending: /* 27 insns */ | |||
287 | 287 | ||
288 | #ifdef DCACHE_ALIASING_POSSIBLE | 288 | #ifdef DCACHE_ALIASING_POSSIBLE |
289 | __cheetah_flush_dcache_page: /* 11 insns */ | 289 | __cheetah_flush_dcache_page: /* 11 insns */ |
290 | sethi %uhi(PAGE_OFFSET), %g1 | 290 | sethi %hi(PAGE_OFFSET), %g1 |
291 | sllx %g1, 32, %g1 | 291 | ldx [%g1 + %lo(PAGE_OFFSET)], %g1 |
292 | sub %o0, %g1, %o0 | 292 | sub %o0, %g1, %o0 |
293 | sethi %hi(PAGE_SIZE), %o4 | 293 | sethi %hi(PAGE_SIZE), %o4 |
294 | 1: subcc %o4, (1 << 5), %o4 | 294 | 1: subcc %o4, (1 << 5), %o4 |