aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-10-11 20:36:34 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-10-11 20:36:34 -0400
commit052db7ec86dff26f734031c3ef5c2c03a94af0af (patch)
treede4ca607d1ede889a3a804e87f25bd38304f9016 /arch/sparc
parentfd9879b9bb3258ebc27a4cc6d2d29f528f71901f (diff)
parentbdcf81b658ebc4c2640c3c2c55c8b31c601b6996 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc
Pull sparc updates from David Miller: 1) Move to 4-level page tables on sparc64 and support up to 53-bits of physical addressing. Kernel static image BSS size reduced by several megabytes. 2) M6/M7 cpu support, from Allan Pais. 3) Move to sparse IRQs, handle hypervisor TLB call errors more gracefully, and add T5 perf_event support. From Bob Picco. 4) Recognize cdroms and compute geometry from capacity in virtual disk driver, also from Allan Pais. 5) Fix memset() return value on sparc32, from Andreas Larsson. 6) Respect gfp flags in dma_alloc_coherent on sparc32, from Daniel Hellstrom. 7) Fix handling of compound pages in virtual disk driver, from Dwight Engen. 8) Fix lockdep warnings in LDC layer by moving IRQ requesting to ldc_alloc() from ldc_bind(). 9) Increase boot string length to 1024 bytes, from Dave Kleikamp. * git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc: (31 commits) sparc64: Fix lockdep warnings on reboot on Ultra-5 sparc64: Increase size of boot string to 1024 bytes sparc64: Kill unnecessary tables and increase MAX_BANKS. sparc64: sparse irq sparc64: Adjust vmalloc region size based upon available virtual address bits. sparc64: Increase MAX_PHYS_ADDRESS_BITS to 53. sparc64: Use kernel page tables for vmemmap. sparc64: Fix physical memory management regressions with large max_phys_bits. sparc64: Adjust KTSB assembler to support larger physical addresses. sparc64: Define VA hole at run time, rather than at compile time. sparc64: Switch to 4-level page tables. sparc64: Fix reversed start/end in flush_tlb_kernel_range() sparc64: Add vio_set_intr() to enable/disable Rx interrupts vio: fix reuse of vio_dring slot sunvdc: limit each sg segment to a page sunvdc: compute vdisk geometry from capacity sunvdc: add cdrom and v1.1 protocol support sparc: VIO protocol version 1.6 sparc64: Fix hibernation code refrence to PAGE_OFFSET. sparc64: Move request_irq() from ldc_bind() to ldc_alloc() ...
Diffstat (limited to 'arch/sparc')
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/sparc/include/asm/dma-mapping.h6
-rw-r--r--arch/sparc/include/asm/hypervisor.h11
-rw-r--r--arch/sparc/include/asm/irq_64.h7
-rw-r--r--arch/sparc/include/asm/ldc.h5
-rw-r--r--arch/sparc/include/asm/page_64.h33
-rw-r--r--arch/sparc/include/asm/pgalloc_64.h28
-rw-r--r--arch/sparc/include/asm/pgtable_64.h100
-rw-r--r--arch/sparc/include/asm/spitfire.h2
-rw-r--r--arch/sparc/include/asm/thread_info_64.h1
-rw-r--r--arch/sparc/include/asm/tsb.h87
-rw-r--r--arch/sparc/include/asm/vio.h16
-rw-r--r--arch/sparc/kernel/cpu.c12
-rw-r--r--arch/sparc/kernel/cpumap.c2
-rw-r--r--arch/sparc/kernel/ds.c4
-rw-r--r--arch/sparc/kernel/head_64.S12
-rw-r--r--arch/sparc/kernel/hvapi.c1
-rw-r--r--arch/sparc/kernel/hvcalls.S16
-rw-r--r--arch/sparc/kernel/ioport.c5
-rw-r--r--arch/sparc/kernel/irq_64.c507
-rw-r--r--arch/sparc/kernel/ktlb.S125
-rw-r--r--arch/sparc/kernel/ldc.c41
-rw-r--r--arch/sparc/kernel/leon_kernel.c31
-rw-r--r--arch/sparc/kernel/pcr.c47
-rw-r--r--arch/sparc/kernel/perf_event.c3
-rw-r--r--arch/sparc/kernel/setup_64.c26
-rw-r--r--arch/sparc/kernel/smp_64.c7
-rw-r--r--arch/sparc/kernel/sun4v_tlb_miss.S35
-rw-r--r--arch/sparc/kernel/traps_64.c15
-rw-r--r--arch/sparc/kernel/vio.c13
-rw-r--r--arch/sparc/kernel/viohs.c4
-rw-r--r--arch/sparc/kernel/vmlinux.lds.S10
-rw-r--r--arch/sparc/lib/memset.S18
-rw-r--r--arch/sparc/mm/fault_64.c3
-rw-r--r--arch/sparc/mm/init_64.c603
-rw-r--r--arch/sparc/mm/init_64.h18
-rw-r--r--arch/sparc/power/hibernate_asm.S4
-rw-r--r--arch/sparc/prom/bootstr_64.c5
-rw-r--r--arch/sparc/prom/p1275.c7
39 files changed, 1117 insertions, 754 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index a537816613f9..96ac69c5eba0 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -67,6 +67,7 @@ config SPARC64
67 select HAVE_SYSCALL_TRACEPOINTS 67 select HAVE_SYSCALL_TRACEPOINTS
68 select HAVE_CONTEXT_TRACKING 68 select HAVE_CONTEXT_TRACKING
69 select HAVE_DEBUG_KMEMLEAK 69 select HAVE_DEBUG_KMEMLEAK
70 select SPARSE_IRQ
70 select RTC_DRV_CMOS 71 select RTC_DRV_CMOS
71 select RTC_DRV_BQ4802 72 select RTC_DRV_BQ4802
72 select RTC_DRV_SUN4V 73 select RTC_DRV_SUN4V
diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h
index 1ee02710b2dc..5b1b52a04ad6 100644
--- a/arch/sparc/include/asm/dma-mapping.h
+++ b/arch/sparc/include/asm/dma-mapping.h
@@ -20,10 +20,12 @@ extern struct bus_type pci_bus_type;
20 20
21static inline struct dma_map_ops *get_dma_ops(struct device *dev) 21static inline struct dma_map_ops *get_dma_ops(struct device *dev)
22{ 22{
23#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) 23#ifdef CONFIG_SPARC_LEON
24 if (sparc_cpu_model == sparc_leon) 24 if (sparc_cpu_model == sparc_leon)
25 return leon_dma_ops; 25 return leon_dma_ops;
26 else if (dev->bus == &pci_bus_type) 26#endif
27#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI)
28 if (dev->bus == &pci_bus_type)
27 return &pci32_dma_ops; 29 return &pci32_dma_ops;
28#endif 30#endif
29 return dma_ops; 31 return dma_ops;
diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index 94b39caea3eb..4f6725ff4c33 100644
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -2947,6 +2947,16 @@ unsigned long sun4v_vt_set_perfreg(unsigned long reg_num,
2947 unsigned long reg_val); 2947 unsigned long reg_val);
2948#endif 2948#endif
2949 2949
2950#define HV_FAST_T5_GET_PERFREG 0x1a8
2951#define HV_FAST_T5_SET_PERFREG 0x1a9
2952
2953#ifndef __ASSEMBLY__
2954unsigned long sun4v_t5_get_perfreg(unsigned long reg_num,
2955 unsigned long *reg_val);
2956unsigned long sun4v_t5_set_perfreg(unsigned long reg_num,
2957 unsigned long reg_val);
2958#endif
2959
2950/* Function numbers for HV_CORE_TRAP. */ 2960/* Function numbers for HV_CORE_TRAP. */
2951#define HV_CORE_SET_VER 0x00 2961#define HV_CORE_SET_VER 0x00
2952#define HV_CORE_PUTCHAR 0x01 2962#define HV_CORE_PUTCHAR 0x01
@@ -2978,6 +2988,7 @@ unsigned long sun4v_vt_set_perfreg(unsigned long reg_num,
2978#define HV_GRP_VF_CPU 0x0205 2988#define HV_GRP_VF_CPU 0x0205
2979#define HV_GRP_KT_CPU 0x0209 2989#define HV_GRP_KT_CPU 0x0209
2980#define HV_GRP_VT_CPU 0x020c 2990#define HV_GRP_VT_CPU 0x020c
2991#define HV_GRP_T5_CPU 0x0211
2981#define HV_GRP_DIAG 0x0300 2992#define HV_GRP_DIAG 0x0300
2982 2993
2983#ifndef __ASSEMBLY__ 2994#ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h
index 91d219381306..3f70f900e834 100644
--- a/arch/sparc/include/asm/irq_64.h
+++ b/arch/sparc/include/asm/irq_64.h
@@ -37,7 +37,7 @@
37 * 37 *
38 * ino_bucket->irq allocation is made during {sun4v_,}build_irq(). 38 * ino_bucket->irq allocation is made during {sun4v_,}build_irq().
39 */ 39 */
40#define NR_IRQS 255 40#define NR_IRQS (2048)
41 41
42void irq_install_pre_handler(int irq, 42void irq_install_pre_handler(int irq,
43 void (*func)(unsigned int, void *, void *), 43 void (*func)(unsigned int, void *, void *),
@@ -57,11 +57,8 @@ unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p,
57 unsigned long iclr_base); 57 unsigned long iclr_base);
58void sun4u_destroy_msi(unsigned int irq); 58void sun4u_destroy_msi(unsigned int irq);
59 59
60unsigned char irq_alloc(unsigned int dev_handle, 60unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino);
61 unsigned int dev_ino);
62#ifdef CONFIG_PCI_MSI
63void irq_free(unsigned int irq); 61void irq_free(unsigned int irq);
64#endif
65 62
66void __init init_IRQ(void); 63void __init init_IRQ(void);
67void fixup_irqs(void); 64void fixup_irqs(void);
diff --git a/arch/sparc/include/asm/ldc.h b/arch/sparc/include/asm/ldc.h
index c8c67f621f4f..58ab64de25d2 100644
--- a/arch/sparc/include/asm/ldc.h
+++ b/arch/sparc/include/asm/ldc.h
@@ -53,13 +53,14 @@ struct ldc_channel;
53/* Allocate state for a channel. */ 53/* Allocate state for a channel. */
54struct ldc_channel *ldc_alloc(unsigned long id, 54struct ldc_channel *ldc_alloc(unsigned long id,
55 const struct ldc_channel_config *cfgp, 55 const struct ldc_channel_config *cfgp,
56 void *event_arg); 56 void *event_arg,
57 const char *name);
57 58
58/* Shut down and free state for a channel. */ 59/* Shut down and free state for a channel. */
59void ldc_free(struct ldc_channel *lp); 60void ldc_free(struct ldc_channel *lp);
60 61
61/* Register TX and RX queues of the link with the hypervisor. */ 62/* Register TX and RX queues of the link with the hypervisor. */
62int ldc_bind(struct ldc_channel *lp, const char *name); 63int ldc_bind(struct ldc_channel *lp);
63 64
64/* For non-RAW protocols we need to complete a handshake before 65/* For non-RAW protocols we need to complete a handshake before
65 * communication can proceed. ldc_connect() does that, if the 66 * communication can proceed. ldc_connect() does that, if the
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index bf109984a032..8c2a8c937540 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -57,18 +57,21 @@ void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topa
57typedef struct { unsigned long pte; } pte_t; 57typedef struct { unsigned long pte; } pte_t;
58typedef struct { unsigned long iopte; } iopte_t; 58typedef struct { unsigned long iopte; } iopte_t;
59typedef struct { unsigned long pmd; } pmd_t; 59typedef struct { unsigned long pmd; } pmd_t;
60typedef struct { unsigned long pud; } pud_t;
60typedef struct { unsigned long pgd; } pgd_t; 61typedef struct { unsigned long pgd; } pgd_t;
61typedef struct { unsigned long pgprot; } pgprot_t; 62typedef struct { unsigned long pgprot; } pgprot_t;
62 63
63#define pte_val(x) ((x).pte) 64#define pte_val(x) ((x).pte)
64#define iopte_val(x) ((x).iopte) 65#define iopte_val(x) ((x).iopte)
65#define pmd_val(x) ((x).pmd) 66#define pmd_val(x) ((x).pmd)
67#define pud_val(x) ((x).pud)
66#define pgd_val(x) ((x).pgd) 68#define pgd_val(x) ((x).pgd)
67#define pgprot_val(x) ((x).pgprot) 69#define pgprot_val(x) ((x).pgprot)
68 70
69#define __pte(x) ((pte_t) { (x) } ) 71#define __pte(x) ((pte_t) { (x) } )
70#define __iopte(x) ((iopte_t) { (x) } ) 72#define __iopte(x) ((iopte_t) { (x) } )
71#define __pmd(x) ((pmd_t) { (x) } ) 73#define __pmd(x) ((pmd_t) { (x) } )
74#define __pud(x) ((pud_t) { (x) } )
72#define __pgd(x) ((pgd_t) { (x) } ) 75#define __pgd(x) ((pgd_t) { (x) } )
73#define __pgprot(x) ((pgprot_t) { (x) } ) 76#define __pgprot(x) ((pgprot_t) { (x) } )
74 77
@@ -77,18 +80,21 @@ typedef struct { unsigned long pgprot; } pgprot_t;
77typedef unsigned long pte_t; 80typedef unsigned long pte_t;
78typedef unsigned long iopte_t; 81typedef unsigned long iopte_t;
79typedef unsigned long pmd_t; 82typedef unsigned long pmd_t;
83typedef unsigned long pud_t;
80typedef unsigned long pgd_t; 84typedef unsigned long pgd_t;
81typedef unsigned long pgprot_t; 85typedef unsigned long pgprot_t;
82 86
83#define pte_val(x) (x) 87#define pte_val(x) (x)
84#define iopte_val(x) (x) 88#define iopte_val(x) (x)
85#define pmd_val(x) (x) 89#define pmd_val(x) (x)
90#define pud_val(x) (x)
86#define pgd_val(x) (x) 91#define pgd_val(x) (x)
87#define pgprot_val(x) (x) 92#define pgprot_val(x) (x)
88 93
89#define __pte(x) (x) 94#define __pte(x) (x)
90#define __iopte(x) (x) 95#define __iopte(x) (x)
91#define __pmd(x) (x) 96#define __pmd(x) (x)
97#define __pud(x) (x)
92#define __pgd(x) (x) 98#define __pgd(x) (x)
93#define __pgprot(x) (x) 99#define __pgprot(x) (x)
94 100
@@ -96,21 +102,14 @@ typedef unsigned long pgprot_t;
96 102
97typedef pte_t *pgtable_t; 103typedef pte_t *pgtable_t;
98 104
99/* These two values define the virtual address space range in which we 105extern unsigned long sparc64_va_hole_top;
100 * must forbid 64-bit user processes from making mappings. It used to 106extern unsigned long sparc64_va_hole_bottom;
101 * represent precisely the virtual address space hole present in most
102 * early sparc64 chips including UltraSPARC-I. But now it also is
103 * further constrained by the limits of our page tables, which is
104 * 43-bits of virtual address.
105 */
106#define SPARC64_VA_HOLE_TOP _AC(0xfffffc0000000000,UL)
107#define SPARC64_VA_HOLE_BOTTOM _AC(0x0000040000000000,UL)
108 107
109/* The next two defines specify the actual exclusion region we 108/* The next two defines specify the actual exclusion region we
110 * enforce, wherein we use a 4GB red zone on each side of the VA hole. 109 * enforce, wherein we use a 4GB red zone on each side of the VA hole.
111 */ 110 */
112#define VA_EXCLUDE_START (SPARC64_VA_HOLE_BOTTOM - (1UL << 32UL)) 111#define VA_EXCLUDE_START (sparc64_va_hole_bottom - (1UL << 32UL))
113#define VA_EXCLUDE_END (SPARC64_VA_HOLE_TOP + (1UL << 32UL)) 112#define VA_EXCLUDE_END (sparc64_va_hole_top + (1UL << 32UL))
114 113
115#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ 114#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \
116 _AC(0x0000000070000000,UL) : \ 115 _AC(0x0000000070000000,UL) : \
@@ -118,20 +117,16 @@ typedef pte_t *pgtable_t;
118 117
119#include <asm-generic/memory_model.h> 118#include <asm-generic/memory_model.h>
120 119
121#define PAGE_OFFSET_BY_BITS(X) (-(_AC(1,UL) << (X)))
122extern unsigned long PAGE_OFFSET; 120extern unsigned long PAGE_OFFSET;
123 121
124#endif /* !(__ASSEMBLY__) */ 122#endif /* !(__ASSEMBLY__) */
125 123
126/* The maximum number of physical memory address bits we support, this 124/* The maximum number of physical memory address bits we support. The
127 * is used to size various tables used to manage kernel TLB misses and 125 * largest value we can support is whatever "KPGD_SHIFT + KPTE_BITS"
128 * also the sparsemem code. 126 * evaluates to.
129 */ 127 */
130#define MAX_PHYS_ADDRESS_BITS 47 128#define MAX_PHYS_ADDRESS_BITS 53
131 129
132/* These two shift counts are used when indexing sparc64_valid_addr_bitmap
133 * and kpte_linear_bitmap.
134 */
135#define ILOG2_4MB 22 130#define ILOG2_4MB 22
136#define ILOG2_256MB 28 131#define ILOG2_256MB 28
137 132
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 39a7ac49b00c..5e3187185b4a 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -15,6 +15,13 @@
15 15
16extern struct kmem_cache *pgtable_cache; 16extern struct kmem_cache *pgtable_cache;
17 17
18static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
19{
20 pgd_set(pgd, pud);
21}
22
23#define pgd_populate(MM, PGD, PUD) __pgd_populate(PGD, PUD)
24
18static inline pgd_t *pgd_alloc(struct mm_struct *mm) 25static inline pgd_t *pgd_alloc(struct mm_struct *mm)
19{ 26{
20 return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); 27 return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
@@ -25,7 +32,23 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
25 kmem_cache_free(pgtable_cache, pgd); 32 kmem_cache_free(pgtable_cache, pgd);
26} 33}
27 34
28#define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) 35static inline void __pud_populate(pud_t *pud, pmd_t *pmd)
36{
37 pud_set(pud, pmd);
38}
39
40#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD)
41
42static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
43{
44 return kmem_cache_alloc(pgtable_cache,
45 GFP_KERNEL|__GFP_REPEAT);
46}
47
48static inline void pud_free(struct mm_struct *mm, pud_t *pud)
49{
50 kmem_cache_free(pgtable_cache, pud);
51}
29 52
30static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 53static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
31{ 54{
@@ -91,4 +114,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
91#define __pmd_free_tlb(tlb, pmd, addr) \ 114#define __pmd_free_tlb(tlb, pmd, addr) \
92 pgtable_free_tlb(tlb, pmd, false) 115 pgtable_free_tlb(tlb, pmd, false)
93 116
117#define __pud_free_tlb(tlb, pud, addr) \
118 pgtable_free_tlb(tlb, pud, false)
119
94#endif /* _SPARC64_PGALLOC_H */ 120#endif /* _SPARC64_PGALLOC_H */
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 3770bf5c6e1b..bfeb626085ac 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -20,8 +20,6 @@
20#include <asm/page.h> 20#include <asm/page.h>
21#include <asm/processor.h> 21#include <asm/processor.h>
22 22
23#include <asm-generic/pgtable-nopud.h>
24
25/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB). 23/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
26 * The page copy blockops can use 0x6000000 to 0x8000000. 24 * The page copy blockops can use 0x6000000 to 0x8000000.
27 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range. 25 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
@@ -42,10 +40,7 @@
42#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL) 40#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL)
43#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL) 41#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL)
44#define VMALLOC_START _AC(0x0000000100000000,UL) 42#define VMALLOC_START _AC(0x0000000100000000,UL)
45#define VMALLOC_END _AC(0x0000010000000000,UL) 43#define VMEMMAP_BASE VMALLOC_END
46#define VMEMMAP_BASE _AC(0x0000010000000000,UL)
47
48#define vmemmap ((struct page *)VMEMMAP_BASE)
49 44
50/* PMD_SHIFT determines the size of the area a second-level page 45/* PMD_SHIFT determines the size of the area a second-level page
51 * table can map 46 * table can map
@@ -55,13 +50,25 @@
55#define PMD_MASK (~(PMD_SIZE-1)) 50#define PMD_MASK (~(PMD_SIZE-1))
56#define PMD_BITS (PAGE_SHIFT - 3) 51#define PMD_BITS (PAGE_SHIFT - 3)
57 52
58/* PGDIR_SHIFT determines what a third-level page table entry can map */ 53/* PUD_SHIFT determines the size of the area a third-level page
59#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS) 54 * table can map
55 */
56#define PUD_SHIFT (PMD_SHIFT + PMD_BITS)
57#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT)
58#define PUD_MASK (~(PUD_SIZE-1))
59#define PUD_BITS (PAGE_SHIFT - 3)
60
61/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
62#define PGDIR_SHIFT (PUD_SHIFT + PUD_BITS)
60#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) 63#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT)
61#define PGDIR_MASK (~(PGDIR_SIZE-1)) 64#define PGDIR_MASK (~(PGDIR_SIZE-1))
62#define PGDIR_BITS (PAGE_SHIFT - 3) 65#define PGDIR_BITS (PAGE_SHIFT - 3)
63 66
64#if (PGDIR_SHIFT + PGDIR_BITS) != 43 67#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS)
68#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
69#endif
70
71#if (PGDIR_SHIFT + PGDIR_BITS) != 53
65#error Page table parameters do not cover virtual address space properly. 72#error Page table parameters do not cover virtual address space properly.
66#endif 73#endif
67 74
@@ -71,28 +78,18 @@
71 78
72#ifndef __ASSEMBLY__ 79#ifndef __ASSEMBLY__
73 80
74#include <linux/sched.h> 81extern unsigned long VMALLOC_END;
75
76extern unsigned long sparc64_valid_addr_bitmap[];
77 82
78/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ 83#define vmemmap ((struct page *)VMEMMAP_BASE)
79static inline bool __kern_addr_valid(unsigned long paddr)
80{
81 if ((paddr >> MAX_PHYS_ADDRESS_BITS) != 0UL)
82 return false;
83 return test_bit(paddr >> ILOG2_4MB, sparc64_valid_addr_bitmap);
84}
85 84
86static inline bool kern_addr_valid(unsigned long addr) 85#include <linux/sched.h>
87{
88 unsigned long paddr = __pa(addr);
89 86
90 return __kern_addr_valid(paddr); 87bool kern_addr_valid(unsigned long addr);
91}
92 88
93/* Entries per page directory level. */ 89/* Entries per page directory level. */
94#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) 90#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
95#define PTRS_PER_PMD (1UL << PMD_BITS) 91#define PTRS_PER_PMD (1UL << PMD_BITS)
92#define PTRS_PER_PUD (1UL << PUD_BITS)
96#define PTRS_PER_PGD (1UL << PGDIR_BITS) 93#define PTRS_PER_PGD (1UL << PGDIR_BITS)
97 94
98/* Kernel has a separate 44bit address space. */ 95/* Kernel has a separate 44bit address space. */
@@ -101,6 +98,9 @@ static inline bool kern_addr_valid(unsigned long addr)
101#define pmd_ERROR(e) \ 98#define pmd_ERROR(e) \
102 pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ 99 pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \
103 __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) 100 __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
101#define pud_ERROR(e) \
102 pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \
103 __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
104#define pgd_ERROR(e) \ 104#define pgd_ERROR(e) \
105 pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ 105 pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \
106 __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) 106 __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
@@ -112,6 +112,7 @@ static inline bool kern_addr_valid(unsigned long addr)
112#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ 112#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
113#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ 113#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
114#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ 114#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */
115#define _PAGE_PUD_HUGE _PAGE_PMD_HUGE
115 116
116/* Advertise support for _PAGE_SPECIAL */ 117/* Advertise support for _PAGE_SPECIAL */
117#define __HAVE_ARCH_PTE_SPECIAL 118#define __HAVE_ARCH_PTE_SPECIAL
@@ -658,26 +659,26 @@ static inline unsigned long pmd_large(pmd_t pmd)
658 return pte_val(pte) & _PAGE_PMD_HUGE; 659 return pte_val(pte) & _PAGE_PMD_HUGE;
659} 660}
660 661
661#ifdef CONFIG_TRANSPARENT_HUGEPAGE 662static inline unsigned long pmd_pfn(pmd_t pmd)
662static inline unsigned long pmd_young(pmd_t pmd)
663{ 663{
664 pte_t pte = __pte(pmd_val(pmd)); 664 pte_t pte = __pte(pmd_val(pmd));
665 665
666 return pte_young(pte); 666 return pte_pfn(pte);
667} 667}
668 668
669static inline unsigned long pmd_write(pmd_t pmd) 669#ifdef CONFIG_TRANSPARENT_HUGEPAGE
670static inline unsigned long pmd_young(pmd_t pmd)
670{ 671{
671 pte_t pte = __pte(pmd_val(pmd)); 672 pte_t pte = __pte(pmd_val(pmd));
672 673
673 return pte_write(pte); 674 return pte_young(pte);
674} 675}
675 676
676static inline unsigned long pmd_pfn(pmd_t pmd) 677static inline unsigned long pmd_write(pmd_t pmd)
677{ 678{
678 pte_t pte = __pte(pmd_val(pmd)); 679 pte_t pte = __pte(pmd_val(pmd));
679 680
680 return pte_pfn(pte); 681 return pte_write(pte);
681} 682}
682 683
683static inline unsigned long pmd_trans_huge(pmd_t pmd) 684static inline unsigned long pmd_trans_huge(pmd_t pmd)
@@ -771,13 +772,15 @@ static inline int pmd_present(pmd_t pmd)
771 * the top bits outside of the range of any physical address size we 772 * the top bits outside of the range of any physical address size we
772 * support are clear as well. We also validate the physical itself. 773 * support are clear as well. We also validate the physical itself.
773 */ 774 */
774#define pmd_bad(pmd) ((pmd_val(pmd) & ~PAGE_MASK) || \ 775#define pmd_bad(pmd) (pmd_val(pmd) & ~PAGE_MASK)
775 !__kern_addr_valid(pmd_val(pmd)))
776 776
777#define pud_none(pud) (!pud_val(pud)) 777#define pud_none(pud) (!pud_val(pud))
778 778
779#define pud_bad(pud) ((pud_val(pud) & ~PAGE_MASK) || \ 779#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK)
780 !__kern_addr_valid(pud_val(pud))) 780
781#define pgd_none(pgd) (!pgd_val(pgd))
782
783#define pgd_bad(pgd) (pgd_val(pgd) & ~PAGE_MASK)
781 784
782#ifdef CONFIG_TRANSPARENT_HUGEPAGE 785#ifdef CONFIG_TRANSPARENT_HUGEPAGE
783void set_pmd_at(struct mm_struct *mm, unsigned long addr, 786void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -815,10 +818,31 @@ static inline unsigned long __pmd_page(pmd_t pmd)
815#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) 818#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
816#define pud_present(pud) (pud_val(pud) != 0U) 819#define pud_present(pud) (pud_val(pud) != 0U)
817#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) 820#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
821#define pgd_page_vaddr(pgd) \
822 ((unsigned long) __va(pgd_val(pgd)))
823#define pgd_present(pgd) (pgd_val(pgd) != 0U)
824#define pgd_clear(pgdp) (pgd_val(*(pgd)) = 0UL)
825
826static inline unsigned long pud_large(pud_t pud)
827{
828 pte_t pte = __pte(pud_val(pud));
829
830 return pte_val(pte) & _PAGE_PMD_HUGE;
831}
832
833static inline unsigned long pud_pfn(pud_t pud)
834{
835 pte_t pte = __pte(pud_val(pud));
836
837 return pte_pfn(pte);
838}
818 839
819/* Same in both SUN4V and SUN4U. */ 840/* Same in both SUN4V and SUN4U. */
820#define pte_none(pte) (!pte_val(pte)) 841#define pte_none(pte) (!pte_val(pte))
821 842
843#define pgd_set(pgdp, pudp) \
844 (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
845
822/* to find an entry in a page-table-directory. */ 846/* to find an entry in a page-table-directory. */
823#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 847#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
824#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) 848#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
@@ -826,6 +850,11 @@ static inline unsigned long __pmd_page(pmd_t pmd)
826/* to find an entry in a kernel page-table-directory */ 850/* to find an entry in a kernel page-table-directory */
827#define pgd_offset_k(address) pgd_offset(&init_mm, address) 851#define pgd_offset_k(address) pgd_offset(&init_mm, address)
828 852
853/* Find an entry in the third-level page table.. */
854#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
855#define pud_offset(pgdp, address) \
856 ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
857
829/* Find an entry in the second-level page table.. */ 858/* Find an entry in the second-level page table.. */
830#define pmd_offset(pudp, address) \ 859#define pmd_offset(pudp, address) \
831 ((pmd_t *) pud_page_vaddr(*(pudp)) + \ 860 ((pmd_t *) pud_page_vaddr(*(pudp)) + \
@@ -898,7 +927,6 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
898#endif 927#endif
899 928
900extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 929extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
901extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD];
902 930
903void paging_init(void); 931void paging_init(void);
904unsigned long find_ecache_flush_span(unsigned long size); 932unsigned long find_ecache_flush_span(unsigned long size);
diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h
index 3fc58691dbd0..56f933816144 100644
--- a/arch/sparc/include/asm/spitfire.h
+++ b/arch/sparc/include/asm/spitfire.h
@@ -45,6 +45,8 @@
45#define SUN4V_CHIP_NIAGARA3 0x03 45#define SUN4V_CHIP_NIAGARA3 0x03
46#define SUN4V_CHIP_NIAGARA4 0x04 46#define SUN4V_CHIP_NIAGARA4 0x04
47#define SUN4V_CHIP_NIAGARA5 0x05 47#define SUN4V_CHIP_NIAGARA5 0x05
48#define SUN4V_CHIP_SPARC_M6 0x06
49#define SUN4V_CHIP_SPARC_M7 0x07
48#define SUN4V_CHIP_SPARC64X 0x8a 50#define SUN4V_CHIP_SPARC64X 0x8a
49#define SUN4V_CHIP_UNKNOWN 0xff 51#define SUN4V_CHIP_UNKNOWN 0xff
50 52
diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h
index a5f01ac6d0f1..f85dc8512ab3 100644
--- a/arch/sparc/include/asm/thread_info_64.h
+++ b/arch/sparc/include/asm/thread_info_64.h
@@ -102,6 +102,7 @@ struct thread_info {
102#define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */ 102#define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */
103#define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */ 103#define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */
104#define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */ 104#define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */
105#define FAULT_CODE_BAD_RA 0x20 /* Bad RA for sun4v */
105 106
106#if PAGE_SHIFT == 13 107#if PAGE_SHIFT == 13
107#define THREAD_SIZE (2*PAGE_SIZE) 108#define THREAD_SIZE (2*PAGE_SIZE)
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index 90916f955cac..ecb49cfa3be9 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -133,9 +133,24 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
133 sub TSB, 0x8, TSB; \ 133 sub TSB, 0x8, TSB; \
134 TSB_STORE(TSB, TAG); 134 TSB_STORE(TSB, TAG);
135 135
136 /* Do a kernel page table walk. Leaves physical PTE pointer in 136 /* Do a kernel page table walk. Leaves valid PTE value in
137 * REG1. Jumps to FAIL_LABEL on early page table walk termination. 137 * REG1. Jumps to FAIL_LABEL on early page table walk
138 * VADDR will not be clobbered, but REG2 will. 138 * termination. VADDR will not be clobbered, but REG2 will.
139 *
140 * There are two masks we must apply to propagate bits from
141 * the virtual address into the PTE physical address field
142 * when dealing with huge pages. This is because the page
143 * table boundaries do not match the huge page size(s) the
144 * hardware supports.
145 *
146 * In these cases we propagate the bits that are below the
147 * page table level where we saw the huge page mapping, but
148 * are still within the relevant physical bits for the huge
149 * page size in question. So for PMD mappings (which fall on
150 * bit 23, for 8MB per PMD) we must propagate bit 22 for a
151 * 4MB huge page. For huge PUDs (which fall on bit 33, for
152 * 8GB per PUD), we have to accomodate 256MB and 2GB huge
153 * pages. So for those we propagate bits 32 to 28.
139 */ 154 */
140#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \ 155#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
141 sethi %hi(swapper_pg_dir), REG1; \ 156 sethi %hi(swapper_pg_dir), REG1; \
@@ -145,15 +160,40 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
145 andn REG2, 0x7, REG2; \ 160 andn REG2, 0x7, REG2; \
146 ldx [REG1 + REG2], REG1; \ 161 ldx [REG1 + REG2], REG1; \
147 brz,pn REG1, FAIL_LABEL; \ 162 brz,pn REG1, FAIL_LABEL; \
148 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 163 sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
149 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 164 srlx REG2, 64 - PAGE_SHIFT, REG2; \
150 andn REG2, 0x7, REG2; \ 165 andn REG2, 0x7, REG2; \
151 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ 166 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
152 brz,pn REG1, FAIL_LABEL; \ 167 brz,pn REG1, FAIL_LABEL; \
153 sllx VADDR, 64 - PMD_SHIFT, REG2; \ 168 sethi %uhi(_PAGE_PUD_HUGE), REG2; \
169 brz,pn REG1, FAIL_LABEL; \
170 sllx REG2, 32, REG2; \
171 andcc REG1, REG2, %g0; \
172 sethi %hi(0xf8000000), REG2; \
173 bne,pt %xcc, 697f; \
174 sllx REG2, 1, REG2; \
175 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
154 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 176 srlx REG2, 64 - PAGE_SHIFT, REG2; \
155 andn REG2, 0x7, REG2; \ 177 andn REG2, 0x7, REG2; \
156 add REG1, REG2, REG1; 178 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
179 sethi %uhi(_PAGE_PMD_HUGE), REG2; \
180 brz,pn REG1, FAIL_LABEL; \
181 sllx REG2, 32, REG2; \
182 andcc REG1, REG2, %g0; \
183 be,pn %xcc, 698f; \
184 sethi %hi(0x400000), REG2; \
185697: brgez,pn REG1, FAIL_LABEL; \
186 andn REG1, REG2, REG1; \
187 and VADDR, REG2, REG2; \
188 ba,pt %xcc, 699f; \
189 or REG1, REG2, REG1; \
190698: sllx VADDR, 64 - PMD_SHIFT, REG2; \
191 srlx REG2, 64 - PAGE_SHIFT, REG2; \
192 andn REG2, 0x7, REG2; \
193 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
194 brgez,pn REG1, FAIL_LABEL; \
195 nop; \
196699:
157 197
158 /* PMD has been loaded into REG1, interpret the value, seeing 198 /* PMD has been loaded into REG1, interpret the value, seeing
159 * if it is a HUGE PMD or a normal one. If it is not valid 199 * if it is a HUGE PMD or a normal one. If it is not valid
@@ -198,6 +238,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
198 andn REG2, 0x7, REG2; \ 238 andn REG2, 0x7, REG2; \
199 ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ 239 ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
200 brz,pn REG1, FAIL_LABEL; \ 240 brz,pn REG1, FAIL_LABEL; \
241 sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
242 srlx REG2, 64 - PAGE_SHIFT, REG2; \
243 andn REG2, 0x7, REG2; \
244 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
245 brz,pn REG1, FAIL_LABEL; \
201 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 246 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
202 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 247 srlx REG2, 64 - PAGE_SHIFT, REG2; \
203 andn REG2, 0x7, REG2; \ 248 andn REG2, 0x7, REG2; \
@@ -246,8 +291,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
246 (KERNEL_TSB_SIZE_BYTES / 16) 291 (KERNEL_TSB_SIZE_BYTES / 16)
247#define KERNEL_TSB4M_NENTRIES 4096 292#define KERNEL_TSB4M_NENTRIES 4096
248 293
249#define KTSB_PHYS_SHIFT 15
250
251 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL 294 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
252 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries 295 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
253 * and the found TTE will be left in REG1. REG3 and REG4 must 296 * and the found TTE will be left in REG1. REG3 and REG4 must
@@ -256,17 +299,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
256 * VADDR and TAG will be preserved and not clobbered by this macro. 299 * VADDR and TAG will be preserved and not clobbered by this macro.
257 */ 300 */
258#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ 301#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
259661: sethi %hi(swapper_tsb), REG1; \ 302661: sethi %uhi(swapper_tsb), REG1; \
260 or REG1, %lo(swapper_tsb), REG1; \ 303 sethi %hi(swapper_tsb), REG2; \
304 or REG1, %ulo(swapper_tsb), REG1; \
305 or REG2, %lo(swapper_tsb), REG2; \
261 .section .swapper_tsb_phys_patch, "ax"; \ 306 .section .swapper_tsb_phys_patch, "ax"; \
262 .word 661b; \ 307 .word 661b; \
263 .previous; \ 308 .previous; \
264661: nop; \ 309 sllx REG1, 32, REG1; \
265 .section .tsb_ldquad_phys_patch, "ax"; \ 310 or REG1, REG2, REG1; \
266 .word 661b; \
267 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
268 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
269 .previous; \
270 srlx VADDR, PAGE_SHIFT, REG2; \ 311 srlx VADDR, PAGE_SHIFT, REG2; \
271 and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ 312 and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
272 sllx REG2, 4, REG2; \ 313 sllx REG2, 4, REG2; \
@@ -281,17 +322,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
281 * we can make use of that for the index computation. 322 * we can make use of that for the index computation.
282 */ 323 */
283#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ 324#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
284661: sethi %hi(swapper_4m_tsb), REG1; \ 325661: sethi %uhi(swapper_4m_tsb), REG1; \
285 or REG1, %lo(swapper_4m_tsb), REG1; \ 326 sethi %hi(swapper_4m_tsb), REG2; \
327 or REG1, %ulo(swapper_4m_tsb), REG1; \
328 or REG2, %lo(swapper_4m_tsb), REG2; \
286 .section .swapper_4m_tsb_phys_patch, "ax"; \ 329 .section .swapper_4m_tsb_phys_patch, "ax"; \
287 .word 661b; \ 330 .word 661b; \
288 .previous; \ 331 .previous; \
289661: nop; \ 332 sllx REG1, 32, REG1; \
290 .section .tsb_ldquad_phys_patch, "ax"; \ 333 or REG1, REG2, REG1; \
291 .word 661b; \
292 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
293 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
294 .previous; \
295 and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ 334 and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
296 sllx REG2, 4, REG2; \ 335 sllx REG2, 4, REG2; \
297 add REG1, REG2, REG2; \ 336 add REG1, REG2, REG2; \
diff --git a/arch/sparc/include/asm/vio.h b/arch/sparc/include/asm/vio.h
index 6b135a8ab07b..d758c8d8f47d 100644
--- a/arch/sparc/include/asm/vio.h
+++ b/arch/sparc/include/asm/vio.h
@@ -121,12 +121,18 @@ struct vio_disk_attr_info {
121 u8 vdisk_type; 121 u8 vdisk_type;
122#define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */ 122#define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */
123#define VD_DISK_TYPE_DISK 0x02 /* Entire block device */ 123#define VD_DISK_TYPE_DISK 0x02 /* Entire block device */
124 u16 resv1; 124 u8 vdisk_mtype; /* v1.1 */
125#define VD_MEDIA_TYPE_FIXED 0x01 /* Fixed device */
126#define VD_MEDIA_TYPE_CD 0x02 /* CD Device */
127#define VD_MEDIA_TYPE_DVD 0x03 /* DVD Device */
128 u8 resv1;
125 u32 vdisk_block_size; 129 u32 vdisk_block_size;
126 u64 operations; 130 u64 operations;
127 u64 vdisk_size; 131 u64 vdisk_size; /* v1.1 */
128 u64 max_xfer_size; 132 u64 max_xfer_size;
129 u64 resv2[2]; 133 u32 phys_block_size; /* v1.2 */
134 u32 resv2;
135 u64 resv3[1];
130}; 136};
131 137
132struct vio_disk_desc { 138struct vio_disk_desc {
@@ -272,7 +278,7 @@ static inline u32 vio_dring_avail(struct vio_dring_state *dr,
272 unsigned int ring_size) 278 unsigned int ring_size)
273{ 279{
274 return (dr->pending - 280 return (dr->pending -
275 ((dr->prod - dr->cons) & (ring_size - 1))); 281 ((dr->prod - dr->cons) & (ring_size - 1)) - 1);
276} 282}
277 283
278#define VIO_MAX_TYPE_LEN 32 284#define VIO_MAX_TYPE_LEN 32
@@ -292,6 +298,7 @@ struct vio_dev {
292 298
293 unsigned int tx_irq; 299 unsigned int tx_irq;
294 unsigned int rx_irq; 300 unsigned int rx_irq;
301 u64 rx_ino;
295 302
296 struct device dev; 303 struct device dev;
297}; 304};
@@ -447,5 +454,6 @@ int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev,
447 char *name); 454 char *name);
448 455
449void vio_port_up(struct vio_driver_state *vio); 456void vio_port_up(struct vio_driver_state *vio);
457int vio_set_intr(unsigned long dev_ino, int state);
450 458
451#endif /* _SPARC64_VIO_H */ 459#endif /* _SPARC64_VIO_H */
diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c
index 82a3a71c451e..dfad8b1aea9f 100644
--- a/arch/sparc/kernel/cpu.c
+++ b/arch/sparc/kernel/cpu.c
@@ -494,6 +494,18 @@ static void __init sun4v_cpu_probe(void)
494 sparc_pmu_type = "niagara5"; 494 sparc_pmu_type = "niagara5";
495 break; 495 break;
496 496
497 case SUN4V_CHIP_SPARC_M6:
498 sparc_cpu_type = "SPARC-M6";
499 sparc_fpu_type = "SPARC-M6 integrated FPU";
500 sparc_pmu_type = "sparc-m6";
501 break;
502
503 case SUN4V_CHIP_SPARC_M7:
504 sparc_cpu_type = "SPARC-M7";
505 sparc_fpu_type = "SPARC-M7 integrated FPU";
506 sparc_pmu_type = "sparc-m7";
507 break;
508
497 case SUN4V_CHIP_SPARC64X: 509 case SUN4V_CHIP_SPARC64X:
498 sparc_cpu_type = "SPARC64-X"; 510 sparc_cpu_type = "SPARC64-X";
499 sparc_fpu_type = "SPARC64-X integrated FPU"; 511 sparc_fpu_type = "SPARC64-X integrated FPU";
diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c
index de1c844dfabc..e69ec0e3f155 100644
--- a/arch/sparc/kernel/cpumap.c
+++ b/arch/sparc/kernel/cpumap.c
@@ -326,6 +326,8 @@ static int iterate_cpu(struct cpuinfo_tree *t, unsigned int root_index)
326 case SUN4V_CHIP_NIAGARA3: 326 case SUN4V_CHIP_NIAGARA3:
327 case SUN4V_CHIP_NIAGARA4: 327 case SUN4V_CHIP_NIAGARA4:
328 case SUN4V_CHIP_NIAGARA5: 328 case SUN4V_CHIP_NIAGARA5:
329 case SUN4V_CHIP_SPARC_M6:
330 case SUN4V_CHIP_SPARC_M7:
329 case SUN4V_CHIP_SPARC64X: 331 case SUN4V_CHIP_SPARC64X:
330 rover_inc_table = niagara_iterate_method; 332 rover_inc_table = niagara_iterate_method;
331 break; 333 break;
diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c
index dff60abbea01..f87a55d77094 100644
--- a/arch/sparc/kernel/ds.c
+++ b/arch/sparc/kernel/ds.c
@@ -1200,14 +1200,14 @@ static int ds_probe(struct vio_dev *vdev, const struct vio_device_id *id)
1200 ds_cfg.tx_irq = vdev->tx_irq; 1200 ds_cfg.tx_irq = vdev->tx_irq;
1201 ds_cfg.rx_irq = vdev->rx_irq; 1201 ds_cfg.rx_irq = vdev->rx_irq;
1202 1202
1203 lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp); 1203 lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp, "DS");
1204 if (IS_ERR(lp)) { 1204 if (IS_ERR(lp)) {
1205 err = PTR_ERR(lp); 1205 err = PTR_ERR(lp);
1206 goto out_free_ds_states; 1206 goto out_free_ds_states;
1207 } 1207 }
1208 dp->lp = lp; 1208 dp->lp = lp;
1209 1209
1210 err = ldc_bind(lp, "DS"); 1210 err = ldc_bind(lp);
1211 if (err) 1211 if (err)
1212 goto out_free_ldc; 1212 goto out_free_ldc;
1213 1213
diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
index 452f04fe8da6..4fdeb8040d4d 100644
--- a/arch/sparc/kernel/head_64.S
+++ b/arch/sparc/kernel/head_64.S
@@ -427,6 +427,12 @@ sun4v_chip_type:
427 cmp %g2, '5' 427 cmp %g2, '5'
428 be,pt %xcc, 5f 428 be,pt %xcc, 5f
429 mov SUN4V_CHIP_NIAGARA5, %g4 429 mov SUN4V_CHIP_NIAGARA5, %g4
430 cmp %g2, '6'
431 be,pt %xcc, 5f
432 mov SUN4V_CHIP_SPARC_M6, %g4
433 cmp %g2, '7'
434 be,pt %xcc, 5f
435 mov SUN4V_CHIP_SPARC_M7, %g4
430 ba,pt %xcc, 49f 436 ba,pt %xcc, 49f
431 nop 437 nop
432 438
@@ -585,6 +591,12 @@ niagara_tlb_fixup:
585 cmp %g1, SUN4V_CHIP_NIAGARA5 591 cmp %g1, SUN4V_CHIP_NIAGARA5
586 be,pt %xcc, niagara4_patch 592 be,pt %xcc, niagara4_patch
587 nop 593 nop
594 cmp %g1, SUN4V_CHIP_SPARC_M6
595 be,pt %xcc, niagara4_patch
596 nop
597 cmp %g1, SUN4V_CHIP_SPARC_M7
598 be,pt %xcc, niagara4_patch
599 nop
588 600
589 call generic_patch_copyops 601 call generic_patch_copyops
590 nop 602 nop
diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c
index c0a2de0fd624..5c55145bfbf0 100644
--- a/arch/sparc/kernel/hvapi.c
+++ b/arch/sparc/kernel/hvapi.c
@@ -46,6 +46,7 @@ static struct api_info api_table[] = {
46 { .group = HV_GRP_VF_CPU, }, 46 { .group = HV_GRP_VF_CPU, },
47 { .group = HV_GRP_KT_CPU, }, 47 { .group = HV_GRP_KT_CPU, },
48 { .group = HV_GRP_VT_CPU, }, 48 { .group = HV_GRP_VT_CPU, },
49 { .group = HV_GRP_T5_CPU, },
49 { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API }, 50 { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API },
50}; 51};
51 52
diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S
index f3ab509b76a8..caedf8320416 100644
--- a/arch/sparc/kernel/hvcalls.S
+++ b/arch/sparc/kernel/hvcalls.S
@@ -821,3 +821,19 @@ ENTRY(sun4v_vt_set_perfreg)
821 retl 821 retl
822 nop 822 nop
823ENDPROC(sun4v_vt_set_perfreg) 823ENDPROC(sun4v_vt_set_perfreg)
824
825ENTRY(sun4v_t5_get_perfreg)
826 mov %o1, %o4
827 mov HV_FAST_T5_GET_PERFREG, %o5
828 ta HV_FAST_TRAP
829 stx %o1, [%o4]
830 retl
831 nop
832ENDPROC(sun4v_t5_get_perfreg)
833
834ENTRY(sun4v_t5_set_perfreg)
835 mov HV_FAST_T5_SET_PERFREG, %o5
836 ta HV_FAST_TRAP
837 retl
838 nop
839ENDPROC(sun4v_t5_set_perfreg)
diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c
index 7f08ec8a7c68..28fed53b13a0 100644
--- a/arch/sparc/kernel/ioport.c
+++ b/arch/sparc/kernel/ioport.c
@@ -278,7 +278,8 @@ static void *sbus_alloc_coherent(struct device *dev, size_t len,
278 } 278 }
279 279
280 order = get_order(len_total); 280 order = get_order(len_total);
281 if ((va = __get_free_pages(GFP_KERNEL|__GFP_COMP, order)) == 0) 281 va = __get_free_pages(gfp, order);
282 if (va == 0)
282 goto err_nopages; 283 goto err_nopages;
283 284
284 if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL) 285 if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL)
@@ -443,7 +444,7 @@ static void *pci32_alloc_coherent(struct device *dev, size_t len,
443 } 444 }
444 445
445 order = get_order(len_total); 446 order = get_order(len_total);
446 va = (void *) __get_free_pages(GFP_KERNEL, order); 447 va = (void *) __get_free_pages(gfp, order);
447 if (va == NULL) { 448 if (va == NULL) {
448 printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT); 449 printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT);
449 goto err_nopages; 450 goto err_nopages;
diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c
index 666193f4e8bb..4033c23bdfa6 100644
--- a/arch/sparc/kernel/irq_64.c
+++ b/arch/sparc/kernel/irq_64.c
@@ -47,8 +47,6 @@
47#include "cpumap.h" 47#include "cpumap.h"
48#include "kstack.h" 48#include "kstack.h"
49 49
50#define NUM_IVECS (IMAP_INR + 1)
51
52struct ino_bucket *ivector_table; 50struct ino_bucket *ivector_table;
53unsigned long ivector_table_pa; 51unsigned long ivector_table_pa;
54 52
@@ -107,55 +105,196 @@ static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq)
107 105
108#define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa) 106#define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa)
109 107
110static struct { 108static unsigned long hvirq_major __initdata;
111 unsigned int dev_handle; 109static int __init early_hvirq_major(char *p)
112 unsigned int dev_ino; 110{
113 unsigned int in_use; 111 int rc = kstrtoul(p, 10, &hvirq_major);
114} irq_table[NR_IRQS]; 112
115static DEFINE_SPINLOCK(irq_alloc_lock); 113 return rc;
114}
115early_param("hvirq", early_hvirq_major);
116
117static int hv_irq_version;
118
119/* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
120 * based interfaces, but:
121 *
122 * 1) Several OSs, Solaris and Linux included, use them even when only
123 * negotiating version 1.0 (or failing to negotiate at all). So the
124 * hypervisor has a workaround that provides the VIRQ interfaces even
125 * when only verion 1.0 of the API is in use.
126 *
127 * 2) Second, and more importantly, with major version 2.0 these VIRQ
128 * interfaces only were actually hooked up for LDC interrupts, even
129 * though the Hypervisor specification clearly stated:
130 *
131 * The new interrupt API functions will be available to a guest
132 * when it negotiates version 2.0 in the interrupt API group 0x2. When
133 * a guest negotiates version 2.0, all interrupt sources will only
134 * support using the cookie interface, and any attempt to use the
135 * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
136 * ENOTSUPPORTED error being returned.
137 *
138 * with an emphasis on "all interrupt sources".
139 *
140 * To correct this, major version 3.0 was created which does actually
141 * support VIRQs for all interrupt sources (not just LDC devices). So
142 * if we want to move completely over the cookie based VIRQs we must
143 * negotiate major version 3.0 or later of HV_GRP_INTR.
144 */
145static bool sun4v_cookie_only_virqs(void)
146{
147 if (hv_irq_version >= 3)
148 return true;
149 return false;
150}
116 151
117unsigned char irq_alloc(unsigned int dev_handle, unsigned int dev_ino) 152static void __init irq_init_hv(void)
118{ 153{
119 unsigned long flags; 154 unsigned long hv_error, major, minor = 0;
120 unsigned char ent; 155
156 if (tlb_type != hypervisor)
157 return;
121 158
122 BUILD_BUG_ON(NR_IRQS >= 256); 159 if (hvirq_major)
160 major = hvirq_major;
161 else
162 major = 3;
123 163
124 spin_lock_irqsave(&irq_alloc_lock, flags); 164 hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor);
165 if (!hv_error)
166 hv_irq_version = major;
167 else
168 hv_irq_version = 1;
125 169
126 for (ent = 1; ent < NR_IRQS; ent++) { 170 pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
127 if (!irq_table[ent].in_use) 171 hv_irq_version,
172 sun4v_cookie_only_virqs() ? "enabled" : "disabled");
173}
174
175/* This function is for the timer interrupt.*/
176int __init arch_probe_nr_irqs(void)
177{
178 return 1;
179}
180
181#define DEFAULT_NUM_IVECS (0xfffU)
182static unsigned int nr_ivec = DEFAULT_NUM_IVECS;
183#define NUM_IVECS (nr_ivec)
184
185static unsigned int __init size_nr_ivec(void)
186{
187 if (tlb_type == hypervisor) {
188 switch (sun4v_chip_type) {
189 /* Athena's devhandle|devino is large.*/
190 case SUN4V_CHIP_SPARC64X:
191 nr_ivec = 0xffff;
128 break; 192 break;
193 }
129 } 194 }
130 if (ent >= NR_IRQS) { 195 return nr_ivec;
131 printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); 196}
132 ent = 0; 197
133 } else { 198struct irq_handler_data {
134 irq_table[ent].dev_handle = dev_handle; 199 union {
135 irq_table[ent].dev_ino = dev_ino; 200 struct {
136 irq_table[ent].in_use = 1; 201 unsigned int dev_handle;
137 } 202 unsigned int dev_ino;
203 };
204 unsigned long sysino;
205 };
206 struct ino_bucket bucket;
207 unsigned long iclr;
208 unsigned long imap;
209};
210
211static inline unsigned int irq_data_to_handle(struct irq_data *data)
212{
213 struct irq_handler_data *ihd = data->handler_data;
214
215 return ihd->dev_handle;
216}
217
218static inline unsigned int irq_data_to_ino(struct irq_data *data)
219{
220 struct irq_handler_data *ihd = data->handler_data;
138 221
139 spin_unlock_irqrestore(&irq_alloc_lock, flags); 222 return ihd->dev_ino;
223}
224
225static inline unsigned long irq_data_to_sysino(struct irq_data *data)
226{
227 struct irq_handler_data *ihd = data->handler_data;
140 228
141 return ent; 229 return ihd->sysino;
142} 230}
143 231
144#ifdef CONFIG_PCI_MSI
145void irq_free(unsigned int irq) 232void irq_free(unsigned int irq)
146{ 233{
147 unsigned long flags; 234 void *data = irq_get_handler_data(irq);
148 235
149 if (irq >= NR_IRQS) 236 kfree(data);
150 return; 237 irq_set_handler_data(irq, NULL);
238 irq_free_descs(irq, 1);
239}
151 240
152 spin_lock_irqsave(&irq_alloc_lock, flags); 241unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino)
242{
243 int irq;
153 244
154 irq_table[irq].in_use = 0; 245 irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL);
246 if (irq <= 0)
247 goto out;
155 248
156 spin_unlock_irqrestore(&irq_alloc_lock, flags); 249 return irq;
250out:
251 return 0;
252}
253
254static unsigned int cookie_exists(u32 devhandle, unsigned int devino)
255{
256 unsigned long hv_err, cookie;
257 struct ino_bucket *bucket;
258 unsigned int irq = 0U;
259
260 hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie);
261 if (hv_err) {
262 pr_err("HV get cookie failed hv_err = %ld\n", hv_err);
263 goto out;
264 }
265
266 if (cookie & ((1UL << 63UL))) {
267 cookie = ~cookie;
268 bucket = (struct ino_bucket *) __va(cookie);
269 irq = bucket->__irq;
270 }
271out:
272 return irq;
273}
274
275static unsigned int sysino_exists(u32 devhandle, unsigned int devino)
276{
277 unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
278 struct ino_bucket *bucket;
279 unsigned int irq;
280
281 bucket = &ivector_table[sysino];
282 irq = bucket_get_irq(__pa(bucket));
283
284 return irq;
285}
286
287void ack_bad_irq(unsigned int irq)
288{
289 pr_crit("BAD IRQ ack %d\n", irq);
290}
291
292void irq_install_pre_handler(int irq,
293 void (*func)(unsigned int, void *, void *),
294 void *arg1, void *arg2)
295{
296 pr_warn("IRQ pre handler NOT supported.\n");
157} 297}
158#endif
159 298
160/* 299/*
161 * /proc/interrupts printing: 300 * /proc/interrupts printing:
@@ -206,15 +345,6 @@ static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
206 return tid; 345 return tid;
207} 346}
208 347
209struct irq_handler_data {
210 unsigned long iclr;
211 unsigned long imap;
212
213 void (*pre_handler)(unsigned int, void *, void *);
214 void *arg1;
215 void *arg2;
216};
217
218#ifdef CONFIG_SMP 348#ifdef CONFIG_SMP
219static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) 349static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity)
220{ 350{
@@ -316,8 +446,8 @@ static void sun4u_irq_eoi(struct irq_data *data)
316 446
317static void sun4v_irq_enable(struct irq_data *data) 447static void sun4v_irq_enable(struct irq_data *data)
318{ 448{
319 unsigned int ino = irq_table[data->irq].dev_ino;
320 unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity); 449 unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity);
450 unsigned int ino = irq_data_to_sysino(data);
321 int err; 451 int err;
322 452
323 err = sun4v_intr_settarget(ino, cpuid); 453 err = sun4v_intr_settarget(ino, cpuid);
@@ -337,8 +467,8 @@ static void sun4v_irq_enable(struct irq_data *data)
337static int sun4v_set_affinity(struct irq_data *data, 467static int sun4v_set_affinity(struct irq_data *data,
338 const struct cpumask *mask, bool force) 468 const struct cpumask *mask, bool force)
339{ 469{
340 unsigned int ino = irq_table[data->irq].dev_ino;
341 unsigned long cpuid = irq_choose_cpu(data->irq, mask); 470 unsigned long cpuid = irq_choose_cpu(data->irq, mask);
471 unsigned int ino = irq_data_to_sysino(data);
342 int err; 472 int err;
343 473
344 err = sun4v_intr_settarget(ino, cpuid); 474 err = sun4v_intr_settarget(ino, cpuid);
@@ -351,7 +481,7 @@ static int sun4v_set_affinity(struct irq_data *data,
351 481
352static void sun4v_irq_disable(struct irq_data *data) 482static void sun4v_irq_disable(struct irq_data *data)
353{ 483{
354 unsigned int ino = irq_table[data->irq].dev_ino; 484 unsigned int ino = irq_data_to_sysino(data);
355 int err; 485 int err;
356 486
357 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); 487 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
@@ -362,7 +492,7 @@ static void sun4v_irq_disable(struct irq_data *data)
362 492
363static void sun4v_irq_eoi(struct irq_data *data) 493static void sun4v_irq_eoi(struct irq_data *data)
364{ 494{
365 unsigned int ino = irq_table[data->irq].dev_ino; 495 unsigned int ino = irq_data_to_sysino(data);
366 int err; 496 int err;
367 497
368 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); 498 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
@@ -373,14 +503,13 @@ static void sun4v_irq_eoi(struct irq_data *data)
373 503
374static void sun4v_virq_enable(struct irq_data *data) 504static void sun4v_virq_enable(struct irq_data *data)
375{ 505{
376 unsigned long cpuid, dev_handle, dev_ino; 506 unsigned long dev_handle = irq_data_to_handle(data);
507 unsigned long dev_ino = irq_data_to_ino(data);
508 unsigned long cpuid;
377 int err; 509 int err;
378 510
379 cpuid = irq_choose_cpu(data->irq, data->affinity); 511 cpuid = irq_choose_cpu(data->irq, data->affinity);
380 512
381 dev_handle = irq_table[data->irq].dev_handle;
382 dev_ino = irq_table[data->irq].dev_ino;
383
384 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); 513 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
385 if (err != HV_EOK) 514 if (err != HV_EOK)
386 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " 515 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
@@ -403,14 +532,13 @@ static void sun4v_virq_enable(struct irq_data *data)
403static int sun4v_virt_set_affinity(struct irq_data *data, 532static int sun4v_virt_set_affinity(struct irq_data *data,
404 const struct cpumask *mask, bool force) 533 const struct cpumask *mask, bool force)
405{ 534{
406 unsigned long cpuid, dev_handle, dev_ino; 535 unsigned long dev_handle = irq_data_to_handle(data);
536 unsigned long dev_ino = irq_data_to_ino(data);
537 unsigned long cpuid;
407 int err; 538 int err;
408 539
409 cpuid = irq_choose_cpu(data->irq, mask); 540 cpuid = irq_choose_cpu(data->irq, mask);
410 541
411 dev_handle = irq_table[data->irq].dev_handle;
412 dev_ino = irq_table[data->irq].dev_ino;
413
414 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); 542 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
415 if (err != HV_EOK) 543 if (err != HV_EOK)
416 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " 544 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
@@ -422,11 +550,10 @@ static int sun4v_virt_set_affinity(struct irq_data *data,
422 550
423static void sun4v_virq_disable(struct irq_data *data) 551static void sun4v_virq_disable(struct irq_data *data)
424{ 552{
425 unsigned long dev_handle, dev_ino; 553 unsigned long dev_handle = irq_data_to_handle(data);
554 unsigned long dev_ino = irq_data_to_ino(data);
426 int err; 555 int err;
427 556
428 dev_handle = irq_table[data->irq].dev_handle;
429 dev_ino = irq_table[data->irq].dev_ino;
430 557
431 err = sun4v_vintr_set_valid(dev_handle, dev_ino, 558 err = sun4v_vintr_set_valid(dev_handle, dev_ino,
432 HV_INTR_DISABLED); 559 HV_INTR_DISABLED);
@@ -438,12 +565,10 @@ static void sun4v_virq_disable(struct irq_data *data)
438 565
439static void sun4v_virq_eoi(struct irq_data *data) 566static void sun4v_virq_eoi(struct irq_data *data)
440{ 567{
441 unsigned long dev_handle, dev_ino; 568 unsigned long dev_handle = irq_data_to_handle(data);
569 unsigned long dev_ino = irq_data_to_ino(data);
442 int err; 570 int err;
443 571
444 dev_handle = irq_table[data->irq].dev_handle;
445 dev_ino = irq_table[data->irq].dev_ino;
446
447 err = sun4v_vintr_set_state(dev_handle, dev_ino, 572 err = sun4v_vintr_set_state(dev_handle, dev_ino,
448 HV_INTR_STATE_IDLE); 573 HV_INTR_STATE_IDLE);
449 if (err != HV_EOK) 574 if (err != HV_EOK)
@@ -479,31 +604,10 @@ static struct irq_chip sun4v_virq = {
479 .flags = IRQCHIP_EOI_IF_HANDLED, 604 .flags = IRQCHIP_EOI_IF_HANDLED,
480}; 605};
481 606
482static void pre_flow_handler(struct irq_data *d)
483{
484 struct irq_handler_data *handler_data = irq_data_get_irq_handler_data(d);
485 unsigned int ino = irq_table[d->irq].dev_ino;
486
487 handler_data->pre_handler(ino, handler_data->arg1, handler_data->arg2);
488}
489
490void irq_install_pre_handler(int irq,
491 void (*func)(unsigned int, void *, void *),
492 void *arg1, void *arg2)
493{
494 struct irq_handler_data *handler_data = irq_get_handler_data(irq);
495
496 handler_data->pre_handler = func;
497 handler_data->arg1 = arg1;
498 handler_data->arg2 = arg2;
499
500 __irq_set_preflow_handler(irq, pre_flow_handler);
501}
502
503unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) 607unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
504{ 608{
505 struct ino_bucket *bucket;
506 struct irq_handler_data *handler_data; 609 struct irq_handler_data *handler_data;
610 struct ino_bucket *bucket;
507 unsigned int irq; 611 unsigned int irq;
508 int ino; 612 int ino;
509 613
@@ -537,119 +641,166 @@ out:
537 return irq; 641 return irq;
538} 642}
539 643
540static unsigned int sun4v_build_common(unsigned long sysino, 644static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino,
541 struct irq_chip *chip) 645 void (*handler_data_init)(struct irq_handler_data *data,
646 u32 devhandle, unsigned int devino),
647 struct irq_chip *chip)
542{ 648{
543 struct ino_bucket *bucket; 649 struct irq_handler_data *data;
544 struct irq_handler_data *handler_data;
545 unsigned int irq; 650 unsigned int irq;
546 651
547 BUG_ON(tlb_type != hypervisor); 652 irq = irq_alloc(devhandle, devino);
653 if (!irq)
654 goto out;
548 655
549 bucket = &ivector_table[sysino]; 656 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
550 irq = bucket_get_irq(__pa(bucket)); 657 if (unlikely(!data)) {
551 if (!irq) { 658 pr_err("IRQ handler data allocation failed.\n");
552 irq = irq_alloc(0, sysino); 659 irq_free(irq);
553 bucket_set_irq(__pa(bucket), irq); 660 irq = 0;
554 irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, 661 goto out;
555 "IVEC");
556 } 662 }
557 663
558 handler_data = irq_get_handler_data(irq); 664 irq_set_handler_data(irq, data);
559 if (unlikely(handler_data)) 665 handler_data_init(data, devhandle, devino);
560 goto out; 666 irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC");
667 data->imap = ~0UL;
668 data->iclr = ~0UL;
669out:
670 return irq;
671}
561 672
562 handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 673static unsigned long cookie_assign(unsigned int irq, u32 devhandle,
563 if (unlikely(!handler_data)) { 674 unsigned int devino)
564 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); 675{
565 prom_halt(); 676 struct irq_handler_data *ihd = irq_get_handler_data(irq);
566 } 677 unsigned long hv_error, cookie;
567 irq_set_handler_data(irq, handler_data);
568 678
569 /* Catch accidental accesses to these things. IMAP/ICLR handling 679 /* handler_irq needs to find the irq. cookie is seen signed in
570 * is done by hypervisor calls on sun4v platforms, not by direct 680 * sun4v_dev_mondo and treated as a non ivector_table delivery.
571 * register accesses.
572 */ 681 */
573 handler_data->imap = ~0UL; 682 ihd->bucket.__irq = irq;
574 handler_data->iclr = ~0UL; 683 cookie = ~__pa(&ihd->bucket);
575 684
576out: 685 hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie);
577 return irq; 686 if (hv_error)
687 pr_err("HV vintr set cookie failed = %ld\n", hv_error);
688
689 return hv_error;
578} 690}
579 691
580unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) 692static void cookie_handler_data(struct irq_handler_data *data,
693 u32 devhandle, unsigned int devino)
581{ 694{
582 unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); 695 data->dev_handle = devhandle;
696 data->dev_ino = devino;
697}
583 698
584 return sun4v_build_common(sysino, &sun4v_irq); 699static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino,
700 struct irq_chip *chip)
701{
702 unsigned long hv_error;
703 unsigned int irq;
704
705 irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip);
706
707 hv_error = cookie_assign(irq, devhandle, devino);
708 if (hv_error) {
709 irq_free(irq);
710 irq = 0;
711 }
712
713 return irq;
585} 714}
586 715
587unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) 716static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino)
588{ 717{
589 struct irq_handler_data *handler_data;
590 unsigned long hv_err, cookie;
591 struct ino_bucket *bucket;
592 unsigned int irq; 718 unsigned int irq;
593 719
594 bucket = kzalloc(sizeof(struct ino_bucket), GFP_ATOMIC); 720 irq = cookie_exists(devhandle, devino);
595 if (unlikely(!bucket)) 721 if (irq)
596 return 0; 722 goto out;
597 723
598 /* The only reference we store to the IRQ bucket is 724 irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
599 * by physical address which kmemleak can't see, tell
600 * it that this object explicitly is not a leak and
601 * should be scanned.
602 */
603 kmemleak_not_leak(bucket);
604 725
605 __flush_dcache_range((unsigned long) bucket, 726out:
606 ((unsigned long) bucket + 727 return irq;
607 sizeof(struct ino_bucket))); 728}
608 729
609 irq = irq_alloc(devhandle, devino); 730static void sysino_set_bucket(unsigned int irq)
731{
732 struct irq_handler_data *ihd = irq_get_handler_data(irq);
733 struct ino_bucket *bucket;
734 unsigned long sysino;
735
736 sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino);
737 BUG_ON(sysino >= nr_ivec);
738 bucket = &ivector_table[sysino];
610 bucket_set_irq(__pa(bucket), irq); 739 bucket_set_irq(__pa(bucket), irq);
740}
611 741
612 irq_set_chip_and_handler_name(irq, &sun4v_virq, handle_fasteoi_irq, 742static void sysino_handler_data(struct irq_handler_data *data,
613 "IVEC"); 743 u32 devhandle, unsigned int devino)
744{
745 unsigned long sysino;
614 746
615 handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 747 sysino = sun4v_devino_to_sysino(devhandle, devino);
616 if (unlikely(!handler_data)) 748 data->sysino = sysino;
617 return 0; 749}
618 750
619 /* In order to make the LDC channel startup sequence easier, 751static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino,
620 * especially wrt. locking, we do not let request_irq() enable 752 struct irq_chip *chip)
621 * the interrupt. 753{
622 */ 754 unsigned int irq;
623 irq_set_status_flags(irq, IRQ_NOAUTOEN);
624 irq_set_handler_data(irq, handler_data);
625 755
626 /* Catch accidental accesses to these things. IMAP/ICLR handling 756 irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip);
627 * is done by hypervisor calls on sun4v platforms, not by direct 757 if (!irq)
628 * register accesses. 758 goto out;
629 */
630 handler_data->imap = ~0UL;
631 handler_data->iclr = ~0UL;
632 759
633 cookie = ~__pa(bucket); 760 sysino_set_bucket(irq);
634 hv_err = sun4v_vintr_set_cookie(devhandle, devino, cookie); 761out:
635 if (hv_err) { 762 return irq;
636 prom_printf("IRQ: Fatal, cannot set cookie for [%x:%x] " 763}
637 "err=%lu\n", devhandle, devino, hv_err);
638 prom_halt();
639 }
640 764
765static int sun4v_build_sysino(u32 devhandle, unsigned int devino)
766{
767 int irq;
768
769 irq = sysino_exists(devhandle, devino);
770 if (irq)
771 goto out;
772
773 irq = sysino_build_irq(devhandle, devino, &sun4v_irq);
774out:
641 return irq; 775 return irq;
642} 776}
643 777
644void ack_bad_irq(unsigned int irq) 778unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
645{ 779{
646 unsigned int ino = irq_table[irq].dev_ino; 780 unsigned int irq;
647 781
648 if (!ino) 782 if (sun4v_cookie_only_virqs())
649 ino = 0xdeadbeef; 783 irq = sun4v_build_cookie(devhandle, devino);
784 else
785 irq = sun4v_build_sysino(devhandle, devino);
650 786
651 printk(KERN_CRIT "Unexpected IRQ from ino[%x] irq[%u]\n", 787 return irq;
652 ino, irq); 788}
789
790unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
791{
792 int irq;
793
794 irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
795 if (!irq)
796 goto out;
797
798 /* This is borrowed from the original function.
799 */
800 irq_set_status_flags(irq, IRQ_NOAUTOEN);
801
802out:
803 return irq;
653} 804}
654 805
655void *hardirq_stack[NR_CPUS]; 806void *hardirq_stack[NR_CPUS];
@@ -720,9 +871,12 @@ void fixup_irqs(void)
720 871
721 for (irq = 0; irq < NR_IRQS; irq++) { 872 for (irq = 0; irq < NR_IRQS; irq++) {
722 struct irq_desc *desc = irq_to_desc(irq); 873 struct irq_desc *desc = irq_to_desc(irq);
723 struct irq_data *data = irq_desc_get_irq_data(desc); 874 struct irq_data *data;
724 unsigned long flags; 875 unsigned long flags;
725 876
877 if (!desc)
878 continue;
879 data = irq_desc_get_irq_data(desc);
726 raw_spin_lock_irqsave(&desc->lock, flags); 880 raw_spin_lock_irqsave(&desc->lock, flags);
727 if (desc->action && !irqd_is_per_cpu(data)) { 881 if (desc->action && !irqd_is_per_cpu(data)) {
728 if (data->chip->irq_set_affinity) 882 if (data->chip->irq_set_affinity)
@@ -922,16 +1076,22 @@ static struct irqaction timer_irq_action = {
922 .name = "timer", 1076 .name = "timer",
923}; 1077};
924 1078
925/* Only invoked on boot processor. */ 1079static void __init irq_ivector_init(void)
926void __init init_IRQ(void)
927{ 1080{
928 unsigned long size; 1081 unsigned long size, order;
1082 unsigned int ivecs;
929 1083
930 map_prom_timers(); 1084 /* If we are doing cookie only VIRQs then we do not need the ivector
931 kill_prom_timer(); 1085 * table to process interrupts.
1086 */
1087 if (sun4v_cookie_only_virqs())
1088 return;
932 1089
933 size = sizeof(struct ino_bucket) * NUM_IVECS; 1090 ivecs = size_nr_ivec();
934 ivector_table = kzalloc(size, GFP_KERNEL); 1091 size = sizeof(struct ino_bucket) * ivecs;
1092 order = get_order(size);
1093 ivector_table = (struct ino_bucket *)
1094 __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
935 if (!ivector_table) { 1095 if (!ivector_table) {
936 prom_printf("Fatal error, cannot allocate ivector_table\n"); 1096 prom_printf("Fatal error, cannot allocate ivector_table\n");
937 prom_halt(); 1097 prom_halt();
@@ -940,6 +1100,15 @@ void __init init_IRQ(void)
940 ((unsigned long) ivector_table) + size); 1100 ((unsigned long) ivector_table) + size);
941 1101
942 ivector_table_pa = __pa(ivector_table); 1102 ivector_table_pa = __pa(ivector_table);
1103}
1104
1105/* Only invoked on boot processor.*/
1106void __init init_IRQ(void)
1107{
1108 irq_init_hv();
1109 irq_ivector_init();
1110 map_prom_timers();
1111 kill_prom_timer();
943 1112
944 if (tlb_type == hypervisor) 1113 if (tlb_type == hypervisor)
945 sun4v_init_mondo_queues(); 1114 sun4v_init_mondo_queues();
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
index 605d49204580..ef0d8e9e1210 100644
--- a/arch/sparc/kernel/ktlb.S
+++ b/arch/sparc/kernel/ktlb.S
@@ -47,14 +47,6 @@ kvmap_itlb_vmalloc_addr:
47 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) 47 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
48 48
49 TSB_LOCK_TAG(%g1, %g2, %g7) 49 TSB_LOCK_TAG(%g1, %g2, %g7)
50
51 /* Load and check PTE. */
52 ldxa [%g5] ASI_PHYS_USE_EC, %g5
53 mov 1, %g7
54 sllx %g7, TSB_TAG_INVALID_BIT, %g7
55 brgez,a,pn %g5, kvmap_itlb_longpath
56 TSB_STORE(%g1, %g7)
57
58 TSB_WRITE(%g1, %g5, %g6) 50 TSB_WRITE(%g1, %g5, %g6)
59 51
60 /* fallthrough to TLB load */ 52 /* fallthrough to TLB load */
@@ -118,6 +110,12 @@ kvmap_dtlb_obp:
118 ba,pt %xcc, kvmap_dtlb_load 110 ba,pt %xcc, kvmap_dtlb_load
119 nop 111 nop
120 112
113kvmap_linear_early:
114 sethi %hi(kern_linear_pte_xor), %g7
115 ldx [%g7 + %lo(kern_linear_pte_xor)], %g2
116 ba,pt %xcc, kvmap_dtlb_tsb4m_load
117 xor %g2, %g4, %g5
118
121 .align 32 119 .align 32
122kvmap_dtlb_tsb4m_load: 120kvmap_dtlb_tsb4m_load:
123 TSB_LOCK_TAG(%g1, %g2, %g7) 121 TSB_LOCK_TAG(%g1, %g2, %g7)
@@ -146,105 +144,17 @@ kvmap_dtlb_4v:
146 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ 144 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
147 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) 145 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
148#endif 146#endif
149 /* TSB entry address left in %g1, lookup linear PTE. 147 /* Linear mapping TSB lookup failed. Fallthrough to kernel
150 * Must preserve %g1 and %g6 (TAG). 148 * page table based lookup.
151 */
152kvmap_dtlb_tsb4m_miss:
153 /* Clear the PAGE_OFFSET top virtual bits, shift
154 * down to get PFN, and make sure PFN is in range.
155 */
156661: sllx %g4, 0, %g5
157 .section .page_offset_shift_patch, "ax"
158 .word 661b
159 .previous
160
161 /* Check to see if we know about valid memory at the 4MB
162 * chunk this physical address will reside within.
163 */ 149 */
164661: srlx %g5, MAX_PHYS_ADDRESS_BITS, %g2
165 .section .page_offset_shift_patch, "ax"
166 .word 661b
167 .previous
168
169 brnz,pn %g2, kvmap_dtlb_longpath
170 nop
171
172 /* This unconditional branch and delay-slot nop gets patched
173 * by the sethi sequence once the bitmap is properly setup.
174 */
175 .globl valid_addr_bitmap_insn
176valid_addr_bitmap_insn:
177 ba,pt %xcc, 2f
178 nop
179 .subsection 2
180 .globl valid_addr_bitmap_patch
181valid_addr_bitmap_patch:
182 sethi %hi(sparc64_valid_addr_bitmap), %g7
183 or %g7, %lo(sparc64_valid_addr_bitmap), %g7
184 .previous
185
186661: srlx %g5, ILOG2_4MB, %g2
187 .section .page_offset_shift_patch, "ax"
188 .word 661b
189 .previous
190
191 srlx %g2, 6, %g5
192 and %g2, 63, %g2
193 sllx %g5, 3, %g5
194 ldx [%g7 + %g5], %g5
195 mov 1, %g7
196 sllx %g7, %g2, %g7
197 andcc %g5, %g7, %g0
198 be,pn %xcc, kvmap_dtlb_longpath
199
2002: sethi %hi(kpte_linear_bitmap), %g2
201
202 /* Get the 256MB physical address index. */
203661: sllx %g4, 0, %g5
204 .section .page_offset_shift_patch, "ax"
205 .word 661b
206 .previous
207
208 or %g2, %lo(kpte_linear_bitmap), %g2
209
210661: srlx %g5, ILOG2_256MB, %g5
211 .section .page_offset_shift_patch, "ax"
212 .word 661b
213 .previous
214
215 and %g5, (32 - 1), %g7
216
217 /* Divide by 32 to get the offset into the bitmask. */
218 srlx %g5, 5, %g5
219 add %g7, %g7, %g7
220 sllx %g5, 3, %g5
221
222 /* kern_linear_pte_xor[(mask >> shift) & 3)] */
223 ldx [%g2 + %g5], %g2
224 srlx %g2, %g7, %g7
225 sethi %hi(kern_linear_pte_xor), %g5
226 and %g7, 3, %g7
227 or %g5, %lo(kern_linear_pte_xor), %g5
228 sllx %g7, 3, %g7
229 ldx [%g5 + %g7], %g2
230
231 .globl kvmap_linear_patch 150 .globl kvmap_linear_patch
232kvmap_linear_patch: 151kvmap_linear_patch:
233 ba,pt %xcc, kvmap_dtlb_tsb4m_load 152 ba,a,pt %xcc, kvmap_linear_early
234 xor %g2, %g4, %g5
235 153
236kvmap_dtlb_vmalloc_addr: 154kvmap_dtlb_vmalloc_addr:
237 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) 155 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
238 156
239 TSB_LOCK_TAG(%g1, %g2, %g7) 157 TSB_LOCK_TAG(%g1, %g2, %g7)
240
241 /* Load and check PTE. */
242 ldxa [%g5] ASI_PHYS_USE_EC, %g5
243 mov 1, %g7
244 sllx %g7, TSB_TAG_INVALID_BIT, %g7
245 brgez,a,pn %g5, kvmap_dtlb_longpath
246 TSB_STORE(%g1, %g7)
247
248 TSB_WRITE(%g1, %g5, %g6) 158 TSB_WRITE(%g1, %g5, %g6)
249 159
250 /* fallthrough to TLB load */ 160 /* fallthrough to TLB load */
@@ -276,13 +186,8 @@ kvmap_dtlb_load:
276 186
277#ifdef CONFIG_SPARSEMEM_VMEMMAP 187#ifdef CONFIG_SPARSEMEM_VMEMMAP
278kvmap_vmemmap: 188kvmap_vmemmap:
279 sub %g4, %g5, %g5 189 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
280 srlx %g5, ILOG2_4MB, %g5 190 ba,a,pt %xcc, kvmap_dtlb_load
281 sethi %hi(vmemmap_table), %g1
282 sllx %g5, 3, %g5
283 or %g1, %lo(vmemmap_table), %g1
284 ba,pt %xcc, kvmap_dtlb_load
285 ldx [%g1 + %g5], %g5
286#endif 191#endif
287 192
288kvmap_dtlb_nonlinear: 193kvmap_dtlb_nonlinear:
@@ -294,8 +199,8 @@ kvmap_dtlb_nonlinear:
294 199
295#ifdef CONFIG_SPARSEMEM_VMEMMAP 200#ifdef CONFIG_SPARSEMEM_VMEMMAP
296 /* Do not use the TSB for vmemmap. */ 201 /* Do not use the TSB for vmemmap. */
297 mov (VMEMMAP_BASE >> 40), %g5 202 sethi %hi(VMEMMAP_BASE), %g5
298 sllx %g5, 40, %g5 203 ldx [%g5 + %lo(VMEMMAP_BASE)], %g5
299 cmp %g4,%g5 204 cmp %g4,%g5
300 bgeu,pn %xcc, kvmap_vmemmap 205 bgeu,pn %xcc, kvmap_vmemmap
301 nop 206 nop
@@ -307,8 +212,8 @@ kvmap_dtlb_tsbmiss:
307 sethi %hi(MODULES_VADDR), %g5 212 sethi %hi(MODULES_VADDR), %g5
308 cmp %g4, %g5 213 cmp %g4, %g5
309 blu,pn %xcc, kvmap_dtlb_longpath 214 blu,pn %xcc, kvmap_dtlb_longpath
310 mov (VMALLOC_END >> 40), %g5 215 sethi %hi(VMALLOC_END), %g5
311 sllx %g5, 40, %g5 216 ldx [%g5 + %lo(VMALLOC_END)], %g5
312 cmp %g4, %g5 217 cmp %g4, %g5
313 bgeu,pn %xcc, kvmap_dtlb_longpath 218 bgeu,pn %xcc, kvmap_dtlb_longpath
314 nop 219 nop
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
index 0af28b984695..4310332872d4 100644
--- a/arch/sparc/kernel/ldc.c
+++ b/arch/sparc/kernel/ldc.c
@@ -1078,7 +1078,8 @@ static void ldc_iommu_release(struct ldc_channel *lp)
1078 1078
1079struct ldc_channel *ldc_alloc(unsigned long id, 1079struct ldc_channel *ldc_alloc(unsigned long id,
1080 const struct ldc_channel_config *cfgp, 1080 const struct ldc_channel_config *cfgp,
1081 void *event_arg) 1081 void *event_arg,
1082 const char *name)
1082{ 1083{
1083 struct ldc_channel *lp; 1084 struct ldc_channel *lp;
1084 const struct ldc_mode_ops *mops; 1085 const struct ldc_mode_ops *mops;
@@ -1093,6 +1094,8 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1093 err = -EINVAL; 1094 err = -EINVAL;
1094 if (!cfgp) 1095 if (!cfgp)
1095 goto out_err; 1096 goto out_err;
1097 if (!name)
1098 goto out_err;
1096 1099
1097 switch (cfgp->mode) { 1100 switch (cfgp->mode) {
1098 case LDC_MODE_RAW: 1101 case LDC_MODE_RAW:
@@ -1185,6 +1188,21 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1185 1188
1186 INIT_HLIST_HEAD(&lp->mh_list); 1189 INIT_HLIST_HEAD(&lp->mh_list);
1187 1190
1191 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1192 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1193
1194 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1195 lp->rx_irq_name, lp);
1196 if (err)
1197 goto out_free_txq;
1198
1199 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1200 lp->tx_irq_name, lp);
1201 if (err) {
1202 free_irq(lp->cfg.rx_irq, lp);
1203 goto out_free_txq;
1204 }
1205
1188 return lp; 1206 return lp;
1189 1207
1190out_free_txq: 1208out_free_txq:
@@ -1237,31 +1255,14 @@ EXPORT_SYMBOL(ldc_free);
1237 * state. This does not initiate a handshake, ldc_connect() does 1255 * state. This does not initiate a handshake, ldc_connect() does
1238 * that. 1256 * that.
1239 */ 1257 */
1240int ldc_bind(struct ldc_channel *lp, const char *name) 1258int ldc_bind(struct ldc_channel *lp)
1241{ 1259{
1242 unsigned long hv_err, flags; 1260 unsigned long hv_err, flags;
1243 int err = -EINVAL; 1261 int err = -EINVAL;
1244 1262
1245 if (!name || 1263 if (lp->state != LDC_STATE_INIT)
1246 (lp->state != LDC_STATE_INIT))
1247 return -EINVAL; 1264 return -EINVAL;
1248 1265
1249 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1250 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1251
1252 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1253 lp->rx_irq_name, lp);
1254 if (err)
1255 return err;
1256
1257 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1258 lp->tx_irq_name, lp);
1259 if (err) {
1260 free_irq(lp->cfg.rx_irq, lp);
1261 return err;
1262 }
1263
1264
1265 spin_lock_irqsave(&lp->lock, flags); 1266 spin_lock_irqsave(&lp->lock, flags);
1266 1267
1267 enable_irq(lp->cfg.rx_irq); 1268 enable_irq(lp->cfg.rx_irq);
diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c
index 683c4af999de..9bbb8f2bbfcc 100644
--- a/arch/sparc/kernel/leon_kernel.c
+++ b/arch/sparc/kernel/leon_kernel.c
@@ -37,6 +37,7 @@ unsigned long amba_system_id;
37static DEFINE_SPINLOCK(leon_irq_lock); 37static DEFINE_SPINLOCK(leon_irq_lock);
38 38
39static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ 39static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */
40static unsigned long leon3_gptimer_ackmask; /* For clearing pending bit */
40unsigned long leon3_gptimer_irq; /* interrupt controller irq number */ 41unsigned long leon3_gptimer_irq; /* interrupt controller irq number */
41unsigned int sparc_leon_eirq; 42unsigned int sparc_leon_eirq;
42#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu]) 43#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu])
@@ -260,11 +261,19 @@ void leon_update_virq_handling(unsigned int virq,
260 261
261static u32 leon_cycles_offset(void) 262static u32 leon_cycles_offset(void)
262{ 263{
263 u32 rld, val, off; 264 u32 rld, val, ctrl, off;
265
264 rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld); 266 rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld);
265 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); 267 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val);
266 off = rld - val; 268 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
267 return rld - val; 269 if (LEON3_GPTIMER_CTRL_ISPENDING(ctrl)) {
270 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val);
271 off = 2 * rld - val;
272 } else {
273 off = rld - val;
274 }
275
276 return off;
268} 277}
269 278
270#ifdef CONFIG_SMP 279#ifdef CONFIG_SMP
@@ -302,6 +311,7 @@ void __init leon_init_timers(void)
302 int ampopts; 311 int ampopts;
303 int err; 312 int err;
304 u32 config; 313 u32 config;
314 u32 ctrl;
305 315
306 sparc_config.get_cycles_offset = leon_cycles_offset; 316 sparc_config.get_cycles_offset = leon_cycles_offset;
307 sparc_config.cs_period = 1000000 / HZ; 317 sparc_config.cs_period = 1000000 / HZ;
@@ -374,6 +384,16 @@ void __init leon_init_timers(void)
374 if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq)) 384 if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq))
375 goto bad; 385 goto bad;
376 386
387 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
388 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl,
389 ctrl | LEON3_GPTIMER_CTRL_PENDING);
390 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
391
392 if ((ctrl & LEON3_GPTIMER_CTRL_PENDING) != 0)
393 leon3_gptimer_ackmask = ~LEON3_GPTIMER_CTRL_PENDING;
394 else
395 leon3_gptimer_ackmask = ~0;
396
377 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); 397 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0);
378 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld, 398 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld,
379 (((1000000 / HZ) - 1))); 399 (((1000000 / HZ) - 1)));
@@ -452,6 +472,11 @@ bad:
452 472
453static void leon_clear_clock_irq(void) 473static void leon_clear_clock_irq(void)
454{ 474{
475 u32 ctrl;
476
477 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
478 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl,
479 ctrl & leon3_gptimer_ackmask);
455} 480}
456 481
457static void leon_load_profile_irq(int cpu, unsigned int limit) 482static void leon_load_profile_irq(int cpu, unsigned int limit)
diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c
index 269af58497aa..7e967c8018c8 100644
--- a/arch/sparc/kernel/pcr.c
+++ b/arch/sparc/kernel/pcr.c
@@ -191,12 +191,41 @@ static const struct pcr_ops n4_pcr_ops = {
191 .pcr_nmi_disable = PCR_N4_PICNPT, 191 .pcr_nmi_disable = PCR_N4_PICNPT,
192}; 192};
193 193
194static u64 n5_pcr_read(unsigned long reg_num)
195{
196 unsigned long val;
197
198 (void) sun4v_t5_get_perfreg(reg_num, &val);
199
200 return val;
201}
202
203static void n5_pcr_write(unsigned long reg_num, u64 val)
204{
205 (void) sun4v_t5_set_perfreg(reg_num, val);
206}
207
208static const struct pcr_ops n5_pcr_ops = {
209 .read_pcr = n5_pcr_read,
210 .write_pcr = n5_pcr_write,
211 .read_pic = n4_pic_read,
212 .write_pic = n4_pic_write,
213 .nmi_picl_value = n4_picl_value,
214 .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE |
215 PCR_N4_UTRACE | PCR_N4_TOE |
216 (26 << PCR_N4_SL_SHIFT)),
217 .pcr_nmi_disable = PCR_N4_PICNPT,
218};
219
220
194static unsigned long perf_hsvc_group; 221static unsigned long perf_hsvc_group;
195static unsigned long perf_hsvc_major; 222static unsigned long perf_hsvc_major;
196static unsigned long perf_hsvc_minor; 223static unsigned long perf_hsvc_minor;
197 224
198static int __init register_perf_hsvc(void) 225static int __init register_perf_hsvc(void)
199{ 226{
227 unsigned long hverror;
228
200 if (tlb_type == hypervisor) { 229 if (tlb_type == hypervisor) {
201 switch (sun4v_chip_type) { 230 switch (sun4v_chip_type) {
202 case SUN4V_CHIP_NIAGARA1: 231 case SUN4V_CHIP_NIAGARA1:
@@ -215,6 +244,10 @@ static int __init register_perf_hsvc(void)
215 perf_hsvc_group = HV_GRP_VT_CPU; 244 perf_hsvc_group = HV_GRP_VT_CPU;
216 break; 245 break;
217 246
247 case SUN4V_CHIP_NIAGARA5:
248 perf_hsvc_group = HV_GRP_T5_CPU;
249 break;
250
218 default: 251 default:
219 return -ENODEV; 252 return -ENODEV;
220 } 253 }
@@ -222,10 +255,12 @@ static int __init register_perf_hsvc(void)
222 255
223 perf_hsvc_major = 1; 256 perf_hsvc_major = 1;
224 perf_hsvc_minor = 0; 257 perf_hsvc_minor = 0;
225 if (sun4v_hvapi_register(perf_hsvc_group, 258 hverror = sun4v_hvapi_register(perf_hsvc_group,
226 perf_hsvc_major, 259 perf_hsvc_major,
227 &perf_hsvc_minor)) { 260 &perf_hsvc_minor);
228 printk("perfmon: Could not register hvapi.\n"); 261 if (hverror) {
262 pr_err("perfmon: Could not register hvapi(0x%lx).\n",
263 hverror);
229 return -ENODEV; 264 return -ENODEV;
230 } 265 }
231 } 266 }
@@ -254,6 +289,10 @@ static int __init setup_sun4v_pcr_ops(void)
254 pcr_ops = &n4_pcr_ops; 289 pcr_ops = &n4_pcr_ops;
255 break; 290 break;
256 291
292 case SUN4V_CHIP_NIAGARA5:
293 pcr_ops = &n5_pcr_ops;
294 break;
295
257 default: 296 default:
258 ret = -ENODEV; 297 ret = -ENODEV;
259 break; 298 break;
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c
index d35c490a91cb..c9759ad3f34a 100644
--- a/arch/sparc/kernel/perf_event.c
+++ b/arch/sparc/kernel/perf_event.c
@@ -1662,7 +1662,8 @@ static bool __init supported_pmu(void)
1662 sparc_pmu = &niagara2_pmu; 1662 sparc_pmu = &niagara2_pmu;
1663 return true; 1663 return true;
1664 } 1664 }
1665 if (!strcmp(sparc_pmu_type, "niagara4")) { 1665 if (!strcmp(sparc_pmu_type, "niagara4") ||
1666 !strcmp(sparc_pmu_type, "niagara5")) {
1666 sparc_pmu = &niagara4_pmu; 1667 sparc_pmu = &niagara4_pmu;
1667 return true; 1668 return true;
1668 } 1669 }
diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c
index 3fdb455e3318..e629b8377587 100644
--- a/arch/sparc/kernel/setup_64.c
+++ b/arch/sparc/kernel/setup_64.c
@@ -141,21 +141,9 @@ static void __init boot_flags_init(char *commands)
141 process_switch(*commands++); 141 process_switch(*commands++);
142 continue; 142 continue;
143 } 143 }
144 if (!strncmp(commands, "mem=", 4)) { 144 if (!strncmp(commands, "mem=", 4))
145 /* 145 cmdline_memory_size = memparse(commands + 4, &commands);
146 * "mem=XXX[kKmM]" overrides the PROM-reported 146
147 * memory size.
148 */
149 cmdline_memory_size = simple_strtoul(commands + 4,
150 &commands, 0);
151 if (*commands == 'K' || *commands == 'k') {
152 cmdline_memory_size <<= 10;
153 commands++;
154 } else if (*commands=='M' || *commands=='m') {
155 cmdline_memory_size <<= 20;
156 commands++;
157 }
158 }
159 while (*commands && *commands != ' ') 147 while (*commands && *commands != ' ')
160 commands++; 148 commands++;
161 } 149 }
@@ -500,12 +488,16 @@ static void __init init_sparc64_elf_hwcap(void)
500 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 488 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
501 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 489 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
502 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 490 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
491 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
492 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
503 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 493 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
504 cap |= HWCAP_SPARC_BLKINIT; 494 cap |= HWCAP_SPARC_BLKINIT;
505 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || 495 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
506 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 496 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
507 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 497 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
508 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 498 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
499 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
500 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
509 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 501 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
510 cap |= HWCAP_SPARC_N2; 502 cap |= HWCAP_SPARC_N2;
511 } 503 }
@@ -533,6 +525,8 @@ static void __init init_sparc64_elf_hwcap(void)
533 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 525 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
534 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 526 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
535 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 527 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
528 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
529 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
536 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 530 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
537 cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 | 531 cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 |
538 AV_SPARC_ASI_BLK_INIT | 532 AV_SPARC_ASI_BLK_INIT |
@@ -540,6 +534,8 @@ static void __init init_sparc64_elf_hwcap(void)
540 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 534 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
541 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 535 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
542 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 536 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
537 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
538 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
543 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 539 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
544 cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC | 540 cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC |
545 AV_SPARC_FMAF); 541 AV_SPARC_FMAF);
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index f7ba87543e5f..c9300bfaee5a 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1467,6 +1467,13 @@ static void __init pcpu_populate_pte(unsigned long addr)
1467 pud_t *pud; 1467 pud_t *pud;
1468 pmd_t *pmd; 1468 pmd_t *pmd;
1469 1469
1470 if (pgd_none(*pgd)) {
1471 pud_t *new;
1472
1473 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1474 pgd_populate(&init_mm, pgd, new);
1475 }
1476
1470 pud = pud_offset(pgd, addr); 1477 pud = pud_offset(pgd, addr);
1471 if (pud_none(*pud)) { 1478 if (pud_none(*pud)) {
1472 pmd_t *new; 1479 pmd_t *new;
diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S
index e0c09bf85610..6179e19bc9b9 100644
--- a/arch/sparc/kernel/sun4v_tlb_miss.S
+++ b/arch/sparc/kernel/sun4v_tlb_miss.S
@@ -195,6 +195,11 @@ sun4v_tsb_miss_common:
195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
196 196
197sun4v_itlb_error: 197sun4v_itlb_error:
198 rdpr %tl, %g1
199 cmp %g1, 1
200 ble,pt %icc, sun4v_bad_ra
201 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
202
198 sethi %hi(sun4v_err_itlb_vaddr), %g1 203 sethi %hi(sun4v_err_itlb_vaddr), %g1
199 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] 204 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
200 sethi %hi(sun4v_err_itlb_ctx), %g1 205 sethi %hi(sun4v_err_itlb_ctx), %g1
@@ -206,15 +211,10 @@ sun4v_itlb_error:
206 sethi %hi(sun4v_err_itlb_error), %g1 211 sethi %hi(sun4v_err_itlb_error), %g1
207 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] 212 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)]
208 213
214 sethi %hi(1f), %g7
209 rdpr %tl, %g4 215 rdpr %tl, %g4
210 cmp %g4, 1
211 ble,pt %icc, 1f
212 sethi %hi(2f), %g7
213 ba,pt %xcc, etraptl1 216 ba,pt %xcc, etraptl1
214 or %g7, %lo(2f), %g7 2171: or %g7, %lo(1f), %g7
215
2161: ba,pt %xcc, etrap
2172: or %g7, %lo(2b), %g7
218 mov %l4, %o1 218 mov %l4, %o1
219 call sun4v_itlb_error_report 219 call sun4v_itlb_error_report
220 add %sp, PTREGS_OFF, %o0 220 add %sp, PTREGS_OFF, %o0
@@ -222,6 +222,11 @@ sun4v_itlb_error:
222 /* NOTREACHED */ 222 /* NOTREACHED */
223 223
224sun4v_dtlb_error: 224sun4v_dtlb_error:
225 rdpr %tl, %g1
226 cmp %g1, 1
227 ble,pt %icc, sun4v_bad_ra
228 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
229
225 sethi %hi(sun4v_err_dtlb_vaddr), %g1 230 sethi %hi(sun4v_err_dtlb_vaddr), %g1
226 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] 231 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
227 sethi %hi(sun4v_err_dtlb_ctx), %g1 232 sethi %hi(sun4v_err_dtlb_ctx), %g1
@@ -233,21 +238,23 @@ sun4v_dtlb_error:
233 sethi %hi(sun4v_err_dtlb_error), %g1 238 sethi %hi(sun4v_err_dtlb_error), %g1
234 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] 239 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
235 240
241 sethi %hi(1f), %g7
236 rdpr %tl, %g4 242 rdpr %tl, %g4
237 cmp %g4, 1
238 ble,pt %icc, 1f
239 sethi %hi(2f), %g7
240 ba,pt %xcc, etraptl1 243 ba,pt %xcc, etraptl1
241 or %g7, %lo(2f), %g7 2441: or %g7, %lo(1f), %g7
242
2431: ba,pt %xcc, etrap
2442: or %g7, %lo(2b), %g7
245 mov %l4, %o1 245 mov %l4, %o1
246 call sun4v_dtlb_error_report 246 call sun4v_dtlb_error_report
247 add %sp, PTREGS_OFF, %o0 247 add %sp, PTREGS_OFF, %o0
248 248
249 /* NOTREACHED */ 249 /* NOTREACHED */
250 250
251sun4v_bad_ra:
252 or %g0, %g4, %g5
253 ba,pt %xcc, sparc64_realfault_common
254 or %g1, %g0, %g4
255
256 /* NOTREACHED */
257
251 /* Instruction Access Exception, tl0. */ 258 /* Instruction Access Exception, tl0. */
252sun4v_iacc: 259sun4v_iacc:
253 ldxa [%g0] ASI_SCRATCHPAD, %g2 260 ldxa [%g0] ASI_SCRATCHPAD, %g2
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index fb6640ec8557..981a769b9558 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -2104,6 +2104,11 @@ void sun4v_nonresum_overflow(struct pt_regs *regs)
2104 atomic_inc(&sun4v_nonresum_oflow_cnt); 2104 atomic_inc(&sun4v_nonresum_oflow_cnt);
2105} 2105}
2106 2106
2107static void sun4v_tlb_error(struct pt_regs *regs)
2108{
2109 die_if_kernel("TLB/TSB error", regs);
2110}
2111
2107unsigned long sun4v_err_itlb_vaddr; 2112unsigned long sun4v_err_itlb_vaddr;
2108unsigned long sun4v_err_itlb_ctx; 2113unsigned long sun4v_err_itlb_ctx;
2109unsigned long sun4v_err_itlb_pte; 2114unsigned long sun4v_err_itlb_pte;
@@ -2111,8 +2116,7 @@ unsigned long sun4v_err_itlb_error;
2111 2116
2112void sun4v_itlb_error_report(struct pt_regs *regs, int tl) 2117void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2113{ 2118{
2114 if (tl > 1) 2119 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2115 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2116 2120
2117 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n", 2121 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2118 regs->tpc, tl); 2122 regs->tpc, tl);
@@ -2125,7 +2129,7 @@ void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2125 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, 2129 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2126 sun4v_err_itlb_pte, sun4v_err_itlb_error); 2130 sun4v_err_itlb_pte, sun4v_err_itlb_error);
2127 2131
2128 prom_halt(); 2132 sun4v_tlb_error(regs);
2129} 2133}
2130 2134
2131unsigned long sun4v_err_dtlb_vaddr; 2135unsigned long sun4v_err_dtlb_vaddr;
@@ -2135,8 +2139,7 @@ unsigned long sun4v_err_dtlb_error;
2135 2139
2136void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) 2140void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2137{ 2141{
2138 if (tl > 1) 2142 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2139 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2140 2143
2141 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n", 2144 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2142 regs->tpc, tl); 2145 regs->tpc, tl);
@@ -2149,7 +2152,7 @@ void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2149 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, 2152 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2150 sun4v_err_dtlb_pte, sun4v_err_dtlb_error); 2153 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2151 2154
2152 prom_halt(); 2155 sun4v_tlb_error(regs);
2153} 2156}
2154 2157
2155void hypervisor_tlbop_error(unsigned long err, unsigned long op) 2158void hypervisor_tlbop_error(unsigned long err, unsigned long op)
diff --git a/arch/sparc/kernel/vio.c b/arch/sparc/kernel/vio.c
index 8647fcc5ca6c..cb5789c9f961 100644
--- a/arch/sparc/kernel/vio.c
+++ b/arch/sparc/kernel/vio.c
@@ -180,8 +180,10 @@ static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp,
180 vdev->tx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); 180 vdev->tx_irq = sun4v_build_virq(cdev_cfg_handle, *irq);
181 181
182 irq = mdesc_get_property(hp, target, "rx-ino", NULL); 182 irq = mdesc_get_property(hp, target, "rx-ino", NULL);
183 if (irq) 183 if (irq) {
184 vdev->rx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); 184 vdev->rx_irq = sun4v_build_virq(cdev_cfg_handle, *irq);
185 vdev->rx_ino = *irq;
186 }
185 187
186 chan_id = mdesc_get_property(hp, target, "id", NULL); 188 chan_id = mdesc_get_property(hp, target, "id", NULL);
187 if (chan_id) 189 if (chan_id)
@@ -189,6 +191,15 @@ static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp,
189 } 191 }
190} 192}
191 193
194int vio_set_intr(unsigned long dev_ino, int state)
195{
196 int err;
197
198 err = sun4v_vintr_set_valid(cdev_cfg_handle, dev_ino, state);
199 return err;
200}
201EXPORT_SYMBOL(vio_set_intr);
202
192static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, 203static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp,
193 struct device *parent) 204 struct device *parent)
194{ 205{
diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c
index 7ef081a185b1..526fcb5d8ce9 100644
--- a/arch/sparc/kernel/viohs.c
+++ b/arch/sparc/kernel/viohs.c
@@ -724,7 +724,7 @@ int vio_ldc_alloc(struct vio_driver_state *vio,
724 cfg.tx_irq = vio->vdev->tx_irq; 724 cfg.tx_irq = vio->vdev->tx_irq;
725 cfg.rx_irq = vio->vdev->rx_irq; 725 cfg.rx_irq = vio->vdev->rx_irq;
726 726
727 lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg); 727 lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg, vio->name);
728 if (IS_ERR(lp)) 728 if (IS_ERR(lp))
729 return PTR_ERR(lp); 729 return PTR_ERR(lp);
730 730
@@ -756,7 +756,7 @@ void vio_port_up(struct vio_driver_state *vio)
756 756
757 err = 0; 757 err = 0;
758 if (state == LDC_STATE_INIT) { 758 if (state == LDC_STATE_INIT) {
759 err = ldc_bind(vio->lp, vio->name); 759 err = ldc_bind(vio->lp);
760 if (err) 760 if (err)
761 printk(KERN_WARNING "%s: Port %lu bind failed, " 761 printk(KERN_WARNING "%s: Port %lu bind failed, "
762 "err=%d\n", 762 "err=%d\n",
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index 932ff90fd760..09243057cb0b 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -35,8 +35,9 @@ jiffies = jiffies_64;
35 35
36SECTIONS 36SECTIONS
37{ 37{
38 /* swapper_low_pmd_dir is sparc64 only */ 38#ifdef CONFIG_SPARC64
39 swapper_low_pmd_dir = 0x0000000000402000; 39 swapper_pg_dir = 0x0000000000402000;
40#endif
40 . = INITIAL_ADDRESS; 41 . = INITIAL_ADDRESS;
41 .text TEXTSTART : 42 .text TEXTSTART :
42 { 43 {
@@ -122,11 +123,6 @@ SECTIONS
122 *(.swapper_4m_tsb_phys_patch) 123 *(.swapper_4m_tsb_phys_patch)
123 __swapper_4m_tsb_phys_patch_end = .; 124 __swapper_4m_tsb_phys_patch_end = .;
124 } 125 }
125 .page_offset_shift_patch : {
126 __page_offset_shift_patch = .;
127 *(.page_offset_shift_patch)
128 __page_offset_shift_patch_end = .;
129 }
130 .popc_3insn_patch : { 126 .popc_3insn_patch : {
131 __popc_3insn_patch = .; 127 __popc_3insn_patch = .;
132 *(.popc_3insn_patch) 128 *(.popc_3insn_patch)
diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S
index 99c017be8719..f75e6906df14 100644
--- a/arch/sparc/lib/memset.S
+++ b/arch/sparc/lib/memset.S
@@ -3,8 +3,9 @@
3 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 3 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) 4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
5 * 5 *
6 * Returns 0, if ok, and number of bytes not yet set if exception 6 * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and
7 * occurs and we were called as clear_user. 7 * number of bytes not yet set if exception occurs and we were called as
8 * clear_user.
8 */ 9 */
9 10
10#include <asm/ptrace.h> 11#include <asm/ptrace.h>
@@ -65,6 +66,8 @@ __bzero_begin:
65 .globl __memset_start, __memset_end 66 .globl __memset_start, __memset_end
66__memset_start: 67__memset_start:
67memset: 68memset:
69 mov %o0, %g1
70 mov 1, %g4
68 and %o1, 0xff, %g3 71 and %o1, 0xff, %g3
69 sll %g3, 8, %g2 72 sll %g3, 8, %g2
70 or %g3, %g2, %g3 73 or %g3, %g2, %g3
@@ -89,6 +92,7 @@ memset:
89 sub %o0, %o2, %o0 92 sub %o0, %o2, %o0
90 93
91__bzero: 94__bzero:
95 clr %g4
92 mov %g0, %g3 96 mov %g0, %g3
931: 971:
94 cmp %o1, 7 98 cmp %o1, 7
@@ -151,8 +155,8 @@ __bzero:
151 bne,a 8f 155 bne,a 8f
152 EX(stb %g3, [%o0], and %o1, 1) 156 EX(stb %g3, [%o0], and %o1, 1)
1538: 1578:
154 retl 158 b 0f
155 clr %o0 159 nop
1567: 1607:
157 be 13b 161 be 13b
158 orcc %o1, 0, %g0 162 orcc %o1, 0, %g0
@@ -164,6 +168,12 @@ __bzero:
164 bne 8b 168 bne 8b
165 EX(stb %g3, [%o0 - 1], add %o1, 1) 169 EX(stb %g3, [%o0 - 1], add %o1, 1)
1660: 1700:
171 andcc %g4, 1, %g0
172 be 5f
173 nop
174 retl
175 mov %g1, %o0
1765:
167 retl 177 retl
168 clr %o0 178 clr %o0
169__memset_end: 179__memset_end:
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 587cd0565128..18fcd7167095 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -346,6 +346,9 @@ retry:
346 down_read(&mm->mmap_sem); 346 down_read(&mm->mmap_sem);
347 } 347 }
348 348
349 if (fault_code & FAULT_CODE_BAD_RA)
350 goto do_sigbus;
351
349 vma = find_vma(mm, address); 352 vma = find_vma(mm, address);
350 if (!vma) 353 if (!vma)
351 goto bad_area; 354 goto bad_area;
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 98ac8e80adae..2d91c62f7f5f 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -75,7 +75,6 @@ unsigned long kern_linear_pte_xor[4] __read_mostly;
75 * 'cpu' properties, but we need to have this table setup before the 75 * 'cpu' properties, but we need to have this table setup before the
76 * MDESC is initialized. 76 * MDESC is initialized.
77 */ 77 */
78unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
79 78
80#ifndef CONFIG_DEBUG_PAGEALLOC 79#ifndef CONFIG_DEBUG_PAGEALLOC
81/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. 80/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
@@ -84,10 +83,11 @@ unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
84 */ 83 */
85extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; 84extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
86#endif 85#endif
86extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
87 87
88static unsigned long cpu_pgsz_mask; 88static unsigned long cpu_pgsz_mask;
89 89
90#define MAX_BANKS 32 90#define MAX_BANKS 1024
91 91
92static struct linux_prom64_registers pavail[MAX_BANKS]; 92static struct linux_prom64_registers pavail[MAX_BANKS];
93static int pavail_ents; 93static int pavail_ents;
@@ -165,10 +165,6 @@ static void __init read_obp_memory(const char *property,
165 cmp_p64, NULL); 165 cmp_p64, NULL);
166} 166}
167 167
168unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
169 sizeof(unsigned long)];
170EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
171
172/* Kernel physical address base and size in bytes. */ 168/* Kernel physical address base and size in bytes. */
173unsigned long kern_base __read_mostly; 169unsigned long kern_base __read_mostly;
174unsigned long kern_size __read_mostly; 170unsigned long kern_size __read_mostly;
@@ -840,7 +836,10 @@ static int find_node(unsigned long addr)
840 if ((addr & p->mask) == p->val) 836 if ((addr & p->mask) == p->val)
841 return i; 837 return i;
842 } 838 }
843 return -1; 839 /* The following condition has been observed on LDOM guests.*/
840 WARN_ONCE(1, "find_node: A physical address doesn't match a NUMA node"
841 " rule. Some physical memory will be owned by node 0.");
842 return 0;
844} 843}
845 844
846static u64 memblock_nid_range(u64 start, u64 end, int *nid) 845static u64 memblock_nid_range(u64 start, u64 end, int *nid)
@@ -1366,9 +1365,144 @@ static unsigned long __init bootmem_init(unsigned long phys_base)
1366static struct linux_prom64_registers pall[MAX_BANKS] __initdata; 1365static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1367static int pall_ents __initdata; 1366static int pall_ents __initdata;
1368 1367
1369#ifdef CONFIG_DEBUG_PAGEALLOC 1368static unsigned long max_phys_bits = 40;
1369
1370bool kern_addr_valid(unsigned long addr)
1371{
1372 pgd_t *pgd;
1373 pud_t *pud;
1374 pmd_t *pmd;
1375 pte_t *pte;
1376
1377 if ((long)addr < 0L) {
1378 unsigned long pa = __pa(addr);
1379
1380 if ((addr >> max_phys_bits) != 0UL)
1381 return false;
1382
1383 return pfn_valid(pa >> PAGE_SHIFT);
1384 }
1385
1386 if (addr >= (unsigned long) KERNBASE &&
1387 addr < (unsigned long)&_end)
1388 return true;
1389
1390 pgd = pgd_offset_k(addr);
1391 if (pgd_none(*pgd))
1392 return 0;
1393
1394 pud = pud_offset(pgd, addr);
1395 if (pud_none(*pud))
1396 return 0;
1397
1398 if (pud_large(*pud))
1399 return pfn_valid(pud_pfn(*pud));
1400
1401 pmd = pmd_offset(pud, addr);
1402 if (pmd_none(*pmd))
1403 return 0;
1404
1405 if (pmd_large(*pmd))
1406 return pfn_valid(pmd_pfn(*pmd));
1407
1408 pte = pte_offset_kernel(pmd, addr);
1409 if (pte_none(*pte))
1410 return 0;
1411
1412 return pfn_valid(pte_pfn(*pte));
1413}
1414EXPORT_SYMBOL(kern_addr_valid);
1415
1416static unsigned long __ref kernel_map_hugepud(unsigned long vstart,
1417 unsigned long vend,
1418 pud_t *pud)
1419{
1420 const unsigned long mask16gb = (1UL << 34) - 1UL;
1421 u64 pte_val = vstart;
1422
1423 /* Each PUD is 8GB */
1424 if ((vstart & mask16gb) ||
1425 (vend - vstart <= mask16gb)) {
1426 pte_val ^= kern_linear_pte_xor[2];
1427 pud_val(*pud) = pte_val | _PAGE_PUD_HUGE;
1428
1429 return vstart + PUD_SIZE;
1430 }
1431
1432 pte_val ^= kern_linear_pte_xor[3];
1433 pte_val |= _PAGE_PUD_HUGE;
1434
1435 vend = vstart + mask16gb + 1UL;
1436 while (vstart < vend) {
1437 pud_val(*pud) = pte_val;
1438
1439 pte_val += PUD_SIZE;
1440 vstart += PUD_SIZE;
1441 pud++;
1442 }
1443 return vstart;
1444}
1445
1446static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,
1447 bool guard)
1448{
1449 if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)
1450 return true;
1451
1452 return false;
1453}
1454
1455static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,
1456 unsigned long vend,
1457 pmd_t *pmd)
1458{
1459 const unsigned long mask256mb = (1UL << 28) - 1UL;
1460 const unsigned long mask2gb = (1UL << 31) - 1UL;
1461 u64 pte_val = vstart;
1462
1463 /* Each PMD is 8MB */
1464 if ((vstart & mask256mb) ||
1465 (vend - vstart <= mask256mb)) {
1466 pte_val ^= kern_linear_pte_xor[0];
1467 pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE;
1468
1469 return vstart + PMD_SIZE;
1470 }
1471
1472 if ((vstart & mask2gb) ||
1473 (vend - vstart <= mask2gb)) {
1474 pte_val ^= kern_linear_pte_xor[1];
1475 pte_val |= _PAGE_PMD_HUGE;
1476 vend = vstart + mask256mb + 1UL;
1477 } else {
1478 pte_val ^= kern_linear_pte_xor[2];
1479 pte_val |= _PAGE_PMD_HUGE;
1480 vend = vstart + mask2gb + 1UL;
1481 }
1482
1483 while (vstart < vend) {
1484 pmd_val(*pmd) = pte_val;
1485
1486 pte_val += PMD_SIZE;
1487 vstart += PMD_SIZE;
1488 pmd++;
1489 }
1490
1491 return vstart;
1492}
1493
1494static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,
1495 bool guard)
1496{
1497 if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)
1498 return true;
1499
1500 return false;
1501}
1502
1370static unsigned long __ref kernel_map_range(unsigned long pstart, 1503static unsigned long __ref kernel_map_range(unsigned long pstart,
1371 unsigned long pend, pgprot_t prot) 1504 unsigned long pend, pgprot_t prot,
1505 bool use_huge)
1372{ 1506{
1373 unsigned long vstart = PAGE_OFFSET + pstart; 1507 unsigned long vstart = PAGE_OFFSET + pstart;
1374 unsigned long vend = PAGE_OFFSET + pend; 1508 unsigned long vend = PAGE_OFFSET + pend;
@@ -1387,19 +1521,34 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
1387 pmd_t *pmd; 1521 pmd_t *pmd;
1388 pte_t *pte; 1522 pte_t *pte;
1389 1523
1524 if (pgd_none(*pgd)) {
1525 pud_t *new;
1526
1527 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1528 alloc_bytes += PAGE_SIZE;
1529 pgd_populate(&init_mm, pgd, new);
1530 }
1390 pud = pud_offset(pgd, vstart); 1531 pud = pud_offset(pgd, vstart);
1391 if (pud_none(*pud)) { 1532 if (pud_none(*pud)) {
1392 pmd_t *new; 1533 pmd_t *new;
1393 1534
1535 if (kernel_can_map_hugepud(vstart, vend, use_huge)) {
1536 vstart = kernel_map_hugepud(vstart, vend, pud);
1537 continue;
1538 }
1394 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1539 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1395 alloc_bytes += PAGE_SIZE; 1540 alloc_bytes += PAGE_SIZE;
1396 pud_populate(&init_mm, pud, new); 1541 pud_populate(&init_mm, pud, new);
1397 } 1542 }
1398 1543
1399 pmd = pmd_offset(pud, vstart); 1544 pmd = pmd_offset(pud, vstart);
1400 if (!pmd_present(*pmd)) { 1545 if (pmd_none(*pmd)) {
1401 pte_t *new; 1546 pte_t *new;
1402 1547
1548 if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {
1549 vstart = kernel_map_hugepmd(vstart, vend, pmd);
1550 continue;
1551 }
1403 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1552 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1404 alloc_bytes += PAGE_SIZE; 1553 alloc_bytes += PAGE_SIZE;
1405 pmd_populate_kernel(&init_mm, pmd, new); 1554 pmd_populate_kernel(&init_mm, pmd, new);
@@ -1422,100 +1571,34 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
1422 return alloc_bytes; 1571 return alloc_bytes;
1423} 1572}
1424 1573
1425extern unsigned int kvmap_linear_patch[1]; 1574static void __init flush_all_kernel_tsbs(void)
1426#endif /* CONFIG_DEBUG_PAGEALLOC */
1427
1428static void __init kpte_set_val(unsigned long index, unsigned long val)
1429{
1430 unsigned long *ptr = kpte_linear_bitmap;
1431
1432 val <<= ((index % (BITS_PER_LONG / 2)) * 2);
1433 ptr += (index / (BITS_PER_LONG / 2));
1434
1435 *ptr |= val;
1436}
1437
1438static const unsigned long kpte_shift_min = 28; /* 256MB */
1439static const unsigned long kpte_shift_max = 34; /* 16GB */
1440static const unsigned long kpte_shift_incr = 3;
1441
1442static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,
1443 unsigned long shift)
1444{ 1575{
1445 unsigned long size = (1UL << shift); 1576 int i;
1446 unsigned long mask = (size - 1UL);
1447 unsigned long remains = end - start;
1448 unsigned long val;
1449
1450 if (remains < size || (start & mask))
1451 return start;
1452
1453 /* VAL maps:
1454 *
1455 * shift 28 --> kern_linear_pte_xor index 1
1456 * shift 31 --> kern_linear_pte_xor index 2
1457 * shift 34 --> kern_linear_pte_xor index 3
1458 */
1459 val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;
1460
1461 remains &= ~mask;
1462 if (shift != kpte_shift_max)
1463 remains = size;
1464
1465 while (remains) {
1466 unsigned long index = start >> kpte_shift_min;
1467 1577
1468 kpte_set_val(index, val); 1578 for (i = 0; i < KERNEL_TSB_NENTRIES; i++) {
1579 struct tsb *ent = &swapper_tsb[i];
1469 1580
1470 start += 1UL << kpte_shift_min; 1581 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1471 remains -= 1UL << kpte_shift_min;
1472 } 1582 }
1583#ifndef CONFIG_DEBUG_PAGEALLOC
1584 for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) {
1585 struct tsb *ent = &swapper_4m_tsb[i];
1473 1586
1474 return start; 1587 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1475}
1476
1477static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
1478{
1479 unsigned long smallest_size, smallest_mask;
1480 unsigned long s;
1481
1482 smallest_size = (1UL << kpte_shift_min);
1483 smallest_mask = (smallest_size - 1UL);
1484
1485 while (start < end) {
1486 unsigned long orig_start = start;
1487
1488 for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {
1489 start = kpte_mark_using_shift(start, end, s);
1490
1491 if (start != orig_start)
1492 break;
1493 }
1494
1495 if (start == orig_start)
1496 start = (start + smallest_size) & ~smallest_mask;
1497 } 1588 }
1589#endif
1498} 1590}
1499 1591
1500static void __init init_kpte_bitmap(void) 1592extern unsigned int kvmap_linear_patch[1];
1501{
1502 unsigned long i;
1503
1504 for (i = 0; i < pall_ents; i++) {
1505 unsigned long phys_start, phys_end;
1506
1507 phys_start = pall[i].phys_addr;
1508 phys_end = phys_start + pall[i].reg_size;
1509
1510 mark_kpte_bitmap(phys_start, phys_end);
1511 }
1512}
1513 1593
1514static void __init kernel_physical_mapping_init(void) 1594static void __init kernel_physical_mapping_init(void)
1515{ 1595{
1516#ifdef CONFIG_DEBUG_PAGEALLOC
1517 unsigned long i, mem_alloced = 0UL; 1596 unsigned long i, mem_alloced = 0UL;
1597 bool use_huge = true;
1518 1598
1599#ifdef CONFIG_DEBUG_PAGEALLOC
1600 use_huge = false;
1601#endif
1519 for (i = 0; i < pall_ents; i++) { 1602 for (i = 0; i < pall_ents; i++) {
1520 unsigned long phys_start, phys_end; 1603 unsigned long phys_start, phys_end;
1521 1604
@@ -1523,7 +1606,7 @@ static void __init kernel_physical_mapping_init(void)
1523 phys_end = phys_start + pall[i].reg_size; 1606 phys_end = phys_start + pall[i].reg_size;
1524 1607
1525 mem_alloced += kernel_map_range(phys_start, phys_end, 1608 mem_alloced += kernel_map_range(phys_start, phys_end,
1526 PAGE_KERNEL); 1609 PAGE_KERNEL, use_huge);
1527 } 1610 }
1528 1611
1529 printk("Allocated %ld bytes for kernel page tables.\n", 1612 printk("Allocated %ld bytes for kernel page tables.\n",
@@ -1532,8 +1615,9 @@ static void __init kernel_physical_mapping_init(void)
1532 kvmap_linear_patch[0] = 0x01000000; /* nop */ 1615 kvmap_linear_patch[0] = 0x01000000; /* nop */
1533 flushi(&kvmap_linear_patch[0]); 1616 flushi(&kvmap_linear_patch[0]);
1534 1617
1618 flush_all_kernel_tsbs();
1619
1535 __flush_tlb_all(); 1620 __flush_tlb_all();
1536#endif
1537} 1621}
1538 1622
1539#ifdef CONFIG_DEBUG_PAGEALLOC 1623#ifdef CONFIG_DEBUG_PAGEALLOC
@@ -1543,7 +1627,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1543 unsigned long phys_end = phys_start + (numpages * PAGE_SIZE); 1627 unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1544 1628
1545 kernel_map_range(phys_start, phys_end, 1629 kernel_map_range(phys_start, phys_end,
1546 (enable ? PAGE_KERNEL : __pgprot(0))); 1630 (enable ? PAGE_KERNEL : __pgprot(0)), false);
1547 1631
1548 flush_tsb_kernel_range(PAGE_OFFSET + phys_start, 1632 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1549 PAGE_OFFSET + phys_end); 1633 PAGE_OFFSET + phys_end);
@@ -1571,76 +1655,56 @@ unsigned long __init find_ecache_flush_span(unsigned long size)
1571unsigned long PAGE_OFFSET; 1655unsigned long PAGE_OFFSET;
1572EXPORT_SYMBOL(PAGE_OFFSET); 1656EXPORT_SYMBOL(PAGE_OFFSET);
1573 1657
1574static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits) 1658unsigned long VMALLOC_END = 0x0000010000000000UL;
1575{ 1659EXPORT_SYMBOL(VMALLOC_END);
1576 unsigned long final_shift;
1577 unsigned int val = *insn;
1578 unsigned int cnt;
1579
1580 /* We are patching in ilog2(max_supported_phys_address), and
1581 * we are doing so in a manner similar to a relocation addend.
1582 * That is, we are adding the shift value to whatever value
1583 * is in the shift instruction count field already.
1584 */
1585 cnt = (val & 0x3f);
1586 val &= ~0x3f;
1587
1588 /* If we are trying to shift >= 64 bits, clear the destination
1589 * register. This can happen when phys_bits ends up being equal
1590 * to MAX_PHYS_ADDRESS_BITS.
1591 */
1592 final_shift = (cnt + (64 - phys_bits));
1593 if (final_shift >= 64) {
1594 unsigned int rd = (val >> 25) & 0x1f;
1595
1596 val = 0x80100000 | (rd << 25);
1597 } else {
1598 val |= final_shift;
1599 }
1600 *insn = val;
1601
1602 __asm__ __volatile__("flush %0"
1603 : /* no outputs */
1604 : "r" (insn));
1605}
1606
1607static void __init page_offset_shift_patch(unsigned long phys_bits)
1608{
1609 extern unsigned int __page_offset_shift_patch;
1610 extern unsigned int __page_offset_shift_patch_end;
1611 unsigned int *p;
1612
1613 p = &__page_offset_shift_patch;
1614 while (p < &__page_offset_shift_patch_end) {
1615 unsigned int *insn = (unsigned int *)(unsigned long)*p;
1616 1660
1617 page_offset_shift_patch_one(insn, phys_bits); 1661unsigned long sparc64_va_hole_top = 0xfffff80000000000UL;
1618 1662unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL;
1619 p++;
1620 }
1621}
1622 1663
1623static void __init setup_page_offset(void) 1664static void __init setup_page_offset(void)
1624{ 1665{
1625 unsigned long max_phys_bits = 40;
1626
1627 if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1666 if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1667 /* Cheetah/Panther support a full 64-bit virtual
1668 * address, so we can use all that our page tables
1669 * support.
1670 */
1671 sparc64_va_hole_top = 0xfff0000000000000UL;
1672 sparc64_va_hole_bottom = 0x0010000000000000UL;
1673
1628 max_phys_bits = 42; 1674 max_phys_bits = 42;
1629 } else if (tlb_type == hypervisor) { 1675 } else if (tlb_type == hypervisor) {
1630 switch (sun4v_chip_type) { 1676 switch (sun4v_chip_type) {
1631 case SUN4V_CHIP_NIAGARA1: 1677 case SUN4V_CHIP_NIAGARA1:
1632 case SUN4V_CHIP_NIAGARA2: 1678 case SUN4V_CHIP_NIAGARA2:
1679 /* T1 and T2 support 48-bit virtual addresses. */
1680 sparc64_va_hole_top = 0xffff800000000000UL;
1681 sparc64_va_hole_bottom = 0x0000800000000000UL;
1682
1633 max_phys_bits = 39; 1683 max_phys_bits = 39;
1634 break; 1684 break;
1635 case SUN4V_CHIP_NIAGARA3: 1685 case SUN4V_CHIP_NIAGARA3:
1686 /* T3 supports 48-bit virtual addresses. */
1687 sparc64_va_hole_top = 0xffff800000000000UL;
1688 sparc64_va_hole_bottom = 0x0000800000000000UL;
1689
1636 max_phys_bits = 43; 1690 max_phys_bits = 43;
1637 break; 1691 break;
1638 case SUN4V_CHIP_NIAGARA4: 1692 case SUN4V_CHIP_NIAGARA4:
1639 case SUN4V_CHIP_NIAGARA5: 1693 case SUN4V_CHIP_NIAGARA5:
1640 case SUN4V_CHIP_SPARC64X: 1694 case SUN4V_CHIP_SPARC64X:
1641 default: 1695 case SUN4V_CHIP_SPARC_M6:
1696 /* T4 and later support 52-bit virtual addresses. */
1697 sparc64_va_hole_top = 0xfff8000000000000UL;
1698 sparc64_va_hole_bottom = 0x0008000000000000UL;
1642 max_phys_bits = 47; 1699 max_phys_bits = 47;
1643 break; 1700 break;
1701 case SUN4V_CHIP_SPARC_M7:
1702 default:
1703 /* M7 and later support 52-bit virtual addresses. */
1704 sparc64_va_hole_top = 0xfff8000000000000UL;
1705 sparc64_va_hole_bottom = 0x0008000000000000UL;
1706 max_phys_bits = 49;
1707 break;
1644 } 1708 }
1645 } 1709 }
1646 1710
@@ -1650,12 +1714,16 @@ static void __init setup_page_offset(void)
1650 prom_halt(); 1714 prom_halt();
1651 } 1715 }
1652 1716
1653 PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits); 1717 PAGE_OFFSET = sparc64_va_hole_top;
1718 VMALLOC_END = ((sparc64_va_hole_bottom >> 1) +
1719 (sparc64_va_hole_bottom >> 2));
1654 1720
1655 pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", 1721 pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
1656 PAGE_OFFSET, max_phys_bits); 1722 PAGE_OFFSET, max_phys_bits);
1657 1723 pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n",
1658 page_offset_shift_patch(max_phys_bits); 1724 VMALLOC_START, VMALLOC_END);
1725 pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n",
1726 VMEMMAP_BASE, VMEMMAP_BASE << 1);
1659} 1727}
1660 1728
1661static void __init tsb_phys_patch(void) 1729static void __init tsb_phys_patch(void)
@@ -1700,21 +1768,42 @@ static void __init tsb_phys_patch(void)
1700#define NUM_KTSB_DESCR 1 1768#define NUM_KTSB_DESCR 1
1701#endif 1769#endif
1702static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; 1770static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1703extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; 1771
1772/* The swapper TSBs are loaded with a base sequence of:
1773 *
1774 * sethi %uhi(SYMBOL), REG1
1775 * sethi %hi(SYMBOL), REG2
1776 * or REG1, %ulo(SYMBOL), REG1
1777 * or REG2, %lo(SYMBOL), REG2
1778 * sllx REG1, 32, REG1
1779 * or REG1, REG2, REG1
1780 *
1781 * When we use physical addressing for the TSB accesses, we patch the
1782 * first four instructions in the above sequence.
1783 */
1704 1784
1705static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) 1785static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
1706{ 1786{
1707 pa >>= KTSB_PHYS_SHIFT; 1787 unsigned long high_bits, low_bits;
1788
1789 high_bits = (pa >> 32) & 0xffffffff;
1790 low_bits = (pa >> 0) & 0xffffffff;
1708 1791
1709 while (start < end) { 1792 while (start < end) {
1710 unsigned int *ia = (unsigned int *)(unsigned long)*start; 1793 unsigned int *ia = (unsigned int *)(unsigned long)*start;
1711 1794
1712 ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); 1795 ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10);
1713 __asm__ __volatile__("flush %0" : : "r" (ia)); 1796 __asm__ __volatile__("flush %0" : : "r" (ia));
1714 1797
1715 ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); 1798 ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10);
1716 __asm__ __volatile__("flush %0" : : "r" (ia + 1)); 1799 __asm__ __volatile__("flush %0" : : "r" (ia + 1));
1717 1800
1801 ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff);
1802 __asm__ __volatile__("flush %0" : : "r" (ia + 2));
1803
1804 ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff);
1805 __asm__ __volatile__("flush %0" : : "r" (ia + 3));
1806
1718 start++; 1807 start++;
1719 } 1808 }
1720} 1809}
@@ -1853,11 +1942,56 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1853/* paging_init() sets up the page tables */ 1942/* paging_init() sets up the page tables */
1854 1943
1855static unsigned long last_valid_pfn; 1944static unsigned long last_valid_pfn;
1856pgd_t swapper_pg_dir[PTRS_PER_PGD];
1857 1945
1858static void sun4u_pgprot_init(void); 1946static void sun4u_pgprot_init(void);
1859static void sun4v_pgprot_init(void); 1947static void sun4v_pgprot_init(void);
1860 1948
1949static phys_addr_t __init available_memory(void)
1950{
1951 phys_addr_t available = 0ULL;
1952 phys_addr_t pa_start, pa_end;
1953 u64 i;
1954
1955 for_each_free_mem_range(i, NUMA_NO_NODE, &pa_start, &pa_end, NULL)
1956 available = available + (pa_end - pa_start);
1957
1958 return available;
1959}
1960
1961/* We need to exclude reserved regions. This exclusion will include
1962 * vmlinux and initrd. To be more precise the initrd size could be used to
1963 * compute a new lower limit because it is freed later during initialization.
1964 */
1965static void __init reduce_memory(phys_addr_t limit_ram)
1966{
1967 phys_addr_t avail_ram = available_memory();
1968 phys_addr_t pa_start, pa_end;
1969 u64 i;
1970
1971 if (limit_ram >= avail_ram)
1972 return;
1973
1974 for_each_free_mem_range(i, NUMA_NO_NODE, &pa_start, &pa_end, NULL) {
1975 phys_addr_t region_size = pa_end - pa_start;
1976 phys_addr_t clip_start = pa_start;
1977
1978 avail_ram = avail_ram - region_size;
1979 /* Are we consuming too much? */
1980 if (avail_ram < limit_ram) {
1981 phys_addr_t give_back = limit_ram - avail_ram;
1982
1983 region_size = region_size - give_back;
1984 clip_start = clip_start + give_back;
1985 }
1986
1987 memblock_remove(clip_start, region_size);
1988
1989 if (avail_ram <= limit_ram)
1990 break;
1991 i = 0UL;
1992 }
1993}
1994
1861void __init paging_init(void) 1995void __init paging_init(void)
1862{ 1996{
1863 unsigned long end_pfn, shift, phys_base; 1997 unsigned long end_pfn, shift, phys_base;
@@ -1937,7 +2071,8 @@ void __init paging_init(void)
1937 2071
1938 find_ramdisk(phys_base); 2072 find_ramdisk(phys_base);
1939 2073
1940 memblock_enforce_memory_limit(cmdline_memory_size); 2074 if (cmdline_memory_size)
2075 reduce_memory(cmdline_memory_size);
1941 2076
1942 memblock_allow_resize(); 2077 memblock_allow_resize();
1943 memblock_dump_all(); 2078 memblock_dump_all();
@@ -1956,16 +2091,10 @@ void __init paging_init(void)
1956 */ 2091 */
1957 init_mm.pgd += ((shift) / (sizeof(pgd_t))); 2092 init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1958 2093
1959 memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir)); 2094 memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
1960 2095
1961 /* Now can init the kernel/bad page tables. */
1962 pud_set(pud_offset(&swapper_pg_dir[0], 0),
1963 swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
1964
1965 inherit_prom_mappings(); 2096 inherit_prom_mappings();
1966 2097
1967 init_kpte_bitmap();
1968
1969 /* Ok, we can use our TLB miss and window trap handlers safely. */ 2098 /* Ok, we can use our TLB miss and window trap handlers safely. */
1970 setup_tba(); 2099 setup_tba();
1971 2100
@@ -2072,70 +2201,6 @@ int page_in_phys_avail(unsigned long paddr)
2072 return 0; 2201 return 0;
2073} 2202}
2074 2203
2075static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
2076static int pavail_rescan_ents __initdata;
2077
2078/* Certain OBP calls, such as fetching "available" properties, can
2079 * claim physical memory. So, along with initializing the valid
2080 * address bitmap, what we do here is refetch the physical available
2081 * memory list again, and make sure it provides at least as much
2082 * memory as 'pavail' does.
2083 */
2084static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
2085{
2086 int i;
2087
2088 read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
2089
2090 for (i = 0; i < pavail_ents; i++) {
2091 unsigned long old_start, old_end;
2092
2093 old_start = pavail[i].phys_addr;
2094 old_end = old_start + pavail[i].reg_size;
2095 while (old_start < old_end) {
2096 int n;
2097
2098 for (n = 0; n < pavail_rescan_ents; n++) {
2099 unsigned long new_start, new_end;
2100
2101 new_start = pavail_rescan[n].phys_addr;
2102 new_end = new_start +
2103 pavail_rescan[n].reg_size;
2104
2105 if (new_start <= old_start &&
2106 new_end >= (old_start + PAGE_SIZE)) {
2107 set_bit(old_start >> ILOG2_4MB, bitmap);
2108 goto do_next_page;
2109 }
2110 }
2111
2112 prom_printf("mem_init: Lost memory in pavail\n");
2113 prom_printf("mem_init: OLD start[%lx] size[%lx]\n",
2114 pavail[i].phys_addr,
2115 pavail[i].reg_size);
2116 prom_printf("mem_init: NEW start[%lx] size[%lx]\n",
2117 pavail_rescan[i].phys_addr,
2118 pavail_rescan[i].reg_size);
2119 prom_printf("mem_init: Cannot continue, aborting.\n");
2120 prom_halt();
2121
2122 do_next_page:
2123 old_start += PAGE_SIZE;
2124 }
2125 }
2126}
2127
2128static void __init patch_tlb_miss_handler_bitmap(void)
2129{
2130 extern unsigned int valid_addr_bitmap_insn[];
2131 extern unsigned int valid_addr_bitmap_patch[];
2132
2133 valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
2134 mb();
2135 valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
2136 flushi(&valid_addr_bitmap_insn[0]);
2137}
2138
2139static void __init register_page_bootmem_info(void) 2204static void __init register_page_bootmem_info(void)
2140{ 2205{
2141#ifdef CONFIG_NEED_MULTIPLE_NODES 2206#ifdef CONFIG_NEED_MULTIPLE_NODES
@@ -2148,18 +2213,6 @@ static void __init register_page_bootmem_info(void)
2148} 2213}
2149void __init mem_init(void) 2214void __init mem_init(void)
2150{ 2215{
2151 unsigned long addr, last;
2152
2153 addr = PAGE_OFFSET + kern_base;
2154 last = PAGE_ALIGN(kern_size) + addr;
2155 while (addr < last) {
2156 set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);
2157 addr += PAGE_SIZE;
2158 }
2159
2160 setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
2161 patch_tlb_miss_handler_bitmap();
2162
2163 high_memory = __va(last_valid_pfn << PAGE_SHIFT); 2216 high_memory = __va(last_valid_pfn << PAGE_SHIFT);
2164 2217
2165 register_page_bootmem_info(); 2218 register_page_bootmem_info();
@@ -2249,18 +2302,9 @@ unsigned long _PAGE_CACHE __read_mostly;
2249EXPORT_SYMBOL(_PAGE_CACHE); 2302EXPORT_SYMBOL(_PAGE_CACHE);
2250 2303
2251#ifdef CONFIG_SPARSEMEM_VMEMMAP 2304#ifdef CONFIG_SPARSEMEM_VMEMMAP
2252unsigned long vmemmap_table[VMEMMAP_SIZE];
2253
2254static long __meminitdata addr_start, addr_end;
2255static int __meminitdata node_start;
2256
2257int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, 2305int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2258 int node) 2306 int node)
2259{ 2307{
2260 unsigned long phys_start = (vstart - VMEMMAP_BASE);
2261 unsigned long phys_end = (vend - VMEMMAP_BASE);
2262 unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
2263 unsigned long end = VMEMMAP_ALIGN(phys_end);
2264 unsigned long pte_base; 2308 unsigned long pte_base;
2265 2309
2266 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U | 2310 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
@@ -2271,47 +2315,52 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2271 _PAGE_CP_4V | _PAGE_CV_4V | 2315 _PAGE_CP_4V | _PAGE_CV_4V |
2272 _PAGE_P_4V | _PAGE_W_4V); 2316 _PAGE_P_4V | _PAGE_W_4V);
2273 2317
2274 for (; addr < end; addr += VMEMMAP_CHUNK) { 2318 pte_base |= _PAGE_PMD_HUGE;
2275 unsigned long *vmem_pp =
2276 vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);
2277 void *block;
2278 2319
2279 if (!(*vmem_pp & _PAGE_VALID)) { 2320 vstart = vstart & PMD_MASK;
2280 block = vmemmap_alloc_block(1UL << ILOG2_4MB, node); 2321 vend = ALIGN(vend, PMD_SIZE);
2281 if (!block) 2322 for (; vstart < vend; vstart += PMD_SIZE) {
2323 pgd_t *pgd = pgd_offset_k(vstart);
2324 unsigned long pte;
2325 pud_t *pud;
2326 pmd_t *pmd;
2327
2328 if (pgd_none(*pgd)) {
2329 pud_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2330
2331 if (!new)
2282 return -ENOMEM; 2332 return -ENOMEM;
2333 pgd_populate(&init_mm, pgd, new);
2334 }
2283 2335
2284 *vmem_pp = pte_base | __pa(block); 2336 pud = pud_offset(pgd, vstart);
2337 if (pud_none(*pud)) {
2338 pmd_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2285 2339
2286 /* check to see if we have contiguous blocks */ 2340 if (!new)
2287 if (addr_end != addr || node_start != node) { 2341 return -ENOMEM;
2288 if (addr_start) 2342 pud_populate(&init_mm, pud, new);
2289 printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
2290 addr_start, addr_end-1, node_start);
2291 addr_start = addr;
2292 node_start = node;
2293 }
2294 addr_end = addr + VMEMMAP_CHUNK;
2295 } 2343 }
2296 }
2297 return 0;
2298}
2299 2344
2300void __meminit vmemmap_populate_print_last(void) 2345 pmd = pmd_offset(pud, vstart);
2301{ 2346
2302 if (addr_start) { 2347 pte = pmd_val(*pmd);
2303 printk(KERN_DEBUG " [%lx-%lx] on node %d\n", 2348 if (!(pte & _PAGE_VALID)) {
2304 addr_start, addr_end-1, node_start); 2349 void *block = vmemmap_alloc_block(PMD_SIZE, node);
2305 addr_start = 0; 2350
2306 addr_end = 0; 2351 if (!block)
2307 node_start = 0; 2352 return -ENOMEM;
2353
2354 pmd_val(*pmd) = pte_base | __pa(block);
2355 }
2308 } 2356 }
2357
2358 return 0;
2309} 2359}
2310 2360
2311void vmemmap_free(unsigned long start, unsigned long end) 2361void vmemmap_free(unsigned long start, unsigned long end)
2312{ 2362{
2313} 2363}
2314
2315#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 2364#endif /* CONFIG_SPARSEMEM_VMEMMAP */
2316 2365
2317static void prot_init_common(unsigned long page_none, 2366static void prot_init_common(unsigned long page_none,
@@ -2787,8 +2836,8 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
2787 do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS); 2836 do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS);
2788 } 2837 }
2789 if (end > HI_OBP_ADDRESS) { 2838 if (end > HI_OBP_ADDRESS) {
2790 flush_tsb_kernel_range(end, HI_OBP_ADDRESS); 2839 flush_tsb_kernel_range(HI_OBP_ADDRESS, end);
2791 do_flush_tlb_kernel_range(end, HI_OBP_ADDRESS); 2840 do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end);
2792 } 2841 }
2793 } else { 2842 } else {
2794 flush_tsb_kernel_range(start, end); 2843 flush_tsb_kernel_range(start, end);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 0668b364f44d..a4c09603b05c 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -8,15 +8,8 @@
8 */ 8 */
9 9
10#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) 10#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS)
11#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
12#define KPTE_BITMAP_BYTES \
13 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4)
14#define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL)
15#define VALID_ADDR_BITMAP_BYTES \
16 ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8)
17 11
18extern unsigned long kern_linear_pte_xor[4]; 12extern unsigned long kern_linear_pte_xor[4];
19extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
20extern unsigned int sparc64_highest_unlocked_tlb_ent; 13extern unsigned int sparc64_highest_unlocked_tlb_ent;
21extern unsigned long sparc64_kern_pri_context; 14extern unsigned long sparc64_kern_pri_context;
22extern unsigned long sparc64_kern_pri_nuc_bits; 15extern unsigned long sparc64_kern_pri_nuc_bits;
@@ -38,15 +31,4 @@ extern unsigned long kern_locked_tte_data;
38 31
39void prom_world(int enter); 32void prom_world(int enter);
40 33
41#ifdef CONFIG_SPARSEMEM_VMEMMAP
42#define VMEMMAP_CHUNK_SHIFT 22
43#define VMEMMAP_CHUNK (1UL << VMEMMAP_CHUNK_SHIFT)
44#define VMEMMAP_CHUNK_MASK ~(VMEMMAP_CHUNK - 1UL)
45#define VMEMMAP_ALIGN(x) (((x)+VMEMMAP_CHUNK-1UL)&VMEMMAP_CHUNK_MASK)
46
47#define VMEMMAP_SIZE ((((1UL << MAX_PHYSADDR_BITS) >> PAGE_SHIFT) * \
48 sizeof(struct page)) >> VMEMMAP_CHUNK_SHIFT)
49extern unsigned long vmemmap_table[VMEMMAP_SIZE];
50#endif
51
52#endif /* _SPARC64_MM_INIT_H */ 34#endif /* _SPARC64_MM_INIT_H */
diff --git a/arch/sparc/power/hibernate_asm.S b/arch/sparc/power/hibernate_asm.S
index 79942166df84..d7d9017dcb15 100644
--- a/arch/sparc/power/hibernate_asm.S
+++ b/arch/sparc/power/hibernate_asm.S
@@ -54,8 +54,8 @@ ENTRY(swsusp_arch_resume)
54 nop 54 nop
55 55
56 /* Write PAGE_OFFSET to %g7 */ 56 /* Write PAGE_OFFSET to %g7 */
57 sethi %uhi(PAGE_OFFSET), %g7 57 sethi %hi(PAGE_OFFSET), %g7
58 sllx %g7, 32, %g7 58 ldx [%g7 + %lo(PAGE_OFFSET)], %g7
59 59
60 setuw (PAGE_SIZE-8), %g3 60 setuw (PAGE_SIZE-8), %g3
61 61
diff --git a/arch/sparc/prom/bootstr_64.c b/arch/sparc/prom/bootstr_64.c
index ab9ccc63b388..7149e77714a4 100644
--- a/arch/sparc/prom/bootstr_64.c
+++ b/arch/sparc/prom/bootstr_64.c
@@ -14,7 +14,10 @@
14 * the .bss section or it will break things. 14 * the .bss section or it will break things.
15 */ 15 */
16 16
17#define BARG_LEN 256 17/* We limit BARG_LEN to 1024 because this is the size of the
18 * 'barg_out' command line buffer in the SILO bootloader.
19 */
20#define BARG_LEN 1024
18struct { 21struct {
19 int bootstr_len; 22 int bootstr_len;
20 int bootstr_valid; 23 int bootstr_valid;
diff --git a/arch/sparc/prom/p1275.c b/arch/sparc/prom/p1275.c
index e58b81726319..b2340f008ae0 100644
--- a/arch/sparc/prom/p1275.c
+++ b/arch/sparc/prom/p1275.c
@@ -9,6 +9,7 @@
9#include <linux/smp.h> 9#include <linux/smp.h>
10#include <linux/string.h> 10#include <linux/string.h>
11#include <linux/spinlock.h> 11#include <linux/spinlock.h>
12#include <linux/irqflags.h>
12 13
13#include <asm/openprom.h> 14#include <asm/openprom.h>
14#include <asm/oplib.h> 15#include <asm/oplib.h>
@@ -36,8 +37,8 @@ void p1275_cmd_direct(unsigned long *args)
36{ 37{
37 unsigned long flags; 38 unsigned long flags;
38 39
39 raw_local_save_flags(flags); 40 local_save_flags(flags);
40 raw_local_irq_restore((unsigned long)PIL_NMI); 41 local_irq_restore((unsigned long)PIL_NMI);
41 raw_spin_lock(&prom_entry_lock); 42 raw_spin_lock(&prom_entry_lock);
42 43
43 prom_world(1); 44 prom_world(1);
@@ -45,7 +46,7 @@ void p1275_cmd_direct(unsigned long *args)
45 prom_world(0); 46 prom_world(0);
46 47
47 raw_spin_unlock(&prom_entry_lock); 48 raw_spin_unlock(&prom_entry_lock);
48 raw_local_irq_restore(flags); 49 local_irq_restore(flags);
49} 50}
50 51
51void prom_cif_init(void *cif_handler, void *cif_stack) 52void prom_cif_init(void *cif_handler, void *cif_stack)
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-28 01:31:33 -0500