aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/sparc/Kconfig1
-rw-r--r--arch/sparc/include/asm/dma-mapping.h6
-rw-r--r--arch/sparc/include/asm/hypervisor.h11
-rw-r--r--arch/sparc/include/asm/irq_64.h7
-rw-r--r--arch/sparc/include/asm/ldc.h5
-rw-r--r--arch/sparc/include/asm/page_64.h33
-rw-r--r--arch/sparc/include/asm/pgalloc_64.h28
-rw-r--r--arch/sparc/include/asm/pgtable_64.h100
-rw-r--r--arch/sparc/include/asm/spitfire.h2
-rw-r--r--arch/sparc/include/asm/thread_info_64.h1
-rw-r--r--arch/sparc/include/asm/tsb.h87
-rw-r--r--arch/sparc/include/asm/vio.h16
-rw-r--r--arch/sparc/kernel/cpu.c12
-rw-r--r--arch/sparc/kernel/cpumap.c2
-rw-r--r--arch/sparc/kernel/ds.c4
-rw-r--r--arch/sparc/kernel/head_64.S12
-rw-r--r--arch/sparc/kernel/hvapi.c1
-rw-r--r--arch/sparc/kernel/hvcalls.S16
-rw-r--r--arch/sparc/kernel/ioport.c5
-rw-r--r--arch/sparc/kernel/irq_64.c507
-rw-r--r--arch/sparc/kernel/ktlb.S125
-rw-r--r--arch/sparc/kernel/ldc.c41
-rw-r--r--arch/sparc/kernel/leon_kernel.c31
-rw-r--r--arch/sparc/kernel/pcr.c47
-rw-r--r--arch/sparc/kernel/perf_event.c3
-rw-r--r--arch/sparc/kernel/setup_64.c26
-rw-r--r--arch/sparc/kernel/smp_64.c7
-rw-r--r--arch/sparc/kernel/sun4v_tlb_miss.S35
-rw-r--r--arch/sparc/kernel/traps_64.c15
-rw-r--r--arch/sparc/kernel/vio.c13
-rw-r--r--arch/sparc/kernel/viohs.c4
-rw-r--r--arch/sparc/kernel/vmlinux.lds.S10
-rw-r--r--arch/sparc/lib/memset.S18
-rw-r--r--arch/sparc/mm/fault_64.c3
-rw-r--r--arch/sparc/mm/init_64.c603
-rw-r--r--arch/sparc/mm/init_64.h18
-rw-r--r--arch/sparc/power/hibernate_asm.S4
-rw-r--r--arch/sparc/prom/bootstr_64.c5
-rw-r--r--arch/sparc/prom/p1275.c7
-rw-r--r--drivers/block/sunvdc.c167
-rw-r--r--drivers/net/ethernet/sun/sunvnet.c4
41 files changed, 1248 insertions, 794 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index a537816613f9..96ac69c5eba0 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -67,6 +67,7 @@ config SPARC64
67 select HAVE_SYSCALL_TRACEPOINTS 67 select HAVE_SYSCALL_TRACEPOINTS
68 select HAVE_CONTEXT_TRACKING 68 select HAVE_CONTEXT_TRACKING
69 select HAVE_DEBUG_KMEMLEAK 69 select HAVE_DEBUG_KMEMLEAK
70 select SPARSE_IRQ
70 select RTC_DRV_CMOS 71 select RTC_DRV_CMOS
71 select RTC_DRV_BQ4802 72 select RTC_DRV_BQ4802
72 select RTC_DRV_SUN4V 73 select RTC_DRV_SUN4V
diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h
index 1ee02710b2dc..5b1b52a04ad6 100644
--- a/arch/sparc/include/asm/dma-mapping.h
+++ b/arch/sparc/include/asm/dma-mapping.h
@@ -20,10 +20,12 @@ extern struct bus_type pci_bus_type;
20 20
21static inline struct dma_map_ops *get_dma_ops(struct device *dev) 21static inline struct dma_map_ops *get_dma_ops(struct device *dev)
22{ 22{
23#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) 23#ifdef CONFIG_SPARC_LEON
24 if (sparc_cpu_model == sparc_leon) 24 if (sparc_cpu_model == sparc_leon)
25 return leon_dma_ops; 25 return leon_dma_ops;
26 else if (dev->bus == &pci_bus_type) 26#endif
27#if defined(CONFIG_SPARC32) && defined(CONFIG_PCI)
28 if (dev->bus == &pci_bus_type)
27 return &pci32_dma_ops; 29 return &pci32_dma_ops;
28#endif 30#endif
29 return dma_ops; 31 return dma_ops;
diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index 94b39caea3eb..4f6725ff4c33 100644
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -2947,6 +2947,16 @@ unsigned long sun4v_vt_set_perfreg(unsigned long reg_num,
2947 unsigned long reg_val); 2947 unsigned long reg_val);
2948#endif 2948#endif
2949 2949
2950#define HV_FAST_T5_GET_PERFREG 0x1a8
2951#define HV_FAST_T5_SET_PERFREG 0x1a9
2952
2953#ifndef __ASSEMBLY__
2954unsigned long sun4v_t5_get_perfreg(unsigned long reg_num,
2955 unsigned long *reg_val);
2956unsigned long sun4v_t5_set_perfreg(unsigned long reg_num,
2957 unsigned long reg_val);
2958#endif
2959
2950/* Function numbers for HV_CORE_TRAP. */ 2960/* Function numbers for HV_CORE_TRAP. */
2951#define HV_CORE_SET_VER 0x00 2961#define HV_CORE_SET_VER 0x00
2952#define HV_CORE_PUTCHAR 0x01 2962#define HV_CORE_PUTCHAR 0x01
@@ -2978,6 +2988,7 @@ unsigned long sun4v_vt_set_perfreg(unsigned long reg_num,
2978#define HV_GRP_VF_CPU 0x0205 2988#define HV_GRP_VF_CPU 0x0205
2979#define HV_GRP_KT_CPU 0x0209 2989#define HV_GRP_KT_CPU 0x0209
2980#define HV_GRP_VT_CPU 0x020c 2990#define HV_GRP_VT_CPU 0x020c
2991#define HV_GRP_T5_CPU 0x0211
2981#define HV_GRP_DIAG 0x0300 2992#define HV_GRP_DIAG 0x0300
2982 2993
2983#ifndef __ASSEMBLY__ 2994#ifndef __ASSEMBLY__
diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h
index 91d219381306..3f70f900e834 100644
--- a/arch/sparc/include/asm/irq_64.h
+++ b/arch/sparc/include/asm/irq_64.h
@@ -37,7 +37,7 @@
37 * 37 *
38 * ino_bucket->irq allocation is made during {sun4v_,}build_irq(). 38 * ino_bucket->irq allocation is made during {sun4v_,}build_irq().
39 */ 39 */
40#define NR_IRQS 255 40#define NR_IRQS (2048)
41 41
42void irq_install_pre_handler(int irq, 42void irq_install_pre_handler(int irq,
43 void (*func)(unsigned int, void *, void *), 43 void (*func)(unsigned int, void *, void *),
@@ -57,11 +57,8 @@ unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p,
57 unsigned long iclr_base); 57 unsigned long iclr_base);
58void sun4u_destroy_msi(unsigned int irq); 58void sun4u_destroy_msi(unsigned int irq);
59 59
60unsigned char irq_alloc(unsigned int dev_handle, 60unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino);
61 unsigned int dev_ino);
62#ifdef CONFIG_PCI_MSI
63void irq_free(unsigned int irq); 61void irq_free(unsigned int irq);
64#endif
65 62
66void __init init_IRQ(void); 63void __init init_IRQ(void);
67void fixup_irqs(void); 64void fixup_irqs(void);
diff --git a/arch/sparc/include/asm/ldc.h b/arch/sparc/include/asm/ldc.h
index c8c67f621f4f..58ab64de25d2 100644
--- a/arch/sparc/include/asm/ldc.h
+++ b/arch/sparc/include/asm/ldc.h
@@ -53,13 +53,14 @@ struct ldc_channel;
53/* Allocate state for a channel. */ 53/* Allocate state for a channel. */
54struct ldc_channel *ldc_alloc(unsigned long id, 54struct ldc_channel *ldc_alloc(unsigned long id,
55 const struct ldc_channel_config *cfgp, 55 const struct ldc_channel_config *cfgp,
56 void *event_arg); 56 void *event_arg,
57 const char *name);
57 58
58/* Shut down and free state for a channel. */ 59/* Shut down and free state for a channel. */
59void ldc_free(struct ldc_channel *lp); 60void ldc_free(struct ldc_channel *lp);
60 61
61/* Register TX and RX queues of the link with the hypervisor. */ 62/* Register TX and RX queues of the link with the hypervisor. */
62int ldc_bind(struct ldc_channel *lp, const char *name); 63int ldc_bind(struct ldc_channel *lp);
63 64
64/* For non-RAW protocols we need to complete a handshake before 65/* For non-RAW protocols we need to complete a handshake before
65 * communication can proceed. ldc_connect() does that, if the 66 * communication can proceed. ldc_connect() does that, if the
diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h
index bf109984a032..8c2a8c937540 100644
--- a/arch/sparc/include/asm/page_64.h
+++ b/arch/sparc/include/asm/page_64.h
@@ -57,18 +57,21 @@ void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topa
57typedef struct { unsigned long pte; } pte_t; 57typedef struct { unsigned long pte; } pte_t;
58typedef struct { unsigned long iopte; } iopte_t; 58typedef struct { unsigned long iopte; } iopte_t;
59typedef struct { unsigned long pmd; } pmd_t; 59typedef struct { unsigned long pmd; } pmd_t;
60typedef struct { unsigned long pud; } pud_t;
60typedef struct { unsigned long pgd; } pgd_t; 61typedef struct { unsigned long pgd; } pgd_t;
61typedef struct { unsigned long pgprot; } pgprot_t; 62typedef struct { unsigned long pgprot; } pgprot_t;
62 63
63#define pte_val(x) ((x).pte) 64#define pte_val(x) ((x).pte)
64#define iopte_val(x) ((x).iopte) 65#define iopte_val(x) ((x).iopte)
65#define pmd_val(x) ((x).pmd) 66#define pmd_val(x) ((x).pmd)
67#define pud_val(x) ((x).pud)
66#define pgd_val(x) ((x).pgd) 68#define pgd_val(x) ((x).pgd)
67#define pgprot_val(x) ((x).pgprot) 69#define pgprot_val(x) ((x).pgprot)
68 70
69#define __pte(x) ((pte_t) { (x) } ) 71#define __pte(x) ((pte_t) { (x) } )
70#define __iopte(x) ((iopte_t) { (x) } ) 72#define __iopte(x) ((iopte_t) { (x) } )
71#define __pmd(x) ((pmd_t) { (x) } ) 73#define __pmd(x) ((pmd_t) { (x) } )
74#define __pud(x) ((pud_t) { (x) } )
72#define __pgd(x) ((pgd_t) { (x) } ) 75#define __pgd(x) ((pgd_t) { (x) } )
73#define __pgprot(x) ((pgprot_t) { (x) } ) 76#define __pgprot(x) ((pgprot_t) { (x) } )
74 77
@@ -77,18 +80,21 @@ typedef struct { unsigned long pgprot; } pgprot_t;
77typedef unsigned long pte_t; 80typedef unsigned long pte_t;
78typedef unsigned long iopte_t; 81typedef unsigned long iopte_t;
79typedef unsigned long pmd_t; 82typedef unsigned long pmd_t;
83typedef unsigned long pud_t;
80typedef unsigned long pgd_t; 84typedef unsigned long pgd_t;
81typedef unsigned long pgprot_t; 85typedef unsigned long pgprot_t;
82 86
83#define pte_val(x) (x) 87#define pte_val(x) (x)
84#define iopte_val(x) (x) 88#define iopte_val(x) (x)
85#define pmd_val(x) (x) 89#define pmd_val(x) (x)
90#define pud_val(x) (x)
86#define pgd_val(x) (x) 91#define pgd_val(x) (x)
87#define pgprot_val(x) (x) 92#define pgprot_val(x) (x)
88 93
89#define __pte(x) (x) 94#define __pte(x) (x)
90#define __iopte(x) (x) 95#define __iopte(x) (x)
91#define __pmd(x) (x) 96#define __pmd(x) (x)
97#define __pud(x) (x)
92#define __pgd(x) (x) 98#define __pgd(x) (x)
93#define __pgprot(x) (x) 99#define __pgprot(x) (x)
94 100
@@ -96,21 +102,14 @@ typedef unsigned long pgprot_t;
96 102
97typedef pte_t *pgtable_t; 103typedef pte_t *pgtable_t;
98 104
99/* These two values define the virtual address space range in which we 105extern unsigned long sparc64_va_hole_top;
100 * must forbid 64-bit user processes from making mappings. It used to 106extern unsigned long sparc64_va_hole_bottom;
101 * represent precisely the virtual address space hole present in most
102 * early sparc64 chips including UltraSPARC-I. But now it also is
103 * further constrained by the limits of our page tables, which is
104 * 43-bits of virtual address.
105 */
106#define SPARC64_VA_HOLE_TOP _AC(0xfffffc0000000000,UL)
107#define SPARC64_VA_HOLE_BOTTOM _AC(0x0000040000000000,UL)
108 107
109/* The next two defines specify the actual exclusion region we 108/* The next two defines specify the actual exclusion region we
110 * enforce, wherein we use a 4GB red zone on each side of the VA hole. 109 * enforce, wherein we use a 4GB red zone on each side of the VA hole.
111 */ 110 */
112#define VA_EXCLUDE_START (SPARC64_VA_HOLE_BOTTOM - (1UL << 32UL)) 111#define VA_EXCLUDE_START (sparc64_va_hole_bottom - (1UL << 32UL))
113#define VA_EXCLUDE_END (SPARC64_VA_HOLE_TOP + (1UL << 32UL)) 112#define VA_EXCLUDE_END (sparc64_va_hole_top + (1UL << 32UL))
114 113
115#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ 114#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \
116 _AC(0x0000000070000000,UL) : \ 115 _AC(0x0000000070000000,UL) : \
@@ -118,20 +117,16 @@ typedef pte_t *pgtable_t;
118 117
119#include <asm-generic/memory_model.h> 118#include <asm-generic/memory_model.h>
120 119
121#define PAGE_OFFSET_BY_BITS(X) (-(_AC(1,UL) << (X)))
122extern unsigned long PAGE_OFFSET; 120extern unsigned long PAGE_OFFSET;
123 121
124#endif /* !(__ASSEMBLY__) */ 122#endif /* !(__ASSEMBLY__) */
125 123
126/* The maximum number of physical memory address bits we support, this 124/* The maximum number of physical memory address bits we support. The
127 * is used to size various tables used to manage kernel TLB misses and 125 * largest value we can support is whatever "KPGD_SHIFT + KPTE_BITS"
128 * also the sparsemem code. 126 * evaluates to.
129 */ 127 */
130#define MAX_PHYS_ADDRESS_BITS 47 128#define MAX_PHYS_ADDRESS_BITS 53
131 129
132/* These two shift counts are used when indexing sparc64_valid_addr_bitmap
133 * and kpte_linear_bitmap.
134 */
135#define ILOG2_4MB 22 130#define ILOG2_4MB 22
136#define ILOG2_256MB 28 131#define ILOG2_256MB 28
137 132
diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h
index 39a7ac49b00c..5e3187185b4a 100644
--- a/arch/sparc/include/asm/pgalloc_64.h
+++ b/arch/sparc/include/asm/pgalloc_64.h
@@ -15,6 +15,13 @@
15 15
16extern struct kmem_cache *pgtable_cache; 16extern struct kmem_cache *pgtable_cache;
17 17
18static inline void __pgd_populate(pgd_t *pgd, pud_t *pud)
19{
20 pgd_set(pgd, pud);
21}
22
23#define pgd_populate(MM, PGD, PUD) __pgd_populate(PGD, PUD)
24
18static inline pgd_t *pgd_alloc(struct mm_struct *mm) 25static inline pgd_t *pgd_alloc(struct mm_struct *mm)
19{ 26{
20 return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); 27 return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
@@ -25,7 +32,23 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
25 kmem_cache_free(pgtable_cache, pgd); 32 kmem_cache_free(pgtable_cache, pgd);
26} 33}
27 34
28#define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) 35static inline void __pud_populate(pud_t *pud, pmd_t *pmd)
36{
37 pud_set(pud, pmd);
38}
39
40#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD)
41
42static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
43{
44 return kmem_cache_alloc(pgtable_cache,
45 GFP_KERNEL|__GFP_REPEAT);
46}
47
48static inline void pud_free(struct mm_struct *mm, pud_t *pud)
49{
50 kmem_cache_free(pgtable_cache, pud);
51}
29 52
30static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 53static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
31{ 54{
@@ -91,4 +114,7 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte,
91#define __pmd_free_tlb(tlb, pmd, addr) \ 114#define __pmd_free_tlb(tlb, pmd, addr) \
92 pgtable_free_tlb(tlb, pmd, false) 115 pgtable_free_tlb(tlb, pmd, false)
93 116
117#define __pud_free_tlb(tlb, pud, addr) \
118 pgtable_free_tlb(tlb, pud, false)
119
94#endif /* _SPARC64_PGALLOC_H */ 120#endif /* _SPARC64_PGALLOC_H */
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 3770bf5c6e1b..bfeb626085ac 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -20,8 +20,6 @@
20#include <asm/page.h> 20#include <asm/page.h>
21#include <asm/processor.h> 21#include <asm/processor.h>
22 22
23#include <asm-generic/pgtable-nopud.h>
24
25/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB). 23/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
26 * The page copy blockops can use 0x6000000 to 0x8000000. 24 * The page copy blockops can use 0x6000000 to 0x8000000.
27 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range. 25 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
@@ -42,10 +40,7 @@
42#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL) 40#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL)
43#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL) 41#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL)
44#define VMALLOC_START _AC(0x0000000100000000,UL) 42#define VMALLOC_START _AC(0x0000000100000000,UL)
45#define VMALLOC_END _AC(0x0000010000000000,UL) 43#define VMEMMAP_BASE VMALLOC_END
46#define VMEMMAP_BASE _AC(0x0000010000000000,UL)
47
48#define vmemmap ((struct page *)VMEMMAP_BASE)
49 44
50/* PMD_SHIFT determines the size of the area a second-level page 45/* PMD_SHIFT determines the size of the area a second-level page
51 * table can map 46 * table can map
@@ -55,13 +50,25 @@
55#define PMD_MASK (~(PMD_SIZE-1)) 50#define PMD_MASK (~(PMD_SIZE-1))
56#define PMD_BITS (PAGE_SHIFT - 3) 51#define PMD_BITS (PAGE_SHIFT - 3)
57 52
58/* PGDIR_SHIFT determines what a third-level page table entry can map */ 53/* PUD_SHIFT determines the size of the area a third-level page
59#define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS) 54 * table can map
55 */
56#define PUD_SHIFT (PMD_SHIFT + PMD_BITS)
57#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT)
58#define PUD_MASK (~(PUD_SIZE-1))
59#define PUD_BITS (PAGE_SHIFT - 3)
60
61/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
62#define PGDIR_SHIFT (PUD_SHIFT + PUD_BITS)
60#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) 63#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT)
61#define PGDIR_MASK (~(PGDIR_SIZE-1)) 64#define PGDIR_MASK (~(PGDIR_SIZE-1))
62#define PGDIR_BITS (PAGE_SHIFT - 3) 65#define PGDIR_BITS (PAGE_SHIFT - 3)
63 66
64#if (PGDIR_SHIFT + PGDIR_BITS) != 43 67#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS)
68#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
69#endif
70
71#if (PGDIR_SHIFT + PGDIR_BITS) != 53
65#error Page table parameters do not cover virtual address space properly. 72#error Page table parameters do not cover virtual address space properly.
66#endif 73#endif
67 74
@@ -71,28 +78,18 @@
71 78
72#ifndef __ASSEMBLY__ 79#ifndef __ASSEMBLY__
73 80
74#include <linux/sched.h> 81extern unsigned long VMALLOC_END;
75
76extern unsigned long sparc64_valid_addr_bitmap[];
77 82
78/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ 83#define vmemmap ((struct page *)VMEMMAP_BASE)
79static inline bool __kern_addr_valid(unsigned long paddr)
80{
81 if ((paddr >> MAX_PHYS_ADDRESS_BITS) != 0UL)
82 return false;
83 return test_bit(paddr >> ILOG2_4MB, sparc64_valid_addr_bitmap);
84}
85 84
86static inline bool kern_addr_valid(unsigned long addr) 85#include <linux/sched.h>
87{
88 unsigned long paddr = __pa(addr);
89 86
90 return __kern_addr_valid(paddr); 87bool kern_addr_valid(unsigned long addr);
91}
92 88
93/* Entries per page directory level. */ 89/* Entries per page directory level. */
94#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) 90#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3))
95#define PTRS_PER_PMD (1UL << PMD_BITS) 91#define PTRS_PER_PMD (1UL << PMD_BITS)
92#define PTRS_PER_PUD (1UL << PUD_BITS)
96#define PTRS_PER_PGD (1UL << PGDIR_BITS) 93#define PTRS_PER_PGD (1UL << PGDIR_BITS)
97 94
98/* Kernel has a separate 44bit address space. */ 95/* Kernel has a separate 44bit address space. */
@@ -101,6 +98,9 @@ static inline bool kern_addr_valid(unsigned long addr)
101#define pmd_ERROR(e) \ 98#define pmd_ERROR(e) \
102 pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ 99 pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \
103 __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) 100 __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
101#define pud_ERROR(e) \
102 pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \
103 __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
104#define pgd_ERROR(e) \ 104#define pgd_ERROR(e) \
105 pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ 105 pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \
106 __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) 106 __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
@@ -112,6 +112,7 @@ static inline bool kern_addr_valid(unsigned long addr)
112#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ 112#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
113#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ 113#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
114#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ 114#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */
115#define _PAGE_PUD_HUGE _PAGE_PMD_HUGE
115 116
116/* Advertise support for _PAGE_SPECIAL */ 117/* Advertise support for _PAGE_SPECIAL */
117#define __HAVE_ARCH_PTE_SPECIAL 118#define __HAVE_ARCH_PTE_SPECIAL
@@ -658,26 +659,26 @@ static inline unsigned long pmd_large(pmd_t pmd)
658 return pte_val(pte) & _PAGE_PMD_HUGE; 659 return pte_val(pte) & _PAGE_PMD_HUGE;
659} 660}
660 661
661#ifdef CONFIG_TRANSPARENT_HUGEPAGE 662static inline unsigned long pmd_pfn(pmd_t pmd)
662static inline unsigned long pmd_young(pmd_t pmd)
663{ 663{
664 pte_t pte = __pte(pmd_val(pmd)); 664 pte_t pte = __pte(pmd_val(pmd));
665 665
666 return pte_young(pte); 666 return pte_pfn(pte);
667} 667}
668 668
669static inline unsigned long pmd_write(pmd_t pmd) 669#ifdef CONFIG_TRANSPARENT_HUGEPAGE
670static inline unsigned long pmd_young(pmd_t pmd)
670{ 671{
671 pte_t pte = __pte(pmd_val(pmd)); 672 pte_t pte = __pte(pmd_val(pmd));
672 673
673 return pte_write(pte); 674 return pte_young(pte);
674} 675}
675 676
676static inline unsigned long pmd_pfn(pmd_t pmd) 677static inline unsigned long pmd_write(pmd_t pmd)
677{ 678{
678 pte_t pte = __pte(pmd_val(pmd)); 679 pte_t pte = __pte(pmd_val(pmd));
679 680
680 return pte_pfn(pte); 681 return pte_write(pte);
681} 682}
682 683
683static inline unsigned long pmd_trans_huge(pmd_t pmd) 684static inline unsigned long pmd_trans_huge(pmd_t pmd)
@@ -771,13 +772,15 @@ static inline int pmd_present(pmd_t pmd)
771 * the top bits outside of the range of any physical address size we 772 * the top bits outside of the range of any physical address size we
772 * support are clear as well. We also validate the physical itself. 773 * support are clear as well. We also validate the physical itself.
773 */ 774 */
774#define pmd_bad(pmd) ((pmd_val(pmd) & ~PAGE_MASK) || \ 775#define pmd_bad(pmd) (pmd_val(pmd) & ~PAGE_MASK)
775 !__kern_addr_valid(pmd_val(pmd)))
776 776
777#define pud_none(pud) (!pud_val(pud)) 777#define pud_none(pud) (!pud_val(pud))
778 778
779#define pud_bad(pud) ((pud_val(pud) & ~PAGE_MASK) || \ 779#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK)
780 !__kern_addr_valid(pud_val(pud))) 780
781#define pgd_none(pgd) (!pgd_val(pgd))
782
783#define pgd_bad(pgd) (pgd_val(pgd) & ~PAGE_MASK)
781 784
782#ifdef CONFIG_TRANSPARENT_HUGEPAGE 785#ifdef CONFIG_TRANSPARENT_HUGEPAGE
783void set_pmd_at(struct mm_struct *mm, unsigned long addr, 786void set_pmd_at(struct mm_struct *mm, unsigned long addr,
@@ -815,10 +818,31 @@ static inline unsigned long __pmd_page(pmd_t pmd)
815#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) 818#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL)
816#define pud_present(pud) (pud_val(pud) != 0U) 819#define pud_present(pud) (pud_val(pud) != 0U)
817#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) 820#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL)
821#define pgd_page_vaddr(pgd) \
822 ((unsigned long) __va(pgd_val(pgd)))
823#define pgd_present(pgd) (pgd_val(pgd) != 0U)
824#define pgd_clear(pgdp) (pgd_val(*(pgd)) = 0UL)
825
826static inline unsigned long pud_large(pud_t pud)
827{
828 pte_t pte = __pte(pud_val(pud));
829
830 return pte_val(pte) & _PAGE_PMD_HUGE;
831}
832
833static inline unsigned long pud_pfn(pud_t pud)
834{
835 pte_t pte = __pte(pud_val(pud));
836
837 return pte_pfn(pte);
838}
818 839
819/* Same in both SUN4V and SUN4U. */ 840/* Same in both SUN4V and SUN4U. */
820#define pte_none(pte) (!pte_val(pte)) 841#define pte_none(pte) (!pte_val(pte))
821 842
843#define pgd_set(pgdp, pudp) \
844 (pgd_val(*(pgdp)) = (__pa((unsigned long) (pudp))))
845
822/* to find an entry in a page-table-directory. */ 846/* to find an entry in a page-table-directory. */
823#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 847#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
824#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) 848#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
@@ -826,6 +850,11 @@ static inline unsigned long __pmd_page(pmd_t pmd)
826/* to find an entry in a kernel page-table-directory */ 850/* to find an entry in a kernel page-table-directory */
827#define pgd_offset_k(address) pgd_offset(&init_mm, address) 851#define pgd_offset_k(address) pgd_offset(&init_mm, address)
828 852
853/* Find an entry in the third-level page table.. */
854#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
855#define pud_offset(pgdp, address) \
856 ((pud_t *) pgd_page_vaddr(*(pgdp)) + pud_index(address))
857
829/* Find an entry in the second-level page table.. */ 858/* Find an entry in the second-level page table.. */
830#define pmd_offset(pudp, address) \ 859#define pmd_offset(pudp, address) \
831 ((pmd_t *) pud_page_vaddr(*(pudp)) + \ 860 ((pmd_t *) pud_page_vaddr(*(pudp)) + \
@@ -898,7 +927,6 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
898#endif 927#endif
899 928
900extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; 929extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
901extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD];
902 930
903void paging_init(void); 931void paging_init(void);
904unsigned long find_ecache_flush_span(unsigned long size); 932unsigned long find_ecache_flush_span(unsigned long size);
diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h
index 3fc58691dbd0..56f933816144 100644
--- a/arch/sparc/include/asm/spitfire.h
+++ b/arch/sparc/include/asm/spitfire.h
@@ -45,6 +45,8 @@
45#define SUN4V_CHIP_NIAGARA3 0x03 45#define SUN4V_CHIP_NIAGARA3 0x03
46#define SUN4V_CHIP_NIAGARA4 0x04 46#define SUN4V_CHIP_NIAGARA4 0x04
47#define SUN4V_CHIP_NIAGARA5 0x05 47#define SUN4V_CHIP_NIAGARA5 0x05
48#define SUN4V_CHIP_SPARC_M6 0x06
49#define SUN4V_CHIP_SPARC_M7 0x07
48#define SUN4V_CHIP_SPARC64X 0x8a 50#define SUN4V_CHIP_SPARC64X 0x8a
49#define SUN4V_CHIP_UNKNOWN 0xff 51#define SUN4V_CHIP_UNKNOWN 0xff
50 52
diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h
index a5f01ac6d0f1..f85dc8512ab3 100644
--- a/arch/sparc/include/asm/thread_info_64.h
+++ b/arch/sparc/include/asm/thread_info_64.h
@@ -102,6 +102,7 @@ struct thread_info {
102#define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */ 102#define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */
103#define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */ 103#define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */
104#define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */ 104#define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */
105#define FAULT_CODE_BAD_RA 0x20 /* Bad RA for sun4v */
105 106
106#if PAGE_SHIFT == 13 107#if PAGE_SHIFT == 13
107#define THREAD_SIZE (2*PAGE_SIZE) 108#define THREAD_SIZE (2*PAGE_SIZE)
diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index 90916f955cac..ecb49cfa3be9 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -133,9 +133,24 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
133 sub TSB, 0x8, TSB; \ 133 sub TSB, 0x8, TSB; \
134 TSB_STORE(TSB, TAG); 134 TSB_STORE(TSB, TAG);
135 135
136 /* Do a kernel page table walk. Leaves physical PTE pointer in 136 /* Do a kernel page table walk. Leaves valid PTE value in
137 * REG1. Jumps to FAIL_LABEL on early page table walk termination. 137 * REG1. Jumps to FAIL_LABEL on early page table walk
138 * VADDR will not be clobbered, but REG2 will. 138 * termination. VADDR will not be clobbered, but REG2 will.
139 *
140 * There are two masks we must apply to propagate bits from
141 * the virtual address into the PTE physical address field
142 * when dealing with huge pages. This is because the page
143 * table boundaries do not match the huge page size(s) the
144 * hardware supports.
145 *
146 * In these cases we propagate the bits that are below the
147 * page table level where we saw the huge page mapping, but
148 * are still within the relevant physical bits for the huge
149 * page size in question. So for PMD mappings (which fall on
150 * bit 23, for 8MB per PMD) we must propagate bit 22 for a
151 * 4MB huge page. For huge PUDs (which fall on bit 33, for
152 * 8GB per PUD), we have to accomodate 256MB and 2GB huge
153 * pages. So for those we propagate bits 32 to 28.
139 */ 154 */
140#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \ 155#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
141 sethi %hi(swapper_pg_dir), REG1; \ 156 sethi %hi(swapper_pg_dir), REG1; \
@@ -145,15 +160,40 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
145 andn REG2, 0x7, REG2; \ 160 andn REG2, 0x7, REG2; \
146 ldx [REG1 + REG2], REG1; \ 161 ldx [REG1 + REG2], REG1; \
147 brz,pn REG1, FAIL_LABEL; \ 162 brz,pn REG1, FAIL_LABEL; \
148 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 163 sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
149 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 164 srlx REG2, 64 - PAGE_SHIFT, REG2; \
150 andn REG2, 0x7, REG2; \ 165 andn REG2, 0x7, REG2; \
151 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ 166 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
152 brz,pn REG1, FAIL_LABEL; \ 167 brz,pn REG1, FAIL_LABEL; \
153 sllx VADDR, 64 - PMD_SHIFT, REG2; \ 168 sethi %uhi(_PAGE_PUD_HUGE), REG2; \
169 brz,pn REG1, FAIL_LABEL; \
170 sllx REG2, 32, REG2; \
171 andcc REG1, REG2, %g0; \
172 sethi %hi(0xf8000000), REG2; \
173 bne,pt %xcc, 697f; \
174 sllx REG2, 1, REG2; \
175 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
154 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 176 srlx REG2, 64 - PAGE_SHIFT, REG2; \
155 andn REG2, 0x7, REG2; \ 177 andn REG2, 0x7, REG2; \
156 add REG1, REG2, REG1; 178 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
179 sethi %uhi(_PAGE_PMD_HUGE), REG2; \
180 brz,pn REG1, FAIL_LABEL; \
181 sllx REG2, 32, REG2; \
182 andcc REG1, REG2, %g0; \
183 be,pn %xcc, 698f; \
184 sethi %hi(0x400000), REG2; \
185697: brgez,pn REG1, FAIL_LABEL; \
186 andn REG1, REG2, REG1; \
187 and VADDR, REG2, REG2; \
188 ba,pt %xcc, 699f; \
189 or REG1, REG2, REG1; \
190698: sllx VADDR, 64 - PMD_SHIFT, REG2; \
191 srlx REG2, 64 - PAGE_SHIFT, REG2; \
192 andn REG2, 0x7, REG2; \
193 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
194 brgez,pn REG1, FAIL_LABEL; \
195 nop; \
196699:
157 197
158 /* PMD has been loaded into REG1, interpret the value, seeing 198 /* PMD has been loaded into REG1, interpret the value, seeing
159 * if it is a HUGE PMD or a normal one. If it is not valid 199 * if it is a HUGE PMD or a normal one. If it is not valid
@@ -198,6 +238,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
198 andn REG2, 0x7, REG2; \ 238 andn REG2, 0x7, REG2; \
199 ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ 239 ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
200 brz,pn REG1, FAIL_LABEL; \ 240 brz,pn REG1, FAIL_LABEL; \
241 sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
242 srlx REG2, 64 - PAGE_SHIFT, REG2; \
243 andn REG2, 0x7, REG2; \
244 ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
245 brz,pn REG1, FAIL_LABEL; \
201 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ 246 sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
202 srlx REG2, 64 - PAGE_SHIFT, REG2; \ 247 srlx REG2, 64 - PAGE_SHIFT, REG2; \
203 andn REG2, 0x7, REG2; \ 248 andn REG2, 0x7, REG2; \
@@ -246,8 +291,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
246 (KERNEL_TSB_SIZE_BYTES / 16) 291 (KERNEL_TSB_SIZE_BYTES / 16)
247#define KERNEL_TSB4M_NENTRIES 4096 292#define KERNEL_TSB4M_NENTRIES 4096
248 293
249#define KTSB_PHYS_SHIFT 15
250
251 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL 294 /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
252 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries 295 * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
253 * and the found TTE will be left in REG1. REG3 and REG4 must 296 * and the found TTE will be left in REG1. REG3 and REG4 must
@@ -256,17 +299,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
256 * VADDR and TAG will be preserved and not clobbered by this macro. 299 * VADDR and TAG will be preserved and not clobbered by this macro.
257 */ 300 */
258#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ 301#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
259661: sethi %hi(swapper_tsb), REG1; \ 302661: sethi %uhi(swapper_tsb), REG1; \
260 or REG1, %lo(swapper_tsb), REG1; \ 303 sethi %hi(swapper_tsb), REG2; \
304 or REG1, %ulo(swapper_tsb), REG1; \
305 or REG2, %lo(swapper_tsb), REG2; \
261 .section .swapper_tsb_phys_patch, "ax"; \ 306 .section .swapper_tsb_phys_patch, "ax"; \
262 .word 661b; \ 307 .word 661b; \
263 .previous; \ 308 .previous; \
264661: nop; \ 309 sllx REG1, 32, REG1; \
265 .section .tsb_ldquad_phys_patch, "ax"; \ 310 or REG1, REG2, REG1; \
266 .word 661b; \
267 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
268 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
269 .previous; \
270 srlx VADDR, PAGE_SHIFT, REG2; \ 311 srlx VADDR, PAGE_SHIFT, REG2; \
271 and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ 312 and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
272 sllx REG2, 4, REG2; \ 313 sllx REG2, 4, REG2; \
@@ -281,17 +322,15 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
281 * we can make use of that for the index computation. 322 * we can make use of that for the index computation.
282 */ 323 */
283#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ 324#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
284661: sethi %hi(swapper_4m_tsb), REG1; \ 325661: sethi %uhi(swapper_4m_tsb), REG1; \
285 or REG1, %lo(swapper_4m_tsb), REG1; \ 326 sethi %hi(swapper_4m_tsb), REG2; \
327 or REG1, %ulo(swapper_4m_tsb), REG1; \
328 or REG2, %lo(swapper_4m_tsb), REG2; \
286 .section .swapper_4m_tsb_phys_patch, "ax"; \ 329 .section .swapper_4m_tsb_phys_patch, "ax"; \
287 .word 661b; \ 330 .word 661b; \
288 .previous; \ 331 .previous; \
289661: nop; \ 332 sllx REG1, 32, REG1; \
290 .section .tsb_ldquad_phys_patch, "ax"; \ 333 or REG1, REG2, REG1; \
291 .word 661b; \
292 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
293 sllx REG1, KTSB_PHYS_SHIFT, REG1; \
294 .previous; \
295 and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ 334 and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
296 sllx REG2, 4, REG2; \ 335 sllx REG2, 4, REG2; \
297 add REG1, REG2, REG2; \ 336 add REG1, REG2, REG2; \
diff --git a/arch/sparc/include/asm/vio.h b/arch/sparc/include/asm/vio.h
index 6b135a8ab07b..d758c8d8f47d 100644
--- a/arch/sparc/include/asm/vio.h
+++ b/arch/sparc/include/asm/vio.h
@@ -121,12 +121,18 @@ struct vio_disk_attr_info {
121 u8 vdisk_type; 121 u8 vdisk_type;
122#define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */ 122#define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */
123#define VD_DISK_TYPE_DISK 0x02 /* Entire block device */ 123#define VD_DISK_TYPE_DISK 0x02 /* Entire block device */
124 u16 resv1; 124 u8 vdisk_mtype; /* v1.1 */
125#define VD_MEDIA_TYPE_FIXED 0x01 /* Fixed device */
126#define VD_MEDIA_TYPE_CD 0x02 /* CD Device */
127#define VD_MEDIA_TYPE_DVD 0x03 /* DVD Device */
128 u8 resv1;
125 u32 vdisk_block_size; 129 u32 vdisk_block_size;
126 u64 operations; 130 u64 operations;
127 u64 vdisk_size; 131 u64 vdisk_size; /* v1.1 */
128 u64 max_xfer_size; 132 u64 max_xfer_size;
129 u64 resv2[2]; 133 u32 phys_block_size; /* v1.2 */
134 u32 resv2;
135 u64 resv3[1];
130}; 136};
131 137
132struct vio_disk_desc { 138struct vio_disk_desc {
@@ -272,7 +278,7 @@ static inline u32 vio_dring_avail(struct vio_dring_state *dr,
272 unsigned int ring_size) 278 unsigned int ring_size)
273{ 279{
274 return (dr->pending - 280 return (dr->pending -
275 ((dr->prod - dr->cons) & (ring_size - 1))); 281 ((dr->prod - dr->cons) & (ring_size - 1)) - 1);
276} 282}
277 283
278#define VIO_MAX_TYPE_LEN 32 284#define VIO_MAX_TYPE_LEN 32
@@ -292,6 +298,7 @@ struct vio_dev {
292 298
293 unsigned int tx_irq; 299 unsigned int tx_irq;
294 unsigned int rx_irq; 300 unsigned int rx_irq;
301 u64 rx_ino;
295 302
296 struct device dev; 303 struct device dev;
297}; 304};
@@ -447,5 +454,6 @@ int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev,
447 char *name); 454 char *name);
448 455
449void vio_port_up(struct vio_driver_state *vio); 456void vio_port_up(struct vio_driver_state *vio);
457int vio_set_intr(unsigned long dev_ino, int state);
450 458
451#endif /* _SPARC64_VIO_H */ 459#endif /* _SPARC64_VIO_H */
diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c
index 82a3a71c451e..dfad8b1aea9f 100644
--- a/arch/sparc/kernel/cpu.c
+++ b/arch/sparc/kernel/cpu.c
@@ -494,6 +494,18 @@ static void __init sun4v_cpu_probe(void)
494 sparc_pmu_type = "niagara5"; 494 sparc_pmu_type = "niagara5";
495 break; 495 break;
496 496
497 case SUN4V_CHIP_SPARC_M6:
498 sparc_cpu_type = "SPARC-M6";
499 sparc_fpu_type = "SPARC-M6 integrated FPU";
500 sparc_pmu_type = "sparc-m6";
501 break;
502
503 case SUN4V_CHIP_SPARC_M7:
504 sparc_cpu_type = "SPARC-M7";
505 sparc_fpu_type = "SPARC-M7 integrated FPU";
506 sparc_pmu_type = "sparc-m7";
507 break;
508
497 case SUN4V_CHIP_SPARC64X: 509 case SUN4V_CHIP_SPARC64X:
498 sparc_cpu_type = "SPARC64-X"; 510 sparc_cpu_type = "SPARC64-X";
499 sparc_fpu_type = "SPARC64-X integrated FPU"; 511 sparc_fpu_type = "SPARC64-X integrated FPU";
diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c
index de1c844dfabc..e69ec0e3f155 100644
--- a/arch/sparc/kernel/cpumap.c
+++ b/arch/sparc/kernel/cpumap.c
@@ -326,6 +326,8 @@ static int iterate_cpu(struct cpuinfo_tree *t, unsigned int root_index)
326 case SUN4V_CHIP_NIAGARA3: 326 case SUN4V_CHIP_NIAGARA3:
327 case SUN4V_CHIP_NIAGARA4: 327 case SUN4V_CHIP_NIAGARA4:
328 case SUN4V_CHIP_NIAGARA5: 328 case SUN4V_CHIP_NIAGARA5:
329 case SUN4V_CHIP_SPARC_M6:
330 case SUN4V_CHIP_SPARC_M7:
329 case SUN4V_CHIP_SPARC64X: 331 case SUN4V_CHIP_SPARC64X:
330 rover_inc_table = niagara_iterate_method; 332 rover_inc_table = niagara_iterate_method;
331 break; 333 break;
diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c
index dff60abbea01..f87a55d77094 100644
--- a/arch/sparc/kernel/ds.c
+++ b/arch/sparc/kernel/ds.c
@@ -1200,14 +1200,14 @@ static int ds_probe(struct vio_dev *vdev, const struct vio_device_id *id)
1200 ds_cfg.tx_irq = vdev->tx_irq; 1200 ds_cfg.tx_irq = vdev->tx_irq;
1201 ds_cfg.rx_irq = vdev->rx_irq; 1201 ds_cfg.rx_irq = vdev->rx_irq;
1202 1202
1203 lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp); 1203 lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp, "DS");
1204 if (IS_ERR(lp)) { 1204 if (IS_ERR(lp)) {
1205 err = PTR_ERR(lp); 1205 err = PTR_ERR(lp);
1206 goto out_free_ds_states; 1206 goto out_free_ds_states;
1207 } 1207 }
1208 dp->lp = lp; 1208 dp->lp = lp;
1209 1209
1210 err = ldc_bind(lp, "DS"); 1210 err = ldc_bind(lp);
1211 if (err) 1211 if (err)
1212 goto out_free_ldc; 1212 goto out_free_ldc;
1213 1213
diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
index 452f04fe8da6..4fdeb8040d4d 100644
--- a/arch/sparc/kernel/head_64.S
+++ b/arch/sparc/kernel/head_64.S
@@ -427,6 +427,12 @@ sun4v_chip_type:
427 cmp %g2, '5' 427 cmp %g2, '5'
428 be,pt %xcc, 5f 428 be,pt %xcc, 5f
429 mov SUN4V_CHIP_NIAGARA5, %g4 429 mov SUN4V_CHIP_NIAGARA5, %g4
430 cmp %g2, '6'
431 be,pt %xcc, 5f
432 mov SUN4V_CHIP_SPARC_M6, %g4
433 cmp %g2, '7'
434 be,pt %xcc, 5f
435 mov SUN4V_CHIP_SPARC_M7, %g4
430 ba,pt %xcc, 49f 436 ba,pt %xcc, 49f
431 nop 437 nop
432 438
@@ -585,6 +591,12 @@ niagara_tlb_fixup:
585 cmp %g1, SUN4V_CHIP_NIAGARA5 591 cmp %g1, SUN4V_CHIP_NIAGARA5
586 be,pt %xcc, niagara4_patch 592 be,pt %xcc, niagara4_patch
587 nop 593 nop
594 cmp %g1, SUN4V_CHIP_SPARC_M6
595 be,pt %xcc, niagara4_patch
596 nop
597 cmp %g1, SUN4V_CHIP_SPARC_M7
598 be,pt %xcc, niagara4_patch
599 nop
588 600
589 call generic_patch_copyops 601 call generic_patch_copyops
590 nop 602 nop
diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c
index c0a2de0fd624..5c55145bfbf0 100644
--- a/arch/sparc/kernel/hvapi.c
+++ b/arch/sparc/kernel/hvapi.c
@@ -46,6 +46,7 @@ static struct api_info api_table[] = {
46 { .group = HV_GRP_VF_CPU, }, 46 { .group = HV_GRP_VF_CPU, },
47 { .group = HV_GRP_KT_CPU, }, 47 { .group = HV_GRP_KT_CPU, },
48 { .group = HV_GRP_VT_CPU, }, 48 { .group = HV_GRP_VT_CPU, },
49 { .group = HV_GRP_T5_CPU, },
49 { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API }, 50 { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API },
50}; 51};
51 52
diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S
index f3ab509b76a8..caedf8320416 100644
--- a/arch/sparc/kernel/hvcalls.S
+++ b/arch/sparc/kernel/hvcalls.S
@@ -821,3 +821,19 @@ ENTRY(sun4v_vt_set_perfreg)
821 retl 821 retl
822 nop 822 nop
823ENDPROC(sun4v_vt_set_perfreg) 823ENDPROC(sun4v_vt_set_perfreg)
824
825ENTRY(sun4v_t5_get_perfreg)
826 mov %o1, %o4
827 mov HV_FAST_T5_GET_PERFREG, %o5
828 ta HV_FAST_TRAP
829 stx %o1, [%o4]
830 retl
831 nop
832ENDPROC(sun4v_t5_get_perfreg)
833
834ENTRY(sun4v_t5_set_perfreg)
835 mov HV_FAST_T5_SET_PERFREG, %o5
836 ta HV_FAST_TRAP
837 retl
838 nop
839ENDPROC(sun4v_t5_set_perfreg)
diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c
index 7f08ec8a7c68..28fed53b13a0 100644
--- a/arch/sparc/kernel/ioport.c
+++ b/arch/sparc/kernel/ioport.c
@@ -278,7 +278,8 @@ static void *sbus_alloc_coherent(struct device *dev, size_t len,
278 } 278 }
279 279
280 order = get_order(len_total); 280 order = get_order(len_total);
281 if ((va = __get_free_pages(GFP_KERNEL|__GFP_COMP, order)) == 0) 281 va = __get_free_pages(gfp, order);
282 if (va == 0)
282 goto err_nopages; 283 goto err_nopages;
283 284
284 if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL) 285 if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL)
@@ -443,7 +444,7 @@ static void *pci32_alloc_coherent(struct device *dev, size_t len,
443 } 444 }
444 445
445 order = get_order(len_total); 446 order = get_order(len_total);
446 va = (void *) __get_free_pages(GFP_KERNEL, order); 447 va = (void *) __get_free_pages(gfp, order);
447 if (va == NULL) { 448 if (va == NULL) {
448 printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT); 449 printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT);
449 goto err_nopages; 450 goto err_nopages;
diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c
index 666193f4e8bb..4033c23bdfa6 100644
--- a/arch/sparc/kernel/irq_64.c
+++ b/arch/sparc/kernel/irq_64.c
@@ -47,8 +47,6 @@
47#include "cpumap.h" 47#include "cpumap.h"
48#include "kstack.h" 48#include "kstack.h"
49 49
50#define NUM_IVECS (IMAP_INR + 1)
51
52struct ino_bucket *ivector_table; 50struct ino_bucket *ivector_table;
53unsigned long ivector_table_pa; 51unsigned long ivector_table_pa;
54 52
@@ -107,55 +105,196 @@ static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq)
107 105
108#define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa) 106#define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa)
109 107
110static struct { 108static unsigned long hvirq_major __initdata;
111 unsigned int dev_handle; 109static int __init early_hvirq_major(char *p)
112 unsigned int dev_ino; 110{
113 unsigned int in_use; 111 int rc = kstrtoul(p, 10, &hvirq_major);
114} irq_table[NR_IRQS]; 112
115static DEFINE_SPINLOCK(irq_alloc_lock); 113 return rc;
114}
115early_param("hvirq", early_hvirq_major);
116
117static int hv_irq_version;
118
119/* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie
120 * based interfaces, but:
121 *
122 * 1) Several OSs, Solaris and Linux included, use them even when only
123 * negotiating version 1.0 (or failing to negotiate at all). So the
124 * hypervisor has a workaround that provides the VIRQ interfaces even
125 * when only verion 1.0 of the API is in use.
126 *
127 * 2) Second, and more importantly, with major version 2.0 these VIRQ
128 * interfaces only were actually hooked up for LDC interrupts, even
129 * though the Hypervisor specification clearly stated:
130 *
131 * The new interrupt API functions will be available to a guest
132 * when it negotiates version 2.0 in the interrupt API group 0x2. When
133 * a guest negotiates version 2.0, all interrupt sources will only
134 * support using the cookie interface, and any attempt to use the
135 * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the
136 * ENOTSUPPORTED error being returned.
137 *
138 * with an emphasis on "all interrupt sources".
139 *
140 * To correct this, major version 3.0 was created which does actually
141 * support VIRQs for all interrupt sources (not just LDC devices). So
142 * if we want to move completely over the cookie based VIRQs we must
143 * negotiate major version 3.0 or later of HV_GRP_INTR.
144 */
145static bool sun4v_cookie_only_virqs(void)
146{
147 if (hv_irq_version >= 3)
148 return true;
149 return false;
150}
116 151
117unsigned char irq_alloc(unsigned int dev_handle, unsigned int dev_ino) 152static void __init irq_init_hv(void)
118{ 153{
119 unsigned long flags; 154 unsigned long hv_error, major, minor = 0;
120 unsigned char ent; 155
156 if (tlb_type != hypervisor)
157 return;
121 158
122 BUILD_BUG_ON(NR_IRQS >= 256); 159 if (hvirq_major)
160 major = hvirq_major;
161 else
162 major = 3;
123 163
124 spin_lock_irqsave(&irq_alloc_lock, flags); 164 hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor);
165 if (!hv_error)
166 hv_irq_version = major;
167 else
168 hv_irq_version = 1;
125 169
126 for (ent = 1; ent < NR_IRQS; ent++) { 170 pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n",
127 if (!irq_table[ent].in_use) 171 hv_irq_version,
172 sun4v_cookie_only_virqs() ? "enabled" : "disabled");
173}
174
175/* This function is for the timer interrupt.*/
176int __init arch_probe_nr_irqs(void)
177{
178 return 1;
179}
180
181#define DEFAULT_NUM_IVECS (0xfffU)
182static unsigned int nr_ivec = DEFAULT_NUM_IVECS;
183#define NUM_IVECS (nr_ivec)
184
185static unsigned int __init size_nr_ivec(void)
186{
187 if (tlb_type == hypervisor) {
188 switch (sun4v_chip_type) {
189 /* Athena's devhandle|devino is large.*/
190 case SUN4V_CHIP_SPARC64X:
191 nr_ivec = 0xffff;
128 break; 192 break;
193 }
129 } 194 }
130 if (ent >= NR_IRQS) { 195 return nr_ivec;
131 printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); 196}
132 ent = 0; 197
133 } else { 198struct irq_handler_data {
134 irq_table[ent].dev_handle = dev_handle; 199 union {
135 irq_table[ent].dev_ino = dev_ino; 200 struct {
136 irq_table[ent].in_use = 1; 201 unsigned int dev_handle;
137 } 202 unsigned int dev_ino;
203 };
204 unsigned long sysino;
205 };
206 struct ino_bucket bucket;
207 unsigned long iclr;
208 unsigned long imap;
209};
210
211static inline unsigned int irq_data_to_handle(struct irq_data *data)
212{
213 struct irq_handler_data *ihd = data->handler_data;
214
215 return ihd->dev_handle;
216}
217
218static inline unsigned int irq_data_to_ino(struct irq_data *data)
219{
220 struct irq_handler_data *ihd = data->handler_data;
138 221
139 spin_unlock_irqrestore(&irq_alloc_lock, flags); 222 return ihd->dev_ino;
223}
224
225static inline unsigned long irq_data_to_sysino(struct irq_data *data)
226{
227 struct irq_handler_data *ihd = data->handler_data;
140 228
141 return ent; 229 return ihd->sysino;
142} 230}
143 231
144#ifdef CONFIG_PCI_MSI
145void irq_free(unsigned int irq) 232void irq_free(unsigned int irq)
146{ 233{
147 unsigned long flags; 234 void *data = irq_get_handler_data(irq);
148 235
149 if (irq >= NR_IRQS) 236 kfree(data);
150 return; 237 irq_set_handler_data(irq, NULL);
238 irq_free_descs(irq, 1);
239}
151 240
152 spin_lock_irqsave(&irq_alloc_lock, flags); 241unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino)
242{
243 int irq;
153 244
154 irq_table[irq].in_use = 0; 245 irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL);
246 if (irq <= 0)
247 goto out;
155 248
156 spin_unlock_irqrestore(&irq_alloc_lock, flags); 249 return irq;
250out:
251 return 0;
252}
253
254static unsigned int cookie_exists(u32 devhandle, unsigned int devino)
255{
256 unsigned long hv_err, cookie;
257 struct ino_bucket *bucket;
258 unsigned int irq = 0U;
259
260 hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie);
261 if (hv_err) {
262 pr_err("HV get cookie failed hv_err = %ld\n", hv_err);
263 goto out;
264 }
265
266 if (cookie & ((1UL << 63UL))) {
267 cookie = ~cookie;
268 bucket = (struct ino_bucket *) __va(cookie);
269 irq = bucket->__irq;
270 }
271out:
272 return irq;
273}
274
275static unsigned int sysino_exists(u32 devhandle, unsigned int devino)
276{
277 unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino);
278 struct ino_bucket *bucket;
279 unsigned int irq;
280
281 bucket = &ivector_table[sysino];
282 irq = bucket_get_irq(__pa(bucket));
283
284 return irq;
285}
286
287void ack_bad_irq(unsigned int irq)
288{
289 pr_crit("BAD IRQ ack %d\n", irq);
290}
291
292void irq_install_pre_handler(int irq,
293 void (*func)(unsigned int, void *, void *),
294 void *arg1, void *arg2)
295{
296 pr_warn("IRQ pre handler NOT supported.\n");
157} 297}
158#endif
159 298
160/* 299/*
161 * /proc/interrupts printing: 300 * /proc/interrupts printing:
@@ -206,15 +345,6 @@ static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
206 return tid; 345 return tid;
207} 346}
208 347
209struct irq_handler_data {
210 unsigned long iclr;
211 unsigned long imap;
212
213 void (*pre_handler)(unsigned int, void *, void *);
214 void *arg1;
215 void *arg2;
216};
217
218#ifdef CONFIG_SMP 348#ifdef CONFIG_SMP
219static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) 349static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity)
220{ 350{
@@ -316,8 +446,8 @@ static void sun4u_irq_eoi(struct irq_data *data)
316 446
317static void sun4v_irq_enable(struct irq_data *data) 447static void sun4v_irq_enable(struct irq_data *data)
318{ 448{
319 unsigned int ino = irq_table[data->irq].dev_ino;
320 unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity); 449 unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity);
450 unsigned int ino = irq_data_to_sysino(data);
321 int err; 451 int err;
322 452
323 err = sun4v_intr_settarget(ino, cpuid); 453 err = sun4v_intr_settarget(ino, cpuid);
@@ -337,8 +467,8 @@ static void sun4v_irq_enable(struct irq_data *data)
337static int sun4v_set_affinity(struct irq_data *data, 467static int sun4v_set_affinity(struct irq_data *data,
338 const struct cpumask *mask, bool force) 468 const struct cpumask *mask, bool force)
339{ 469{
340 unsigned int ino = irq_table[data->irq].dev_ino;
341 unsigned long cpuid = irq_choose_cpu(data->irq, mask); 470 unsigned long cpuid = irq_choose_cpu(data->irq, mask);
471 unsigned int ino = irq_data_to_sysino(data);
342 int err; 472 int err;
343 473
344 err = sun4v_intr_settarget(ino, cpuid); 474 err = sun4v_intr_settarget(ino, cpuid);
@@ -351,7 +481,7 @@ static int sun4v_set_affinity(struct irq_data *data,
351 481
352static void sun4v_irq_disable(struct irq_data *data) 482static void sun4v_irq_disable(struct irq_data *data)
353{ 483{
354 unsigned int ino = irq_table[data->irq].dev_ino; 484 unsigned int ino = irq_data_to_sysino(data);
355 int err; 485 int err;
356 486
357 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); 487 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
@@ -362,7 +492,7 @@ static void sun4v_irq_disable(struct irq_data *data)
362 492
363static void sun4v_irq_eoi(struct irq_data *data) 493static void sun4v_irq_eoi(struct irq_data *data)
364{ 494{
365 unsigned int ino = irq_table[data->irq].dev_ino; 495 unsigned int ino = irq_data_to_sysino(data);
366 int err; 496 int err;
367 497
368 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); 498 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
@@ -373,14 +503,13 @@ static void sun4v_irq_eoi(struct irq_data *data)
373 503
374static void sun4v_virq_enable(struct irq_data *data) 504static void sun4v_virq_enable(struct irq_data *data)
375{ 505{
376 unsigned long cpuid, dev_handle, dev_ino; 506 unsigned long dev_handle = irq_data_to_handle(data);
507 unsigned long dev_ino = irq_data_to_ino(data);
508 unsigned long cpuid;
377 int err; 509 int err;
378 510
379 cpuid = irq_choose_cpu(data->irq, data->affinity); 511 cpuid = irq_choose_cpu(data->irq, data->affinity);
380 512
381 dev_handle = irq_table[data->irq].dev_handle;
382 dev_ino = irq_table[data->irq].dev_ino;
383
384 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); 513 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
385 if (err != HV_EOK) 514 if (err != HV_EOK)
386 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " 515 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
@@ -403,14 +532,13 @@ static void sun4v_virq_enable(struct irq_data *data)
403static int sun4v_virt_set_affinity(struct irq_data *data, 532static int sun4v_virt_set_affinity(struct irq_data *data,
404 const struct cpumask *mask, bool force) 533 const struct cpumask *mask, bool force)
405{ 534{
406 unsigned long cpuid, dev_handle, dev_ino; 535 unsigned long dev_handle = irq_data_to_handle(data);
536 unsigned long dev_ino = irq_data_to_ino(data);
537 unsigned long cpuid;
407 int err; 538 int err;
408 539
409 cpuid = irq_choose_cpu(data->irq, mask); 540 cpuid = irq_choose_cpu(data->irq, mask);
410 541
411 dev_handle = irq_table[data->irq].dev_handle;
412 dev_ino = irq_table[data->irq].dev_ino;
413
414 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); 542 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid);
415 if (err != HV_EOK) 543 if (err != HV_EOK)
416 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " 544 printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): "
@@ -422,11 +550,10 @@ static int sun4v_virt_set_affinity(struct irq_data *data,
422 550
423static void sun4v_virq_disable(struct irq_data *data) 551static void sun4v_virq_disable(struct irq_data *data)
424{ 552{
425 unsigned long dev_handle, dev_ino; 553 unsigned long dev_handle = irq_data_to_handle(data);
554 unsigned long dev_ino = irq_data_to_ino(data);
426 int err; 555 int err;
427 556
428 dev_handle = irq_table[data->irq].dev_handle;
429 dev_ino = irq_table[data->irq].dev_ino;
430 557
431 err = sun4v_vintr_set_valid(dev_handle, dev_ino, 558 err = sun4v_vintr_set_valid(dev_handle, dev_ino,
432 HV_INTR_DISABLED); 559 HV_INTR_DISABLED);
@@ -438,12 +565,10 @@ static void sun4v_virq_disable(struct irq_data *data)
438 565
439static void sun4v_virq_eoi(struct irq_data *data) 566static void sun4v_virq_eoi(struct irq_data *data)
440{ 567{
441 unsigned long dev_handle, dev_ino; 568 unsigned long dev_handle = irq_data_to_handle(data);
569 unsigned long dev_ino = irq_data_to_ino(data);
442 int err; 570 int err;
443 571
444 dev_handle = irq_table[data->irq].dev_handle;
445 dev_ino = irq_table[data->irq].dev_ino;
446
447 err = sun4v_vintr_set_state(dev_handle, dev_ino, 572 err = sun4v_vintr_set_state(dev_handle, dev_ino,
448 HV_INTR_STATE_IDLE); 573 HV_INTR_STATE_IDLE);
449 if (err != HV_EOK) 574 if (err != HV_EOK)
@@ -479,31 +604,10 @@ static struct irq_chip sun4v_virq = {
479 .flags = IRQCHIP_EOI_IF_HANDLED, 604 .flags = IRQCHIP_EOI_IF_HANDLED,
480}; 605};
481 606
482static void pre_flow_handler(struct irq_data *d)
483{
484 struct irq_handler_data *handler_data = irq_data_get_irq_handler_data(d);
485 unsigned int ino = irq_table[d->irq].dev_ino;
486
487 handler_data->pre_handler(ino, handler_data->arg1, handler_data->arg2);
488}
489
490void irq_install_pre_handler(int irq,
491 void (*func)(unsigned int, void *, void *),
492 void *arg1, void *arg2)
493{
494 struct irq_handler_data *handler_data = irq_get_handler_data(irq);
495
496 handler_data->pre_handler = func;
497 handler_data->arg1 = arg1;
498 handler_data->arg2 = arg2;
499
500 __irq_set_preflow_handler(irq, pre_flow_handler);
501}
502
503unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) 607unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
504{ 608{
505 struct ino_bucket *bucket;
506 struct irq_handler_data *handler_data; 609 struct irq_handler_data *handler_data;
610 struct ino_bucket *bucket;
507 unsigned int irq; 611 unsigned int irq;
508 int ino; 612 int ino;
509 613
@@ -537,119 +641,166 @@ out:
537 return irq; 641 return irq;
538} 642}
539 643
540static unsigned int sun4v_build_common(unsigned long sysino, 644static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino,
541 struct irq_chip *chip) 645 void (*handler_data_init)(struct irq_handler_data *data,
646 u32 devhandle, unsigned int devino),
647 struct irq_chip *chip)
542{ 648{
543 struct ino_bucket *bucket; 649 struct irq_handler_data *data;
544 struct irq_handler_data *handler_data;
545 unsigned int irq; 650 unsigned int irq;
546 651
547 BUG_ON(tlb_type != hypervisor); 652 irq = irq_alloc(devhandle, devino);
653 if (!irq)
654 goto out;
548 655
549 bucket = &ivector_table[sysino]; 656 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
550 irq = bucket_get_irq(__pa(bucket)); 657 if (unlikely(!data)) {
551 if (!irq) { 658 pr_err("IRQ handler data allocation failed.\n");
552 irq = irq_alloc(0, sysino); 659 irq_free(irq);
553 bucket_set_irq(__pa(bucket), irq); 660 irq = 0;
554 irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, 661 goto out;
555 "IVEC");
556 } 662 }
557 663
558 handler_data = irq_get_handler_data(irq); 664 irq_set_handler_data(irq, data);
559 if (unlikely(handler_data)) 665 handler_data_init(data, devhandle, devino);
560 goto out; 666 irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC");
667 data->imap = ~0UL;
668 data->iclr = ~0UL;
669out:
670 return irq;
671}
561 672
562 handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 673static unsigned long cookie_assign(unsigned int irq, u32 devhandle,
563 if (unlikely(!handler_data)) { 674 unsigned int devino)
564 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); 675{
565 prom_halt(); 676 struct irq_handler_data *ihd = irq_get_handler_data(irq);
566 } 677 unsigned long hv_error, cookie;
567 irq_set_handler_data(irq, handler_data);
568 678
569 /* Catch accidental accesses to these things. IMAP/ICLR handling 679 /* handler_irq needs to find the irq. cookie is seen signed in
570 * is done by hypervisor calls on sun4v platforms, not by direct 680 * sun4v_dev_mondo and treated as a non ivector_table delivery.
571 * register accesses.
572 */ 681 */
573 handler_data->imap = ~0UL; 682 ihd->bucket.__irq = irq;
574 handler_data->iclr = ~0UL; 683 cookie = ~__pa(&ihd->bucket);
575 684
576out: 685 hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie);
577 return irq; 686 if (hv_error)
687 pr_err("HV vintr set cookie failed = %ld\n", hv_error);
688
689 return hv_error;
578} 690}
579 691
580unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) 692static void cookie_handler_data(struct irq_handler_data *data,
693 u32 devhandle, unsigned int devino)
581{ 694{
582 unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); 695 data->dev_handle = devhandle;
696 data->dev_ino = devino;
697}
583 698
584 return sun4v_build_common(sysino, &sun4v_irq); 699static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino,
700 struct irq_chip *chip)
701{
702 unsigned long hv_error;
703 unsigned int irq;
704
705 irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip);
706
707 hv_error = cookie_assign(irq, devhandle, devino);
708 if (hv_error) {
709 irq_free(irq);
710 irq = 0;
711 }
712
713 return irq;
585} 714}
586 715
587unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) 716static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino)
588{ 717{
589 struct irq_handler_data *handler_data;
590 unsigned long hv_err, cookie;
591 struct ino_bucket *bucket;
592 unsigned int irq; 718 unsigned int irq;
593 719
594 bucket = kzalloc(sizeof(struct ino_bucket), GFP_ATOMIC); 720 irq = cookie_exists(devhandle, devino);
595 if (unlikely(!bucket)) 721 if (irq)
596 return 0; 722 goto out;
597 723
598 /* The only reference we store to the IRQ bucket is 724 irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
599 * by physical address which kmemleak can't see, tell
600 * it that this object explicitly is not a leak and
601 * should be scanned.
602 */
603 kmemleak_not_leak(bucket);
604 725
605 __flush_dcache_range((unsigned long) bucket, 726out:
606 ((unsigned long) bucket + 727 return irq;
607 sizeof(struct ino_bucket))); 728}
608 729
609 irq = irq_alloc(devhandle, devino); 730static void sysino_set_bucket(unsigned int irq)
731{
732 struct irq_handler_data *ihd = irq_get_handler_data(irq);
733 struct ino_bucket *bucket;
734 unsigned long sysino;
735
736 sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino);
737 BUG_ON(sysino >= nr_ivec);
738 bucket = &ivector_table[sysino];
610 bucket_set_irq(__pa(bucket), irq); 739 bucket_set_irq(__pa(bucket), irq);
740}
611 741
612 irq_set_chip_and_handler_name(irq, &sun4v_virq, handle_fasteoi_irq, 742static void sysino_handler_data(struct irq_handler_data *data,
613 "IVEC"); 743 u32 devhandle, unsigned int devino)
744{
745 unsigned long sysino;
614 746
615 handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 747 sysino = sun4v_devino_to_sysino(devhandle, devino);
616 if (unlikely(!handler_data)) 748 data->sysino = sysino;
617 return 0; 749}
618 750
619 /* In order to make the LDC channel startup sequence easier, 751static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino,
620 * especially wrt. locking, we do not let request_irq() enable 752 struct irq_chip *chip)
621 * the interrupt. 753{
622 */ 754 unsigned int irq;
623 irq_set_status_flags(irq, IRQ_NOAUTOEN);
624 irq_set_handler_data(irq, handler_data);
625 755
626 /* Catch accidental accesses to these things. IMAP/ICLR handling 756 irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip);
627 * is done by hypervisor calls on sun4v platforms, not by direct 757 if (!irq)
628 * register accesses. 758 goto out;
629 */
630 handler_data->imap = ~0UL;
631 handler_data->iclr = ~0UL;
632 759
633 cookie = ~__pa(bucket); 760 sysino_set_bucket(irq);
634 hv_err = sun4v_vintr_set_cookie(devhandle, devino, cookie); 761out:
635 if (hv_err) { 762 return irq;
636 prom_printf("IRQ: Fatal, cannot set cookie for [%x:%x] " 763}
637 "err=%lu\n", devhandle, devino, hv_err);
638 prom_halt();
639 }
640 764
765static int sun4v_build_sysino(u32 devhandle, unsigned int devino)
766{
767 int irq;
768
769 irq = sysino_exists(devhandle, devino);
770 if (irq)
771 goto out;
772
773 irq = sysino_build_irq(devhandle, devino, &sun4v_irq);
774out:
641 return irq; 775 return irq;
642} 776}
643 777
644void ack_bad_irq(unsigned int irq) 778unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
645{ 779{
646 unsigned int ino = irq_table[irq].dev_ino; 780 unsigned int irq;
647 781
648 if (!ino) 782 if (sun4v_cookie_only_virqs())
649 ino = 0xdeadbeef; 783 irq = sun4v_build_cookie(devhandle, devino);
784 else
785 irq = sun4v_build_sysino(devhandle, devino);
650 786
651 printk(KERN_CRIT "Unexpected IRQ from ino[%x] irq[%u]\n", 787 return irq;
652 ino, irq); 788}
789
790unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino)
791{
792 int irq;
793
794 irq = cookie_build_irq(devhandle, devino, &sun4v_virq);
795 if (!irq)
796 goto out;
797
798 /* This is borrowed from the original function.
799 */
800 irq_set_status_flags(irq, IRQ_NOAUTOEN);
801
802out:
803 return irq;
653} 804}
654 805
655void *hardirq_stack[NR_CPUS]; 806void *hardirq_stack[NR_CPUS];
@@ -720,9 +871,12 @@ void fixup_irqs(void)
720 871
721 for (irq = 0; irq < NR_IRQS; irq++) { 872 for (irq = 0; irq < NR_IRQS; irq++) {
722 struct irq_desc *desc = irq_to_desc(irq); 873 struct irq_desc *desc = irq_to_desc(irq);
723 struct irq_data *data = irq_desc_get_irq_data(desc); 874 struct irq_data *data;
724 unsigned long flags; 875 unsigned long flags;
725 876
877 if (!desc)
878 continue;
879 data = irq_desc_get_irq_data(desc);
726 raw_spin_lock_irqsave(&desc->lock, flags); 880 raw_spin_lock_irqsave(&desc->lock, flags);
727 if (desc->action && !irqd_is_per_cpu(data)) { 881 if (desc->action && !irqd_is_per_cpu(data)) {
728 if (data->chip->irq_set_affinity) 882 if (data->chip->irq_set_affinity)
@@ -922,16 +1076,22 @@ static struct irqaction timer_irq_action = {
922 .name = "timer", 1076 .name = "timer",
923}; 1077};
924 1078
925/* Only invoked on boot processor. */ 1079static void __init irq_ivector_init(void)
926void __init init_IRQ(void)
927{ 1080{
928 unsigned long size; 1081 unsigned long size, order;
1082 unsigned int ivecs;
929 1083
930 map_prom_timers(); 1084 /* If we are doing cookie only VIRQs then we do not need the ivector
931 kill_prom_timer(); 1085 * table to process interrupts.
1086 */
1087 if (sun4v_cookie_only_virqs())
1088 return;
932 1089
933 size = sizeof(struct ino_bucket) * NUM_IVECS; 1090 ivecs = size_nr_ivec();
934 ivector_table = kzalloc(size, GFP_KERNEL); 1091 size = sizeof(struct ino_bucket) * ivecs;
1092 order = get_order(size);
1093 ivector_table = (struct ino_bucket *)
1094 __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
935 if (!ivector_table) { 1095 if (!ivector_table) {
936 prom_printf("Fatal error, cannot allocate ivector_table\n"); 1096 prom_printf("Fatal error, cannot allocate ivector_table\n");
937 prom_halt(); 1097 prom_halt();
@@ -940,6 +1100,15 @@ void __init init_IRQ(void)
940 ((unsigned long) ivector_table) + size); 1100 ((unsigned long) ivector_table) + size);
941 1101
942 ivector_table_pa = __pa(ivector_table); 1102 ivector_table_pa = __pa(ivector_table);
1103}
1104
1105/* Only invoked on boot processor.*/
1106void __init init_IRQ(void)
1107{
1108 irq_init_hv();
1109 irq_ivector_init();
1110 map_prom_timers();
1111 kill_prom_timer();
943 1112
944 if (tlb_type == hypervisor) 1113 if (tlb_type == hypervisor)
945 sun4v_init_mondo_queues(); 1114 sun4v_init_mondo_queues();
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
index 605d49204580..ef0d8e9e1210 100644
--- a/arch/sparc/kernel/ktlb.S
+++ b/arch/sparc/kernel/ktlb.S
@@ -47,14 +47,6 @@ kvmap_itlb_vmalloc_addr:
47 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) 47 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
48 48
49 TSB_LOCK_TAG(%g1, %g2, %g7) 49 TSB_LOCK_TAG(%g1, %g2, %g7)
50
51 /* Load and check PTE. */
52 ldxa [%g5] ASI_PHYS_USE_EC, %g5
53 mov 1, %g7
54 sllx %g7, TSB_TAG_INVALID_BIT, %g7
55 brgez,a,pn %g5, kvmap_itlb_longpath
56 TSB_STORE(%g1, %g7)
57
58 TSB_WRITE(%g1, %g5, %g6) 50 TSB_WRITE(%g1, %g5, %g6)
59 51
60 /* fallthrough to TLB load */ 52 /* fallthrough to TLB load */
@@ -118,6 +110,12 @@ kvmap_dtlb_obp:
118 ba,pt %xcc, kvmap_dtlb_load 110 ba,pt %xcc, kvmap_dtlb_load
119 nop 111 nop
120 112
113kvmap_linear_early:
114 sethi %hi(kern_linear_pte_xor), %g7
115 ldx [%g7 + %lo(kern_linear_pte_xor)], %g2
116 ba,pt %xcc, kvmap_dtlb_tsb4m_load
117 xor %g2, %g4, %g5
118
121 .align 32 119 .align 32
122kvmap_dtlb_tsb4m_load: 120kvmap_dtlb_tsb4m_load:
123 TSB_LOCK_TAG(%g1, %g2, %g7) 121 TSB_LOCK_TAG(%g1, %g2, %g7)
@@ -146,105 +144,17 @@ kvmap_dtlb_4v:
146 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ 144 /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
147 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) 145 KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
148#endif 146#endif
149 /* TSB entry address left in %g1, lookup linear PTE. 147 /* Linear mapping TSB lookup failed. Fallthrough to kernel
150 * Must preserve %g1 and %g6 (TAG). 148 * page table based lookup.
151 */
152kvmap_dtlb_tsb4m_miss:
153 /* Clear the PAGE_OFFSET top virtual bits, shift
154 * down to get PFN, and make sure PFN is in range.
155 */
156661: sllx %g4, 0, %g5
157 .section .page_offset_shift_patch, "ax"
158 .word 661b
159 .previous
160
161 /* Check to see if we know about valid memory at the 4MB
162 * chunk this physical address will reside within.
163 */ 149 */
164661: srlx %g5, MAX_PHYS_ADDRESS_BITS, %g2
165 .section .page_offset_shift_patch, "ax"
166 .word 661b
167 .previous
168
169 brnz,pn %g2, kvmap_dtlb_longpath
170 nop
171
172 /* This unconditional branch and delay-slot nop gets patched
173 * by the sethi sequence once the bitmap is properly setup.
174 */
175 .globl valid_addr_bitmap_insn
176valid_addr_bitmap_insn:
177 ba,pt %xcc, 2f
178 nop
179 .subsection 2
180 .globl valid_addr_bitmap_patch
181valid_addr_bitmap_patch:
182 sethi %hi(sparc64_valid_addr_bitmap), %g7
183 or %g7, %lo(sparc64_valid_addr_bitmap), %g7
184 .previous
185
186661: srlx %g5, ILOG2_4MB, %g2
187 .section .page_offset_shift_patch, "ax"
188 .word 661b
189 .previous
190
191 srlx %g2, 6, %g5
192 and %g2, 63, %g2
193 sllx %g5, 3, %g5
194 ldx [%g7 + %g5], %g5
195 mov 1, %g7
196 sllx %g7, %g2, %g7
197 andcc %g5, %g7, %g0
198 be,pn %xcc, kvmap_dtlb_longpath
199
2002: sethi %hi(kpte_linear_bitmap), %g2
201
202 /* Get the 256MB physical address index. */
203661: sllx %g4, 0, %g5
204 .section .page_offset_shift_patch, "ax"
205 .word 661b
206 .previous
207
208 or %g2, %lo(kpte_linear_bitmap), %g2
209
210661: srlx %g5, ILOG2_256MB, %g5
211 .section .page_offset_shift_patch, "ax"
212 .word 661b
213 .previous
214
215 and %g5, (32 - 1), %g7
216
217 /* Divide by 32 to get the offset into the bitmask. */
218 srlx %g5, 5, %g5
219 add %g7, %g7, %g7
220 sllx %g5, 3, %g5
221
222 /* kern_linear_pte_xor[(mask >> shift) & 3)] */
223 ldx [%g2 + %g5], %g2
224 srlx %g2, %g7, %g7
225 sethi %hi(kern_linear_pte_xor), %g5
226 and %g7, 3, %g7
227 or %g5, %lo(kern_linear_pte_xor), %g5
228 sllx %g7, 3, %g7
229 ldx [%g5 + %g7], %g2
230
231 .globl kvmap_linear_patch 150 .globl kvmap_linear_patch
232kvmap_linear_patch: 151kvmap_linear_patch:
233 ba,pt %xcc, kvmap_dtlb_tsb4m_load 152 ba,a,pt %xcc, kvmap_linear_early
234 xor %g2, %g4, %g5
235 153
236kvmap_dtlb_vmalloc_addr: 154kvmap_dtlb_vmalloc_addr:
237 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) 155 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
238 156
239 TSB_LOCK_TAG(%g1, %g2, %g7) 157 TSB_LOCK_TAG(%g1, %g2, %g7)
240
241 /* Load and check PTE. */
242 ldxa [%g5] ASI_PHYS_USE_EC, %g5
243 mov 1, %g7
244 sllx %g7, TSB_TAG_INVALID_BIT, %g7
245 brgez,a,pn %g5, kvmap_dtlb_longpath
246 TSB_STORE(%g1, %g7)
247
248 TSB_WRITE(%g1, %g5, %g6) 158 TSB_WRITE(%g1, %g5, %g6)
249 159
250 /* fallthrough to TLB load */ 160 /* fallthrough to TLB load */
@@ -276,13 +186,8 @@ kvmap_dtlb_load:
276 186
277#ifdef CONFIG_SPARSEMEM_VMEMMAP 187#ifdef CONFIG_SPARSEMEM_VMEMMAP
278kvmap_vmemmap: 188kvmap_vmemmap:
279 sub %g4, %g5, %g5 189 KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
280 srlx %g5, ILOG2_4MB, %g5 190 ba,a,pt %xcc, kvmap_dtlb_load
281 sethi %hi(vmemmap_table), %g1
282 sllx %g5, 3, %g5
283 or %g1, %lo(vmemmap_table), %g1
284 ba,pt %xcc, kvmap_dtlb_load
285 ldx [%g1 + %g5], %g5
286#endif 191#endif
287 192
288kvmap_dtlb_nonlinear: 193kvmap_dtlb_nonlinear:
@@ -294,8 +199,8 @@ kvmap_dtlb_nonlinear:
294 199
295#ifdef CONFIG_SPARSEMEM_VMEMMAP 200#ifdef CONFIG_SPARSEMEM_VMEMMAP
296 /* Do not use the TSB for vmemmap. */ 201 /* Do not use the TSB for vmemmap. */
297 mov (VMEMMAP_BASE >> 40), %g5 202 sethi %hi(VMEMMAP_BASE), %g5
298 sllx %g5, 40, %g5 203 ldx [%g5 + %lo(VMEMMAP_BASE)], %g5
299 cmp %g4,%g5 204 cmp %g4,%g5
300 bgeu,pn %xcc, kvmap_vmemmap 205 bgeu,pn %xcc, kvmap_vmemmap
301 nop 206 nop
@@ -307,8 +212,8 @@ kvmap_dtlb_tsbmiss:
307 sethi %hi(MODULES_VADDR), %g5 212 sethi %hi(MODULES_VADDR), %g5
308 cmp %g4, %g5 213 cmp %g4, %g5
309 blu,pn %xcc, kvmap_dtlb_longpath 214 blu,pn %xcc, kvmap_dtlb_longpath
310 mov (VMALLOC_END >> 40), %g5 215 sethi %hi(VMALLOC_END), %g5
311 sllx %g5, 40, %g5 216 ldx [%g5 + %lo(VMALLOC_END)], %g5
312 cmp %g4, %g5 217 cmp %g4, %g5
313 bgeu,pn %xcc, kvmap_dtlb_longpath 218 bgeu,pn %xcc, kvmap_dtlb_longpath
314 nop 219 nop
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
index 0af28b984695..4310332872d4 100644
--- a/arch/sparc/kernel/ldc.c
+++ b/arch/sparc/kernel/ldc.c
@@ -1078,7 +1078,8 @@ static void ldc_iommu_release(struct ldc_channel *lp)
1078 1078
1079struct ldc_channel *ldc_alloc(unsigned long id, 1079struct ldc_channel *ldc_alloc(unsigned long id,
1080 const struct ldc_channel_config *cfgp, 1080 const struct ldc_channel_config *cfgp,
1081 void *event_arg) 1081 void *event_arg,
1082 const char *name)
1082{ 1083{
1083 struct ldc_channel *lp; 1084 struct ldc_channel *lp;
1084 const struct ldc_mode_ops *mops; 1085 const struct ldc_mode_ops *mops;
@@ -1093,6 +1094,8 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1093 err = -EINVAL; 1094 err = -EINVAL;
1094 if (!cfgp) 1095 if (!cfgp)
1095 goto out_err; 1096 goto out_err;
1097 if (!name)
1098 goto out_err;
1096 1099
1097 switch (cfgp->mode) { 1100 switch (cfgp->mode) {
1098 case LDC_MODE_RAW: 1101 case LDC_MODE_RAW:
@@ -1185,6 +1188,21 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1185 1188
1186 INIT_HLIST_HEAD(&lp->mh_list); 1189 INIT_HLIST_HEAD(&lp->mh_list);
1187 1190
1191 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1192 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1193
1194 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1195 lp->rx_irq_name, lp);
1196 if (err)
1197 goto out_free_txq;
1198
1199 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1200 lp->tx_irq_name, lp);
1201 if (err) {
1202 free_irq(lp->cfg.rx_irq, lp);
1203 goto out_free_txq;
1204 }
1205
1188 return lp; 1206 return lp;
1189 1207
1190out_free_txq: 1208out_free_txq:
@@ -1237,31 +1255,14 @@ EXPORT_SYMBOL(ldc_free);
1237 * state. This does not initiate a handshake, ldc_connect() does 1255 * state. This does not initiate a handshake, ldc_connect() does
1238 * that. 1256 * that.
1239 */ 1257 */
1240int ldc_bind(struct ldc_channel *lp, const char *name) 1258int ldc_bind(struct ldc_channel *lp)
1241{ 1259{
1242 unsigned long hv_err, flags; 1260 unsigned long hv_err, flags;
1243 int err = -EINVAL; 1261 int err = -EINVAL;
1244 1262
1245 if (!name || 1263 if (lp->state != LDC_STATE_INIT)
1246 (lp->state != LDC_STATE_INIT))
1247 return -EINVAL; 1264 return -EINVAL;
1248 1265
1249 snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1250 snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1251
1252 err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1253 lp->rx_irq_name, lp);
1254 if (err)
1255 return err;
1256
1257 err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1258 lp->tx_irq_name, lp);
1259 if (err) {
1260 free_irq(lp->cfg.rx_irq, lp);
1261 return err;
1262 }
1263
1264
1265 spin_lock_irqsave(&lp->lock, flags); 1266 spin_lock_irqsave(&lp->lock, flags);
1266 1267
1267 enable_irq(lp->cfg.rx_irq); 1268 enable_irq(lp->cfg.rx_irq);
diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c
index 683c4af999de..9bbb8f2bbfcc 100644
--- a/arch/sparc/kernel/leon_kernel.c
+++ b/arch/sparc/kernel/leon_kernel.c
@@ -37,6 +37,7 @@ unsigned long amba_system_id;
37static DEFINE_SPINLOCK(leon_irq_lock); 37static DEFINE_SPINLOCK(leon_irq_lock);
38 38
39static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ 39static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */
40static unsigned long leon3_gptimer_ackmask; /* For clearing pending bit */
40unsigned long leon3_gptimer_irq; /* interrupt controller irq number */ 41unsigned long leon3_gptimer_irq; /* interrupt controller irq number */
41unsigned int sparc_leon_eirq; 42unsigned int sparc_leon_eirq;
42#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu]) 43#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu])
@@ -260,11 +261,19 @@ void leon_update_virq_handling(unsigned int virq,
260 261
261static u32 leon_cycles_offset(void) 262static u32 leon_cycles_offset(void)
262{ 263{
263 u32 rld, val, off; 264 u32 rld, val, ctrl, off;
265
264 rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld); 266 rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld);
265 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); 267 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val);
266 off = rld - val; 268 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
267 return rld - val; 269 if (LEON3_GPTIMER_CTRL_ISPENDING(ctrl)) {
270 val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val);
271 off = 2 * rld - val;
272 } else {
273 off = rld - val;
274 }
275
276 return off;
268} 277}
269 278
270#ifdef CONFIG_SMP 279#ifdef CONFIG_SMP
@@ -302,6 +311,7 @@ void __init leon_init_timers(void)
302 int ampopts; 311 int ampopts;
303 int err; 312 int err;
304 u32 config; 313 u32 config;
314 u32 ctrl;
305 315
306 sparc_config.get_cycles_offset = leon_cycles_offset; 316 sparc_config.get_cycles_offset = leon_cycles_offset;
307 sparc_config.cs_period = 1000000 / HZ; 317 sparc_config.cs_period = 1000000 / HZ;
@@ -374,6 +384,16 @@ void __init leon_init_timers(void)
374 if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq)) 384 if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq))
375 goto bad; 385 goto bad;
376 386
387 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
388 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl,
389 ctrl | LEON3_GPTIMER_CTRL_PENDING);
390 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
391
392 if ((ctrl & LEON3_GPTIMER_CTRL_PENDING) != 0)
393 leon3_gptimer_ackmask = ~LEON3_GPTIMER_CTRL_PENDING;
394 else
395 leon3_gptimer_ackmask = ~0;
396
377 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); 397 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0);
378 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld, 398 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld,
379 (((1000000 / HZ) - 1))); 399 (((1000000 / HZ) - 1)));
@@ -452,6 +472,11 @@ bad:
452 472
453static void leon_clear_clock_irq(void) 473static void leon_clear_clock_irq(void)
454{ 474{
475 u32 ctrl;
476
477 ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl);
478 LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl,
479 ctrl & leon3_gptimer_ackmask);
455} 480}
456 481
457static void leon_load_profile_irq(int cpu, unsigned int limit) 482static void leon_load_profile_irq(int cpu, unsigned int limit)
diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c
index 269af58497aa..7e967c8018c8 100644
--- a/arch/sparc/kernel/pcr.c
+++ b/arch/sparc/kernel/pcr.c
@@ -191,12 +191,41 @@ static const struct pcr_ops n4_pcr_ops = {
191 .pcr_nmi_disable = PCR_N4_PICNPT, 191 .pcr_nmi_disable = PCR_N4_PICNPT,
192}; 192};
193 193
194static u64 n5_pcr_read(unsigned long reg_num)
195{
196 unsigned long val;
197
198 (void) sun4v_t5_get_perfreg(reg_num, &val);
199
200 return val;
201}
202
203static void n5_pcr_write(unsigned long reg_num, u64 val)
204{
205 (void) sun4v_t5_set_perfreg(reg_num, val);
206}
207
208static const struct pcr_ops n5_pcr_ops = {
209 .read_pcr = n5_pcr_read,
210 .write_pcr = n5_pcr_write,
211 .read_pic = n4_pic_read,
212 .write_pic = n4_pic_write,
213 .nmi_picl_value = n4_picl_value,
214 .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE |
215 PCR_N4_UTRACE | PCR_N4_TOE |
216 (26 << PCR_N4_SL_SHIFT)),
217 .pcr_nmi_disable = PCR_N4_PICNPT,
218};
219
220
194static unsigned long perf_hsvc_group; 221static unsigned long perf_hsvc_group;
195static unsigned long perf_hsvc_major; 222static unsigned long perf_hsvc_major;
196static unsigned long perf_hsvc_minor; 223static unsigned long perf_hsvc_minor;
197 224
198static int __init register_perf_hsvc(void) 225static int __init register_perf_hsvc(void)
199{ 226{
227 unsigned long hverror;
228
200 if (tlb_type == hypervisor) { 229 if (tlb_type == hypervisor) {
201 switch (sun4v_chip_type) { 230 switch (sun4v_chip_type) {
202 case SUN4V_CHIP_NIAGARA1: 231 case SUN4V_CHIP_NIAGARA1:
@@ -215,6 +244,10 @@ static int __init register_perf_hsvc(void)
215 perf_hsvc_group = HV_GRP_VT_CPU; 244 perf_hsvc_group = HV_GRP_VT_CPU;
216 break; 245 break;
217 246
247 case SUN4V_CHIP_NIAGARA5:
248 perf_hsvc_group = HV_GRP_T5_CPU;
249 break;
250
218 default: 251 default:
219 return -ENODEV; 252 return -ENODEV;
220 } 253 }
@@ -222,10 +255,12 @@ static int __init register_perf_hsvc(void)
222 255
223 perf_hsvc_major = 1; 256 perf_hsvc_major = 1;
224 perf_hsvc_minor = 0; 257 perf_hsvc_minor = 0;
225 if (sun4v_hvapi_register(perf_hsvc_group, 258 hverror = sun4v_hvapi_register(perf_hsvc_group,
226 perf_hsvc_major, 259 perf_hsvc_major,
227 &perf_hsvc_minor)) { 260 &perf_hsvc_minor);
228 printk("perfmon: Could not register hvapi.\n"); 261 if (hverror) {
262 pr_err("perfmon: Could not register hvapi(0x%lx).\n",
263 hverror);
229 return -ENODEV; 264 return -ENODEV;
230 } 265 }
231 } 266 }
@@ -254,6 +289,10 @@ static int __init setup_sun4v_pcr_ops(void)
254 pcr_ops = &n4_pcr_ops; 289 pcr_ops = &n4_pcr_ops;
255 break; 290 break;
256 291
292 case SUN4V_CHIP_NIAGARA5:
293 pcr_ops = &n5_pcr_ops;
294 break;
295
257 default: 296 default:
258 ret = -ENODEV; 297 ret = -ENODEV;
259 break; 298 break;
diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c
index d35c490a91cb..c9759ad3f34a 100644
--- a/arch/sparc/kernel/perf_event.c
+++ b/arch/sparc/kernel/perf_event.c
@@ -1662,7 +1662,8 @@ static bool __init supported_pmu(void)
1662 sparc_pmu = &niagara2_pmu; 1662 sparc_pmu = &niagara2_pmu;
1663 return true; 1663 return true;
1664 } 1664 }
1665 if (!strcmp(sparc_pmu_type, "niagara4")) { 1665 if (!strcmp(sparc_pmu_type, "niagara4") ||
1666 !strcmp(sparc_pmu_type, "niagara5")) {
1666 sparc_pmu = &niagara4_pmu; 1667 sparc_pmu = &niagara4_pmu;
1667 return true; 1668 return true;
1668 } 1669 }
diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c
index 3fdb455e3318..e629b8377587 100644
--- a/arch/sparc/kernel/setup_64.c
+++ b/arch/sparc/kernel/setup_64.c
@@ -141,21 +141,9 @@ static void __init boot_flags_init(char *commands)
141 process_switch(*commands++); 141 process_switch(*commands++);
142 continue; 142 continue;
143 } 143 }
144 if (!strncmp(commands, "mem=", 4)) { 144 if (!strncmp(commands, "mem=", 4))
145 /* 145 cmdline_memory_size = memparse(commands + 4, &commands);
146 * "mem=XXX[kKmM]" overrides the PROM-reported 146
147 * memory size.
148 */
149 cmdline_memory_size = simple_strtoul(commands + 4,
150 &commands, 0);
151 if (*commands == 'K' || *commands == 'k') {
152 cmdline_memory_size <<= 10;
153 commands++;
154 } else if (*commands=='M' || *commands=='m') {
155 cmdline_memory_size <<= 20;
156 commands++;
157 }
158 }
159 while (*commands && *commands != ' ') 147 while (*commands && *commands != ' ')
160 commands++; 148 commands++;
161 } 149 }
@@ -500,12 +488,16 @@ static void __init init_sparc64_elf_hwcap(void)
500 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 488 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
501 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 489 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
502 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 490 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
491 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
492 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
503 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 493 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
504 cap |= HWCAP_SPARC_BLKINIT; 494 cap |= HWCAP_SPARC_BLKINIT;
505 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || 495 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
506 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 496 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
507 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 497 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
508 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 498 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
499 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
500 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
509 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 501 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
510 cap |= HWCAP_SPARC_N2; 502 cap |= HWCAP_SPARC_N2;
511 } 503 }
@@ -533,6 +525,8 @@ static void __init init_sparc64_elf_hwcap(void)
533 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 525 sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
534 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 526 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
535 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 527 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
528 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
529 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
536 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 530 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
537 cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 | 531 cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 |
538 AV_SPARC_ASI_BLK_INIT | 532 AV_SPARC_ASI_BLK_INIT |
@@ -540,6 +534,8 @@ static void __init init_sparc64_elf_hwcap(void)
540 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || 534 if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
541 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || 535 sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
542 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || 536 sun4v_chip_type == SUN4V_CHIP_NIAGARA5 ||
537 sun4v_chip_type == SUN4V_CHIP_SPARC_M6 ||
538 sun4v_chip_type == SUN4V_CHIP_SPARC_M7 ||
543 sun4v_chip_type == SUN4V_CHIP_SPARC64X) 539 sun4v_chip_type == SUN4V_CHIP_SPARC64X)
544 cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC | 540 cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC |
545 AV_SPARC_FMAF); 541 AV_SPARC_FMAF);
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index f7ba87543e5f..c9300bfaee5a 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -1467,6 +1467,13 @@ static void __init pcpu_populate_pte(unsigned long addr)
1467 pud_t *pud; 1467 pud_t *pud;
1468 pmd_t *pmd; 1468 pmd_t *pmd;
1469 1469
1470 if (pgd_none(*pgd)) {
1471 pud_t *new;
1472
1473 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1474 pgd_populate(&init_mm, pgd, new);
1475 }
1476
1470 pud = pud_offset(pgd, addr); 1477 pud = pud_offset(pgd, addr);
1471 if (pud_none(*pud)) { 1478 if (pud_none(*pud)) {
1472 pmd_t *new; 1479 pmd_t *new;
diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S
index e0c09bf85610..6179e19bc9b9 100644
--- a/arch/sparc/kernel/sun4v_tlb_miss.S
+++ b/arch/sparc/kernel/sun4v_tlb_miss.S
@@ -195,6 +195,11 @@ sun4v_tsb_miss_common:
195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 195 ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
196 196
197sun4v_itlb_error: 197sun4v_itlb_error:
198 rdpr %tl, %g1
199 cmp %g1, 1
200 ble,pt %icc, sun4v_bad_ra
201 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
202
198 sethi %hi(sun4v_err_itlb_vaddr), %g1 203 sethi %hi(sun4v_err_itlb_vaddr), %g1
199 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] 204 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
200 sethi %hi(sun4v_err_itlb_ctx), %g1 205 sethi %hi(sun4v_err_itlb_ctx), %g1
@@ -206,15 +211,10 @@ sun4v_itlb_error:
206 sethi %hi(sun4v_err_itlb_error), %g1 211 sethi %hi(sun4v_err_itlb_error), %g1
207 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] 212 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)]
208 213
214 sethi %hi(1f), %g7
209 rdpr %tl, %g4 215 rdpr %tl, %g4
210 cmp %g4, 1
211 ble,pt %icc, 1f
212 sethi %hi(2f), %g7
213 ba,pt %xcc, etraptl1 216 ba,pt %xcc, etraptl1
214 or %g7, %lo(2f), %g7 2171: or %g7, %lo(1f), %g7
215
2161: ba,pt %xcc, etrap
2172: or %g7, %lo(2b), %g7
218 mov %l4, %o1 218 mov %l4, %o1
219 call sun4v_itlb_error_report 219 call sun4v_itlb_error_report
220 add %sp, PTREGS_OFF, %o0 220 add %sp, PTREGS_OFF, %o0
@@ -222,6 +222,11 @@ sun4v_itlb_error:
222 /* NOTREACHED */ 222 /* NOTREACHED */
223 223
224sun4v_dtlb_error: 224sun4v_dtlb_error:
225 rdpr %tl, %g1
226 cmp %g1, 1
227 ble,pt %icc, sun4v_bad_ra
228 or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
229
225 sethi %hi(sun4v_err_dtlb_vaddr), %g1 230 sethi %hi(sun4v_err_dtlb_vaddr), %g1
226 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] 231 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
227 sethi %hi(sun4v_err_dtlb_ctx), %g1 232 sethi %hi(sun4v_err_dtlb_ctx), %g1
@@ -233,21 +238,23 @@ sun4v_dtlb_error:
233 sethi %hi(sun4v_err_dtlb_error), %g1 238 sethi %hi(sun4v_err_dtlb_error), %g1
234 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] 239 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)]
235 240
241 sethi %hi(1f), %g7
236 rdpr %tl, %g4 242 rdpr %tl, %g4
237 cmp %g4, 1
238 ble,pt %icc, 1f
239 sethi %hi(2f), %g7
240 ba,pt %xcc, etraptl1 243 ba,pt %xcc, etraptl1
241 or %g7, %lo(2f), %g7 2441: or %g7, %lo(1f), %g7
242
2431: ba,pt %xcc, etrap
2442: or %g7, %lo(2b), %g7
245 mov %l4, %o1 245 mov %l4, %o1
246 call sun4v_dtlb_error_report 246 call sun4v_dtlb_error_report
247 add %sp, PTREGS_OFF, %o0 247 add %sp, PTREGS_OFF, %o0
248 248
249 /* NOTREACHED */ 249 /* NOTREACHED */
250 250
251sun4v_bad_ra:
252 or %g0, %g4, %g5
253 ba,pt %xcc, sparc64_realfault_common
254 or %g1, %g0, %g4
255
256 /* NOTREACHED */
257
251 /* Instruction Access Exception, tl0. */ 258 /* Instruction Access Exception, tl0. */
252sun4v_iacc: 259sun4v_iacc:
253 ldxa [%g0] ASI_SCRATCHPAD, %g2 260 ldxa [%g0] ASI_SCRATCHPAD, %g2
diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c
index fb6640ec8557..981a769b9558 100644
--- a/arch/sparc/kernel/traps_64.c
+++ b/arch/sparc/kernel/traps_64.c
@@ -2104,6 +2104,11 @@ void sun4v_nonresum_overflow(struct pt_regs *regs)
2104 atomic_inc(&sun4v_nonresum_oflow_cnt); 2104 atomic_inc(&sun4v_nonresum_oflow_cnt);
2105} 2105}
2106 2106
2107static void sun4v_tlb_error(struct pt_regs *regs)
2108{
2109 die_if_kernel("TLB/TSB error", regs);
2110}
2111
2107unsigned long sun4v_err_itlb_vaddr; 2112unsigned long sun4v_err_itlb_vaddr;
2108unsigned long sun4v_err_itlb_ctx; 2113unsigned long sun4v_err_itlb_ctx;
2109unsigned long sun4v_err_itlb_pte; 2114unsigned long sun4v_err_itlb_pte;
@@ -2111,8 +2116,7 @@ unsigned long sun4v_err_itlb_error;
2111 2116
2112void sun4v_itlb_error_report(struct pt_regs *regs, int tl) 2117void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2113{ 2118{
2114 if (tl > 1) 2119 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2115 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2116 2120
2117 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n", 2121 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2118 regs->tpc, tl); 2122 regs->tpc, tl);
@@ -2125,7 +2129,7 @@ void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2125 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, 2129 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2126 sun4v_err_itlb_pte, sun4v_err_itlb_error); 2130 sun4v_err_itlb_pte, sun4v_err_itlb_error);
2127 2131
2128 prom_halt(); 2132 sun4v_tlb_error(regs);
2129} 2133}
2130 2134
2131unsigned long sun4v_err_dtlb_vaddr; 2135unsigned long sun4v_err_dtlb_vaddr;
@@ -2135,8 +2139,7 @@ unsigned long sun4v_err_dtlb_error;
2135 2139
2136void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) 2140void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2137{ 2141{
2138 if (tl > 1) 2142 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2139 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2140 2143
2141 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n", 2144 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2142 regs->tpc, tl); 2145 regs->tpc, tl);
@@ -2149,7 +2152,7 @@ void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2149 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, 2152 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2150 sun4v_err_dtlb_pte, sun4v_err_dtlb_error); 2153 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2151 2154
2152 prom_halt(); 2155 sun4v_tlb_error(regs);
2153} 2156}
2154 2157
2155void hypervisor_tlbop_error(unsigned long err, unsigned long op) 2158void hypervisor_tlbop_error(unsigned long err, unsigned long op)
diff --git a/arch/sparc/kernel/vio.c b/arch/sparc/kernel/vio.c
index 8647fcc5ca6c..cb5789c9f961 100644
--- a/arch/sparc/kernel/vio.c
+++ b/arch/sparc/kernel/vio.c
@@ -180,8 +180,10 @@ static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp,
180 vdev->tx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); 180 vdev->tx_irq = sun4v_build_virq(cdev_cfg_handle, *irq);
181 181
182 irq = mdesc_get_property(hp, target, "rx-ino", NULL); 182 irq = mdesc_get_property(hp, target, "rx-ino", NULL);
183 if (irq) 183 if (irq) {
184 vdev->rx_irq = sun4v_build_virq(cdev_cfg_handle, *irq); 184 vdev->rx_irq = sun4v_build_virq(cdev_cfg_handle, *irq);
185 vdev->rx_ino = *irq;
186 }
185 187
186 chan_id = mdesc_get_property(hp, target, "id", NULL); 188 chan_id = mdesc_get_property(hp, target, "id", NULL);
187 if (chan_id) 189 if (chan_id)
@@ -189,6 +191,15 @@ static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp,
189 } 191 }
190} 192}
191 193
194int vio_set_intr(unsigned long dev_ino, int state)
195{
196 int err;
197
198 err = sun4v_vintr_set_valid(cdev_cfg_handle, dev_ino, state);
199 return err;
200}
201EXPORT_SYMBOL(vio_set_intr);
202
192static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, 203static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp,
193 struct device *parent) 204 struct device *parent)
194{ 205{
diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c
index 7ef081a185b1..526fcb5d8ce9 100644
--- a/arch/sparc/kernel/viohs.c
+++ b/arch/sparc/kernel/viohs.c
@@ -724,7 +724,7 @@ int vio_ldc_alloc(struct vio_driver_state *vio,
724 cfg.tx_irq = vio->vdev->tx_irq; 724 cfg.tx_irq = vio->vdev->tx_irq;
725 cfg.rx_irq = vio->vdev->rx_irq; 725 cfg.rx_irq = vio->vdev->rx_irq;
726 726
727 lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg); 727 lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg, vio->name);
728 if (IS_ERR(lp)) 728 if (IS_ERR(lp))
729 return PTR_ERR(lp); 729 return PTR_ERR(lp);
730 730
@@ -756,7 +756,7 @@ void vio_port_up(struct vio_driver_state *vio)
756 756
757 err = 0; 757 err = 0;
758 if (state == LDC_STATE_INIT) { 758 if (state == LDC_STATE_INIT) {
759 err = ldc_bind(vio->lp, vio->name); 759 err = ldc_bind(vio->lp);
760 if (err) 760 if (err)
761 printk(KERN_WARNING "%s: Port %lu bind failed, " 761 printk(KERN_WARNING "%s: Port %lu bind failed, "
762 "err=%d\n", 762 "err=%d\n",
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index 932ff90fd760..09243057cb0b 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -35,8 +35,9 @@ jiffies = jiffies_64;
35 35
36SECTIONS 36SECTIONS
37{ 37{
38 /* swapper_low_pmd_dir is sparc64 only */ 38#ifdef CONFIG_SPARC64
39 swapper_low_pmd_dir = 0x0000000000402000; 39 swapper_pg_dir = 0x0000000000402000;
40#endif
40 . = INITIAL_ADDRESS; 41 . = INITIAL_ADDRESS;
41 .text TEXTSTART : 42 .text TEXTSTART :
42 { 43 {
@@ -122,11 +123,6 @@ SECTIONS
122 *(.swapper_4m_tsb_phys_patch) 123 *(.swapper_4m_tsb_phys_patch)
123 __swapper_4m_tsb_phys_patch_end = .; 124 __swapper_4m_tsb_phys_patch_end = .;
124 } 125 }
125 .page_offset_shift_patch : {
126 __page_offset_shift_patch = .;
127 *(.page_offset_shift_patch)
128 __page_offset_shift_patch_end = .;
129 }
130 .popc_3insn_patch : { 126 .popc_3insn_patch : {
131 __popc_3insn_patch = .; 127 __popc_3insn_patch = .;
132 *(.popc_3insn_patch) 128 *(.popc_3insn_patch)
diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S
index 99c017be8719..f75e6906df14 100644
--- a/arch/sparc/lib/memset.S
+++ b/arch/sparc/lib/memset.S
@@ -3,8 +3,9 @@
3 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 3 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) 4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
5 * 5 *
6 * Returns 0, if ok, and number of bytes not yet set if exception 6 * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and
7 * occurs and we were called as clear_user. 7 * number of bytes not yet set if exception occurs and we were called as
8 * clear_user.
8 */ 9 */
9 10
10#include <asm/ptrace.h> 11#include <asm/ptrace.h>
@@ -65,6 +66,8 @@ __bzero_begin:
65 .globl __memset_start, __memset_end 66 .globl __memset_start, __memset_end
66__memset_start: 67__memset_start:
67memset: 68memset:
69 mov %o0, %g1
70 mov 1, %g4
68 and %o1, 0xff, %g3 71 and %o1, 0xff, %g3
69 sll %g3, 8, %g2 72 sll %g3, 8, %g2
70 or %g3, %g2, %g3 73 or %g3, %g2, %g3
@@ -89,6 +92,7 @@ memset:
89 sub %o0, %o2, %o0 92 sub %o0, %o2, %o0
90 93
91__bzero: 94__bzero:
95 clr %g4
92 mov %g0, %g3 96 mov %g0, %g3
931: 971:
94 cmp %o1, 7 98 cmp %o1, 7
@@ -151,8 +155,8 @@ __bzero:
151 bne,a 8f 155 bne,a 8f
152 EX(stb %g3, [%o0], and %o1, 1) 156 EX(stb %g3, [%o0], and %o1, 1)
1538: 1578:
154 retl 158 b 0f
155 clr %o0 159 nop
1567: 1607:
157 be 13b 161 be 13b
158 orcc %o1, 0, %g0 162 orcc %o1, 0, %g0
@@ -164,6 +168,12 @@ __bzero:
164 bne 8b 168 bne 8b
165 EX(stb %g3, [%o0 - 1], add %o1, 1) 169 EX(stb %g3, [%o0 - 1], add %o1, 1)
1660: 1700:
171 andcc %g4, 1, %g0
172 be 5f
173 nop
174 retl
175 mov %g1, %o0
1765:
167 retl 177 retl
168 clr %o0 178 clr %o0
169__memset_end: 179__memset_end:
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 587cd0565128..18fcd7167095 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -346,6 +346,9 @@ retry:
346 down_read(&mm->mmap_sem); 346 down_read(&mm->mmap_sem);
347 } 347 }
348 348
349 if (fault_code & FAULT_CODE_BAD_RA)
350 goto do_sigbus;
351
349 vma = find_vma(mm, address); 352 vma = find_vma(mm, address);
350 if (!vma) 353 if (!vma)
351 goto bad_area; 354 goto bad_area;
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 98ac8e80adae..2d91c62f7f5f 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -75,7 +75,6 @@ unsigned long kern_linear_pte_xor[4] __read_mostly;
75 * 'cpu' properties, but we need to have this table setup before the 75 * 'cpu' properties, but we need to have this table setup before the
76 * MDESC is initialized. 76 * MDESC is initialized.
77 */ 77 */
78unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
79 78
80#ifndef CONFIG_DEBUG_PAGEALLOC 79#ifndef CONFIG_DEBUG_PAGEALLOC
81/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. 80/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
@@ -84,10 +83,11 @@ unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
84 */ 83 */
85extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; 84extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
86#endif 85#endif
86extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
87 87
88static unsigned long cpu_pgsz_mask; 88static unsigned long cpu_pgsz_mask;
89 89
90#define MAX_BANKS 32 90#define MAX_BANKS 1024
91 91
92static struct linux_prom64_registers pavail[MAX_BANKS]; 92static struct linux_prom64_registers pavail[MAX_BANKS];
93static int pavail_ents; 93static int pavail_ents;
@@ -165,10 +165,6 @@ static void __init read_obp_memory(const char *property,
165 cmp_p64, NULL); 165 cmp_p64, NULL);
166} 166}
167 167
168unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
169 sizeof(unsigned long)];
170EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
171
172/* Kernel physical address base and size in bytes. */ 168/* Kernel physical address base and size in bytes. */
173unsigned long kern_base __read_mostly; 169unsigned long kern_base __read_mostly;
174unsigned long kern_size __read_mostly; 170unsigned long kern_size __read_mostly;
@@ -840,7 +836,10 @@ static int find_node(unsigned long addr)
840 if ((addr & p->mask) == p->val) 836 if ((addr & p->mask) == p->val)
841 return i; 837 return i;
842 } 838 }
843 return -1; 839 /* The following condition has been observed on LDOM guests.*/
840 WARN_ONCE(1, "find_node: A physical address doesn't match a NUMA node"
841 " rule. Some physical memory will be owned by node 0.");
842 return 0;
844} 843}
845 844
846static u64 memblock_nid_range(u64 start, u64 end, int *nid) 845static u64 memblock_nid_range(u64 start, u64 end, int *nid)
@@ -1366,9 +1365,144 @@ static unsigned long __init bootmem_init(unsigned long phys_base)
1366static struct linux_prom64_registers pall[MAX_BANKS] __initdata; 1365static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1367static int pall_ents __initdata; 1366static int pall_ents __initdata;
1368 1367
1369#ifdef CONFIG_DEBUG_PAGEALLOC 1368static unsigned long max_phys_bits = 40;
1369
1370bool kern_addr_valid(unsigned long addr)
1371{
1372 pgd_t *pgd;
1373 pud_t *pud;
1374 pmd_t *pmd;
1375 pte_t *pte;
1376
1377 if ((long)addr < 0L) {
1378 unsigned long pa = __pa(addr);
1379
1380 if ((addr >> max_phys_bits) != 0UL)
1381 return false;
1382
1383 return pfn_valid(pa >> PAGE_SHIFT);
1384 }
1385
1386 if (addr >= (unsigned long) KERNBASE &&
1387 addr < (unsigned long)&_end)
1388 return true;
1389
1390 pgd = pgd_offset_k(addr);
1391 if (pgd_none(*pgd))
1392 return 0;
1393
1394 pud = pud_offset(pgd, addr);
1395 if (pud_none(*pud))
1396 return 0;
1397
1398 if (pud_large(*pud))
1399 return pfn_valid(pud_pfn(*pud));
1400
1401 pmd = pmd_offset(pud, addr);
1402 if (pmd_none(*pmd))
1403 return 0;
1404
1405 if (pmd_large(*pmd))
1406 return pfn_valid(pmd_pfn(*pmd));
1407
1408 pte = pte_offset_kernel(pmd, addr);
1409 if (pte_none(*pte))
1410 return 0;
1411
1412 return pfn_valid(pte_pfn(*pte));
1413}
1414EXPORT_SYMBOL(kern_addr_valid);
1415
1416static unsigned long __ref kernel_map_hugepud(unsigned long vstart,
1417 unsigned long vend,
1418 pud_t *pud)
1419{
1420 const unsigned long mask16gb = (1UL << 34) - 1UL;
1421 u64 pte_val = vstart;
1422
1423 /* Each PUD is 8GB */
1424 if ((vstart & mask16gb) ||
1425 (vend - vstart <= mask16gb)) {
1426 pte_val ^= kern_linear_pte_xor[2];
1427 pud_val(*pud) = pte_val | _PAGE_PUD_HUGE;
1428
1429 return vstart + PUD_SIZE;
1430 }
1431
1432 pte_val ^= kern_linear_pte_xor[3];
1433 pte_val |= _PAGE_PUD_HUGE;
1434
1435 vend = vstart + mask16gb + 1UL;
1436 while (vstart < vend) {
1437 pud_val(*pud) = pte_val;
1438
1439 pte_val += PUD_SIZE;
1440 vstart += PUD_SIZE;
1441 pud++;
1442 }
1443 return vstart;
1444}
1445
1446static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,
1447 bool guard)
1448{
1449 if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)
1450 return true;
1451
1452 return false;
1453}
1454
1455static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,
1456 unsigned long vend,
1457 pmd_t *pmd)
1458{
1459 const unsigned long mask256mb = (1UL << 28) - 1UL;
1460 const unsigned long mask2gb = (1UL << 31) - 1UL;
1461 u64 pte_val = vstart;
1462
1463 /* Each PMD is 8MB */
1464 if ((vstart & mask256mb) ||
1465 (vend - vstart <= mask256mb)) {
1466 pte_val ^= kern_linear_pte_xor[0];
1467 pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE;
1468
1469 return vstart + PMD_SIZE;
1470 }
1471
1472 if ((vstart & mask2gb) ||
1473 (vend - vstart <= mask2gb)) {
1474 pte_val ^= kern_linear_pte_xor[1];
1475 pte_val |= _PAGE_PMD_HUGE;
1476 vend = vstart + mask256mb + 1UL;
1477 } else {
1478 pte_val ^= kern_linear_pte_xor[2];
1479 pte_val |= _PAGE_PMD_HUGE;
1480 vend = vstart + mask2gb + 1UL;
1481 }
1482
1483 while (vstart < vend) {
1484 pmd_val(*pmd) = pte_val;
1485
1486 pte_val += PMD_SIZE;
1487 vstart += PMD_SIZE;
1488 pmd++;
1489 }
1490
1491 return vstart;
1492}
1493
1494static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,
1495 bool guard)
1496{
1497 if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)
1498 return true;
1499
1500 return false;
1501}
1502
1370static unsigned long __ref kernel_map_range(unsigned long pstart, 1503static unsigned long __ref kernel_map_range(unsigned long pstart,
1371 unsigned long pend, pgprot_t prot) 1504 unsigned long pend, pgprot_t prot,
1505 bool use_huge)
1372{ 1506{
1373 unsigned long vstart = PAGE_OFFSET + pstart; 1507 unsigned long vstart = PAGE_OFFSET + pstart;
1374 unsigned long vend = PAGE_OFFSET + pend; 1508 unsigned long vend = PAGE_OFFSET + pend;
@@ -1387,19 +1521,34 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
1387 pmd_t *pmd; 1521 pmd_t *pmd;
1388 pte_t *pte; 1522 pte_t *pte;
1389 1523
1524 if (pgd_none(*pgd)) {
1525 pud_t *new;
1526
1527 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1528 alloc_bytes += PAGE_SIZE;
1529 pgd_populate(&init_mm, pgd, new);
1530 }
1390 pud = pud_offset(pgd, vstart); 1531 pud = pud_offset(pgd, vstart);
1391 if (pud_none(*pud)) { 1532 if (pud_none(*pud)) {
1392 pmd_t *new; 1533 pmd_t *new;
1393 1534
1535 if (kernel_can_map_hugepud(vstart, vend, use_huge)) {
1536 vstart = kernel_map_hugepud(vstart, vend, pud);
1537 continue;
1538 }
1394 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1539 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1395 alloc_bytes += PAGE_SIZE; 1540 alloc_bytes += PAGE_SIZE;
1396 pud_populate(&init_mm, pud, new); 1541 pud_populate(&init_mm, pud, new);
1397 } 1542 }
1398 1543
1399 pmd = pmd_offset(pud, vstart); 1544 pmd = pmd_offset(pud, vstart);
1400 if (!pmd_present(*pmd)) { 1545 if (pmd_none(*pmd)) {
1401 pte_t *new; 1546 pte_t *new;
1402 1547
1548 if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {
1549 vstart = kernel_map_hugepmd(vstart, vend, pmd);
1550 continue;
1551 }
1403 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1552 new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1404 alloc_bytes += PAGE_SIZE; 1553 alloc_bytes += PAGE_SIZE;
1405 pmd_populate_kernel(&init_mm, pmd, new); 1554 pmd_populate_kernel(&init_mm, pmd, new);
@@ -1422,100 +1571,34 @@ static unsigned long __ref kernel_map_range(unsigned long pstart,
1422 return alloc_bytes; 1571 return alloc_bytes;
1423} 1572}
1424 1573
1425extern unsigned int kvmap_linear_patch[1]; 1574static void __init flush_all_kernel_tsbs(void)
1426#endif /* CONFIG_DEBUG_PAGEALLOC */
1427
1428static void __init kpte_set_val(unsigned long index, unsigned long val)
1429{
1430 unsigned long *ptr = kpte_linear_bitmap;
1431
1432 val <<= ((index % (BITS_PER_LONG / 2)) * 2);
1433 ptr += (index / (BITS_PER_LONG / 2));
1434
1435 *ptr |= val;
1436}
1437
1438static const unsigned long kpte_shift_min = 28; /* 256MB */
1439static const unsigned long kpte_shift_max = 34; /* 16GB */
1440static const unsigned long kpte_shift_incr = 3;
1441
1442static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,
1443 unsigned long shift)
1444{ 1575{
1445 unsigned long size = (1UL << shift); 1576 int i;
1446 unsigned long mask = (size - 1UL);
1447 unsigned long remains = end - start;
1448 unsigned long val;
1449
1450 if (remains < size || (start & mask))
1451 return start;
1452
1453 /* VAL maps:
1454 *
1455 * shift 28 --> kern_linear_pte_xor index 1
1456 * shift 31 --> kern_linear_pte_xor index 2
1457 * shift 34 --> kern_linear_pte_xor index 3
1458 */
1459 val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;
1460
1461 remains &= ~mask;
1462 if (shift != kpte_shift_max)
1463 remains = size;
1464
1465 while (remains) {
1466 unsigned long index = start >> kpte_shift_min;
1467 1577
1468 kpte_set_val(index, val); 1578 for (i = 0; i < KERNEL_TSB_NENTRIES; i++) {
1579 struct tsb *ent = &swapper_tsb[i];
1469 1580
1470 start += 1UL << kpte_shift_min; 1581 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1471 remains -= 1UL << kpte_shift_min;
1472 } 1582 }
1583#ifndef CONFIG_DEBUG_PAGEALLOC
1584 for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) {
1585 struct tsb *ent = &swapper_4m_tsb[i];
1473 1586
1474 return start; 1587 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1475}
1476
1477static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
1478{
1479 unsigned long smallest_size, smallest_mask;
1480 unsigned long s;
1481
1482 smallest_size = (1UL << kpte_shift_min);
1483 smallest_mask = (smallest_size - 1UL);
1484
1485 while (start < end) {
1486 unsigned long orig_start = start;
1487
1488 for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {
1489 start = kpte_mark_using_shift(start, end, s);
1490
1491 if (start != orig_start)
1492 break;
1493 }
1494
1495 if (start == orig_start)
1496 start = (start + smallest_size) & ~smallest_mask;
1497 } 1588 }
1589#endif
1498} 1590}
1499 1591
1500static void __init init_kpte_bitmap(void) 1592extern unsigned int kvmap_linear_patch[1];
1501{
1502 unsigned long i;
1503
1504 for (i = 0; i < pall_ents; i++) {
1505 unsigned long phys_start, phys_end;
1506
1507 phys_start = pall[i].phys_addr;
1508 phys_end = phys_start + pall[i].reg_size;
1509
1510 mark_kpte_bitmap(phys_start, phys_end);
1511 }
1512}
1513 1593
1514static void __init kernel_physical_mapping_init(void) 1594static void __init kernel_physical_mapping_init(void)
1515{ 1595{
1516#ifdef CONFIG_DEBUG_PAGEALLOC
1517 unsigned long i, mem_alloced = 0UL; 1596 unsigned long i, mem_alloced = 0UL;
1597 bool use_huge = true;
1518 1598
1599#ifdef CONFIG_DEBUG_PAGEALLOC
1600 use_huge = false;
1601#endif
1519 for (i = 0; i < pall_ents; i++) { 1602 for (i = 0; i < pall_ents; i++) {
1520 unsigned long phys_start, phys_end; 1603 unsigned long phys_start, phys_end;
1521 1604
@@ -1523,7 +1606,7 @@ static void __init kernel_physical_mapping_init(void)
1523 phys_end = phys_start + pall[i].reg_size; 1606 phys_end = phys_start + pall[i].reg_size;
1524 1607
1525 mem_alloced += kernel_map_range(phys_start, phys_end, 1608 mem_alloced += kernel_map_range(phys_start, phys_end,
1526 PAGE_KERNEL); 1609 PAGE_KERNEL, use_huge);
1527 } 1610 }
1528 1611
1529 printk("Allocated %ld bytes for kernel page tables.\n", 1612 printk("Allocated %ld bytes for kernel page tables.\n",
@@ -1532,8 +1615,9 @@ static void __init kernel_physical_mapping_init(void)
1532 kvmap_linear_patch[0] = 0x01000000; /* nop */ 1615 kvmap_linear_patch[0] = 0x01000000; /* nop */
1533 flushi(&kvmap_linear_patch[0]); 1616 flushi(&kvmap_linear_patch[0]);
1534 1617
1618 flush_all_kernel_tsbs();
1619
1535 __flush_tlb_all(); 1620 __flush_tlb_all();
1536#endif
1537} 1621}
1538 1622
1539#ifdef CONFIG_DEBUG_PAGEALLOC 1623#ifdef CONFIG_DEBUG_PAGEALLOC
@@ -1543,7 +1627,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
1543 unsigned long phys_end = phys_start + (numpages * PAGE_SIZE); 1627 unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1544 1628
1545 kernel_map_range(phys_start, phys_end, 1629 kernel_map_range(phys_start, phys_end,
1546 (enable ? PAGE_KERNEL : __pgprot(0))); 1630 (enable ? PAGE_KERNEL : __pgprot(0)), false);
1547 1631
1548 flush_tsb_kernel_range(PAGE_OFFSET + phys_start, 1632 flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1549 PAGE_OFFSET + phys_end); 1633 PAGE_OFFSET + phys_end);
@@ -1571,76 +1655,56 @@ unsigned long __init find_ecache_flush_span(unsigned long size)
1571unsigned long PAGE_OFFSET; 1655unsigned long PAGE_OFFSET;
1572EXPORT_SYMBOL(PAGE_OFFSET); 1656EXPORT_SYMBOL(PAGE_OFFSET);
1573 1657
1574static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits) 1658unsigned long VMALLOC_END = 0x0000010000000000UL;
1575{ 1659EXPORT_SYMBOL(VMALLOC_END);
1576 unsigned long final_shift;
1577 unsigned int val = *insn;
1578 unsigned int cnt;
1579
1580 /* We are patching in ilog2(max_supported_phys_address), and
1581 * we are doing so in a manner similar to a relocation addend.
1582 * That is, we are adding the shift value to whatever value
1583 * is in the shift instruction count field already.
1584 */
1585 cnt = (val & 0x3f);
1586 val &= ~0x3f;
1587
1588 /* If we are trying to shift >= 64 bits, clear the destination
1589 * register. This can happen when phys_bits ends up being equal
1590 * to MAX_PHYS_ADDRESS_BITS.
1591 */
1592 final_shift = (cnt + (64 - phys_bits));
1593 if (final_shift >= 64) {
1594 unsigned int rd = (val >> 25) & 0x1f;
1595
1596 val = 0x80100000 | (rd << 25);
1597 } else {
1598 val |= final_shift;
1599 }
1600 *insn = val;
1601
1602 __asm__ __volatile__("flush %0"
1603 : /* no outputs */
1604 : "r" (insn));
1605}
1606
1607static void __init page_offset_shift_patch(unsigned long phys_bits)
1608{
1609 extern unsigned int __page_offset_shift_patch;
1610 extern unsigned int __page_offset_shift_patch_end;
1611 unsigned int *p;
1612
1613 p = &__page_offset_shift_patch;
1614 while (p < &__page_offset_shift_patch_end) {
1615 unsigned int *insn = (unsigned int *)(unsigned long)*p;
1616 1660
1617 page_offset_shift_patch_one(insn, phys_bits); 1661unsigned long sparc64_va_hole_top = 0xfffff80000000000UL;
1618 1662unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL;
1619 p++;
1620 }
1621}
1622 1663
1623static void __init setup_page_offset(void) 1664static void __init setup_page_offset(void)
1624{ 1665{
1625 unsigned long max_phys_bits = 40;
1626
1627 if (tlb_type == cheetah || tlb_type == cheetah_plus) { 1666 if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1667 /* Cheetah/Panther support a full 64-bit virtual
1668 * address, so we can use all that our page tables
1669 * support.
1670 */
1671 sparc64_va_hole_top = 0xfff0000000000000UL;
1672 sparc64_va_hole_bottom = 0x0010000000000000UL;
1673
1628 max_phys_bits = 42; 1674 max_phys_bits = 42;
1629 } else if (tlb_type == hypervisor) { 1675 } else if (tlb_type == hypervisor) {
1630 switch (sun4v_chip_type) { 1676 switch (sun4v_chip_type) {
1631 case SUN4V_CHIP_NIAGARA1: 1677 case SUN4V_CHIP_NIAGARA1:
1632 case SUN4V_CHIP_NIAGARA2: 1678 case SUN4V_CHIP_NIAGARA2:
1679 /* T1 and T2 support 48-bit virtual addresses. */
1680 sparc64_va_hole_top = 0xffff800000000000UL;
1681 sparc64_va_hole_bottom = 0x0000800000000000UL;
1682
1633 max_phys_bits = 39; 1683 max_phys_bits = 39;
1634 break; 1684 break;
1635 case SUN4V_CHIP_NIAGARA3: 1685 case SUN4V_CHIP_NIAGARA3:
1686 /* T3 supports 48-bit virtual addresses. */
1687 sparc64_va_hole_top = 0xffff800000000000UL;
1688 sparc64_va_hole_bottom = 0x0000800000000000UL;
1689
1636 max_phys_bits = 43; 1690 max_phys_bits = 43;
1637 break; 1691 break;
1638 case SUN4V_CHIP_NIAGARA4: 1692 case SUN4V_CHIP_NIAGARA4:
1639 case SUN4V_CHIP_NIAGARA5: 1693 case SUN4V_CHIP_NIAGARA5:
1640 case SUN4V_CHIP_SPARC64X: 1694 case SUN4V_CHIP_SPARC64X:
1641 default: 1695 case SUN4V_CHIP_SPARC_M6:
1696 /* T4 and later support 52-bit virtual addresses. */
1697 sparc64_va_hole_top = 0xfff8000000000000UL;
1698 sparc64_va_hole_bottom = 0x0008000000000000UL;
1642 max_phys_bits = 47; 1699 max_phys_bits = 47;
1643 break; 1700 break;
1701 case SUN4V_CHIP_SPARC_M7:
1702 default:
1703 /* M7 and later support 52-bit virtual addresses. */
1704 sparc64_va_hole_top = 0xfff8000000000000UL;
1705 sparc64_va_hole_bottom = 0x0008000000000000UL;
1706 max_phys_bits = 49;
1707 break;
1644 } 1708 }
1645 } 1709 }
1646 1710
@@ -1650,12 +1714,16 @@ static void __init setup_page_offset(void)
1650 prom_halt(); 1714 prom_halt();
1651 } 1715 }
1652 1716
1653 PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits); 1717 PAGE_OFFSET = sparc64_va_hole_top;
1718 VMALLOC_END = ((sparc64_va_hole_bottom >> 1) +
1719 (sparc64_va_hole_bottom >> 2));
1654 1720
1655 pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", 1721 pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
1656 PAGE_OFFSET, max_phys_bits); 1722 PAGE_OFFSET, max_phys_bits);
1657 1723 pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n",
1658 page_offset_shift_patch(max_phys_bits); 1724 VMALLOC_START, VMALLOC_END);
1725 pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n",
1726 VMEMMAP_BASE, VMEMMAP_BASE << 1);
1659} 1727}
1660 1728
1661static void __init tsb_phys_patch(void) 1729static void __init tsb_phys_patch(void)
@@ -1700,21 +1768,42 @@ static void __init tsb_phys_patch(void)
1700#define NUM_KTSB_DESCR 1 1768#define NUM_KTSB_DESCR 1
1701#endif 1769#endif
1702static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; 1770static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1703extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; 1771
1772/* The swapper TSBs are loaded with a base sequence of:
1773 *
1774 * sethi %uhi(SYMBOL), REG1
1775 * sethi %hi(SYMBOL), REG2
1776 * or REG1, %ulo(SYMBOL), REG1
1777 * or REG2, %lo(SYMBOL), REG2
1778 * sllx REG1, 32, REG1
1779 * or REG1, REG2, REG1
1780 *
1781 * When we use physical addressing for the TSB accesses, we patch the
1782 * first four instructions in the above sequence.
1783 */
1704 1784
1705static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) 1785static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
1706{ 1786{
1707 pa >>= KTSB_PHYS_SHIFT; 1787 unsigned long high_bits, low_bits;
1788
1789 high_bits = (pa >> 32) & 0xffffffff;
1790 low_bits = (pa >> 0) & 0xffffffff;
1708 1791
1709 while (start < end) { 1792 while (start < end) {
1710 unsigned int *ia = (unsigned int *)(unsigned long)*start; 1793 unsigned int *ia = (unsigned int *)(unsigned long)*start;
1711 1794
1712 ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); 1795 ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10);
1713 __asm__ __volatile__("flush %0" : : "r" (ia)); 1796 __asm__ __volatile__("flush %0" : : "r" (ia));
1714 1797
1715 ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); 1798 ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10);
1716 __asm__ __volatile__("flush %0" : : "r" (ia + 1)); 1799 __asm__ __volatile__("flush %0" : : "r" (ia + 1));
1717 1800
1801 ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff);
1802 __asm__ __volatile__("flush %0" : : "r" (ia + 2));
1803
1804 ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff);
1805 __asm__ __volatile__("flush %0" : : "r" (ia + 3));
1806
1718 start++; 1807 start++;
1719 } 1808 }
1720} 1809}
@@ -1853,11 +1942,56 @@ static void __init sun4v_linear_pte_xor_finalize(void)
1853/* paging_init() sets up the page tables */ 1942/* paging_init() sets up the page tables */
1854 1943
1855static unsigned long last_valid_pfn; 1944static unsigned long last_valid_pfn;
1856pgd_t swapper_pg_dir[PTRS_PER_PGD];
1857 1945
1858static void sun4u_pgprot_init(void); 1946static void sun4u_pgprot_init(void);
1859static void sun4v_pgprot_init(void); 1947static void sun4v_pgprot_init(void);
1860 1948
1949static phys_addr_t __init available_memory(void)
1950{
1951 phys_addr_t available = 0ULL;
1952 phys_addr_t pa_start, pa_end;
1953 u64 i;
1954
1955 for_each_free_mem_range(i, NUMA_NO_NODE, &pa_start, &pa_end, NULL)
1956 available = available + (pa_end - pa_start);
1957
1958 return available;
1959}
1960
1961/* We need to exclude reserved regions. This exclusion will include
1962 * vmlinux and initrd. To be more precise the initrd size could be used to
1963 * compute a new lower limit because it is freed later during initialization.
1964 */
1965static void __init reduce_memory(phys_addr_t limit_ram)
1966{
1967 phys_addr_t avail_ram = available_memory();
1968 phys_addr_t pa_start, pa_end;
1969 u64 i;
1970
1971 if (limit_ram >= avail_ram)
1972 return;
1973
1974 for_each_free_mem_range(i, NUMA_NO_NODE, &pa_start, &pa_end, NULL) {
1975 phys_addr_t region_size = pa_end - pa_start;
1976 phys_addr_t clip_start = pa_start;
1977
1978 avail_ram = avail_ram - region_size;
1979 /* Are we consuming too much? */
1980 if (avail_ram < limit_ram) {
1981 phys_addr_t give_back = limit_ram - avail_ram;
1982
1983 region_size = region_size - give_back;
1984 clip_start = clip_start + give_back;
1985 }
1986
1987 memblock_remove(clip_start, region_size);
1988
1989 if (avail_ram <= limit_ram)
1990 break;
1991 i = 0UL;
1992 }
1993}
1994
1861void __init paging_init(void) 1995void __init paging_init(void)
1862{ 1996{
1863 unsigned long end_pfn, shift, phys_base; 1997 unsigned long end_pfn, shift, phys_base;
@@ -1937,7 +2071,8 @@ void __init paging_init(void)
1937 2071
1938 find_ramdisk(phys_base); 2072 find_ramdisk(phys_base);
1939 2073
1940 memblock_enforce_memory_limit(cmdline_memory_size); 2074 if (cmdline_memory_size)
2075 reduce_memory(cmdline_memory_size);
1941 2076
1942 memblock_allow_resize(); 2077 memblock_allow_resize();
1943 memblock_dump_all(); 2078 memblock_dump_all();
@@ -1956,16 +2091,10 @@ void __init paging_init(void)
1956 */ 2091 */
1957 init_mm.pgd += ((shift) / (sizeof(pgd_t))); 2092 init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1958 2093
1959 memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir)); 2094 memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
1960 2095
1961 /* Now can init the kernel/bad page tables. */
1962 pud_set(pud_offset(&swapper_pg_dir[0], 0),
1963 swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
1964
1965 inherit_prom_mappings(); 2096 inherit_prom_mappings();
1966 2097
1967 init_kpte_bitmap();
1968
1969 /* Ok, we can use our TLB miss and window trap handlers safely. */ 2098 /* Ok, we can use our TLB miss and window trap handlers safely. */
1970 setup_tba(); 2099 setup_tba();
1971 2100
@@ -2072,70 +2201,6 @@ int page_in_phys_avail(unsigned long paddr)
2072 return 0; 2201 return 0;
2073} 2202}
2074 2203
2075static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
2076static int pavail_rescan_ents __initdata;
2077
2078/* Certain OBP calls, such as fetching "available" properties, can
2079 * claim physical memory. So, along with initializing the valid
2080 * address bitmap, what we do here is refetch the physical available
2081 * memory list again, and make sure it provides at least as much
2082 * memory as 'pavail' does.
2083 */
2084static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
2085{
2086 int i;
2087
2088 read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
2089
2090 for (i = 0; i < pavail_ents; i++) {
2091 unsigned long old_start, old_end;
2092
2093 old_start = pavail[i].phys_addr;
2094 old_end = old_start + pavail[i].reg_size;
2095 while (old_start < old_end) {
2096 int n;
2097
2098 for (n = 0; n < pavail_rescan_ents; n++) {
2099 unsigned long new_start, new_end;
2100
2101 new_start = pavail_rescan[n].phys_addr;
2102 new_end = new_start +
2103 pavail_rescan[n].reg_size;
2104
2105 if (new_start <= old_start &&
2106 new_end >= (old_start + PAGE_SIZE)) {
2107 set_bit(old_start >> ILOG2_4MB, bitmap);
2108 goto do_next_page;
2109 }
2110 }
2111
2112 prom_printf("mem_init: Lost memory in pavail\n");
2113 prom_printf("mem_init: OLD start[%lx] size[%lx]\n",
2114 pavail[i].phys_addr,
2115 pavail[i].reg_size);
2116 prom_printf("mem_init: NEW start[%lx] size[%lx]\n",
2117 pavail_rescan[i].phys_addr,
2118 pavail_rescan[i].reg_size);
2119 prom_printf("mem_init: Cannot continue, aborting.\n");
2120 prom_halt();
2121
2122 do_next_page:
2123 old_start += PAGE_SIZE;
2124 }
2125 }
2126}
2127
2128static void __init patch_tlb_miss_handler_bitmap(void)
2129{
2130 extern unsigned int valid_addr_bitmap_insn[];
2131 extern unsigned int valid_addr_bitmap_patch[];
2132
2133 valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
2134 mb();
2135 valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
2136 flushi(&valid_addr_bitmap_insn[0]);
2137}
2138
2139static void __init register_page_bootmem_info(void) 2204static void __init register_page_bootmem_info(void)
2140{ 2205{
2141#ifdef CONFIG_NEED_MULTIPLE_NODES 2206#ifdef CONFIG_NEED_MULTIPLE_NODES
@@ -2148,18 +2213,6 @@ static void __init register_page_bootmem_info(void)
2148} 2213}
2149void __init mem_init(void) 2214void __init mem_init(void)
2150{ 2215{
2151 unsigned long addr, last;
2152
2153 addr = PAGE_OFFSET + kern_base;
2154 last = PAGE_ALIGN(kern_size) + addr;
2155 while (addr < last) {
2156 set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);
2157 addr += PAGE_SIZE;
2158 }
2159
2160 setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
2161 patch_tlb_miss_handler_bitmap();
2162
2163 high_memory = __va(last_valid_pfn << PAGE_SHIFT); 2216 high_memory = __va(last_valid_pfn << PAGE_SHIFT);
2164 2217
2165 register_page_bootmem_info(); 2218 register_page_bootmem_info();
@@ -2249,18 +2302,9 @@ unsigned long _PAGE_CACHE __read_mostly;
2249EXPORT_SYMBOL(_PAGE_CACHE); 2302EXPORT_SYMBOL(_PAGE_CACHE);
2250 2303
2251#ifdef CONFIG_SPARSEMEM_VMEMMAP 2304#ifdef CONFIG_SPARSEMEM_VMEMMAP
2252unsigned long vmemmap_table[VMEMMAP_SIZE];
2253
2254static long __meminitdata addr_start, addr_end;
2255static int __meminitdata node_start;
2256
2257int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, 2305int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2258 int node) 2306 int node)
2259{ 2307{
2260 unsigned long phys_start = (vstart - VMEMMAP_BASE);
2261 unsigned long phys_end = (vend - VMEMMAP_BASE);
2262 unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
2263 unsigned long end = VMEMMAP_ALIGN(phys_end);
2264 unsigned long pte_base; 2308 unsigned long pte_base;
2265 2309
2266 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U | 2310 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
@@ -2271,47 +2315,52 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2271 _PAGE_CP_4V | _PAGE_CV_4V | 2315 _PAGE_CP_4V | _PAGE_CV_4V |
2272 _PAGE_P_4V | _PAGE_W_4V); 2316 _PAGE_P_4V | _PAGE_W_4V);
2273 2317
2274 for (; addr < end; addr += VMEMMAP_CHUNK) { 2318 pte_base |= _PAGE_PMD_HUGE;
2275 unsigned long *vmem_pp =
2276 vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);
2277 void *block;
2278 2319
2279 if (!(*vmem_pp & _PAGE_VALID)) { 2320 vstart = vstart & PMD_MASK;
2280 block = vmemmap_alloc_block(1UL << ILOG2_4MB, node); 2321 vend = ALIGN(vend, PMD_SIZE);
2281 if (!block) 2322 for (; vstart < vend; vstart += PMD_SIZE) {
2323 pgd_t *pgd = pgd_offset_k(vstart);
2324 unsigned long pte;
2325 pud_t *pud;
2326 pmd_t *pmd;
2327
2328 if (pgd_none(*pgd)) {
2329 pud_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2330
2331 if (!new)
2282 return -ENOMEM; 2332 return -ENOMEM;
2333 pgd_populate(&init_mm, pgd, new);
2334 }
2283 2335
2284 *vmem_pp = pte_base | __pa(block); 2336 pud = pud_offset(pgd, vstart);
2337 if (pud_none(*pud)) {
2338 pmd_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2285 2339
2286 /* check to see if we have contiguous blocks */ 2340 if (!new)
2287 if (addr_end != addr || node_start != node) { 2341 return -ENOMEM;
2288 if (addr_start) 2342 pud_populate(&init_mm, pud, new);
2289 printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
2290 addr_start, addr_end-1, node_start);
2291 addr_start = addr;
2292 node_start = node;
2293 }
2294 addr_end = addr + VMEMMAP_CHUNK;
2295 } 2343 }
2296 }
2297 return 0;
2298}
2299 2344
2300void __meminit vmemmap_populate_print_last(void) 2345 pmd = pmd_offset(pud, vstart);
2301{ 2346
2302 if (addr_start) { 2347 pte = pmd_val(*pmd);
2303 printk(KERN_DEBUG " [%lx-%lx] on node %d\n", 2348 if (!(pte & _PAGE_VALID)) {
2304 addr_start, addr_end-1, node_start); 2349 void *block = vmemmap_alloc_block(PMD_SIZE, node);
2305 addr_start = 0; 2350
2306 addr_end = 0; 2351 if (!block)
2307 node_start = 0; 2352 return -ENOMEM;
2353
2354 pmd_val(*pmd) = pte_base | __pa(block);
2355 }
2308 } 2356 }
2357
2358 return 0;
2309} 2359}
2310 2360
2311void vmemmap_free(unsigned long start, unsigned long end) 2361void vmemmap_free(unsigned long start, unsigned long end)
2312{ 2362{
2313} 2363}
2314
2315#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 2364#endif /* CONFIG_SPARSEMEM_VMEMMAP */
2316 2365
2317static void prot_init_common(unsigned long page_none, 2366static void prot_init_common(unsigned long page_none,
@@ -2787,8 +2836,8 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
2787 do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS); 2836 do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS);
2788 } 2837 }
2789 if (end > HI_OBP_ADDRESS) { 2838 if (end > HI_OBP_ADDRESS) {
2790 flush_tsb_kernel_range(end, HI_OBP_ADDRESS); 2839 flush_tsb_kernel_range(HI_OBP_ADDRESS, end);
2791 do_flush_tlb_kernel_range(end, HI_OBP_ADDRESS); 2840 do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end);
2792 } 2841 }
2793 } else { 2842 } else {
2794 flush_tsb_kernel_range(start, end); 2843 flush_tsb_kernel_range(start, end);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 0668b364f44d..a4c09603b05c 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -8,15 +8,8 @@
8 */ 8 */
9 9
10#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) 10#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS)
11#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
12#define KPTE_BITMAP_BYTES \
13 ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4)
14#define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL)
15#define VALID_ADDR_BITMAP_BYTES \
16 ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8)
17 11
18extern unsigned long kern_linear_pte_xor[4]; 12extern unsigned long kern_linear_pte_xor[4];
19extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
20extern unsigned int sparc64_highest_unlocked_tlb_ent; 13extern unsigned int sparc64_highest_unlocked_tlb_ent;
21extern unsigned long sparc64_kern_pri_context; 14extern unsigned long sparc64_kern_pri_context;
22extern unsigned long sparc64_kern_pri_nuc_bits; 15extern unsigned long sparc64_kern_pri_nuc_bits;
@@ -38,15 +31,4 @@ extern unsigned long kern_locked_tte_data;
38 31
39void prom_world(int enter); 32void prom_world(int enter);
40 33
41#ifdef CONFIG_SPARSEMEM_VMEMMAP
42#define VMEMMAP_CHUNK_SHIFT 22
43#define VMEMMAP_CHUNK (1UL << VMEMMAP_CHUNK_SHIFT)
44#define VMEMMAP_CHUNK_MASK ~(VMEMMAP_CHUNK - 1UL)
45#define VMEMMAP_ALIGN(x) (((x)+VMEMMAP_CHUNK-1UL)&VMEMMAP_CHUNK_MASK)
46
47#define VMEMMAP_SIZE ((((1UL << MAX_PHYSADDR_BITS) >> PAGE_SHIFT) * \
48 sizeof(struct page)) >> VMEMMAP_CHUNK_SHIFT)
49extern unsigned long vmemmap_table[VMEMMAP_SIZE];
50#endif
51
52#endif /* _SPARC64_MM_INIT_H */ 34#endif /* _SPARC64_MM_INIT_H */
diff --git a/arch/sparc/power/hibernate_asm.S b/arch/sparc/power/hibernate_asm.S
index 79942166df84..d7d9017dcb15 100644
--- a/arch/sparc/power/hibernate_asm.S
+++ b/arch/sparc/power/hibernate_asm.S
@@ -54,8 +54,8 @@ ENTRY(swsusp_arch_resume)
54 nop 54 nop
55 55
56 /* Write PAGE_OFFSET to %g7 */ 56 /* Write PAGE_OFFSET to %g7 */
57 sethi %uhi(PAGE_OFFSET), %g7 57 sethi %hi(PAGE_OFFSET), %g7
58 sllx %g7, 32, %g7 58 ldx [%g7 + %lo(PAGE_OFFSET)], %g7
59 59
60 setuw (PAGE_SIZE-8), %g3 60 setuw (PAGE_SIZE-8), %g3
61 61
diff --git a/arch/sparc/prom/bootstr_64.c b/arch/sparc/prom/bootstr_64.c
index ab9ccc63b388..7149e77714a4 100644
--- a/arch/sparc/prom/bootstr_64.c
+++ b/arch/sparc/prom/bootstr_64.c
@@ -14,7 +14,10 @@
14 * the .bss section or it will break things. 14 * the .bss section or it will break things.
15 */ 15 */
16 16
17#define BARG_LEN 256 17/* We limit BARG_LEN to 1024 because this is the size of the
18 * 'barg_out' command line buffer in the SILO bootloader.
19 */
20#define BARG_LEN 1024
18struct { 21struct {
19 int bootstr_len; 22 int bootstr_len;
20 int bootstr_valid; 23 int bootstr_valid;
diff --git a/arch/sparc/prom/p1275.c b/arch/sparc/prom/p1275.c
index e58b81726319..b2340f008ae0 100644
--- a/arch/sparc/prom/p1275.c
+++ b/arch/sparc/prom/p1275.c
@@ -9,6 +9,7 @@
9#include <linux/smp.h> 9#include <linux/smp.h>
10#include <linux/string.h> 10#include <linux/string.h>
11#include <linux/spinlock.h> 11#include <linux/spinlock.h>
12#include <linux/irqflags.h>
12 13
13#include <asm/openprom.h> 14#include <asm/openprom.h>
14#include <asm/oplib.h> 15#include <asm/oplib.h>
@@ -36,8 +37,8 @@ void p1275_cmd_direct(unsigned long *args)
36{ 37{
37 unsigned long flags; 38 unsigned long flags;
38 39
39 raw_local_save_flags(flags); 40 local_save_flags(flags);
40 raw_local_irq_restore((unsigned long)PIL_NMI); 41 local_irq_restore((unsigned long)PIL_NMI);
41 raw_spin_lock(&prom_entry_lock); 42 raw_spin_lock(&prom_entry_lock);
42 43
43 prom_world(1); 44 prom_world(1);
@@ -45,7 +46,7 @@ void p1275_cmd_direct(unsigned long *args)
45 prom_world(0); 46 prom_world(0);
46 47
47 raw_spin_unlock(&prom_entry_lock); 48 raw_spin_unlock(&prom_entry_lock);
48 raw_local_irq_restore(flags); 49 local_irq_restore(flags);
49} 50}
50 51
51void prom_cif_init(void *cif_handler, void *cif_stack) 52void prom_cif_init(void *cif_handler, void *cif_stack)
diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c
index 5814deb6963d..756b8ec00f16 100644
--- a/drivers/block/sunvdc.c
+++ b/drivers/block/sunvdc.c
@@ -9,6 +9,7 @@
9#include <linux/blkdev.h> 9#include <linux/blkdev.h>
10#include <linux/hdreg.h> 10#include <linux/hdreg.h>
11#include <linux/genhd.h> 11#include <linux/genhd.h>
12#include <linux/cdrom.h>
12#include <linux/slab.h> 13#include <linux/slab.h>
13#include <linux/spinlock.h> 14#include <linux/spinlock.h>
14#include <linux/completion.h> 15#include <linux/completion.h>
@@ -22,8 +23,8 @@
22 23
23#define DRV_MODULE_NAME "sunvdc" 24#define DRV_MODULE_NAME "sunvdc"
24#define PFX DRV_MODULE_NAME ": " 25#define PFX DRV_MODULE_NAME ": "
25#define DRV_MODULE_VERSION "1.0" 26#define DRV_MODULE_VERSION "1.1"
26#define DRV_MODULE_RELDATE "June 25, 2007" 27#define DRV_MODULE_RELDATE "February 13, 2013"
27 28
28static char version[] = 29static char version[] =
29 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 30 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
@@ -32,7 +33,7 @@ MODULE_DESCRIPTION("Sun LDOM virtual disk client driver");
32MODULE_LICENSE("GPL"); 33MODULE_LICENSE("GPL");
33MODULE_VERSION(DRV_MODULE_VERSION); 34MODULE_VERSION(DRV_MODULE_VERSION);
34 35
35#define VDC_TX_RING_SIZE 256 36#define VDC_TX_RING_SIZE 512
36 37
37#define WAITING_FOR_LINK_UP 0x01 38#define WAITING_FOR_LINK_UP 0x01
38#define WAITING_FOR_TX_SPACE 0x02 39#define WAITING_FOR_TX_SPACE 0x02
@@ -65,10 +66,10 @@ struct vdc_port {
65 u64 operations; 66 u64 operations;
66 u32 vdisk_size; 67 u32 vdisk_size;
67 u8 vdisk_type; 68 u8 vdisk_type;
69 u8 vdisk_mtype;
68 70
69 char disk_name[32]; 71 char disk_name[32];
70 72
71 struct vio_disk_geom geom;
72 struct vio_disk_vtoc label; 73 struct vio_disk_vtoc label;
73}; 74};
74 75
@@ -79,9 +80,16 @@ static inline struct vdc_port *to_vdc_port(struct vio_driver_state *vio)
79 80
80/* Ordered from largest major to lowest */ 81/* Ordered from largest major to lowest */
81static struct vio_version vdc_versions[] = { 82static struct vio_version vdc_versions[] = {
83 { .major = 1, .minor = 1 },
82 { .major = 1, .minor = 0 }, 84 { .major = 1, .minor = 0 },
83}; 85};
84 86
87static inline int vdc_version_supported(struct vdc_port *port,
88 u16 major, u16 minor)
89{
90 return port->vio.ver.major == major && port->vio.ver.minor >= minor;
91}
92
85#define VDCBLK_NAME "vdisk" 93#define VDCBLK_NAME "vdisk"
86static int vdc_major; 94static int vdc_major;
87#define PARTITION_SHIFT 3 95#define PARTITION_SHIFT 3
@@ -94,18 +102,54 @@ static inline u32 vdc_tx_dring_avail(struct vio_dring_state *dr)
94static int vdc_getgeo(struct block_device *bdev, struct hd_geometry *geo) 102static int vdc_getgeo(struct block_device *bdev, struct hd_geometry *geo)
95{ 103{
96 struct gendisk *disk = bdev->bd_disk; 104 struct gendisk *disk = bdev->bd_disk;
97 struct vdc_port *port = disk->private_data; 105 sector_t nsect = get_capacity(disk);
106 sector_t cylinders = nsect;
98 107
99 geo->heads = (u8) port->geom.num_hd; 108 geo->heads = 0xff;
100 geo->sectors = (u8) port->geom.num_sec; 109 geo->sectors = 0x3f;
101 geo->cylinders = port->geom.num_cyl; 110 sector_div(cylinders, geo->heads * geo->sectors);
111 geo->cylinders = cylinders;
112 if ((sector_t)(geo->cylinders + 1) * geo->heads * geo->sectors < nsect)
113 geo->cylinders = 0xffff;
102 114
103 return 0; 115 return 0;
104} 116}
105 117
118/* Add ioctl/CDROM_GET_CAPABILITY to support cdrom_id in udev
119 * when vdisk_mtype is VD_MEDIA_TYPE_CD or VD_MEDIA_TYPE_DVD.
120 * Needed to be able to install inside an ldom from an iso image.
121 */
122static int vdc_ioctl(struct block_device *bdev, fmode_t mode,
123 unsigned command, unsigned long argument)
124{
125 int i;
126 struct gendisk *disk;
127
128 switch (command) {
129 case CDROMMULTISESSION:
130 pr_debug(PFX "Multisession CDs not supported\n");
131 for (i = 0; i < sizeof(struct cdrom_multisession); i++)
132 if (put_user(0, (char __user *)(argument + i)))
133 return -EFAULT;
134 return 0;
135
136 case CDROM_GET_CAPABILITY:
137 disk = bdev->bd_disk;
138
139 if (bdev->bd_disk && (disk->flags & GENHD_FL_CD))
140 return 0;
141 return -EINVAL;
142
143 default:
144 pr_debug(PFX "ioctl %08x not supported\n", command);
145 return -EINVAL;
146 }
147}
148
106static const struct block_device_operations vdc_fops = { 149static const struct block_device_operations vdc_fops = {
107 .owner = THIS_MODULE, 150 .owner = THIS_MODULE,
108 .getgeo = vdc_getgeo, 151 .getgeo = vdc_getgeo,
152 .ioctl = vdc_ioctl,
109}; 153};
110 154
111static void vdc_finish(struct vio_driver_state *vio, int err, int waiting_for) 155static void vdc_finish(struct vio_driver_state *vio, int err, int waiting_for)
@@ -165,9 +209,9 @@ static int vdc_handle_attr(struct vio_driver_state *vio, void *arg)
165 struct vio_disk_attr_info *pkt = arg; 209 struct vio_disk_attr_info *pkt = arg;
166 210
167 viodbg(HS, "GOT ATTR stype[0x%x] ops[%llx] disk_size[%llu] disk_type[%x] " 211 viodbg(HS, "GOT ATTR stype[0x%x] ops[%llx] disk_size[%llu] disk_type[%x] "
168 "xfer_mode[0x%x] blksz[%u] max_xfer[%llu]\n", 212 "mtype[0x%x] xfer_mode[0x%x] blksz[%u] max_xfer[%llu]\n",
169 pkt->tag.stype, pkt->operations, 213 pkt->tag.stype, pkt->operations,
170 pkt->vdisk_size, pkt->vdisk_type, 214 pkt->vdisk_size, pkt->vdisk_type, pkt->vdisk_mtype,
171 pkt->xfer_mode, pkt->vdisk_block_size, 215 pkt->xfer_mode, pkt->vdisk_block_size,
172 pkt->max_xfer_size); 216 pkt->max_xfer_size);
173 217
@@ -192,8 +236,11 @@ static int vdc_handle_attr(struct vio_driver_state *vio, void *arg)
192 } 236 }
193 237
194 port->operations = pkt->operations; 238 port->operations = pkt->operations;
195 port->vdisk_size = pkt->vdisk_size;
196 port->vdisk_type = pkt->vdisk_type; 239 port->vdisk_type = pkt->vdisk_type;
240 if (vdc_version_supported(port, 1, 1)) {
241 port->vdisk_size = pkt->vdisk_size;
242 port->vdisk_mtype = pkt->vdisk_mtype;
243 }
197 if (pkt->max_xfer_size < port->max_xfer_size) 244 if (pkt->max_xfer_size < port->max_xfer_size)
198 port->max_xfer_size = pkt->max_xfer_size; 245 port->max_xfer_size = pkt->max_xfer_size;
199 port->vdisk_block_size = pkt->vdisk_block_size; 246 port->vdisk_block_size = pkt->vdisk_block_size;
@@ -236,7 +283,9 @@ static void vdc_end_one(struct vdc_port *port, struct vio_dring_state *dr,
236 283
237 __blk_end_request(req, (desc->status ? -EIO : 0), desc->size); 284 __blk_end_request(req, (desc->status ? -EIO : 0), desc->size);
238 285
239 if (blk_queue_stopped(port->disk->queue)) 286 /* restart blk queue when ring is half emptied */
287 if (blk_queue_stopped(port->disk->queue) &&
288 vdc_tx_dring_avail(dr) * 100 / VDC_TX_RING_SIZE >= 50)
240 blk_start_queue(port->disk->queue); 289 blk_start_queue(port->disk->queue);
241} 290}
242 291
@@ -388,12 +437,6 @@ static int __send_request(struct request *req)
388 for (i = 0; i < nsg; i++) 437 for (i = 0; i < nsg; i++)
389 len += sg[i].length; 438 len += sg[i].length;
390 439
391 if (unlikely(vdc_tx_dring_avail(dr) < 1)) {
392 blk_stop_queue(port->disk->queue);
393 err = -ENOMEM;
394 goto out;
395 }
396
397 desc = vio_dring_cur(dr); 440 desc = vio_dring_cur(dr);
398 441
399 err = ldc_map_sg(port->vio.lp, sg, nsg, 442 err = ldc_map_sg(port->vio.lp, sg, nsg,
@@ -433,21 +476,32 @@ static int __send_request(struct request *req)
433 port->req_id++; 476 port->req_id++;
434 dr->prod = (dr->prod + 1) & (VDC_TX_RING_SIZE - 1); 477 dr->prod = (dr->prod + 1) & (VDC_TX_RING_SIZE - 1);
435 } 478 }
436out:
437 479
438 return err; 480 return err;
439} 481}
440 482
441static void do_vdc_request(struct request_queue *q) 483static void do_vdc_request(struct request_queue *rq)
442{ 484{
443 while (1) { 485 struct request *req;
444 struct request *req = blk_fetch_request(q);
445 486
446 if (!req) 487 while ((req = blk_peek_request(rq)) != NULL) {
447 break; 488 struct vdc_port *port;
489 struct vio_dring_state *dr;
490
491 port = req->rq_disk->private_data;
492 dr = &port->vio.drings[VIO_DRIVER_TX_RING];
493 if (unlikely(vdc_tx_dring_avail(dr) < 1))
494 goto wait;
495
496 blk_start_request(req);
448 497
449 if (__send_request(req) < 0) 498 if (__send_request(req) < 0) {
450 __blk_end_request_all(req, -EIO); 499 blk_requeue_request(rq, req);
500wait:
501 /* Avoid pointless unplugs. */
502 blk_stop_queue(rq);
503 break;
504 }
451 } 505 }
452} 506}
453 507
@@ -663,18 +717,27 @@ static int probe_disk(struct vdc_port *port)
663 return err; 717 return err;
664 } 718 }
665 719
666 err = generic_request(port, VD_OP_GET_DISKGEOM, 720 if (vdc_version_supported(port, 1, 1)) {
667 &port->geom, sizeof(port->geom)); 721 /* vdisk_size should be set during the handshake, if it wasn't
668 if (err < 0) { 722 * then the underlying disk is reserved by another system
669 printk(KERN_ERR PFX "VD_OP_GET_DISKGEOM returns " 723 */
670 "error %d\n", err); 724 if (port->vdisk_size == -1)
671 return err; 725 return -ENODEV;
726 } else {
727 struct vio_disk_geom geom;
728
729 err = generic_request(port, VD_OP_GET_DISKGEOM,
730 &geom, sizeof(geom));
731 if (err < 0) {
732 printk(KERN_ERR PFX "VD_OP_GET_DISKGEOM returns "
733 "error %d\n", err);
734 return err;
735 }
736 port->vdisk_size = ((u64)geom.num_cyl *
737 (u64)geom.num_hd *
738 (u64)geom.num_sec);
672 } 739 }
673 740
674 port->vdisk_size = ((u64)port->geom.num_cyl *
675 (u64)port->geom.num_hd *
676 (u64)port->geom.num_sec);
677
678 q = blk_init_queue(do_vdc_request, &port->vio.lock); 741 q = blk_init_queue(do_vdc_request, &port->vio.lock);
679 if (!q) { 742 if (!q) {
680 printk(KERN_ERR PFX "%s: Could not allocate queue.\n", 743 printk(KERN_ERR PFX "%s: Could not allocate queue.\n",
@@ -691,6 +754,10 @@ static int probe_disk(struct vdc_port *port)
691 754
692 port->disk = g; 755 port->disk = g;
693 756
757 /* Each segment in a request is up to an aligned page in size. */
758 blk_queue_segment_boundary(q, PAGE_SIZE - 1);
759 blk_queue_max_segment_size(q, PAGE_SIZE);
760
694 blk_queue_max_segments(q, port->ring_cookies); 761 blk_queue_max_segments(q, port->ring_cookies);
695 blk_queue_max_hw_sectors(q, port->max_xfer_size); 762 blk_queue_max_hw_sectors(q, port->max_xfer_size);
696 g->major = vdc_major; 763 g->major = vdc_major;
@@ -704,9 +771,32 @@ static int probe_disk(struct vdc_port *port)
704 771
705 set_capacity(g, port->vdisk_size); 772 set_capacity(g, port->vdisk_size);
706 773
707 printk(KERN_INFO PFX "%s: %u sectors (%u MB)\n", 774 if (vdc_version_supported(port, 1, 1)) {
775 switch (port->vdisk_mtype) {
776 case VD_MEDIA_TYPE_CD:
777 pr_info(PFX "Virtual CDROM %s\n", port->disk_name);
778 g->flags |= GENHD_FL_CD;
779 g->flags |= GENHD_FL_REMOVABLE;
780 set_disk_ro(g, 1);
781 break;
782
783 case VD_MEDIA_TYPE_DVD:
784 pr_info(PFX "Virtual DVD %s\n", port->disk_name);
785 g->flags |= GENHD_FL_CD;
786 g->flags |= GENHD_FL_REMOVABLE;
787 set_disk_ro(g, 1);
788 break;
789
790 case VD_MEDIA_TYPE_FIXED:
791 pr_info(PFX "Virtual Hard disk %s\n", port->disk_name);
792 break;
793 }
794 }
795
796 pr_info(PFX "%s: %u sectors (%u MB) protocol %d.%d\n",
708 g->disk_name, 797 g->disk_name,
709 port->vdisk_size, (port->vdisk_size >> (20 - 9))); 798 port->vdisk_size, (port->vdisk_size >> (20 - 9)),
799 port->vio.ver.major, port->vio.ver.minor);
710 800
711 add_disk(g); 801 add_disk(g);
712 802
@@ -765,6 +855,7 @@ static int vdc_port_probe(struct vio_dev *vdev, const struct vio_device_id *id)
765 else 855 else
766 snprintf(port->disk_name, sizeof(port->disk_name), 856 snprintf(port->disk_name, sizeof(port->disk_name),
767 VDCBLK_NAME "%c", 'a' + ((int)vdev->dev_no % 26)); 857 VDCBLK_NAME "%c", 'a' + ((int)vdev->dev_no % 26));
858 port->vdisk_size = -1;
768 859
769 err = vio_driver_init(&port->vio, vdev, VDEV_DISK, 860 err = vio_driver_init(&port->vio, vdev, VDEV_DISK,
770 vdc_versions, ARRAY_SIZE(vdc_versions), 861 vdc_versions, ARRAY_SIZE(vdc_versions),
diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c
index 15396720f489..3652afd3ec78 100644
--- a/drivers/net/ethernet/sun/sunvnet.c
+++ b/drivers/net/ethernet/sun/sunvnet.c
@@ -954,7 +954,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev)
954 spin_lock_irqsave(&port->vio.lock, flags); 954 spin_lock_irqsave(&port->vio.lock, flags);
955 955
956 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 956 dr = &port->vio.drings[VIO_DRIVER_TX_RING];
957 if (unlikely(vnet_tx_dring_avail(dr) < 2)) { 957 if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
958 if (!netif_queue_stopped(dev)) { 958 if (!netif_queue_stopped(dev)) {
959 netif_stop_queue(dev); 959 netif_stop_queue(dev);
960 960
@@ -1049,7 +1049,7 @@ ldc_start_done:
1049 dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1049 dev->stats.tx_bytes += port->tx_bufs[txi].skb->len;
1050 1050
1051 dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1051 dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);
1052 if (unlikely(vnet_tx_dring_avail(dr) < 2)) { 1052 if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
1053 netif_stop_queue(dev); 1053 netif_stop_queue(dev);
1054 if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1054 if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr))
1055 netif_wake_queue(dev); 1055 netif_wake_queue(dev);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-28 03:42:12 -0500