aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/sparc/kernel/ebus.c1
-rw-r--r--arch/sparc/mm/init.c3
-rw-r--r--arch/sparc/mm/io-unit.c18
-rw-r--r--arch/sparc/mm/iommu.c12
-rw-r--r--arch/sparc/mm/sun4c.c2
-rw-r--r--arch/sparc64/kernel/head.S14
-rw-r--r--arch/sparc64/kernel/mdesc.c38
-rw-r--r--arch/sparc64/kernel/trampoline.S7
-rw-r--r--arch/sparc64/lib/Makefile2
-rw-r--r--arch/sparc64/lib/NG2copy_from_user.S40
-rw-r--r--arch/sparc64/lib/NG2copy_to_user.S49
-rw-r--r--arch/sparc64/lib/NG2memcpy.S520
-rw-r--r--arch/sparc64/lib/NG2page.S61
-rw-r--r--arch/sparc64/lib/NG2patch.S33
-rw-r--r--arch/sparc64/lib/NGpage.S1
15 files changed, 776 insertions, 25 deletions
diff --git a/arch/sparc/kernel/ebus.c b/arch/sparc/kernel/ebus.c
index ac352eb6dff3..e2d02fd13f35 100644
--- a/arch/sparc/kernel/ebus.c
+++ b/arch/sparc/kernel/ebus.c
@@ -238,6 +238,7 @@ void __init fill_ebus_device(struct device_node *dp, struct linux_ebus_device *d
238 sd = &dev->ofdev.dev.archdata; 238 sd = &dev->ofdev.dev.archdata;
239 sd->prom_node = dp; 239 sd->prom_node = dp;
240 sd->op = &dev->ofdev; 240 sd->op = &dev->ofdev;
241 sd->iommu = dev->bus->ofdev.dev.parent->archdata.iommu;
241 242
242 dev->ofdev.node = dp; 243 dev->ofdev.node = dp;
243 dev->ofdev.dev.parent = &dev->bus->ofdev.dev; 244 dev->ofdev.dev.parent = &dev->bus->ofdev.dev;
diff --git a/arch/sparc/mm/init.c b/arch/sparc/mm/init.c
index a1bef07755a9..c13e6cd279ac 100644
--- a/arch/sparc/mm/init.c
+++ b/arch/sparc/mm/init.c
@@ -206,8 +206,7 @@ unsigned long __init bootmem_init(unsigned long *pages_avail)
206#ifdef CONFIG_BLK_DEV_INITRD 206#ifdef CONFIG_BLK_DEV_INITRD
207 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */ 207 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
208 if (sparc_ramdisk_image) { 208 if (sparc_ramdisk_image) {
209 if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE) 209 sparc_ramdisk_image -= KERNBASE;
210 sparc_ramdisk_image -= KERNBASE;
211 initrd_start = sparc_ramdisk_image + phys_base; 210 initrd_start = sparc_ramdisk_image + phys_base;
212 initrd_end = initrd_start + sparc_ramdisk_size; 211 initrd_end = initrd_start + sparc_ramdisk_size;
213 if (initrd_end > end_of_phys_memory) { 212 if (initrd_end > end_of_phys_memory) {
diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c
index 4ccda77d08d6..7c89893b1fe8 100644
--- a/arch/sparc/mm/io-unit.c
+++ b/arch/sparc/mm/io-unit.c
@@ -66,7 +66,7 @@ iounit_init(int sbi_node, int io_node, struct sbus_bus *sbus)
66 } 66 }
67 if(!xpt) panic("Cannot map External Page Table."); 67 if(!xpt) panic("Cannot map External Page Table.");
68 68
69 sbus->iommu = (struct iommu_struct *)iounit; 69 sbus->ofdev.dev.archdata.iommu = iounit;
70 iounit->page_table = xpt; 70 iounit->page_table = xpt;
71 spin_lock_init(&iounit->lock); 71 spin_lock_init(&iounit->lock);
72 72
@@ -127,7 +127,7 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan);
127static __u32 iounit_get_scsi_one(char *vaddr, unsigned long len, struct sbus_bus *sbus) 127static __u32 iounit_get_scsi_one(char *vaddr, unsigned long len, struct sbus_bus *sbus)
128{ 128{
129 unsigned long ret, flags; 129 unsigned long ret, flags;
130 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 130 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
131 131
132 spin_lock_irqsave(&iounit->lock, flags); 132 spin_lock_irqsave(&iounit->lock, flags);
133 ret = iounit_get_area(iounit, (unsigned long)vaddr, len); 133 ret = iounit_get_area(iounit, (unsigned long)vaddr, len);
@@ -138,7 +138,7 @@ static __u32 iounit_get_scsi_one(char *vaddr, unsigned long len, struct sbus_bus
138static void iounit_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus) 138static void iounit_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
139{ 139{
140 unsigned long flags; 140 unsigned long flags;
141 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 141 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
142 142
143 /* FIXME: Cache some resolved pages - often several sg entries are to the same page */ 143 /* FIXME: Cache some resolved pages - often several sg entries are to the same page */
144 spin_lock_irqsave(&iounit->lock, flags); 144 spin_lock_irqsave(&iounit->lock, flags);
@@ -153,7 +153,7 @@ static void iounit_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus
153static void iounit_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus) 153static void iounit_release_scsi_one(__u32 vaddr, unsigned long len, struct sbus_bus *sbus)
154{ 154{
155 unsigned long flags; 155 unsigned long flags;
156 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 156 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
157 157
158 spin_lock_irqsave(&iounit->lock, flags); 158 spin_lock_irqsave(&iounit->lock, flags);
159 len = ((vaddr & ~PAGE_MASK) + len + (PAGE_SIZE-1)) >> PAGE_SHIFT; 159 len = ((vaddr & ~PAGE_MASK) + len + (PAGE_SIZE-1)) >> PAGE_SHIFT;
@@ -168,7 +168,7 @@ static void iounit_release_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_
168{ 168{
169 unsigned long flags; 169 unsigned long flags;
170 unsigned long vaddr, len; 170 unsigned long vaddr, len;
171 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 171 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
172 172
173 spin_lock_irqsave(&iounit->lock, flags); 173 spin_lock_irqsave(&iounit->lock, flags);
174 while (sz != 0) { 174 while (sz != 0) {
@@ -211,7 +211,7 @@ static int iounit_map_dma_area(dma_addr_t *pba, unsigned long va, __u32 addr, in
211 i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); 211 i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT);
212 212
213 for_each_sbus(sbus) { 213 for_each_sbus(sbus) {
214 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 214 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
215 215
216 iopte = (iopte_t *)(iounit->page_table + i); 216 iopte = (iopte_t *)(iounit->page_table + i);
217 *iopte = MKIOPTE(__pa(page)); 217 *iopte = MKIOPTE(__pa(page));
@@ -235,7 +235,7 @@ static void iounit_unmap_dma_area(unsigned long addr, int len)
235static struct page *iounit_translate_dvma(unsigned long addr) 235static struct page *iounit_translate_dvma(unsigned long addr)
236{ 236{
237 struct sbus_bus *sbus = sbus_root; /* They are all the same */ 237 struct sbus_bus *sbus = sbus_root; /* They are all the same */
238 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 238 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
239 int i; 239 int i;
240 iopte_t *iopte; 240 iopte_t *iopte;
241 241
@@ -279,7 +279,7 @@ __u32 iounit_map_dma_init(struct sbus_bus *sbus, int size)
279 unsigned long rotor, scan, limit; 279 unsigned long rotor, scan, limit;
280 unsigned long flags; 280 unsigned long flags;
281 __u32 ret; 281 __u32 ret;
282 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 282 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
283 283
284 npages = (size + (PAGE_SIZE-1)) >> PAGE_SHIFT; 284 npages = (size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
285 i = 0x0213; 285 i = 0x0213;
@@ -315,7 +315,7 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan);
315__u32 iounit_map_dma_page(__u32 vaddr, void *addr, struct sbus_bus *sbus) 315__u32 iounit_map_dma_page(__u32 vaddr, void *addr, struct sbus_bus *sbus)
316{ 316{
317 int scan = (vaddr - IOUNIT_DMA_BASE) >> PAGE_SHIFT; 317 int scan = (vaddr - IOUNIT_DMA_BASE) >> PAGE_SHIFT;
318 struct iounit_struct *iounit = (struct iounit_struct *)sbus->iommu; 318 struct iounit_struct *iounit = sbus->ofdev.dev.archdata.iommu;
319 319
320 iounit->page_table[scan] = MKIOPTE(__pa(((unsigned long)addr) & PAGE_MASK)); 320 iounit->page_table[scan] = MKIOPTE(__pa(((unsigned long)addr) & PAGE_MASK));
321 return vaddr + (((unsigned long)addr) & ~PAGE_MASK); 321 return vaddr + (((unsigned long)addr) & ~PAGE_MASK);
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index be042efd1ba4..52e907af9d29 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -132,7 +132,7 @@ iommu_init(int iommund, struct sbus_bus *sbus)
132 impl, vers, iommu->page_table, 132 impl, vers, iommu->page_table,
133 (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); 133 (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
134 134
135 sbus->iommu = iommu; 135 sbus->ofdev.dev.archdata.iommu = iommu;
136} 136}
137 137
138/* This begs to be btfixup-ed by srmmu. */ 138/* This begs to be btfixup-ed by srmmu. */
@@ -166,7 +166,7 @@ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
166 166
167static u32 iommu_get_one(struct page *page, int npages, struct sbus_bus *sbus) 167static u32 iommu_get_one(struct page *page, int npages, struct sbus_bus *sbus)
168{ 168{
169 struct iommu_struct *iommu = sbus->iommu; 169 struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
170 int ioptex; 170 int ioptex;
171 iopte_t *iopte, *iopte0; 171 iopte_t *iopte, *iopte0;
172 unsigned int busa, busa0; 172 unsigned int busa, busa0;
@@ -291,7 +291,7 @@ static void iommu_get_scsi_sgl_pflush(struct scatterlist *sg, int sz, struct sbu
291 291
292static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus) 292static void iommu_release_one(u32 busa, int npages, struct sbus_bus *sbus)
293{ 293{
294 struct iommu_struct *iommu = sbus->iommu; 294 struct iommu_struct *iommu = sbus->ofdev.dev.archdata.iommu;
295 int ioptex; 295 int ioptex;
296 int i; 296 int i;
297 297
@@ -334,7 +334,7 @@ static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
334 unsigned long addr, int len) 334 unsigned long addr, int len)
335{ 335{
336 unsigned long page, end; 336 unsigned long page, end;
337 struct iommu_struct *iommu = sbus_root->iommu; 337 struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
338 iopte_t *iopte = iommu->page_table; 338 iopte_t *iopte = iommu->page_table;
339 iopte_t *first; 339 iopte_t *first;
340 int ioptex; 340 int ioptex;
@@ -399,7 +399,7 @@ static int iommu_map_dma_area(dma_addr_t *pba, unsigned long va,
399 399
400static void iommu_unmap_dma_area(unsigned long busa, int len) 400static void iommu_unmap_dma_area(unsigned long busa, int len)
401{ 401{
402 struct iommu_struct *iommu = sbus_root->iommu; 402 struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
403 iopte_t *iopte = iommu->page_table; 403 iopte_t *iopte = iommu->page_table;
404 unsigned long end; 404 unsigned long end;
405 int ioptex = (busa - iommu->start) >> PAGE_SHIFT; 405 int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
@@ -420,7 +420,7 @@ static void iommu_unmap_dma_area(unsigned long busa, int len)
420 420
421static struct page *iommu_translate_dvma(unsigned long busa) 421static struct page *iommu_translate_dvma(unsigned long busa)
422{ 422{
423 struct iommu_struct *iommu = sbus_root->iommu; 423 struct iommu_struct *iommu = sbus_root->ofdev.dev.archdata.iommu;
424 iopte_t *iopte = iommu->page_table; 424 iopte_t *iopte = iommu->page_table;
425 425
426 iopte += ((busa - iommu->start) >> PAGE_SHIFT); 426 iopte += ((busa - iommu->start) >> PAGE_SHIFT);
diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c
index 79d60d86f6f8..005a3e72d4f2 100644
--- a/arch/sparc/mm/sun4c.c
+++ b/arch/sparc/mm/sun4c.c
@@ -268,7 +268,6 @@ static inline void sun4c_init_clean_mmu(unsigned long kernel_end)
268 unsigned char savectx, ctx; 268 unsigned char savectx, ctx;
269 269
270 savectx = sun4c_get_context(); 270 savectx = sun4c_get_context();
271 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
272 for (ctx = 0; ctx < num_contexts; ctx++) { 271 for (ctx = 0; ctx < num_contexts; ctx++) {
273 sun4c_set_context(ctx); 272 sun4c_set_context(ctx);
274 for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE) 273 for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE)
@@ -2064,7 +2063,6 @@ void __init sun4c_paging_init(void)
2064 unsigned long end_pfn, pages_avail; 2063 unsigned long end_pfn, pages_avail;
2065 2064
2066 kernel_end = (unsigned long) &end; 2065 kernel_end = (unsigned long) &end;
2067 kernel_end += (SUN4C_REAL_PGDIR_SIZE * 4);
2068 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end); 2066 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
2069 2067
2070 pages_avail = 0; 2068 pages_avail = 0;
diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index ac18bd8e273f..63144ad476f6 100644
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -501,7 +501,7 @@ niagara_tlb_fixup:
501 cmp %g1, SUN4V_CHIP_NIAGARA1 501 cmp %g1, SUN4V_CHIP_NIAGARA1
502 be,pt %xcc, niagara_patch 502 be,pt %xcc, niagara_patch
503 cmp %g1, SUN4V_CHIP_NIAGARA2 503 cmp %g1, SUN4V_CHIP_NIAGARA2
504 be,pt %xcc, niagara_patch 504 be,pt %xcc, niagara2_patch
505 nop 505 nop
506 506
507 call generic_patch_copyops 507 call generic_patch_copyops
@@ -512,6 +512,15 @@ niagara_tlb_fixup:
512 nop 512 nop
513 513
514 ba,a,pt %xcc, 80f 514 ba,a,pt %xcc, 80f
515niagara2_patch:
516 call niagara2_patch_copyops
517 nop
518 call niagara_patch_bzero
519 nop
520 call niagara2_patch_pageops
521 nop
522
523 ba,a,pt %xcc, 80f
515 524
516niagara_patch: 525niagara_patch:
517 call niagara_patch_copyops 526 call niagara_patch_copyops
@@ -706,12 +715,13 @@ setup_trap_table:
706 715
707 membar #Sync 716 membar #Sync
708 717
718 BRANCH_IF_SUN4V(o2, 1f)
719
709 /* Kill PROM timer */ 720 /* Kill PROM timer */
710 sethi %hi(0x80000000), %o2 721 sethi %hi(0x80000000), %o2
711 sllx %o2, 32, %o2 722 sllx %o2, 32, %o2
712 wr %o2, 0, %tick_cmpr 723 wr %o2, 0, %tick_cmpr
713 724
714 BRANCH_IF_SUN4V(o2, 1f)
715 BRANCH_IF_ANY_CHEETAH(o2, o3, 1f) 725 BRANCH_IF_ANY_CHEETAH(o2, o3, 1f)
716 726
717 ba,pt %xcc, 2f 727 ba,pt %xcc, 2f
diff --git a/arch/sparc64/kernel/mdesc.c b/arch/sparc64/kernel/mdesc.c
index 95059c2ec414..9f22e4ff6015 100644
--- a/arch/sparc64/kernel/mdesc.c
+++ b/arch/sparc64/kernel/mdesc.c
@@ -9,6 +9,7 @@
9#include <linux/list.h> 9#include <linux/list.h>
10#include <linux/slab.h> 10#include <linux/slab.h>
11#include <linux/mm.h> 11#include <linux/mm.h>
12#include <linux/miscdevice.h>
12 13
13#include <asm/hypervisor.h> 14#include <asm/hypervisor.h>
14#include <asm/mdesc.h> 15#include <asm/mdesc.h>
@@ -836,6 +837,43 @@ void __devinit mdesc_fill_in_cpu_data(cpumask_t mask)
836 mdesc_release(hp); 837 mdesc_release(hp);
837} 838}
838 839
840static ssize_t mdesc_read(struct file *file, char __user *buf,
841 size_t len, loff_t *offp)
842{
843 struct mdesc_handle *hp = mdesc_grab();
844 int err;
845
846 if (!hp)
847 return -ENODEV;
848
849 err = hp->handle_size;
850 if (len < hp->handle_size)
851 err = -EMSGSIZE;
852 else if (copy_to_user(buf, &hp->mdesc, hp->handle_size))
853 err = -EFAULT;
854 mdesc_release(hp);
855
856 return err;
857}
858
859static const struct file_operations mdesc_fops = {
860 .read = mdesc_read,
861 .owner = THIS_MODULE,
862};
863
864static struct miscdevice mdesc_misc = {
865 .minor = MISC_DYNAMIC_MINOR,
866 .name = "mdesc",
867 .fops = &mdesc_fops,
868};
869
870static int __init mdesc_misc_init(void)
871{
872 return misc_register(&mdesc_misc);
873}
874
875__initcall(mdesc_misc_init);
876
839void __init sun4v_mdesc_init(void) 877void __init sun4v_mdesc_init(void)
840{ 878{
841 struct mdesc_handle *hp; 879 struct mdesc_handle *hp;
diff --git a/arch/sparc64/kernel/trampoline.S b/arch/sparc64/kernel/trampoline.S
index 9448595f9063..9533a25ce5d2 100644
--- a/arch/sparc64/kernel/trampoline.S
+++ b/arch/sparc64/kernel/trampoline.S
@@ -95,14 +95,13 @@ spitfire_startup:
95 membar #Sync 95 membar #Sync
96 96
97startup_continue: 97startup_continue:
98 mov %o0, %l0
99 BRANCH_IF_SUN4V(g1, niagara_lock_tlb)
100
98 sethi %hi(0x80000000), %g2 101 sethi %hi(0x80000000), %g2
99 sllx %g2, 32, %g2 102 sllx %g2, 32, %g2
100 wr %g2, 0, %tick_cmpr 103 wr %g2, 0, %tick_cmpr
101 104
102 mov %o0, %l0
103
104 BRANCH_IF_SUN4V(g1, niagara_lock_tlb)
105
106 /* Call OBP by hand to lock KERNBASE into i/d tlbs. 105 /* Call OBP by hand to lock KERNBASE into i/d tlbs.
107 * We lock 2 consequetive entries if we are 'bigkernel'. 106 * We lock 2 consequetive entries if we are 'bigkernel'.
108 */ 107 */
diff --git a/arch/sparc64/lib/Makefile b/arch/sparc64/lib/Makefile
index f95fbfa3eeb8..f095e13910bc 100644
--- a/arch/sparc64/lib/Makefile
+++ b/arch/sparc64/lib/Makefile
@@ -13,6 +13,8 @@ lib-y := PeeCeeI.o copy_page.o clear_page.o strlen.o strncmp.o \
13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \ 13 U3memcpy.o U3copy_from_user.o U3copy_to_user.o U3patch.o \
14 NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \ 14 NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o NGpatch.o \
15 NGpage.o NGbzero.o \ 15 NGpage.o NGbzero.o \
16 NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o NG2patch.o \
17 NG2page.o \
16 GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \ 18 GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o GENpatch.o \
17 GENpage.o GENbzero.o \ 19 GENpage.o GENbzero.o \
18 copy_in_user.o user_fixup.o memmove.o \ 20 copy_in_user.o user_fixup.o memmove.o \
diff --git a/arch/sparc64/lib/NG2copy_from_user.S b/arch/sparc64/lib/NG2copy_from_user.S
new file mode 100644
index 000000000000..c77ef5f22102
--- /dev/null
+++ b/arch/sparc64/lib/NG2copy_from_user.S
@@ -0,0 +1,40 @@
1/* NG2copy_from_user.S: Niagara-2 optimized copy from userspace.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#define EX_LD(x) \
798: x; \
8 .section .fixup; \
9 .align 4; \
1099: wr %g0, ASI_AIUS, %asi;\
11 retl; \
12 mov 1, %o0; \
13 .section __ex_table,"a";\
14 .align 4; \
15 .word 98b, 99b; \
16 .text; \
17 .align 4;
18
19#ifndef ASI_AIUS
20#define ASI_AIUS 0x11
21#endif
22
23#ifndef ASI_BLK_AIUS_4V
24#define ASI_BLK_AIUS_4V 0x17
25#endif
26
27#define FUNC_NAME NG2copy_from_user
28#define LOAD(type,addr,dest) type##a [addr] %asi, dest
29#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS_4V, dest
30#define EX_RETVAL(x) 0
31
32#ifdef __KERNEL__
33#define PREAMBLE \
34 rd %asi, %g1; \
35 cmp %g1, ASI_AIUS; \
36 bne,pn %icc, memcpy_user_stub; \
37 nop
38#endif
39
40#include "NG2memcpy.S"
diff --git a/arch/sparc64/lib/NG2copy_to_user.S b/arch/sparc64/lib/NG2copy_to_user.S
new file mode 100644
index 000000000000..4bd4093acbbd
--- /dev/null
+++ b/arch/sparc64/lib/NG2copy_to_user.S
@@ -0,0 +1,49 @@
1/* NG2copy_to_user.S: Niagara-2 optimized copy to userspace.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#define EX_ST(x) \
798: x; \
8 .section .fixup; \
9 .align 4; \
1099: wr %g0, ASI_AIUS, %asi;\
11 retl; \
12 mov 1, %o0; \
13 .section __ex_table,"a";\
14 .align 4; \
15 .word 98b, 99b; \
16 .text; \
17 .align 4;
18
19#ifndef ASI_AIUS
20#define ASI_AIUS 0x11
21#endif
22
23#ifndef ASI_BLK_AIUS_4V
24#define ASI_BLK_AIUS_4V 0x17
25#endif
26
27#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS
28#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23
29#endif
30
31#define FUNC_NAME NG2copy_to_user
32#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS
33#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS
34#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS_4V
35#define EX_RETVAL(x) 0
36
37#ifdef __KERNEL__
38 /* Writing to %asi is _expensive_ so we hardcode it.
39 * Reading %asi to check for KERNEL_DS is comparatively
40 * cheap.
41 */
42#define PREAMBLE \
43 rd %asi, %g1; \
44 cmp %g1, ASI_AIUS; \
45 bne,pn %icc, memcpy_user_stub; \
46 nop
47#endif
48
49#include "NG2memcpy.S"
diff --git a/arch/sparc64/lib/NG2memcpy.S b/arch/sparc64/lib/NG2memcpy.S
new file mode 100644
index 000000000000..0aed75653b50
--- /dev/null
+++ b/arch/sparc64/lib/NG2memcpy.S
@@ -0,0 +1,520 @@
1/* NG2memcpy.S: Niagara-2 optimized memcpy.
2 *
3 * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
4 */
5
6#ifdef __KERNEL__
7#include <asm/visasm.h>
8#include <asm/asi.h>
9#define GLOBAL_SPARE %g7
10#else
11#define ASI_PNF 0x82
12#define ASI_BLK_P 0xf0
13#define ASI_BLK_INIT_QUAD_LDD_P 0xe2
14#define FPRS_FEF 0x04
15#ifdef MEMCPY_DEBUG
16#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
17 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
18#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
19#else
20#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
21#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
22#endif
23#define GLOBAL_SPARE %g5
24#endif
25
26#ifndef STORE_ASI
27#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA
28#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P
29#else
30#define STORE_ASI 0x80 /* ASI_P */
31#endif
32#endif
33
34#ifndef EX_LD
35#define EX_LD(x) x
36#endif
37
38#ifndef EX_ST
39#define EX_ST(x) x
40#endif
41
42#ifndef EX_RETVAL
43#define EX_RETVAL(x) x
44#endif
45
46#ifndef LOAD
47#define LOAD(type,addr,dest) type [addr], dest
48#endif
49
50#ifndef LOAD_BLK
51#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
52#endif
53
54#ifndef STORE
55#ifndef MEMCPY_DEBUG
56#define STORE(type,src,addr) type src, [addr]
57#else
58#define STORE(type,src,addr) type##a src, [addr] 0x80
59#endif
60#endif
61
62#ifndef STORE_BLK
63#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
64#endif
65
66#ifndef STORE_INIT
67#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI
68#endif
69
70#ifndef FUNC_NAME
71#define FUNC_NAME NG2memcpy
72#endif
73
74#ifndef PREAMBLE
75#define PREAMBLE
76#endif
77
78#ifndef XCC
79#define XCC xcc
80#endif
81
82#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
83 faligndata %x0, %x1, %f0; \
84 faligndata %x1, %x2, %f2; \
85 faligndata %x2, %x3, %f4; \
86 faligndata %x3, %x4, %f6; \
87 faligndata %x4, %x5, %f8; \
88 faligndata %x5, %x6, %f10; \
89 faligndata %x6, %x7, %f12; \
90 faligndata %x7, %x8, %f14;
91
92#define FREG_MOVE_1(x0) \
93 fmovd %x0, %f0;
94#define FREG_MOVE_2(x0, x1) \
95 fmovd %x0, %f0; \
96 fmovd %x1, %f2;
97#define FREG_MOVE_3(x0, x1, x2) \
98 fmovd %x0, %f0; \
99 fmovd %x1, %f2; \
100 fmovd %x2, %f4;
101#define FREG_MOVE_4(x0, x1, x2, x3) \
102 fmovd %x0, %f0; \
103 fmovd %x1, %f2; \
104 fmovd %x2, %f4; \
105 fmovd %x3, %f6;
106#define FREG_MOVE_5(x0, x1, x2, x3, x4) \
107 fmovd %x0, %f0; \
108 fmovd %x1, %f2; \
109 fmovd %x2, %f4; \
110 fmovd %x3, %f6; \
111 fmovd %x4, %f8;
112#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
113 fmovd %x0, %f0; \
114 fmovd %x1, %f2; \
115 fmovd %x2, %f4; \
116 fmovd %x3, %f6; \
117 fmovd %x4, %f8; \
118 fmovd %x5, %f10;
119#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
120 fmovd %x0, %f0; \
121 fmovd %x1, %f2; \
122 fmovd %x2, %f4; \
123 fmovd %x3, %f6; \
124 fmovd %x4, %f8; \
125 fmovd %x5, %f10; \
126 fmovd %x6, %f12;
127#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
128 fmovd %x0, %f0; \
129 fmovd %x1, %f2; \
130 fmovd %x2, %f4; \
131 fmovd %x3, %f6; \
132 fmovd %x4, %f8; \
133 fmovd %x5, %f10; \
134 fmovd %x6, %f12; \
135 fmovd %x7, %f14;
136#define FREG_LOAD_1(base, x0) \
137 EX_LD(LOAD(ldd, base + 0x00, %x0))
138#define FREG_LOAD_2(base, x0, x1) \
139 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
140 EX_LD(LOAD(ldd, base + 0x08, %x1));
141#define FREG_LOAD_3(base, x0, x1, x2) \
142 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
143 EX_LD(LOAD(ldd, base + 0x08, %x1)); \
144 EX_LD(LOAD(ldd, base + 0x10, %x2));
145#define FREG_LOAD_4(base, x0, x1, x2, x3) \
146 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
147 EX_LD(LOAD(ldd, base + 0x08, %x1)); \
148 EX_LD(LOAD(ldd, base + 0x10, %x2)); \
149 EX_LD(LOAD(ldd, base + 0x18, %x3));
150#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
151 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
152 EX_LD(LOAD(ldd, base + 0x08, %x1)); \
153 EX_LD(LOAD(ldd, base + 0x10, %x2)); \
154 EX_LD(LOAD(ldd, base + 0x18, %x3)); \
155 EX_LD(LOAD(ldd, base + 0x20, %x4));
156#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
157 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
158 EX_LD(LOAD(ldd, base + 0x08, %x1)); \
159 EX_LD(LOAD(ldd, base + 0x10, %x2)); \
160 EX_LD(LOAD(ldd, base + 0x18, %x3)); \
161 EX_LD(LOAD(ldd, base + 0x20, %x4)); \
162 EX_LD(LOAD(ldd, base + 0x28, %x5));
163#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
164 EX_LD(LOAD(ldd, base + 0x00, %x0)); \
165 EX_LD(LOAD(ldd, base + 0x08, %x1)); \
166 EX_LD(LOAD(ldd, base + 0x10, %x2)); \
167 EX_LD(LOAD(ldd, base + 0x18, %x3)); \
168 EX_LD(LOAD(ldd, base + 0x20, %x4)); \
169 EX_LD(LOAD(ldd, base + 0x28, %x5)); \
170 EX_LD(LOAD(ldd, base + 0x30, %x6));
171
172 .register %g2,#scratch
173 .register %g3,#scratch
174
175 .text
176 .align 64
177
178 .globl FUNC_NAME
179 .type FUNC_NAME,#function
180FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
181 srlx %o2, 31, %g2
182 cmp %g2, 0
183 tne %xcc, 5
184 PREAMBLE
185 mov %o0, GLOBAL_SPARE
186 cmp %o2, 0
187 be,pn %XCC, 85f
188 or %o0, %o1, %o3
189 cmp %o2, 16
190 blu,a,pn %XCC, 80f
191 or %o3, %o2, %o3
192
193 /* 2 blocks (128 bytes) is the minimum we can do the block
194 * copy with. We need to ensure that we'll iterate at least
195 * once in the block copy loop. At worst we'll need to align
196 * the destination to a 64-byte boundary which can chew up
197 * to (64 - 1) bytes from the length before we perform the
198 * block copy loop.
199 *
200 * However, the cut-off point, performance wise, is around
201 * 4 64-byte blocks.
202 */
203 cmp %o2, (4 * 64)
204 blu,pt %XCC, 75f
205 andcc %o3, 0x7, %g0
206
207 /* %o0: dst
208 * %o1: src
209 * %o2: len (known to be >= 128)
210 *
211 * The block copy loops can use %o4, %g2, %g3 as
212 * temporaries while copying the data. %o5 must
213 * be preserved between VISEntryHalf and VISExitHalf
214 */
215
216 LOAD(prefetch, %o1 + 0x000, #one_read)
217 LOAD(prefetch, %o1 + 0x040, #one_read)
218 LOAD(prefetch, %o1 + 0x080, #one_read)
219
220 /* Align destination on 64-byte boundary. */
221 andcc %o0, (64 - 1), %o4
222 be,pt %XCC, 2f
223 sub %o4, 64, %o4
224 sub %g0, %o4, %o4 ! bytes to align dst
225 sub %o2, %o4, %o2
2261: subcc %o4, 1, %o4
227 EX_LD(LOAD(ldub, %o1, %g1))
228 EX_ST(STORE(stb, %g1, %o0))
229 add %o1, 1, %o1
230 bne,pt %XCC, 1b
231 add %o0, 1, %o0
232
2332:
234 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve
235 * o5 from here until we hit VISExitHalf.
236 */
237 VISEntryHalf
238
239 alignaddr %o1, %g0, %g0
240
241 add %o1, (64 - 1), %o4
242 andn %o4, (64 - 1), %o4
243 andn %o2, (64 - 1), %g1
244 sub %o2, %g1, %o2
245
246 and %o1, (64 - 1), %g2
247 add %o1, %g1, %o1
248 sub %o0, %o4, %g3
249 brz,pt %g2, 190f
250 cmp %g2, 32
251 blu,a 5f
252 cmp %g2, 16
253 cmp %g2, 48
254 blu,a 4f
255 cmp %g2, 40
256 cmp %g2, 56
257 blu 170f
258 nop
259 ba,a,pt %xcc, 180f
260
2614: /* 32 <= low bits < 48 */
262 blu 150f
263 nop
264 ba,a,pt %xcc, 160f
2655: /* 0 < low bits < 32 */
266 blu,a 6f
267 cmp %g2, 8
268 cmp %g2, 24
269 blu 130f
270 nop
271 ba,a,pt %xcc, 140f
2726: /* 0 < low bits < 16 */
273 bgeu 120f
274 nop
275 /* fall through for 0 < low bits < 8 */
276110: sub %o4, 64, %g2
277 EX_LD(LOAD_BLK(%g2, %f0))
2781: EX_ST(STORE_INIT(%g0, %o4 + %g3))
279 EX_LD(LOAD_BLK(%o4, %f16))
280 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
281 EX_ST(STORE_BLK(%f0, %o4 + %g3))
282 FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
283 subcc %g1, 64, %g1
284 add %o4, 64, %o4
285 bne,pt %xcc, 1b
286 LOAD(prefetch, %o4 + 64, #one_read)
287 ba,pt %xcc, 195f
288 nop
289
290120: sub %o4, 56, %g2
291 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
2921: EX_ST(STORE_INIT(%g0, %o4 + %g3))
293 EX_LD(LOAD_BLK(%o4, %f16))
294 FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
295 EX_ST(STORE_BLK(%f0, %o4 + %g3))
296 FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
297 subcc %g1, 64, %g1
298 add %o4, 64, %o4
299 bne,pt %xcc, 1b
300 LOAD(prefetch, %o4 + 64, #one_read)
301 ba,pt %xcc, 195f
302 nop
303
304130: sub %o4, 48, %g2
305 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
3061: EX_ST(STORE_INIT(%g0, %o4 + %g3))
307 EX_LD(LOAD_BLK(%o4, %f16))
308 FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
309 EX_ST(STORE_BLK(%f0, %o4 + %g3))
310 FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
311 subcc %g1, 64, %g1
312 add %o4, 64, %o4
313 bne,pt %xcc, 1b
314 LOAD(prefetch, %o4 + 64, #one_read)
315 ba,pt %xcc, 195f
316 nop
317
318140: sub %o4, 40, %g2
319 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
3201: EX_ST(STORE_INIT(%g0, %o4 + %g3))
321 EX_LD(LOAD_BLK(%o4, %f16))
322 FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
323 EX_ST(STORE_BLK(%f0, %o4 + %g3))
324 FREG_MOVE_5(f22, f24, f26, f28, f30)
325 subcc %g1, 64, %g1
326 add %o4, 64, %o4
327 bne,pt %xcc, 1b
328 LOAD(prefetch, %o4 + 64, #one_read)
329 ba,pt %xcc, 195f
330 nop
331
332150: sub %o4, 32, %g2
333 FREG_LOAD_4(%g2, f0, f2, f4, f6)
3341: EX_ST(STORE_INIT(%g0, %o4 + %g3))
335 EX_LD(LOAD_BLK(%o4, %f16))
336 FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
337 EX_ST(STORE_BLK(%f0, %o4 + %g3))
338 FREG_MOVE_4(f24, f26, f28, f30)
339 subcc %g1, 64, %g1
340 add %o4, 64, %o4
341 bne,pt %xcc, 1b
342 LOAD(prefetch, %o4 + 64, #one_read)
343 ba,pt %xcc, 195f
344 nop
345
346160: sub %o4, 24, %g2
347 FREG_LOAD_3(%g2, f0, f2, f4)
3481: EX_ST(STORE_INIT(%g0, %o4 + %g3))
349 EX_LD(LOAD_BLK(%o4, %f16))
350 FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
351 EX_ST(STORE_BLK(%f0, %o4 + %g3))
352 FREG_MOVE_3(f26, f28, f30)
353 subcc %g1, 64, %g1
354 add %o4, 64, %o4
355 bne,pt %xcc, 1b
356 LOAD(prefetch, %o4 + 64, #one_read)
357 ba,pt %xcc, 195f
358 nop
359
360170: sub %o4, 16, %g2
361 FREG_LOAD_2(%g2, f0, f2)
3621: EX_ST(STORE_INIT(%g0, %o4 + %g3))
363 EX_LD(LOAD_BLK(%o4, %f16))
364 FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
365 EX_ST(STORE_BLK(%f0, %o4 + %g3))
366 FREG_MOVE_2(f28, f30)
367 subcc %g1, 64, %g1
368 add %o4, 64, %o4
369 bne,pt %xcc, 1b
370 LOAD(prefetch, %o4 + 64, #one_read)
371 ba,pt %xcc, 195f
372 nop
373
374180: sub %o4, 8, %g2
375 FREG_LOAD_1(%g2, f0)
3761: EX_ST(STORE_INIT(%g0, %o4 + %g3))
377 EX_LD(LOAD_BLK(%o4, %f16))
378 FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
379 EX_ST(STORE_BLK(%f0, %o4 + %g3))
380 FREG_MOVE_1(f30)
381 subcc %g1, 64, %g1
382 add %o4, 64, %o4
383 bne,pt %xcc, 1b
384 LOAD(prefetch, %o4 + 64, #one_read)
385 ba,pt %xcc, 195f
386 nop
387
388190:
3891: EX_ST(STORE_INIT(%g0, %o4 + %g3))
390 subcc %g1, 64, %g1
391 EX_LD(LOAD_BLK(%o4, %f0))
392 EX_ST(STORE_BLK(%f0, %o4 + %g3))
393 add %o4, 64, %o4
394 bne,pt %xcc, 1b
395 LOAD(prefetch, %o4 + 64, #one_read)
396
397195:
398 add %o4, %g3, %o0
399 membar #Sync
400
401 VISExitHalf
402
403 /* %o2 contains any final bytes still needed to be copied
404 * over. If anything is left, we copy it one byte at a time.
405 */
406 brz,pt %o2, 85f
407 sub %o0, %o1, %o3
408 ba,a,pt %XCC, 90f
409
410 .align 64
41175: /* 16 < len <= 64 */
412 bne,pn %XCC, 75f
413 sub %o0, %o1, %o3
414
41572:
416 andn %o2, 0xf, %o4
417 and %o2, 0xf, %o2
4181: subcc %o4, 0x10, %o4
419 EX_LD(LOAD(ldx, %o1, %o5))
420 add %o1, 0x08, %o1
421 EX_LD(LOAD(ldx, %o1, %g1))
422 sub %o1, 0x08, %o1
423 EX_ST(STORE(stx, %o5, %o1 + %o3))
424 add %o1, 0x8, %o1
425 EX_ST(STORE(stx, %g1, %o1 + %o3))
426 bgu,pt %XCC, 1b
427 add %o1, 0x8, %o1
42873: andcc %o2, 0x8, %g0
429 be,pt %XCC, 1f
430 nop
431 sub %o2, 0x8, %o2
432 EX_LD(LOAD(ldx, %o1, %o5))
433 EX_ST(STORE(stx, %o5, %o1 + %o3))
434 add %o1, 0x8, %o1
4351: andcc %o2, 0x4, %g0
436 be,pt %XCC, 1f
437 nop
438 sub %o2, 0x4, %o2
439 EX_LD(LOAD(lduw, %o1, %o5))
440 EX_ST(STORE(stw, %o5, %o1 + %o3))
441 add %o1, 0x4, %o1
4421: cmp %o2, 0
443 be,pt %XCC, 85f
444 nop
445 ba,pt %xcc, 90f
446 nop
447
44875:
449 andcc %o0, 0x7, %g1
450 sub %g1, 0x8, %g1
451 be,pn %icc, 2f
452 sub %g0, %g1, %g1
453 sub %o2, %g1, %o2
454
4551: subcc %g1, 1, %g1
456 EX_LD(LOAD(ldub, %o1, %o5))
457 EX_ST(STORE(stb, %o5, %o1 + %o3))
458 bgu,pt %icc, 1b
459 add %o1, 1, %o1
460
4612: add %o1, %o3, %o0
462 andcc %o1, 0x7, %g1
463 bne,pt %icc, 8f
464 sll %g1, 3, %g1
465
466 cmp %o2, 16
467 bgeu,pt %icc, 72b
468 nop
469 ba,a,pt %xcc, 73b
470
4718: mov 64, %o3
472 andn %o1, 0x7, %o1
473 EX_LD(LOAD(ldx, %o1, %g2))
474 sub %o3, %g1, %o3
475 andn %o2, 0x7, %o4
476 sllx %g2, %g1, %g2
4771: add %o1, 0x8, %o1
478 EX_LD(LOAD(ldx, %o1, %g3))
479 subcc %o4, 0x8, %o4
480 srlx %g3, %o3, %o5
481 or %o5, %g2, %o5
482 EX_ST(STORE(stx, %o5, %o0))
483 add %o0, 0x8, %o0
484 bgu,pt %icc, 1b
485 sllx %g3, %g1, %g2
486
487 srl %g1, 3, %g1
488 andcc %o2, 0x7, %o2
489 be,pn %icc, 85f
490 add %o1, %g1, %o1
491 ba,pt %xcc, 90f
492 sub %o0, %o1, %o3
493
494 .align 64
49580: /* 0 < len <= 16 */
496 andcc %o3, 0x3, %g0
497 bne,pn %XCC, 90f
498 sub %o0, %o1, %o3
499
5001:
501 subcc %o2, 4, %o2
502 EX_LD(LOAD(lduw, %o1, %g1))
503 EX_ST(STORE(stw, %g1, %o1 + %o3))
504 bgu,pt %XCC, 1b
505 add %o1, 4, %o1
506
50785: retl
508 mov EX_RETVAL(GLOBAL_SPARE), %o0
509
510 .align 32
51190:
512 subcc %o2, 1, %o2
513 EX_LD(LOAD(ldub, %o1, %g1))
514 EX_ST(STORE(stb, %g1, %o1 + %o3))
515 bgu,pt %XCC, 90b
516 add %o1, 1, %o1
517 retl
518 mov EX_RETVAL(GLOBAL_SPARE), %o0
519
520 .size FUNC_NAME, .-FUNC_NAME
diff --git a/arch/sparc64/lib/NG2page.S b/arch/sparc64/lib/NG2page.S
new file mode 100644
index 000000000000..73b6b7c72cbf
--- /dev/null
+++ b/arch/sparc64/lib/NG2page.S
@@ -0,0 +1,61 @@
1/* NG2page.S: Niagara-2 optimized clear and copy page.
2 *
3 * Copyright (C) 2007 (davem@davemloft.net)
4 */
5
6#include <asm/asi.h>
7#include <asm/page.h>
8#include <asm/visasm.h>
9
10 .text
11 .align 32
12
13 /* This is heavily simplified from the sun4u variants
14 * because Niagara-2 does not have any D-cache aliasing issues.
15 */
16NG2copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
17 prefetch [%o1 + 0x00], #one_read
18 prefetch [%o1 + 0x40], #one_read
19 VISEntryHalf
20 set PAGE_SIZE, %g7
21 sub %o0, %o1, %g3
221: stxa %g0, [%o1 + %g3] ASI_BLK_INIT_QUAD_LDD_P
23 subcc %g7, 64, %g7
24 ldda [%o1] ASI_BLK_P, %f0
25 stda %f0, [%o1 + %g3] ASI_BLK_P
26 add %o1, 64, %o1
27 bne,pt %xcc, 1b
28 prefetch [%o1 + 0x40], #one_read
29 membar #Sync
30 VISExitHalf
31 retl
32 nop
33
34#define BRANCH_ALWAYS 0x10680000
35#define NOP 0x01000000
36#define NG_DO_PATCH(OLD, NEW) \
37 sethi %hi(NEW), %g1; \
38 or %g1, %lo(NEW), %g1; \
39 sethi %hi(OLD), %g2; \
40 or %g2, %lo(OLD), %g2; \
41 sub %g1, %g2, %g1; \
42 sethi %hi(BRANCH_ALWAYS), %g3; \
43 sll %g1, 11, %g1; \
44 srl %g1, 11 + 2, %g1; \
45 or %g3, %lo(BRANCH_ALWAYS), %g3; \
46 or %g3, %g1, %g3; \
47 stw %g3, [%g2]; \
48 sethi %hi(NOP), %g3; \
49 or %g3, %lo(NOP), %g3; \
50 stw %g3, [%g2 + 0x4]; \
51 flush %g2;
52
53 .globl niagara2_patch_pageops
54 .type niagara2_patch_pageops,#function
55niagara2_patch_pageops:
56 NG_DO_PATCH(copy_user_page, NG2copy_user_page)
57 NG_DO_PATCH(_clear_page, NGclear_page)
58 NG_DO_PATCH(clear_user_page, NGclear_user_page)
59 retl
60 nop
61 .size niagara2_patch_pageops,.-niagara2_patch_pageops
diff --git a/arch/sparc64/lib/NG2patch.S b/arch/sparc64/lib/NG2patch.S
new file mode 100644
index 000000000000..28c36f06a6d1
--- /dev/null
+++ b/arch/sparc64/lib/NG2patch.S
@@ -0,0 +1,33 @@
1/* NG2patch.S: Patch Ultra-I routines with Niagara-2 variant.
2 *
3 * Copyright (C) 2007 David S. Miller <davem@davemloft.net>
4 */
5
6#define BRANCH_ALWAYS 0x10680000
7#define NOP 0x01000000
8#define NG_DO_PATCH(OLD, NEW) \
9 sethi %hi(NEW), %g1; \
10 or %g1, %lo(NEW), %g1; \
11 sethi %hi(OLD), %g2; \
12 or %g2, %lo(OLD), %g2; \
13 sub %g1, %g2, %g1; \
14 sethi %hi(BRANCH_ALWAYS), %g3; \
15 sll %g1, 11, %g1; \
16 srl %g1, 11 + 2, %g1; \
17 or %g3, %lo(BRANCH_ALWAYS), %g3; \
18 or %g3, %g1, %g3; \
19 stw %g3, [%g2]; \
20 sethi %hi(NOP), %g3; \
21 or %g3, %lo(NOP), %g3; \
22 stw %g3, [%g2 + 0x4]; \
23 flush %g2;
24
25 .globl niagara2_patch_copyops
26 .type niagara2_patch_copyops,#function
27niagara2_patch_copyops:
28 NG_DO_PATCH(memcpy, NG2memcpy)
29 NG_DO_PATCH(___copy_from_user, NG2copy_from_user)
30 NG_DO_PATCH(___copy_to_user, NG2copy_to_user)
31 retl
32 nop
33 .size niagara2_patch_copyops,.-niagara2_patch_copyops
diff --git a/arch/sparc64/lib/NGpage.S b/arch/sparc64/lib/NGpage.S
index 8ce3a0c9c537..428920de05ba 100644
--- a/arch/sparc64/lib/NGpage.S
+++ b/arch/sparc64/lib/NGpage.S
@@ -45,6 +45,7 @@ NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
45 retl 45 retl
46 nop 46 nop
47 47
48 .globl NGclear_page, NGclear_user_page
48NGclear_page: /* %o0=dest */ 49NGclear_page: /* %o0=dest */
49NGclear_user_page: /* %o0=dest, %o1=vaddr */ 50NGclear_user_page: /* %o0=dest, %o1=vaddr */
50 mov 8, %g1 51 mov 8, %g1
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-30 07:53:04 -0500