aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@sunset.davemloft.net>2006-03-19 02:55:11 -0500
committerDavid S. Miller <davem@sunset.davemloft.net>2006-03-20 04:16:41 -0500
commitbb8646d8340fa7c1b66a037428e39f85f8738f0a (patch)
tree931d4505a0ba65124b662f0f8b5935e0b154bd66
parent88d7079458f87d6f2d2261b2f87b7b9416019f5e (diff)
[SPARC64]: Optimized TSB table initialization.
We only need to write an invalid tag every 16 bytes, so taking advantage of this can save many instructions compared to the simple memset() call we make now. A prefetching implementation is implemented for sun4u and a block-init store version if implemented for Niagara. The next trick is to be able to perform an init and a copy_tsb() in parallel when growing a TSB table. Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--arch/sparc64/kernel/tsb.S69
-rw-r--r--arch/sparc64/lib/NGbzero.S1
-rw-r--r--arch/sparc64/mm/tsb.c2
-rw-r--r--include/asm-sparc64/mmu.h1
4 files changed, 72 insertions, 1 deletions
diff --git a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
index 1b154c863628..118baea44f69 100644
--- a/arch/sparc64/kernel/tsb.S
+++ b/arch/sparc64/kernel/tsb.S
@@ -371,3 +371,72 @@ copy_tsb: /* %o0=old_tsb_base, %o1=old_tsb_size
371 retl 371 retl
372 TSB_MEMBAR 372 TSB_MEMBAR
373 .size copy_tsb, .-copy_tsb 373 .size copy_tsb, .-copy_tsb
374
375 /* Set the invalid bit in all TSB entries. */
376 .align 32
377 .globl tsb_init
378 .type tsb_init,#function
379tsb_init: /* %o0 = TSB vaddr, %o1 = size in bytes */
380 prefetch [%o0 + 0x000], #n_writes
381 mov 1, %g1
382 prefetch [%o0 + 0x040], #n_writes
383 sllx %g1, TSB_TAG_INVALID_BIT, %g1
384 prefetch [%o0 + 0x080], #n_writes
3851: prefetch [%o0 + 0x0c0], #n_writes
386 stx %g1, [%o0 + 0x00]
387 stx %g1, [%o0 + 0x10]
388 stx %g1, [%o0 + 0x20]
389 stx %g1, [%o0 + 0x30]
390 prefetch [%o0 + 0x100], #n_writes
391 stx %g1, [%o0 + 0x40]
392 stx %g1, [%o0 + 0x50]
393 stx %g1, [%o0 + 0x60]
394 stx %g1, [%o0 + 0x70]
395 prefetch [%o0 + 0x140], #n_writes
396 stx %g1, [%o0 + 0x80]
397 stx %g1, [%o0 + 0x90]
398 stx %g1, [%o0 + 0xa0]
399 stx %g1, [%o0 + 0xb0]
400 prefetch [%o0 + 0x180], #n_writes
401 stx %g1, [%o0 + 0xc0]
402 stx %g1, [%o0 + 0xd0]
403 stx %g1, [%o0 + 0xe0]
404 stx %g1, [%o0 + 0xf0]
405 subcc %o1, 0x100, %o1
406 bne,pt %xcc, 1b
407 add %o0, 0x100, %o0
408 retl
409 nop
410 nop
411 nop
412 .size tsb_init, .-tsb_init
413
414 .globl NGtsb_init
415 .type NGtsb_init,#function
416NGtsb_init:
417 rd %asi, %g2
418 mov 1, %g1
419 wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
420 sllx %g1, TSB_TAG_INVALID_BIT, %g1
4211: stxa %g1, [%o0 + 0x00] %asi
422 stxa %g1, [%o0 + 0x10] %asi
423 stxa %g1, [%o0 + 0x20] %asi
424 stxa %g1, [%o0 + 0x30] %asi
425 stxa %g1, [%o0 + 0x40] %asi
426 stxa %g1, [%o0 + 0x50] %asi
427 stxa %g1, [%o0 + 0x60] %asi
428 stxa %g1, [%o0 + 0x70] %asi
429 stxa %g1, [%o0 + 0x80] %asi
430 stxa %g1, [%o0 + 0x90] %asi
431 stxa %g1, [%o0 + 0xa0] %asi
432 stxa %g1, [%o0 + 0xb0] %asi
433 stxa %g1, [%o0 + 0xc0] %asi
434 stxa %g1, [%o0 + 0xd0] %asi
435 stxa %g1, [%o0 + 0xe0] %asi
436 stxa %g1, [%o0 + 0xf0] %asi
437 subcc %o1, 0x100, %o1
438 bne,pt %xcc, 1b
439 add %o0, 0x100, %o0
440 retl
441 wr %g2, 0x0, %asi
442 .size NGtsb_init, .-NGtsb_init
diff --git a/arch/sparc64/lib/NGbzero.S b/arch/sparc64/lib/NGbzero.S
index fef584f745dc..e86baece5cc8 100644
--- a/arch/sparc64/lib/NGbzero.S
+++ b/arch/sparc64/lib/NGbzero.S
@@ -157,6 +157,7 @@ niagara_patch_bzero:
157 NG_DO_PATCH(memset, NGmemset) 157 NG_DO_PATCH(memset, NGmemset)
158 NG_DO_PATCH(__bzero, NGbzero) 158 NG_DO_PATCH(__bzero, NGbzero)
159 NG_DO_PATCH(__clear_user, NGclear_user) 159 NG_DO_PATCH(__clear_user, NGclear_user)
160 NG_DO_PATCH(tsb_init, NGtsb_init)
160 retl 161 retl
161 nop 162 nop
162 .size niagara_patch_bzero,.-niagara_patch_bzero 163 .size niagara_patch_bzero,.-niagara_patch_bzero
diff --git a/arch/sparc64/mm/tsb.c b/arch/sparc64/mm/tsb.c
index 1af797a0a092..b2064e2a44d6 100644
--- a/arch/sparc64/mm/tsb.c
+++ b/arch/sparc64/mm/tsb.c
@@ -313,7 +313,7 @@ retry_tsb_alloc:
313 } 313 }
314 314
315 /* Mark all tags as invalid. */ 315 /* Mark all tags as invalid. */
316 memset(new_tsb, 0x40, new_size); 316 tsb_init(new_tsb, new_size);
317 317
318 /* Ok, we are about to commit the changes. If we are 318 /* Ok, we are about to commit the changes. If we are
319 * growing an existing TSB the locking is very tricky, 319 * growing an existing TSB the locking is very tricky,
diff --git a/include/asm-sparc64/mmu.h b/include/asm-sparc64/mmu.h
index da14a9bf0ed6..230ba678d3b0 100644
--- a/include/asm-sparc64/mmu.h
+++ b/include/asm-sparc64/mmu.h
@@ -88,6 +88,7 @@ struct tsb {
88 88
89extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); 89extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte);
90extern void tsb_flush(unsigned long ent, unsigned long tag); 90extern void tsb_flush(unsigned long ent, unsigned long tag);
91extern void tsb_init(struct tsb *tsb, unsigned long size);
91 92
92typedef struct { 93typedef struct {
93 spinlock_t lock; 94 spinlock_t lock;