diff options
| author | Heiko Carstens <heiko.carstens@de.ibm.com> | 2013-11-19 08:25:17 -0500 |
|---|---|---|
| committer | Martin Schwidefsky <schwidefsky@de.ibm.com> | 2013-11-20 03:04:55 -0500 |
| commit | dba6bb60043ed73abca8990f237db63a8cea6c50 (patch) | |
| tree | cdd10045cafcd15567c00faaa66bad74164aa507 /arch/s390 | |
| parent | 26a35f373fbe6f21e8ad5ca4de1c01021e38fe2f (diff) | |
s390/mm: optimize copy_page
Always use the mvcl instruction to copy a page instead of mvpg or a
couple of mvc instructions.
Copying a huge page is 25% faster this way. Also bypass caches when
copying pages since only parts of a page will be used afterwards.
Especially when copying a huge page this would kick everything out
of the L1 and L2 data caches on a zEC12 machine.
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Diffstat (limited to 'arch/s390')
| -rw-r--r-- | arch/s390/include/asm/page.h | 38 |
1 files changed, 13 insertions, 25 deletions
diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index 316c8503a3b4..114258eeaacd 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h | |||
| @@ -48,33 +48,21 @@ static inline void clear_page(void *page) | |||
| 48 | : "memory", "cc"); | 48 | : "memory", "cc"); |
| 49 | } | 49 | } |
| 50 | 50 | ||
| 51 | /* | ||
| 52 | * copy_page uses the mvcl instruction with 0xb0 padding byte in order to | ||
| 53 | * bypass caches when copying a page. Especially when copying huge pages | ||
| 54 | * this keeps L1 and L2 data caches alive. | ||
| 55 | */ | ||
| 51 | static inline void copy_page(void *to, void *from) | 56 | static inline void copy_page(void *to, void *from) |
| 52 | { | 57 | { |
| 53 | if (MACHINE_HAS_MVPG) { | 58 | register void *reg2 asm ("2") = to; |
| 54 | register unsigned long reg0 asm ("0") = 0; | 59 | register unsigned long reg3 asm ("3") = 0x1000; |
| 55 | asm volatile( | 60 | register void *reg4 asm ("4") = from; |
| 56 | " mvpg %0,%1" | 61 | register unsigned long reg5 asm ("5") = 0xb0001000; |
| 57 | : : "a" (to), "a" (from), "d" (reg0) | 62 | asm volatile( |
| 58 | : "memory", "cc"); | 63 | " mvcl 2,4" |
| 59 | } else | 64 | : "+d" (reg2), "+d" (reg3), "+d" (reg4), "+d" (reg5) |
| 60 | asm volatile( | 65 | : : "memory", "cc"); |
| 61 | " mvc 0(256,%0),0(%1)\n" | ||
| 62 | " mvc 256(256,%0),256(%1)\n" | ||
| 63 | " mvc 512(256,%0),512(%1)\n" | ||
| 64 | " mvc 768(256,%0),768(%1)\n" | ||
| 65 | " mvc 1024(256,%0),1024(%1)\n" | ||
| 66 | " mvc 1280(256,%0),1280(%1)\n" | ||
| 67 | " mvc 1536(256,%0),1536(%1)\n" | ||
| 68 | " mvc 1792(256,%0),1792(%1)\n" | ||
| 69 | " mvc 2048(256,%0),2048(%1)\n" | ||
| 70 | " mvc 2304(256,%0),2304(%1)\n" | ||
| 71 | " mvc 2560(256,%0),2560(%1)\n" | ||
| 72 | " mvc 2816(256,%0),2816(%1)\n" | ||
| 73 | " mvc 3072(256,%0),3072(%1)\n" | ||
| 74 | " mvc 3328(256,%0),3328(%1)\n" | ||
| 75 | " mvc 3584(256,%0),3584(%1)\n" | ||
| 76 | " mvc 3840(256,%0),3840(%1)\n" | ||
| 77 | : : "a" (to), "a" (from) : "memory"); | ||
| 78 | } | 66 | } |
| 79 | 67 | ||
| 80 | #define clear_user_page(page, vaddr, pg) clear_page(page) | 68 | #define clear_user_page(page, vaddr, pg) clear_page(page) |