diff options
author | Lachlan McIlroy <lachlan@redback.melbourne.sgi.com> | 2008-02-25 22:26:14 -0500 |
---|---|---|
committer | Lachlan McIlroy <lachlan@redback.melbourne.sgi.com> | 2008-02-25 22:26:14 -0500 |
commit | 91e229bbad6524aabaac8717b2f559283670c37a (patch) | |
tree | 84a55e4ac2dcf23add97bd9fde3e9cb232c12b30 /arch/x86/lib/csum-wrappers_64.c | |
parent | 6e5e93424dc66542c548dfaa3bfebe30d46d50dd (diff) | |
parent | bfa274e2436fc7ef72ef51c878083647f1cfd429 (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6 into for-linus
Diffstat (limited to 'arch/x86/lib/csum-wrappers_64.c')
-rw-r--r-- | arch/x86/lib/csum-wrappers_64.c | 147 |
1 files changed, 81 insertions, 66 deletions
diff --git a/arch/x86/lib/csum-wrappers_64.c b/arch/x86/lib/csum-wrappers_64.c index fd42a4a095fc..459b58a8a15c 100644 --- a/arch/x86/lib/csum-wrappers_64.c +++ b/arch/x86/lib/csum-wrappers_64.c | |||
@@ -1,117 +1,129 @@ | |||
1 | /* Copyright 2002,2003 Andi Kleen, SuSE Labs. | 1 | /* |
2 | * Copyright 2002, 2003 Andi Kleen, SuSE Labs. | ||
2 | * Subject to the GNU Public License v.2 | 3 | * Subject to the GNU Public License v.2 |
3 | * | 4 | * |
4 | * Wrappers of assembly checksum functions for x86-64. | 5 | * Wrappers of assembly checksum functions for x86-64. |
5 | */ | 6 | */ |
6 | |||
7 | #include <asm/checksum.h> | 7 | #include <asm/checksum.h> |
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | 9 | ||
10 | /** | 10 | /** |
11 | * csum_partial_copy_from_user - Copy and checksum from user space. | 11 | * csum_partial_copy_from_user - Copy and checksum from user space. |
12 | * @src: source address (user space) | 12 | * @src: source address (user space) |
13 | * @dst: destination address | 13 | * @dst: destination address |
14 | * @len: number of bytes to be copied. | 14 | * @len: number of bytes to be copied. |
15 | * @isum: initial sum that is added into the result (32bit unfolded) | 15 | * @isum: initial sum that is added into the result (32bit unfolded) |
16 | * @errp: set to -EFAULT for an bad source address. | 16 | * @errp: set to -EFAULT for an bad source address. |
17 | * | 17 | * |
18 | * Returns an 32bit unfolded checksum of the buffer. | 18 | * Returns an 32bit unfolded checksum of the buffer. |
19 | * src and dst are best aligned to 64bits. | 19 | * src and dst are best aligned to 64bits. |
20 | */ | 20 | */ |
21 | __wsum | 21 | __wsum |
22 | csum_partial_copy_from_user(const void __user *src, void *dst, | 22 | csum_partial_copy_from_user(const void __user *src, void *dst, |
23 | int len, __wsum isum, int *errp) | 23 | int len, __wsum isum, int *errp) |
24 | { | 24 | { |
25 | might_sleep(); | 25 | might_sleep(); |
26 | *errp = 0; | 26 | *errp = 0; |
27 | if (likely(access_ok(VERIFY_READ,src, len))) { | 27 | |
28 | /* Why 6, not 7? To handle odd addresses aligned we | 28 | if (!likely(access_ok(VERIFY_READ, src, len))) |
29 | would need to do considerable complications to fix the | 29 | goto out_err; |
30 | checksum which is defined as an 16bit accumulator. The | 30 | |
31 | fix alignment code is primarily for performance | 31 | /* |
32 | compatibility with 32bit and that will handle odd | 32 | * Why 6, not 7? To handle odd addresses aligned we |
33 | addresses slowly too. */ | 33 | * would need to do considerable complications to fix the |
34 | if (unlikely((unsigned long)src & 6)) { | 34 | * checksum which is defined as an 16bit accumulator. The |
35 | while (((unsigned long)src & 6) && len >= 2) { | 35 | * fix alignment code is primarily for performance |
36 | __u16 val16; | 36 | * compatibility with 32bit and that will handle odd |
37 | *errp = __get_user(val16, (const __u16 __user *)src); | 37 | * addresses slowly too. |
38 | if (*errp) | 38 | */ |
39 | return isum; | 39 | if (unlikely((unsigned long)src & 6)) { |
40 | *(__u16 *)dst = val16; | 40 | while (((unsigned long)src & 6) && len >= 2) { |
41 | isum = (__force __wsum)add32_with_carry( | 41 | __u16 val16; |
42 | (__force unsigned)isum, val16); | 42 | |
43 | src += 2; | 43 | *errp = __get_user(val16, (const __u16 __user *)src); |
44 | dst += 2; | 44 | if (*errp) |
45 | len -= 2; | 45 | return isum; |
46 | } | 46 | |
47 | *(__u16 *)dst = val16; | ||
48 | isum = (__force __wsum)add32_with_carry( | ||
49 | (__force unsigned)isum, val16); | ||
50 | src += 2; | ||
51 | dst += 2; | ||
52 | len -= 2; | ||
47 | } | 53 | } |
48 | isum = csum_partial_copy_generic((__force const void *)src, | 54 | } |
49 | dst, len, isum, errp, NULL); | 55 | isum = csum_partial_copy_generic((__force const void *)src, |
50 | if (likely(*errp == 0)) | 56 | dst, len, isum, errp, NULL); |
51 | return isum; | 57 | if (unlikely(*errp)) |
52 | } | 58 | goto out_err; |
59 | |||
60 | return isum; | ||
61 | |||
62 | out_err: | ||
53 | *errp = -EFAULT; | 63 | *errp = -EFAULT; |
54 | memset(dst,0,len); | 64 | memset(dst, 0, len); |
55 | return isum; | ||
56 | } | ||
57 | 65 | ||
66 | return isum; | ||
67 | } | ||
58 | EXPORT_SYMBOL(csum_partial_copy_from_user); | 68 | EXPORT_SYMBOL(csum_partial_copy_from_user); |
59 | 69 | ||
60 | /** | 70 | /** |
61 | * csum_partial_copy_to_user - Copy and checksum to user space. | 71 | * csum_partial_copy_to_user - Copy and checksum to user space. |
62 | * @src: source address | 72 | * @src: source address |
63 | * @dst: destination address (user space) | 73 | * @dst: destination address (user space) |
64 | * @len: number of bytes to be copied. | 74 | * @len: number of bytes to be copied. |
65 | * @isum: initial sum that is added into the result (32bit unfolded) | 75 | * @isum: initial sum that is added into the result (32bit unfolded) |
66 | * @errp: set to -EFAULT for an bad destination address. | 76 | * @errp: set to -EFAULT for an bad destination address. |
67 | * | 77 | * |
68 | * Returns an 32bit unfolded checksum of the buffer. | 78 | * Returns an 32bit unfolded checksum of the buffer. |
69 | * src and dst are best aligned to 64bits. | 79 | * src and dst are best aligned to 64bits. |
70 | */ | 80 | */ |
71 | __wsum | 81 | __wsum |
72 | csum_partial_copy_to_user(const void *src, void __user *dst, | 82 | csum_partial_copy_to_user(const void *src, void __user *dst, |
73 | int len, __wsum isum, int *errp) | 83 | int len, __wsum isum, int *errp) |
74 | { | 84 | { |
75 | might_sleep(); | 85 | might_sleep(); |
86 | |||
76 | if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) { | 87 | if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) { |
77 | *errp = -EFAULT; | 88 | *errp = -EFAULT; |
78 | return 0; | 89 | return 0; |
79 | } | 90 | } |
80 | 91 | ||
81 | if (unlikely((unsigned long)dst & 6)) { | 92 | if (unlikely((unsigned long)dst & 6)) { |
82 | while (((unsigned long)dst & 6) && len >= 2) { | 93 | while (((unsigned long)dst & 6) && len >= 2) { |
83 | __u16 val16 = *(__u16 *)src; | 94 | __u16 val16 = *(__u16 *)src; |
95 | |||
84 | isum = (__force __wsum)add32_with_carry( | 96 | isum = (__force __wsum)add32_with_carry( |
85 | (__force unsigned)isum, val16); | 97 | (__force unsigned)isum, val16); |
86 | *errp = __put_user(val16, (__u16 __user *)dst); | 98 | *errp = __put_user(val16, (__u16 __user *)dst); |
87 | if (*errp) | 99 | if (*errp) |
88 | return isum; | 100 | return isum; |
89 | src += 2; | 101 | src += 2; |
90 | dst += 2; | 102 | dst += 2; |
91 | len -= 2; | 103 | len -= 2; |
92 | } | 104 | } |
93 | } | 105 | } |
94 | 106 | ||
95 | *errp = 0; | 107 | *errp = 0; |
96 | return csum_partial_copy_generic(src, (void __force *)dst,len,isum,NULL,errp); | 108 | return csum_partial_copy_generic(src, (void __force *)dst, |
97 | } | 109 | len, isum, NULL, errp); |
98 | 110 | } | |
99 | EXPORT_SYMBOL(csum_partial_copy_to_user); | 111 | EXPORT_SYMBOL(csum_partial_copy_to_user); |
100 | 112 | ||
101 | /** | 113 | /** |
102 | * csum_partial_copy_nocheck - Copy and checksum. | 114 | * csum_partial_copy_nocheck - Copy and checksum. |
103 | * @src: source address | 115 | * @src: source address |
104 | * @dst: destination address | 116 | * @dst: destination address |
105 | * @len: number of bytes to be copied. | 117 | * @len: number of bytes to be copied. |
106 | * @isum: initial sum that is added into the result (32bit unfolded) | 118 | * @isum: initial sum that is added into the result (32bit unfolded) |
107 | * | 119 | * |
108 | * Returns an 32bit unfolded checksum of the buffer. | 120 | * Returns an 32bit unfolded checksum of the buffer. |
109 | */ | 121 | */ |
110 | __wsum | 122 | __wsum |
111 | csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) | 123 | csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) |
112 | { | 124 | { |
113 | return csum_partial_copy_generic(src,dst,len,sum,NULL,NULL); | 125 | return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL); |
114 | } | 126 | } |
115 | EXPORT_SYMBOL(csum_partial_copy_nocheck); | 127 | EXPORT_SYMBOL(csum_partial_copy_nocheck); |
116 | 128 | ||
117 | __sum16 csum_ipv6_magic(const struct in6_addr *saddr, | 129 | __sum16 csum_ipv6_magic(const struct in6_addr *saddr, |
@@ -119,17 +131,20 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, | |||
119 | __u32 len, unsigned short proto, __wsum sum) | 131 | __u32 len, unsigned short proto, __wsum sum) |
120 | { | 132 | { |
121 | __u64 rest, sum64; | 133 | __u64 rest, sum64; |
122 | 134 | ||
123 | rest = (__force __u64)htonl(len) + (__force __u64)htons(proto) + | 135 | rest = (__force __u64)htonl(len) + (__force __u64)htons(proto) + |
124 | (__force __u64)sum; | 136 | (__force __u64)sum; |
125 | asm(" addq (%[saddr]),%[sum]\n" | ||
126 | " adcq 8(%[saddr]),%[sum]\n" | ||
127 | " adcq (%[daddr]),%[sum]\n" | ||
128 | " adcq 8(%[daddr]),%[sum]\n" | ||
129 | " adcq $0,%[sum]\n" | ||
130 | : [sum] "=r" (sum64) | ||
131 | : "[sum]" (rest),[saddr] "r" (saddr), [daddr] "r" (daddr)); | ||
132 | return csum_fold((__force __wsum)add32_with_carry(sum64 & 0xffffffff, sum64>>32)); | ||
133 | } | ||
134 | 137 | ||
138 | asm(" addq (%[saddr]),%[sum]\n" | ||
139 | " adcq 8(%[saddr]),%[sum]\n" | ||
140 | " adcq (%[daddr]),%[sum]\n" | ||
141 | " adcq 8(%[daddr]),%[sum]\n" | ||
142 | " adcq $0,%[sum]\n" | ||
143 | |||
144 | : [sum] "=r" (sum64) | ||
145 | : "[sum]" (rest), [saddr] "r" (saddr), [daddr] "r" (daddr)); | ||
146 | |||
147 | return csum_fold( | ||
148 | (__force __wsum)add32_with_carry(sum64 & 0xffffffff, sum64>>32)); | ||
149 | } | ||
135 | EXPORT_SYMBOL(csum_ipv6_magic); | 150 | EXPORT_SYMBOL(csum_ipv6_magic); |