aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/lib/csum-wrappers_64.c
diff options
context:
space:
mode:
authorLachlan McIlroy <lachlan@redback.melbourne.sgi.com>2008-02-25 22:26:14 -0500
committerLachlan McIlroy <lachlan@redback.melbourne.sgi.com>2008-02-25 22:26:14 -0500
commit91e229bbad6524aabaac8717b2f559283670c37a (patch)
tree84a55e4ac2dcf23add97bd9fde3e9cb232c12b30 /arch/x86/lib/csum-wrappers_64.c
parent6e5e93424dc66542c548dfaa3bfebe30d46d50dd (diff)
parentbfa274e2436fc7ef72ef51c878083647f1cfd429 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6 into for-linus
Diffstat (limited to 'arch/x86/lib/csum-wrappers_64.c')
-rw-r--r--arch/x86/lib/csum-wrappers_64.c147
1 files changed, 81 insertions, 66 deletions
diff --git a/arch/x86/lib/csum-wrappers_64.c b/arch/x86/lib/csum-wrappers_64.c
index fd42a4a095fc..459b58a8a15c 100644
--- a/arch/x86/lib/csum-wrappers_64.c
+++ b/arch/x86/lib/csum-wrappers_64.c
@@ -1,117 +1,129 @@
1/* Copyright 2002,2003 Andi Kleen, SuSE Labs. 1/*
2 * Copyright 2002, 2003 Andi Kleen, SuSE Labs.
2 * Subject to the GNU Public License v.2 3 * Subject to the GNU Public License v.2
3 * 4 *
4 * Wrappers of assembly checksum functions for x86-64. 5 * Wrappers of assembly checksum functions for x86-64.
5 */ 6 */
6
7#include <asm/checksum.h> 7#include <asm/checksum.h>
8#include <linux/module.h> 8#include <linux/module.h>
9 9
10/** 10/**
11 * csum_partial_copy_from_user - Copy and checksum from user space. 11 * csum_partial_copy_from_user - Copy and checksum from user space.
12 * @src: source address (user space) 12 * @src: source address (user space)
13 * @dst: destination address 13 * @dst: destination address
14 * @len: number of bytes to be copied. 14 * @len: number of bytes to be copied.
15 * @isum: initial sum that is added into the result (32bit unfolded) 15 * @isum: initial sum that is added into the result (32bit unfolded)
16 * @errp: set to -EFAULT for an bad source address. 16 * @errp: set to -EFAULT for an bad source address.
17 * 17 *
18 * Returns an 32bit unfolded checksum of the buffer. 18 * Returns an 32bit unfolded checksum of the buffer.
19 * src and dst are best aligned to 64bits. 19 * src and dst are best aligned to 64bits.
20 */ 20 */
21__wsum 21__wsum
22csum_partial_copy_from_user(const void __user *src, void *dst, 22csum_partial_copy_from_user(const void __user *src, void *dst,
23 int len, __wsum isum, int *errp) 23 int len, __wsum isum, int *errp)
24{ 24{
25 might_sleep(); 25 might_sleep();
26 *errp = 0; 26 *errp = 0;
27 if (likely(access_ok(VERIFY_READ,src, len))) { 27
28 /* Why 6, not 7? To handle odd addresses aligned we 28 if (!likely(access_ok(VERIFY_READ, src, len)))
29 would need to do considerable complications to fix the 29 goto out_err;
30 checksum which is defined as an 16bit accumulator. The 30
31 fix alignment code is primarily for performance 31 /*
32 compatibility with 32bit and that will handle odd 32 * Why 6, not 7? To handle odd addresses aligned we
33 addresses slowly too. */ 33 * would need to do considerable complications to fix the
34 if (unlikely((unsigned long)src & 6)) { 34 * checksum which is defined as an 16bit accumulator. The
35 while (((unsigned long)src & 6) && len >= 2) { 35 * fix alignment code is primarily for performance
36 __u16 val16; 36 * compatibility with 32bit and that will handle odd
37 *errp = __get_user(val16, (const __u16 __user *)src); 37 * addresses slowly too.
38 if (*errp) 38 */
39 return isum; 39 if (unlikely((unsigned long)src & 6)) {
40 *(__u16 *)dst = val16; 40 while (((unsigned long)src & 6) && len >= 2) {
41 isum = (__force __wsum)add32_with_carry( 41 __u16 val16;
42 (__force unsigned)isum, val16); 42
43 src += 2; 43 *errp = __get_user(val16, (const __u16 __user *)src);
44 dst += 2; 44 if (*errp)
45 len -= 2; 45 return isum;
46 } 46
47 *(__u16 *)dst = val16;
48 isum = (__force __wsum)add32_with_carry(
49 (__force unsigned)isum, val16);
50 src += 2;
51 dst += 2;
52 len -= 2;
47 } 53 }
48 isum = csum_partial_copy_generic((__force const void *)src, 54 }
49 dst, len, isum, errp, NULL); 55 isum = csum_partial_copy_generic((__force const void *)src,
50 if (likely(*errp == 0)) 56 dst, len, isum, errp, NULL);
51 return isum; 57 if (unlikely(*errp))
52 } 58 goto out_err;
59
60 return isum;
61
62out_err:
53 *errp = -EFAULT; 63 *errp = -EFAULT;
54 memset(dst,0,len); 64 memset(dst, 0, len);
55 return isum;
56}
57 65
66 return isum;
67}
58EXPORT_SYMBOL(csum_partial_copy_from_user); 68EXPORT_SYMBOL(csum_partial_copy_from_user);
59 69
60/** 70/**
61 * csum_partial_copy_to_user - Copy and checksum to user space. 71 * csum_partial_copy_to_user - Copy and checksum to user space.
62 * @src: source address 72 * @src: source address
63 * @dst: destination address (user space) 73 * @dst: destination address (user space)
64 * @len: number of bytes to be copied. 74 * @len: number of bytes to be copied.
65 * @isum: initial sum that is added into the result (32bit unfolded) 75 * @isum: initial sum that is added into the result (32bit unfolded)
66 * @errp: set to -EFAULT for an bad destination address. 76 * @errp: set to -EFAULT for an bad destination address.
67 * 77 *
68 * Returns an 32bit unfolded checksum of the buffer. 78 * Returns an 32bit unfolded checksum of the buffer.
69 * src and dst are best aligned to 64bits. 79 * src and dst are best aligned to 64bits.
70 */ 80 */
71__wsum 81__wsum
72csum_partial_copy_to_user(const void *src, void __user *dst, 82csum_partial_copy_to_user(const void *src, void __user *dst,
73 int len, __wsum isum, int *errp) 83 int len, __wsum isum, int *errp)
74{ 84{
75 might_sleep(); 85 might_sleep();
86
76 if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) { 87 if (unlikely(!access_ok(VERIFY_WRITE, dst, len))) {
77 *errp = -EFAULT; 88 *errp = -EFAULT;
78 return 0; 89 return 0;
79 } 90 }
80 91
81 if (unlikely((unsigned long)dst & 6)) { 92 if (unlikely((unsigned long)dst & 6)) {
82 while (((unsigned long)dst & 6) && len >= 2) { 93 while (((unsigned long)dst & 6) && len >= 2) {
83 __u16 val16 = *(__u16 *)src; 94 __u16 val16 = *(__u16 *)src;
95
84 isum = (__force __wsum)add32_with_carry( 96 isum = (__force __wsum)add32_with_carry(
85 (__force unsigned)isum, val16); 97 (__force unsigned)isum, val16);
86 *errp = __put_user(val16, (__u16 __user *)dst); 98 *errp = __put_user(val16, (__u16 __user *)dst);
87 if (*errp) 99 if (*errp)
88 return isum; 100 return isum;
89 src += 2; 101 src += 2;
90 dst += 2; 102 dst += 2;
91 len -= 2; 103 len -= 2;
92 } 104 }
93 } 105 }
94 106
95 *errp = 0; 107 *errp = 0;
96 return csum_partial_copy_generic(src, (void __force *)dst,len,isum,NULL,errp); 108 return csum_partial_copy_generic(src, (void __force *)dst,
97} 109 len, isum, NULL, errp);
98 110}
99EXPORT_SYMBOL(csum_partial_copy_to_user); 111EXPORT_SYMBOL(csum_partial_copy_to_user);
100 112
101/** 113/**
102 * csum_partial_copy_nocheck - Copy and checksum. 114 * csum_partial_copy_nocheck - Copy and checksum.
103 * @src: source address 115 * @src: source address
104 * @dst: destination address 116 * @dst: destination address
105 * @len: number of bytes to be copied. 117 * @len: number of bytes to be copied.
106 * @isum: initial sum that is added into the result (32bit unfolded) 118 * @isum: initial sum that is added into the result (32bit unfolded)
107 * 119 *
108 * Returns an 32bit unfolded checksum of the buffer. 120 * Returns an 32bit unfolded checksum of the buffer.
109 */ 121 */
110__wsum 122__wsum
111csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) 123csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
112{ 124{
113 return csum_partial_copy_generic(src,dst,len,sum,NULL,NULL); 125 return csum_partial_copy_generic(src, dst, len, sum, NULL, NULL);
114} 126}
115EXPORT_SYMBOL(csum_partial_copy_nocheck); 127EXPORT_SYMBOL(csum_partial_copy_nocheck);
116 128
117__sum16 csum_ipv6_magic(const struct in6_addr *saddr, 129__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
@@ -119,17 +131,20 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
119 __u32 len, unsigned short proto, __wsum sum) 131 __u32 len, unsigned short proto, __wsum sum)
120{ 132{
121 __u64 rest, sum64; 133 __u64 rest, sum64;
122 134
123 rest = (__force __u64)htonl(len) + (__force __u64)htons(proto) + 135 rest = (__force __u64)htonl(len) + (__force __u64)htons(proto) +
124 (__force __u64)sum; 136 (__force __u64)sum;
125 asm(" addq (%[saddr]),%[sum]\n"
126 " adcq 8(%[saddr]),%[sum]\n"
127 " adcq (%[daddr]),%[sum]\n"
128 " adcq 8(%[daddr]),%[sum]\n"
129 " adcq $0,%[sum]\n"
130 : [sum] "=r" (sum64)
131 : "[sum]" (rest),[saddr] "r" (saddr), [daddr] "r" (daddr));
132 return csum_fold((__force __wsum)add32_with_carry(sum64 & 0xffffffff, sum64>>32));
133}
134 137
138 asm(" addq (%[saddr]),%[sum]\n"
139 " adcq 8(%[saddr]),%[sum]\n"
140 " adcq (%[daddr]),%[sum]\n"
141 " adcq 8(%[daddr]),%[sum]\n"
142 " adcq $0,%[sum]\n"
143
144 : [sum] "=r" (sum64)
145 : "[sum]" (rest), [saddr] "r" (saddr), [daddr] "r" (daddr));
146
147 return csum_fold(
148 (__force __wsum)add32_with_carry(sum64 & 0xffffffff, sum64>>32));
149}
135EXPORT_SYMBOL(csum_ipv6_magic); 150EXPORT_SYMBOL(csum_ipv6_magic);