summaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2015-03-20 18:51:09 -0400
committerDavid S. Miller <davem@davemloft.net>2015-03-20 18:51:09 -0400
commit0fa74a4be48e0f810d3dc6ddbc9d6ac7e86cbee8 (patch)
treeccfee93ede4e36d6d355e00e485d3d1c0fec0bdd /lib
parent6626af692692b52c8f9e20ad8201a3255e5ab25b (diff)
parent4de930efc23b92ddf88ce91c405ee645fe6e27ea (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net
Conflicts: drivers/net/ethernet/emulex/benet/be_main.c net/core/sysctl_net_core.c net/ipv4/inet_diag.c The be_main.c conflict resolution was really tricky. The conflict hunks generated by GIT were very unhelpful, to say the least. It split functions in half and moved them around, when the real actual conflict only existed solely inside of one function, that being be_map_pci_bars(). So instead, to resolve this, I checked out be_main.c from the top of net-next, then I applied the be_main.c changes from 'net' since the last time I merged. And this worked beautifully. The inet_diag.c and sysctl_net_core.c conflicts were simple overlapping changes, and were easily to resolve. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'lib')
-rw-r--r--lib/Makefile2
-rw-r--r--lib/iov_iter.c768
2 files changed, 769 insertions, 1 deletions
diff --git a/lib/Makefile b/lib/Makefile
index 87eb3bffc283..58f74d2dd396 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -24,7 +24,7 @@ obj-y += lockref.o
24 24
25obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \ 25obj-y += bcd.o div64.o sort.o parser.o halfmd4.o debug_locks.o random32.o \
26 bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \ 26 bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \
27 gcd.o lcm.o list_sort.o uuid.o flex_array.o clz_ctz.o \ 27 gcd.o lcm.o list_sort.o uuid.o flex_array.o iov_iter.o clz_ctz.o \
28 bsearch.o find_last_bit.o find_next_bit.o llist.o memweight.o kfifo.o \ 28 bsearch.o find_last_bit.o find_next_bit.o llist.o memweight.o kfifo.o \
29 percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o 29 percpu-refcount.o percpu_ida.o rhashtable.o reciprocal_div.o
30obj-y += string_helpers.o 30obj-y += string_helpers.o
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
new file mode 100644
index 000000000000..9d96e283520c
--- /dev/null
+++ b/lib/iov_iter.c
@@ -0,0 +1,768 @@
1#include <linux/export.h>
2#include <linux/uio.h>
3#include <linux/pagemap.h>
4#include <linux/slab.h>
5#include <linux/vmalloc.h>
6#include <net/checksum.h>
7
8#define iterate_iovec(i, n, __v, __p, skip, STEP) { \
9 size_t left; \
10 size_t wanted = n; \
11 __p = i->iov; \
12 __v.iov_len = min(n, __p->iov_len - skip); \
13 if (likely(__v.iov_len)) { \
14 __v.iov_base = __p->iov_base + skip; \
15 left = (STEP); \
16 __v.iov_len -= left; \
17 skip += __v.iov_len; \
18 n -= __v.iov_len; \
19 } else { \
20 left = 0; \
21 } \
22 while (unlikely(!left && n)) { \
23 __p++; \
24 __v.iov_len = min(n, __p->iov_len); \
25 if (unlikely(!__v.iov_len)) \
26 continue; \
27 __v.iov_base = __p->iov_base; \
28 left = (STEP); \
29 __v.iov_len -= left; \
30 skip = __v.iov_len; \
31 n -= __v.iov_len; \
32 } \
33 n = wanted - n; \
34}
35
36#define iterate_kvec(i, n, __v, __p, skip, STEP) { \
37 size_t wanted = n; \
38 __p = i->kvec; \
39 __v.iov_len = min(n, __p->iov_len - skip); \
40 if (likely(__v.iov_len)) { \
41 __v.iov_base = __p->iov_base + skip; \
42 (void)(STEP); \
43 skip += __v.iov_len; \
44 n -= __v.iov_len; \
45 } \
46 while (unlikely(n)) { \
47 __p++; \
48 __v.iov_len = min(n, __p->iov_len); \
49 if (unlikely(!__v.iov_len)) \
50 continue; \
51 __v.iov_base = __p->iov_base; \
52 (void)(STEP); \
53 skip = __v.iov_len; \
54 n -= __v.iov_len; \
55 } \
56 n = wanted; \
57}
58
59#define iterate_bvec(i, n, __v, __p, skip, STEP) { \
60 size_t wanted = n; \
61 __p = i->bvec; \
62 __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \
63 if (likely(__v.bv_len)) { \
64 __v.bv_page = __p->bv_page; \
65 __v.bv_offset = __p->bv_offset + skip; \
66 (void)(STEP); \
67 skip += __v.bv_len; \
68 n -= __v.bv_len; \
69 } \
70 while (unlikely(n)) { \
71 __p++; \
72 __v.bv_len = min_t(size_t, n, __p->bv_len); \
73 if (unlikely(!__v.bv_len)) \
74 continue; \
75 __v.bv_page = __p->bv_page; \
76 __v.bv_offset = __p->bv_offset; \
77 (void)(STEP); \
78 skip = __v.bv_len; \
79 n -= __v.bv_len; \
80 } \
81 n = wanted; \
82}
83
84#define iterate_all_kinds(i, n, v, I, B, K) { \
85 size_t skip = i->iov_offset; \
86 if (unlikely(i->type & ITER_BVEC)) { \
87 const struct bio_vec *bvec; \
88 struct bio_vec v; \
89 iterate_bvec(i, n, v, bvec, skip, (B)) \
90 } else if (unlikely(i->type & ITER_KVEC)) { \
91 const struct kvec *kvec; \
92 struct kvec v; \
93 iterate_kvec(i, n, v, kvec, skip, (K)) \
94 } else { \
95 const struct iovec *iov; \
96 struct iovec v; \
97 iterate_iovec(i, n, v, iov, skip, (I)) \
98 } \
99}
100
101#define iterate_and_advance(i, n, v, I, B, K) { \
102 size_t skip = i->iov_offset; \
103 if (unlikely(i->type & ITER_BVEC)) { \
104 const struct bio_vec *bvec; \
105 struct bio_vec v; \
106 iterate_bvec(i, n, v, bvec, skip, (B)) \
107 if (skip == bvec->bv_len) { \
108 bvec++; \
109 skip = 0; \
110 } \
111 i->nr_segs -= bvec - i->bvec; \
112 i->bvec = bvec; \
113 } else if (unlikely(i->type & ITER_KVEC)) { \
114 const struct kvec *kvec; \
115 struct kvec v; \
116 iterate_kvec(i, n, v, kvec, skip, (K)) \
117 if (skip == kvec->iov_len) { \
118 kvec++; \
119 skip = 0; \
120 } \
121 i->nr_segs -= kvec - i->kvec; \
122 i->kvec = kvec; \
123 } else { \
124 const struct iovec *iov; \
125 struct iovec v; \
126 iterate_iovec(i, n, v, iov, skip, (I)) \
127 if (skip == iov->iov_len) { \
128 iov++; \
129 skip = 0; \
130 } \
131 i->nr_segs -= iov - i->iov; \
132 i->iov = iov; \
133 } \
134 i->count -= n; \
135 i->iov_offset = skip; \
136}
137
138static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
139 struct iov_iter *i)
140{
141 size_t skip, copy, left, wanted;
142 const struct iovec *iov;
143 char __user *buf;
144 void *kaddr, *from;
145
146 if (unlikely(bytes > i->count))
147 bytes = i->count;
148
149 if (unlikely(!bytes))
150 return 0;
151
152 wanted = bytes;
153 iov = i->iov;
154 skip = i->iov_offset;
155 buf = iov->iov_base + skip;
156 copy = min(bytes, iov->iov_len - skip);
157
158 if (!fault_in_pages_writeable(buf, copy)) {
159 kaddr = kmap_atomic(page);
160 from = kaddr + offset;
161
162 /* first chunk, usually the only one */
163 left = __copy_to_user_inatomic(buf, from, copy);
164 copy -= left;
165 skip += copy;
166 from += copy;
167 bytes -= copy;
168
169 while (unlikely(!left && bytes)) {
170 iov++;
171 buf = iov->iov_base;
172 copy = min(bytes, iov->iov_len);
173 left = __copy_to_user_inatomic(buf, from, copy);
174 copy -= left;
175 skip = copy;
176 from += copy;
177 bytes -= copy;
178 }
179 if (likely(!bytes)) {
180 kunmap_atomic(kaddr);
181 goto done;
182 }
183 offset = from - kaddr;
184 buf += copy;
185 kunmap_atomic(kaddr);
186 copy = min(bytes, iov->iov_len - skip);
187 }
188 /* Too bad - revert to non-atomic kmap */
189 kaddr = kmap(page);
190 from = kaddr + offset;
191 left = __copy_to_user(buf, from, copy);
192 copy -= left;
193 skip += copy;
194 from += copy;
195 bytes -= copy;
196 while (unlikely(!left && bytes)) {
197 iov++;
198 buf = iov->iov_base;
199 copy = min(bytes, iov->iov_len);
200 left = __copy_to_user(buf, from, copy);
201 copy -= left;
202 skip = copy;
203 from += copy;
204 bytes -= copy;
205 }
206 kunmap(page);
207done:
208 if (skip == iov->iov_len) {
209 iov++;
210 skip = 0;
211 }
212 i->count -= wanted - bytes;
213 i->nr_segs -= iov - i->iov;
214 i->iov = iov;
215 i->iov_offset = skip;
216 return wanted - bytes;
217}
218
219static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
220 struct iov_iter *i)
221{
222 size_t skip, copy, left, wanted;
223 const struct iovec *iov;
224 char __user *buf;
225 void *kaddr, *to;
226
227 if (unlikely(bytes > i->count))
228 bytes = i->count;
229
230 if (unlikely(!bytes))
231 return 0;
232
233 wanted = bytes;
234 iov = i->iov;
235 skip = i->iov_offset;
236 buf = iov->iov_base + skip;
237 copy = min(bytes, iov->iov_len - skip);
238
239 if (!fault_in_pages_readable(buf, copy)) {
240 kaddr = kmap_atomic(page);
241 to = kaddr + offset;
242
243 /* first chunk, usually the only one */
244 left = __copy_from_user_inatomic(to, buf, copy);
245 copy -= left;
246 skip += copy;
247 to += copy;
248 bytes -= copy;
249
250 while (unlikely(!left && bytes)) {
251 iov++;
252 buf = iov->iov_base;
253 copy = min(bytes, iov->iov_len);
254 left = __copy_from_user_inatomic(to, buf, copy);
255 copy -= left;
256 skip = copy;
257 to += copy;
258 bytes -= copy;
259 }
260 if (likely(!bytes)) {
261 kunmap_atomic(kaddr);
262 goto done;
263 }
264 offset = to - kaddr;
265 buf += copy;
266 kunmap_atomic(kaddr);
267 copy = min(bytes, iov->iov_len - skip);
268 }
269 /* Too bad - revert to non-atomic kmap */
270 kaddr = kmap(page);
271 to = kaddr + offset;
272 left = __copy_from_user(to, buf, copy);
273 copy -= left;
274 skip += copy;
275 to += copy;
276 bytes -= copy;
277 while (unlikely(!left && bytes)) {
278 iov++;
279 buf = iov->iov_base;
280 copy = min(bytes, iov->iov_len);
281 left = __copy_from_user(to, buf, copy);
282 copy -= left;
283 skip = copy;
284 to += copy;
285 bytes -= copy;
286 }
287 kunmap(page);
288done:
289 if (skip == iov->iov_len) {
290 iov++;
291 skip = 0;
292 }
293 i->count -= wanted - bytes;
294 i->nr_segs -= iov - i->iov;
295 i->iov = iov;
296 i->iov_offset = skip;
297 return wanted - bytes;
298}
299
300/*
301 * Fault in the first iovec of the given iov_iter, to a maximum length
302 * of bytes. Returns 0 on success, or non-zero if the memory could not be
303 * accessed (ie. because it is an invalid address).
304 *
305 * writev-intensive code may want this to prefault several iovecs -- that
306 * would be possible (callers must not rely on the fact that _only_ the
307 * first iovec will be faulted with the current implementation).
308 */
309int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
310{
311 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
312 char __user *buf = i->iov->iov_base + i->iov_offset;
313 bytes = min(bytes, i->iov->iov_len - i->iov_offset);
314 return fault_in_pages_readable(buf, bytes);
315 }
316 return 0;
317}
318EXPORT_SYMBOL(iov_iter_fault_in_readable);
319
320void iov_iter_init(struct iov_iter *i, int direction,
321 const struct iovec *iov, unsigned long nr_segs,
322 size_t count)
323{
324 /* It will get better. Eventually... */
325 if (segment_eq(get_fs(), KERNEL_DS)) {
326 direction |= ITER_KVEC;
327 i->type = direction;
328 i->kvec = (struct kvec *)iov;
329 } else {
330 i->type = direction;
331 i->iov = iov;
332 }
333 i->nr_segs = nr_segs;
334 i->iov_offset = 0;
335 i->count = count;
336}
337EXPORT_SYMBOL(iov_iter_init);
338
339static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
340{
341 char *from = kmap_atomic(page);
342 memcpy(to, from + offset, len);
343 kunmap_atomic(from);
344}
345
346static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
347{
348 char *to = kmap_atomic(page);
349 memcpy(to + offset, from, len);
350 kunmap_atomic(to);
351}
352
353static void memzero_page(struct page *page, size_t offset, size_t len)
354{
355 char *addr = kmap_atomic(page);
356 memset(addr + offset, 0, len);
357 kunmap_atomic(addr);
358}
359
360size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i)
361{
362 char *from = addr;
363 if (unlikely(bytes > i->count))
364 bytes = i->count;
365
366 if (unlikely(!bytes))
367 return 0;
368
369 iterate_and_advance(i, bytes, v,
370 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
371 v.iov_len),
372 memcpy_to_page(v.bv_page, v.bv_offset,
373 (from += v.bv_len) - v.bv_len, v.bv_len),
374 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
375 )
376
377 return bytes;
378}
379EXPORT_SYMBOL(copy_to_iter);
380
381size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
382{
383 char *to = addr;
384 if (unlikely(bytes > i->count))
385 bytes = i->count;
386
387 if (unlikely(!bytes))
388 return 0;
389
390 iterate_and_advance(i, bytes, v,
391 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
392 v.iov_len),
393 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
394 v.bv_offset, v.bv_len),
395 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
396 )
397
398 return bytes;
399}
400EXPORT_SYMBOL(copy_from_iter);
401
402size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
403{
404 char *to = addr;
405 if (unlikely(bytes > i->count))
406 bytes = i->count;
407
408 if (unlikely(!bytes))
409 return 0;
410
411 iterate_and_advance(i, bytes, v,
412 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
413 v.iov_base, v.iov_len),
414 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
415 v.bv_offset, v.bv_len),
416 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
417 )
418
419 return bytes;
420}
421EXPORT_SYMBOL(copy_from_iter_nocache);
422
423size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
424 struct iov_iter *i)
425{
426 if (i->type & (ITER_BVEC|ITER_KVEC)) {
427 void *kaddr = kmap_atomic(page);
428 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
429 kunmap_atomic(kaddr);
430 return wanted;
431 } else
432 return copy_page_to_iter_iovec(page, offset, bytes, i);
433}
434EXPORT_SYMBOL(copy_page_to_iter);
435
436size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
437 struct iov_iter *i)
438{
439 if (i->type & (ITER_BVEC|ITER_KVEC)) {
440 void *kaddr = kmap_atomic(page);
441 size_t wanted = copy_from_iter(kaddr + offset, bytes, i);
442 kunmap_atomic(kaddr);
443 return wanted;
444 } else
445 return copy_page_from_iter_iovec(page, offset, bytes, i);
446}
447EXPORT_SYMBOL(copy_page_from_iter);
448
449size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
450{
451 if (unlikely(bytes > i->count))
452 bytes = i->count;
453
454 if (unlikely(!bytes))
455 return 0;
456
457 iterate_and_advance(i, bytes, v,
458 __clear_user(v.iov_base, v.iov_len),
459 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
460 memset(v.iov_base, 0, v.iov_len)
461 )
462
463 return bytes;
464}
465EXPORT_SYMBOL(iov_iter_zero);
466
467size_t iov_iter_copy_from_user_atomic(struct page *page,
468 struct iov_iter *i, unsigned long offset, size_t bytes)
469{
470 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
471 iterate_all_kinds(i, bytes, v,
472 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len,
473 v.iov_base, v.iov_len),
474 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
475 v.bv_offset, v.bv_len),
476 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
477 )
478 kunmap_atomic(kaddr);
479 return bytes;
480}
481EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
482
483void iov_iter_advance(struct iov_iter *i, size_t size)
484{
485 iterate_and_advance(i, size, v, 0, 0, 0)
486}
487EXPORT_SYMBOL(iov_iter_advance);
488
489/*
490 * Return the count of just the current iov_iter segment.
491 */
492size_t iov_iter_single_seg_count(const struct iov_iter *i)
493{
494 if (i->nr_segs == 1)
495 return i->count;
496 else if (i->type & ITER_BVEC)
497 return min(i->count, i->bvec->bv_len - i->iov_offset);
498 else
499 return min(i->count, i->iov->iov_len - i->iov_offset);
500}
501EXPORT_SYMBOL(iov_iter_single_seg_count);
502
503void iov_iter_kvec(struct iov_iter *i, int direction,
504 const struct kvec *kvec, unsigned long nr_segs,
505 size_t count)
506{
507 BUG_ON(!(direction & ITER_KVEC));
508 i->type = direction;
509 i->kvec = kvec;
510 i->nr_segs = nr_segs;
511 i->iov_offset = 0;
512 i->count = count;
513}
514EXPORT_SYMBOL(iov_iter_kvec);
515
516void iov_iter_bvec(struct iov_iter *i, int direction,
517 const struct bio_vec *bvec, unsigned long nr_segs,
518 size_t count)
519{
520 BUG_ON(!(direction & ITER_BVEC));
521 i->type = direction;
522 i->bvec = bvec;
523 i->nr_segs = nr_segs;
524 i->iov_offset = 0;
525 i->count = count;
526}
527EXPORT_SYMBOL(iov_iter_bvec);
528
529unsigned long iov_iter_alignment(const struct iov_iter *i)
530{
531 unsigned long res = 0;
532 size_t size = i->count;
533
534 if (!size)
535 return 0;
536
537 iterate_all_kinds(i, size, v,
538 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
539 res |= v.bv_offset | v.bv_len,
540 res |= (unsigned long)v.iov_base | v.iov_len
541 )
542 return res;
543}
544EXPORT_SYMBOL(iov_iter_alignment);
545
546ssize_t iov_iter_get_pages(struct iov_iter *i,
547 struct page **pages, size_t maxsize, unsigned maxpages,
548 size_t *start)
549{
550 if (maxsize > i->count)
551 maxsize = i->count;
552
553 if (!maxsize)
554 return 0;
555
556 iterate_all_kinds(i, maxsize, v, ({
557 unsigned long addr = (unsigned long)v.iov_base;
558 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
559 int n;
560 int res;
561
562 if (len > maxpages * PAGE_SIZE)
563 len = maxpages * PAGE_SIZE;
564 addr &= ~(PAGE_SIZE - 1);
565 n = DIV_ROUND_UP(len, PAGE_SIZE);
566 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
567 if (unlikely(res < 0))
568 return res;
569 return (res == n ? len : res * PAGE_SIZE) - *start;
570 0;}),({
571 /* can't be more than PAGE_SIZE */
572 *start = v.bv_offset;
573 get_page(*pages = v.bv_page);
574 return v.bv_len;
575 }),({
576 return -EFAULT;
577 })
578 )
579 return 0;
580}
581EXPORT_SYMBOL(iov_iter_get_pages);
582
583static struct page **get_pages_array(size_t n)
584{
585 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
586 if (!p)
587 p = vmalloc(n * sizeof(struct page *));
588 return p;
589}
590
591ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
592 struct page ***pages, size_t maxsize,
593 size_t *start)
594{
595 struct page **p;
596
597 if (maxsize > i->count)
598 maxsize = i->count;
599
600 if (!maxsize)
601 return 0;
602
603 iterate_all_kinds(i, maxsize, v, ({
604 unsigned long addr = (unsigned long)v.iov_base;
605 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
606 int n;
607 int res;
608
609 addr &= ~(PAGE_SIZE - 1);
610 n = DIV_ROUND_UP(len, PAGE_SIZE);
611 p = get_pages_array(n);
612 if (!p)
613 return -ENOMEM;
614 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
615 if (unlikely(res < 0)) {
616 kvfree(p);
617 return res;
618 }
619 *pages = p;
620 return (res == n ? len : res * PAGE_SIZE) - *start;
621 0;}),({
622 /* can't be more than PAGE_SIZE */
623 *start = v.bv_offset;
624 *pages = p = get_pages_array(1);
625 if (!p)
626 return -ENOMEM;
627 get_page(*p = v.bv_page);
628 return v.bv_len;
629 }),({
630 return -EFAULT;
631 })
632 )
633 return 0;
634}
635EXPORT_SYMBOL(iov_iter_get_pages_alloc);
636
637size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
638 struct iov_iter *i)
639{
640 char *to = addr;
641 __wsum sum, next;
642 size_t off = 0;
643 if (unlikely(bytes > i->count))
644 bytes = i->count;
645
646 if (unlikely(!bytes))
647 return 0;
648
649 sum = *csum;
650 iterate_and_advance(i, bytes, v, ({
651 int err = 0;
652 next = csum_and_copy_from_user(v.iov_base,
653 (to += v.iov_len) - v.iov_len,
654 v.iov_len, 0, &err);
655 if (!err) {
656 sum = csum_block_add(sum, next, off);
657 off += v.iov_len;
658 }
659 err ? v.iov_len : 0;
660 }), ({
661 char *p = kmap_atomic(v.bv_page);
662 next = csum_partial_copy_nocheck(p + v.bv_offset,
663 (to += v.bv_len) - v.bv_len,
664 v.bv_len, 0);
665 kunmap_atomic(p);
666 sum = csum_block_add(sum, next, off);
667 off += v.bv_len;
668 }),({
669 next = csum_partial_copy_nocheck(v.iov_base,
670 (to += v.iov_len) - v.iov_len,
671 v.iov_len, 0);
672 sum = csum_block_add(sum, next, off);
673 off += v.iov_len;
674 })
675 )
676 *csum = sum;
677 return bytes;
678}
679EXPORT_SYMBOL(csum_and_copy_from_iter);
680
681size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum,
682 struct iov_iter *i)
683{
684 char *from = addr;
685 __wsum sum, next;
686 size_t off = 0;
687 if (unlikely(bytes > i->count))
688 bytes = i->count;
689
690 if (unlikely(!bytes))
691 return 0;
692
693 sum = *csum;
694 iterate_and_advance(i, bytes, v, ({
695 int err = 0;
696 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
697 v.iov_base,
698 v.iov_len, 0, &err);
699 if (!err) {
700 sum = csum_block_add(sum, next, off);
701 off += v.iov_len;
702 }
703 err ? v.iov_len : 0;
704 }), ({
705 char *p = kmap_atomic(v.bv_page);
706 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len,
707 p + v.bv_offset,
708 v.bv_len, 0);
709 kunmap_atomic(p);
710 sum = csum_block_add(sum, next, off);
711 off += v.bv_len;
712 }),({
713 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len,
714 v.iov_base,
715 v.iov_len, 0);
716 sum = csum_block_add(sum, next, off);
717 off += v.iov_len;
718 })
719 )
720 *csum = sum;
721 return bytes;
722}
723EXPORT_SYMBOL(csum_and_copy_to_iter);
724
725int iov_iter_npages(const struct iov_iter *i, int maxpages)
726{
727 size_t size = i->count;
728 int npages = 0;
729
730 if (!size)
731 return 0;
732
733 iterate_all_kinds(i, size, v, ({
734 unsigned long p = (unsigned long)v.iov_base;
735 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
736 - p / PAGE_SIZE;
737 if (npages >= maxpages)
738 return maxpages;
739 0;}),({
740 npages++;
741 if (npages >= maxpages)
742 return maxpages;
743 }),({
744 unsigned long p = (unsigned long)v.iov_base;
745 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
746 - p / PAGE_SIZE;
747 if (npages >= maxpages)
748 return maxpages;
749 })
750 )
751 return npages;
752}
753EXPORT_SYMBOL(iov_iter_npages);
754
755const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
756{
757 *new = *old;
758 if (new->type & ITER_BVEC)
759 return new->bvec = kmemdup(new->bvec,
760 new->nr_segs * sizeof(struct bio_vec),
761 flags);
762 else
763 /* iovec and kvec have identical layout */
764 return new->iov = kmemdup(new->iov,
765 new->nr_segs * sizeof(struct iovec),
766 flags);
767}
768EXPORT_SYMBOL(dup_iter);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 12:14:15 -0500