aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/iov_iter.c104
1 files changed, 36 insertions, 68 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 5fecddc32b1b..015061e49236 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -99,40 +99,44 @@
99} 99}
100 100
101#define iterate_and_advance(i, n, v, I, B, K) { \ 101#define iterate_and_advance(i, n, v, I, B, K) { \
102 size_t skip = i->iov_offset; \ 102 if (unlikely(i->count < n)) \
103 if (unlikely(i->type & ITER_BVEC)) { \ 103 n = i->count; \
104 const struct bio_vec *bvec; \ 104 if (n) { \
105 struct bio_vec v; \ 105 size_t skip = i->iov_offset; \
106 iterate_bvec(i, n, v, bvec, skip, (B)) \ 106 if (unlikely(i->type & ITER_BVEC)) { \
107 if (skip == bvec->bv_len) { \ 107 const struct bio_vec *bvec; \
108 bvec++; \ 108 struct bio_vec v; \
109 skip = 0; \ 109 iterate_bvec(i, n, v, bvec, skip, (B)) \
110 } \ 110 if (skip == bvec->bv_len) { \
111 i->nr_segs -= bvec - i->bvec; \ 111 bvec++; \
112 i->bvec = bvec; \ 112 skip = 0; \
113 } else if (unlikely(i->type & ITER_KVEC)) { \ 113 } \
114 const struct kvec *kvec; \ 114 i->nr_segs -= bvec - i->bvec; \
115 struct kvec v; \ 115 i->bvec = bvec; \
116 iterate_kvec(i, n, v, kvec, skip, (K)) \ 116 } else if (unlikely(i->type & ITER_KVEC)) { \
117 if (skip == kvec->iov_len) { \ 117 const struct kvec *kvec; \
118 kvec++; \ 118 struct kvec v; \
119 skip = 0; \ 119 iterate_kvec(i, n, v, kvec, skip, (K)) \
120 } \ 120 if (skip == kvec->iov_len) { \
121 i->nr_segs -= kvec - i->kvec; \ 121 kvec++; \
122 i->kvec = kvec; \ 122 skip = 0; \
123 } else { \ 123 } \
124 const struct iovec *iov; \ 124 i->nr_segs -= kvec - i->kvec; \
125 struct iovec v; \ 125 i->kvec = kvec; \
126 iterate_iovec(i, n, v, iov, skip, (I)) \ 126 } else { \
127 if (skip == iov->iov_len) { \ 127 const struct iovec *iov; \
128 iov++; \ 128 struct iovec v; \
129 skip = 0; \ 129 iterate_iovec(i, n, v, iov, skip, (I)) \
130 if (skip == iov->iov_len) { \
131 iov++; \
132 skip = 0; \
133 } \
134 i->nr_segs -= iov - i->iov; \
135 i->iov = iov; \
130 } \ 136 } \
131 i->nr_segs -= iov - i->iov; \ 137 i->count -= n; \
132 i->iov = iov; \ 138 i->iov_offset = skip; \
133 } \ 139 } \
134 i->count -= n; \
135 i->iov_offset = skip; \
136} 140}
137 141
138static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 142static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
@@ -386,12 +390,6 @@ static void memzero_page(struct page *page, size_t offset, size_t len)
386size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 390size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
387{ 391{
388 const char *from = addr; 392 const char *from = addr;
389 if (unlikely(bytes > i->count))
390 bytes = i->count;
391
392 if (unlikely(!bytes))
393 return 0;
394
395 iterate_and_advance(i, bytes, v, 393 iterate_and_advance(i, bytes, v,
396 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len, 394 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len,
397 v.iov_len), 395 v.iov_len),
@@ -407,12 +405,6 @@ EXPORT_SYMBOL(copy_to_iter);
407size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 405size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
408{ 406{
409 char *to = addr; 407 char *to = addr;
410 if (unlikely(bytes > i->count))
411 bytes = i->count;
412
413 if (unlikely(!bytes))
414 return 0;
415
416 iterate_and_advance(i, bytes, v, 408 iterate_and_advance(i, bytes, v,
417 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base, 409 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base,
418 v.iov_len), 410 v.iov_len),
@@ -428,12 +420,6 @@ EXPORT_SYMBOL(copy_from_iter);
428size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 420size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
429{ 421{
430 char *to = addr; 422 char *to = addr;
431 if (unlikely(bytes > i->count))
432 bytes = i->count;
433
434 if (unlikely(!bytes))
435 return 0;
436
437 iterate_and_advance(i, bytes, v, 423 iterate_and_advance(i, bytes, v,
438 __copy_from_user_nocache((to += v.iov_len) - v.iov_len, 424 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
439 v.iov_base, v.iov_len), 425 v.iov_base, v.iov_len),
@@ -474,12 +460,6 @@ EXPORT_SYMBOL(copy_page_from_iter);
474 460
475size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 461size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
476{ 462{
477 if (unlikely(bytes > i->count))
478 bytes = i->count;
479
480 if (unlikely(!bytes))
481 return 0;
482
483 iterate_and_advance(i, bytes, v, 463 iterate_and_advance(i, bytes, v,
484 __clear_user(v.iov_base, v.iov_len), 464 __clear_user(v.iov_base, v.iov_len),
485 memzero_page(v.bv_page, v.bv_offset, v.bv_len), 465 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
@@ -666,12 +646,6 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
666 char *to = addr; 646 char *to = addr;
667 __wsum sum, next; 647 __wsum sum, next;
668 size_t off = 0; 648 size_t off = 0;
669 if (unlikely(bytes > i->count))
670 bytes = i->count;
671
672 if (unlikely(!bytes))
673 return 0;
674
675 sum = *csum; 649 sum = *csum;
676 iterate_and_advance(i, bytes, v, ({ 650 iterate_and_advance(i, bytes, v, ({
677 int err = 0; 651 int err = 0;
@@ -710,12 +684,6 @@ size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
710 const char *from = addr; 684 const char *from = addr;
711 __wsum sum, next; 685 __wsum sum, next;
712 size_t off = 0; 686 size_t off = 0;
713 if (unlikely(bytes > i->count))
714 bytes = i->count;
715
716 if (unlikely(!bytes))
717 return 0;
718
719 sum = *csum; 687 sum = *csum;
720 iterate_and_advance(i, bytes, v, ({ 688 iterate_and_advance(i, bytes, v, ({
721 int err = 0; 689 int err = 0;
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-29 05:55:38 -0400