aboutsummaryrefslogtreecommitdiffstats
path: root/block/bio.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/bio.c')
-rw-r--r--block/bio.c146
1 files changed, 67 insertions, 79 deletions
diff --git a/block/bio.c b/block/bio.c
index 0895f694f440..7d8c6555e3f3 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -28,7 +28,6 @@
28#include <linux/mempool.h> 28#include <linux/mempool.h>
29#include <linux/workqueue.h> 29#include <linux/workqueue.h>
30#include <linux/cgroup.h> 30#include <linux/cgroup.h>
31#include <scsi/sg.h> /* for struct sg_iovec */
32 31
33#include <trace/events/block.h> 32#include <trace/events/block.h>
34 33
@@ -1022,21 +1021,11 @@ void bio_copy_data(struct bio *dst, struct bio *src)
1022EXPORT_SYMBOL(bio_copy_data); 1021EXPORT_SYMBOL(bio_copy_data);
1023 1022
1024struct bio_map_data { 1023struct bio_map_data {
1025 int nr_sgvecs;
1026 int is_our_pages; 1024 int is_our_pages;
1027 struct sg_iovec sgvecs[]; 1025 struct iov_iter iter;
1026 struct iovec iov[];
1028}; 1027};
1029 1028
1030static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
1031 const struct sg_iovec *iov, int iov_count,
1032 int is_our_pages)
1033{
1034 memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
1035 bmd->nr_sgvecs = iov_count;
1036 bmd->is_our_pages = is_our_pages;
1037 bio->bi_private = bmd;
1038}
1039
1040static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count, 1029static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
1041 gfp_t gfp_mask) 1030 gfp_t gfp_mask)
1042{ 1031{
@@ -1044,36 +1033,33 @@ static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
1044 return NULL; 1033 return NULL;
1045 1034
1046 return kmalloc(sizeof(struct bio_map_data) + 1035 return kmalloc(sizeof(struct bio_map_data) +
1047 sizeof(struct sg_iovec) * iov_count, gfp_mask); 1036 sizeof(struct iovec) * iov_count, gfp_mask);
1048} 1037}
1049 1038
1050static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_count, 1039static int __bio_copy_iov(struct bio *bio, const struct iov_iter *iter,
1051 int to_user, int from_user) 1040 int to_user, int from_user)
1052{ 1041{
1053 int ret = 0, i; 1042 int ret = 0, i;
1054 struct bio_vec *bvec; 1043 struct bio_vec *bvec;
1055 int iov_idx = 0; 1044 struct iov_iter iov_iter = *iter;
1056 unsigned int iov_off = 0;
1057 1045
1058 bio_for_each_segment_all(bvec, bio, i) { 1046 bio_for_each_segment_all(bvec, bio, i) {
1059 char *bv_addr = page_address(bvec->bv_page); 1047 char *bv_addr = page_address(bvec->bv_page);
1060 unsigned int bv_len = bvec->bv_len; 1048 unsigned int bv_len = bvec->bv_len;
1061 1049
1062 while (bv_len && iov_idx < iov_count) { 1050 while (bv_len && iov_iter.count) {
1063 unsigned int bytes; 1051 struct iovec iov = iov_iter_iovec(&iov_iter);
1064 char __user *iov_addr; 1052 unsigned int bytes = min_t(unsigned int, bv_len,
1065 1053 iov.iov_len);
1066 bytes = min_t(unsigned int,
1067 iov[iov_idx].iov_len - iov_off, bv_len);
1068 iov_addr = iov[iov_idx].iov_base + iov_off;
1069 1054
1070 if (!ret) { 1055 if (!ret) {
1071 if (to_user) 1056 if (to_user)
1072 ret = copy_to_user(iov_addr, bv_addr, 1057 ret = copy_to_user(iov.iov_base,
1073 bytes); 1058 bv_addr, bytes);
1074 1059
1075 if (from_user) 1060 if (from_user)
1076 ret = copy_from_user(bv_addr, iov_addr, 1061 ret = copy_from_user(bv_addr,
1062 iov.iov_base,
1077 bytes); 1063 bytes);
1078 1064
1079 if (ret) 1065 if (ret)
@@ -1082,13 +1068,7 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_c
1082 1068
1083 bv_len -= bytes; 1069 bv_len -= bytes;
1084 bv_addr += bytes; 1070 bv_addr += bytes;
1085 iov_addr += bytes; 1071 iov_iter_advance(&iov_iter, bytes);
1086 iov_off += bytes;
1087
1088 if (iov[iov_idx].iov_len == iov_off) {
1089 iov_idx++;
1090 iov_off = 0;
1091 }
1092 } 1072 }
1093 } 1073 }
1094 1074
@@ -1122,7 +1102,7 @@ int bio_uncopy_user(struct bio *bio)
1122 * don't copy into a random user address space, just free. 1102 * don't copy into a random user address space, just free.
1123 */ 1103 */
1124 if (current->mm) 1104 if (current->mm)
1125 ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs, 1105 ret = __bio_copy_iov(bio, &bmd->iter,
1126 bio_data_dir(bio) == READ, 0); 1106 bio_data_dir(bio) == READ, 0);
1127 if (bmd->is_our_pages) 1107 if (bmd->is_our_pages)
1128 bio_free_pages(bio); 1108 bio_free_pages(bio);
@@ -1135,12 +1115,10 @@ EXPORT_SYMBOL(bio_uncopy_user);
1135 1115
1136/** 1116/**
1137 * bio_copy_user_iov - copy user data to bio 1117 * bio_copy_user_iov - copy user data to bio
1138 * @q: destination block queue 1118 * @q: destination block queue
1139 * @map_data: pointer to the rq_map_data holding pages (if necessary) 1119 * @map_data: pointer to the rq_map_data holding pages (if necessary)
1140 * @iov: the iovec. 1120 * @iter: iovec iterator
1141 * @iov_count: number of elements in the iovec 1121 * @gfp_mask: memory allocation flags
1142 * @write_to_vm: bool indicating writing to pages or not
1143 * @gfp_mask: memory allocation flags
1144 * 1122 *
1145 * Prepares and returns a bio for indirect user io, bouncing data 1123 * Prepares and returns a bio for indirect user io, bouncing data
1146 * to/from kernel pages as necessary. Must be paired with 1124 * to/from kernel pages as necessary. Must be paired with
@@ -1148,24 +1126,25 @@ EXPORT_SYMBOL(bio_uncopy_user);
1148 */ 1126 */
1149struct bio *bio_copy_user_iov(struct request_queue *q, 1127struct bio *bio_copy_user_iov(struct request_queue *q,
1150 struct rq_map_data *map_data, 1128 struct rq_map_data *map_data,
1151 const struct sg_iovec *iov, int iov_count, 1129 const struct iov_iter *iter,
1152 int write_to_vm, gfp_t gfp_mask) 1130 gfp_t gfp_mask)
1153{ 1131{
1154 struct bio_map_data *bmd; 1132 struct bio_map_data *bmd;
1155 struct page *page; 1133 struct page *page;
1156 struct bio *bio; 1134 struct bio *bio;
1157 int i, ret; 1135 int i, ret;
1158 int nr_pages = 0; 1136 int nr_pages = 0;
1159 unsigned int len = 0; 1137 unsigned int len = iter->count;
1160 unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0; 1138 unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
1161 1139
1162 for (i = 0; i < iov_count; i++) { 1140 for (i = 0; i < iter->nr_segs; i++) {
1163 unsigned long uaddr; 1141 unsigned long uaddr;
1164 unsigned long end; 1142 unsigned long end;
1165 unsigned long start; 1143 unsigned long start;
1166 1144
1167 uaddr = (unsigned long)iov[i].iov_base; 1145 uaddr = (unsigned long) iter->iov[i].iov_base;
1168 end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1146 end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1)
1147 >> PAGE_SHIFT;
1169 start = uaddr >> PAGE_SHIFT; 1148 start = uaddr >> PAGE_SHIFT;
1170 1149
1171 /* 1150 /*
@@ -1175,22 +1154,31 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
1175 return ERR_PTR(-EINVAL); 1154 return ERR_PTR(-EINVAL);
1176 1155
1177 nr_pages += end - start; 1156 nr_pages += end - start;
1178 len += iov[i].iov_len;
1179 } 1157 }
1180 1158
1181 if (offset) 1159 if (offset)
1182 nr_pages++; 1160 nr_pages++;
1183 1161
1184 bmd = bio_alloc_map_data(iov_count, gfp_mask); 1162 bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask);
1185 if (!bmd) 1163 if (!bmd)
1186 return ERR_PTR(-ENOMEM); 1164 return ERR_PTR(-ENOMEM);
1187 1165
1166 /*
1167 * We need to do a deep copy of the iov_iter including the iovecs.
1168 * The caller provided iov might point to an on-stack or otherwise
1169 * shortlived one.
1170 */
1171 bmd->is_our_pages = map_data ? 0 : 1;
1172 memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs);
1173 iov_iter_init(&bmd->iter, iter->type, bmd->iov,
1174 iter->nr_segs, iter->count);
1175
1188 ret = -ENOMEM; 1176 ret = -ENOMEM;
1189 bio = bio_kmalloc(gfp_mask, nr_pages); 1177 bio = bio_kmalloc(gfp_mask, nr_pages);
1190 if (!bio) 1178 if (!bio)
1191 goto out_bmd; 1179 goto out_bmd;
1192 1180
1193 if (!write_to_vm) 1181 if (iter->type & WRITE)
1194 bio->bi_rw |= REQ_WRITE; 1182 bio->bi_rw |= REQ_WRITE;
1195 1183
1196 ret = 0; 1184 ret = 0;
@@ -1238,14 +1226,14 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
1238 /* 1226 /*
1239 * success 1227 * success
1240 */ 1228 */
1241 if ((!write_to_vm && (!map_data || !map_data->null_mapped)) || 1229 if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) ||
1242 (map_data && map_data->from_user)) { 1230 (map_data && map_data->from_user)) {
1243 ret = __bio_copy_iov(bio, iov, iov_count, 0, 1); 1231 ret = __bio_copy_iov(bio, iter, 0, 1);
1244 if (ret) 1232 if (ret)
1245 goto cleanup; 1233 goto cleanup;
1246 } 1234 }
1247 1235
1248 bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1); 1236 bio->bi_private = bmd;
1249 return bio; 1237 return bio;
1250cleanup: 1238cleanup:
1251 if (!map_data) 1239 if (!map_data)
@@ -1258,19 +1246,21 @@ out_bmd:
1258 1246
1259static struct bio *__bio_map_user_iov(struct request_queue *q, 1247static struct bio *__bio_map_user_iov(struct request_queue *q,
1260 struct block_device *bdev, 1248 struct block_device *bdev,
1261 const struct sg_iovec *iov, int iov_count, 1249 const struct iov_iter *iter,
1262 int write_to_vm, gfp_t gfp_mask) 1250 gfp_t gfp_mask)
1263{ 1251{
1264 int i, j; 1252 int j;
1265 int nr_pages = 0; 1253 int nr_pages = 0;
1266 struct page **pages; 1254 struct page **pages;
1267 struct bio *bio; 1255 struct bio *bio;
1268 int cur_page = 0; 1256 int cur_page = 0;
1269 int ret, offset; 1257 int ret, offset;
1258 struct iov_iter i;
1259 struct iovec iov;
1270 1260
1271 for (i = 0; i < iov_count; i++) { 1261 iov_for_each(iov, i, *iter) {
1272 unsigned long uaddr = (unsigned long)iov[i].iov_base; 1262 unsigned long uaddr = (unsigned long) iov.iov_base;
1273 unsigned long len = iov[i].iov_len; 1263 unsigned long len = iov.iov_len;
1274 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1264 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1275 unsigned long start = uaddr >> PAGE_SHIFT; 1265 unsigned long start = uaddr >> PAGE_SHIFT;
1276 1266
@@ -1300,16 +1290,17 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
1300 if (!pages) 1290 if (!pages)
1301 goto out; 1291 goto out;
1302 1292
1303 for (i = 0; i < iov_count; i++) { 1293 iov_for_each(iov, i, *iter) {
1304 unsigned long uaddr = (unsigned long)iov[i].iov_base; 1294 unsigned long uaddr = (unsigned long) iov.iov_base;
1305 unsigned long len = iov[i].iov_len; 1295 unsigned long len = iov.iov_len;
1306 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1296 unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1307 unsigned long start = uaddr >> PAGE_SHIFT; 1297 unsigned long start = uaddr >> PAGE_SHIFT;
1308 const int local_nr_pages = end - start; 1298 const int local_nr_pages = end - start;
1309 const int page_limit = cur_page + local_nr_pages; 1299 const int page_limit = cur_page + local_nr_pages;
1310 1300
1311 ret = get_user_pages_fast(uaddr, local_nr_pages, 1301 ret = get_user_pages_fast(uaddr, local_nr_pages,
1312 write_to_vm, &pages[cur_page]); 1302 (iter->type & WRITE) != WRITE,
1303 &pages[cur_page]);
1313 if (ret < local_nr_pages) { 1304 if (ret < local_nr_pages) {
1314 ret = -EFAULT; 1305 ret = -EFAULT;
1315 goto out_unmap; 1306 goto out_unmap;
@@ -1349,7 +1340,7 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
1349 /* 1340 /*
1350 * set data direction, and check if mapped pages need bouncing 1341 * set data direction, and check if mapped pages need bouncing
1351 */ 1342 */
1352 if (!write_to_vm) 1343 if (iter->type & WRITE)
1353 bio->bi_rw |= REQ_WRITE; 1344 bio->bi_rw |= REQ_WRITE;
1354 1345
1355 bio->bi_bdev = bdev; 1346 bio->bi_bdev = bdev;
@@ -1357,10 +1348,10 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
1357 return bio; 1348 return bio;
1358 1349
1359 out_unmap: 1350 out_unmap:
1360 for (i = 0; i < nr_pages; i++) { 1351 for (j = 0; j < nr_pages; j++) {
1361 if(!pages[i]) 1352 if (!pages[j])
1362 break; 1353 break;
1363 page_cache_release(pages[i]); 1354 page_cache_release(pages[j]);
1364 } 1355 }
1365 out: 1356 out:
1366 kfree(pages); 1357 kfree(pages);
@@ -1369,25 +1360,22 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
1369} 1360}
1370 1361
1371/** 1362/**
1372 * bio_map_user_iov - map user sg_iovec table into bio 1363 * bio_map_user_iov - map user iovec into bio
1373 * @q: the struct request_queue for the bio 1364 * @q: the struct request_queue for the bio
1374 * @bdev: destination block device 1365 * @bdev: destination block device
1375 * @iov: the iovec. 1366 * @iter: iovec iterator
1376 * @iov_count: number of elements in the iovec 1367 * @gfp_mask: memory allocation flags
1377 * @write_to_vm: bool indicating writing to pages or not
1378 * @gfp_mask: memory allocation flags
1379 * 1368 *
1380 * Map the user space address into a bio suitable for io to a block 1369 * Map the user space address into a bio suitable for io to a block
1381 * device. Returns an error pointer in case of error. 1370 * device. Returns an error pointer in case of error.
1382 */ 1371 */
1383struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev, 1372struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
1384 const struct sg_iovec *iov, int iov_count, 1373 const struct iov_iter *iter,
1385 int write_to_vm, gfp_t gfp_mask) 1374 gfp_t gfp_mask)
1386{ 1375{
1387 struct bio *bio; 1376 struct bio *bio;
1388 1377
1389 bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm, 1378 bio = __bio_map_user_iov(q, bdev, iter, gfp_mask);
1390 gfp_mask);
1391 if (IS_ERR(bio)) 1379 if (IS_ERR(bio))
1392 return bio; 1380 return bio;
1393 1381