diff options
author | Shane Michael Matthews <shane.matthews@intel.com> | 2011-02-10 08:51:24 -0500 |
---|---|---|
committer | Matthew Wilcox <matthew.r.wilcox@intel.com> | 2011-11-04 15:52:56 -0400 |
commit | e025344c56e08b155f43ea09647969286c78377c (patch) | |
tree | d43db5810c67445944d871ae2103e5d8e0b85b8e /drivers/block/nvme.c | |
parent | 51882d00f07da9601cc962a3596e48aafb4f4163 (diff) |
NVMe: Initial PRP List support
Add a pointer to the nvme_req_info to hold a new data structure
(nvme_prps) which contains a list of the pages allocated to this
particular request for holding PRP list entries. nvme_setup_prps()
now returns this pointer.
To allocate and free the memory used for PRP lists, we need a struct
device, so we need to pass the nvme_queue pointer to many functions
which didn't use to need it.
Signed-off-by: Matthew Wilcox <matthew.r.wilcox@intel.com>
Diffstat (limited to 'drivers/block/nvme.c')
-rw-r--r-- | drivers/block/nvme.c | 104 |
1 files changed, 91 insertions, 13 deletions
diff --git a/drivers/block/nvme.c b/drivers/block/nvme.c index 903e7f15b60d..b1e8445985a2 100644 --- a/drivers/block/nvme.c +++ b/drivers/block/nvme.c | |||
@@ -247,21 +247,55 @@ static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) | |||
247 | return 0; | 247 | return 0; |
248 | } | 248 | } |
249 | 249 | ||
250 | static __le64 *alloc_prp_list(struct nvme_queue *nvmeq, int length, | ||
251 | dma_addr_t *addr) | ||
252 | { | ||
253 | return dma_alloc_coherent(nvmeq->q_dmadev, PAGE_SIZE, addr, GFP_ATOMIC); | ||
254 | } | ||
255 | |||
256 | struct nvme_prps { | ||
257 | int npages; | ||
258 | dma_addr_t first_dma; | ||
259 | __le64 *list[0]; | ||
260 | }; | ||
261 | |||
262 | static void nvme_free_prps(struct nvme_queue *nvmeq, struct nvme_prps *prps) | ||
263 | { | ||
264 | const int last_prp = PAGE_SIZE / 8 - 1; | ||
265 | int i; | ||
266 | dma_addr_t prp_dma; | ||
267 | |||
268 | if (!prps) | ||
269 | return; | ||
270 | |||
271 | prp_dma = prps->first_dma; | ||
272 | for (i = 0; i < prps->npages; i++) { | ||
273 | __le64 *prp_list = prps->list[i]; | ||
274 | dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]); | ||
275 | dma_free_coherent(nvmeq->q_dmadev, PAGE_SIZE, prp_list, | ||
276 | prp_dma); | ||
277 | prp_dma = next_prp_dma; | ||
278 | } | ||
279 | kfree(prps); | ||
280 | } | ||
281 | |||
250 | struct nvme_req_info { | 282 | struct nvme_req_info { |
251 | struct bio *bio; | 283 | struct bio *bio; |
252 | int nents; | 284 | int nents; |
285 | struct nvme_prps *prps; | ||
253 | struct scatterlist sg[0]; | 286 | struct scatterlist sg[0]; |
254 | }; | 287 | }; |
255 | 288 | ||
256 | /* XXX: use a mempool */ | 289 | /* XXX: use a mempool */ |
257 | static struct nvme_req_info *alloc_info(unsigned nseg, gfp_t gfp) | 290 | static struct nvme_req_info *alloc_info(unsigned nseg, gfp_t gfp) |
258 | { | 291 | { |
259 | return kmalloc(sizeof(struct nvme_req_info) + | 292 | return kzalloc(sizeof(struct nvme_req_info) + |
260 | sizeof(struct scatterlist) * nseg, gfp); | 293 | sizeof(struct scatterlist) * nseg, gfp); |
261 | } | 294 | } |
262 | 295 | ||
263 | static void free_info(struct nvme_req_info *info) | 296 | static void free_info(struct nvme_queue *nvmeq, struct nvme_req_info *info) |
264 | { | 297 | { |
298 | nvme_free_prps(nvmeq, info->prps); | ||
265 | kfree(info); | 299 | kfree(info); |
266 | } | 300 | } |
267 | 301 | ||
@@ -274,7 +308,7 @@ static void bio_completion(struct nvme_queue *nvmeq, void *ctx, | |||
274 | 308 | ||
275 | dma_unmap_sg(nvmeq->q_dmadev, info->sg, info->nents, | 309 | dma_unmap_sg(nvmeq->q_dmadev, info->sg, info->nents, |
276 | bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); | 310 | bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); |
277 | free_info(info); | 311 | free_info(nvmeq, info); |
278 | bio_endio(bio, status ? -EIO : 0); | 312 | bio_endio(bio, status ? -EIO : 0); |
279 | bio = bio_list_pop(&nvmeq->sq_cong); | 313 | bio = bio_list_pop(&nvmeq->sq_cong); |
280 | if (bio) | 314 | if (bio) |
@@ -282,17 +316,22 @@ static void bio_completion(struct nvme_queue *nvmeq, void *ctx, | |||
282 | } | 316 | } |
283 | 317 | ||
284 | /* length is in bytes */ | 318 | /* length is in bytes */ |
285 | static void nvme_setup_prps(struct nvme_common_command *cmd, | 319 | static struct nvme_prps *nvme_setup_prps(struct nvme_queue *nvmeq, |
320 | struct nvme_common_command *cmd, | ||
286 | struct scatterlist *sg, int length) | 321 | struct scatterlist *sg, int length) |
287 | { | 322 | { |
288 | int dma_len = sg_dma_len(sg); | 323 | int dma_len = sg_dma_len(sg); |
289 | u64 dma_addr = sg_dma_address(sg); | 324 | u64 dma_addr = sg_dma_address(sg); |
290 | int offset = offset_in_page(dma_addr); | 325 | int offset = offset_in_page(dma_addr); |
326 | __le64 *prp_list; | ||
327 | dma_addr_t prp_dma; | ||
328 | int nprps, npages, i, prp_page; | ||
329 | struct nvme_prps *prps = NULL; | ||
291 | 330 | ||
292 | cmd->prp1 = cpu_to_le64(dma_addr); | 331 | cmd->prp1 = cpu_to_le64(dma_addr); |
293 | length -= (PAGE_SIZE - offset); | 332 | length -= (PAGE_SIZE - offset); |
294 | if (length <= 0) | 333 | if (length <= 0) |
295 | return; | 334 | return prps; |
296 | 335 | ||
297 | dma_len -= (PAGE_SIZE - offset); | 336 | dma_len -= (PAGE_SIZE - offset); |
298 | if (dma_len) { | 337 | if (dma_len) { |
@@ -305,10 +344,42 @@ static void nvme_setup_prps(struct nvme_common_command *cmd, | |||
305 | 344 | ||
306 | if (length <= PAGE_SIZE) { | 345 | if (length <= PAGE_SIZE) { |
307 | cmd->prp2 = cpu_to_le64(dma_addr); | 346 | cmd->prp2 = cpu_to_le64(dma_addr); |
308 | return; | 347 | return prps; |
309 | } | 348 | } |
310 | 349 | ||
311 | /* XXX: support PRP lists */ | 350 | nprps = DIV_ROUND_UP(length, PAGE_SIZE); |
351 | npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE); | ||
352 | prps = kmalloc(sizeof(*prps) + sizeof(__le64 *) * npages, GFP_ATOMIC); | ||
353 | prps->npages = npages; | ||
354 | prp_page = 0; | ||
355 | prp_list = alloc_prp_list(nvmeq, length, &prp_dma); | ||
356 | prps->list[prp_page++] = prp_list; | ||
357 | prps->first_dma = prp_dma; | ||
358 | cmd->prp2 = cpu_to_le64(prp_dma); | ||
359 | i = 0; | ||
360 | for (;;) { | ||
361 | if (i == PAGE_SIZE / 8 - 1) { | ||
362 | __le64 *old_prp_list = prp_list; | ||
363 | prp_list = alloc_prp_list(nvmeq, length, &prp_dma); | ||
364 | prps->list[prp_page++] = prp_list; | ||
365 | old_prp_list[i] = cpu_to_le64(prp_dma); | ||
366 | i = 0; | ||
367 | } | ||
368 | prp_list[i++] = cpu_to_le64(dma_addr); | ||
369 | dma_len -= PAGE_SIZE; | ||
370 | dma_addr += PAGE_SIZE; | ||
371 | length -= PAGE_SIZE; | ||
372 | if (length <= 0) | ||
373 | break; | ||
374 | if (dma_len > 0) | ||
375 | continue; | ||
376 | BUG_ON(dma_len < 0); | ||
377 | sg = sg_next(sg); | ||
378 | dma_addr = sg_dma_address(sg); | ||
379 | dma_len = sg_dma_len(sg); | ||
380 | } | ||
381 | |||
382 | return prps; | ||
312 | } | 383 | } |
313 | 384 | ||
314 | static int nvme_map_bio(struct device *dev, struct nvme_req_info *info, | 385 | static int nvme_map_bio(struct device *dev, struct nvme_req_info *info, |
@@ -378,7 +449,8 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, | |||
378 | cmnd->rw.flags = 1; | 449 | cmnd->rw.flags = 1; |
379 | cmnd->rw.command_id = cmdid; | 450 | cmnd->rw.command_id = cmdid; |
380 | cmnd->rw.nsid = cpu_to_le32(ns->ns_id); | 451 | cmnd->rw.nsid = cpu_to_le32(ns->ns_id); |
381 | nvme_setup_prps(&cmnd->common, info->sg, bio->bi_size); | 452 | info->prps = nvme_setup_prps(nvmeq, &cmnd->common, info->sg, |
453 | bio->bi_size); | ||
382 | cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); | 454 | cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9)); |
383 | cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1); | 455 | cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1); |
384 | cmnd->rw.control = cpu_to_le16(control); | 456 | cmnd->rw.control = cpu_to_le16(control); |
@@ -393,7 +465,7 @@ static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, | |||
393 | return 0; | 465 | return 0; |
394 | 466 | ||
395 | free_info: | 467 | free_info: |
396 | free_info(info); | 468 | free_info(nvmeq, info); |
397 | congestion: | 469 | congestion: |
398 | return -EBUSY; | 470 | return -EBUSY; |
399 | } | 471 | } |
@@ -852,13 +924,15 @@ static int nvme_submit_user_admin_command(struct nvme_dev *dev, | |||
852 | { | 924 | { |
853 | int err, nents; | 925 | int err, nents; |
854 | struct scatterlist *sg; | 926 | struct scatterlist *sg; |
927 | struct nvme_prps *prps; | ||
855 | 928 | ||
856 | nents = nvme_map_user_pages(dev, 0, addr, length, &sg); | 929 | nents = nvme_map_user_pages(dev, 0, addr, length, &sg); |
857 | if (nents < 0) | 930 | if (nents < 0) |
858 | return nents; | 931 | return nents; |
859 | nvme_setup_prps(&cmd->common, sg, length); | 932 | prps = nvme_setup_prps(dev->queues[0], &cmd->common, sg, length); |
860 | err = nvme_submit_admin_cmd(dev, cmd, NULL); | 933 | err = nvme_submit_admin_cmd(dev, cmd, NULL); |
861 | nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); | 934 | nvme_unmap_user_pages(dev, 0, addr, length, sg, nents); |
935 | nvme_free_prps(dev->queues[0], prps); | ||
862 | return err ? -EIO : 0; | 936 | return err ? -EIO : 0; |
863 | } | 937 | } |
864 | 938 | ||
@@ -896,6 +970,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) | |||
896 | u32 result; | 970 | u32 result; |
897 | int nents, status; | 971 | int nents, status; |
898 | struct scatterlist *sg; | 972 | struct scatterlist *sg; |
973 | struct nvme_prps *prps; | ||
899 | 974 | ||
900 | if (copy_from_user(&io, uio, sizeof(io))) | 975 | if (copy_from_user(&io, uio, sizeof(io))) |
901 | return -EFAULT; | 976 | return -EFAULT; |
@@ -915,10 +990,10 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) | |||
915 | c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */ | 990 | c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */ |
916 | c.rw.apptag = cpu_to_le16(io.apptag); | 991 | c.rw.apptag = cpu_to_le16(io.apptag); |
917 | c.rw.appmask = cpu_to_le16(io.appmask); | 992 | c.rw.appmask = cpu_to_le16(io.appmask); |
993 | nvmeq = get_nvmeq(ns); | ||
918 | /* XXX: metadata */ | 994 | /* XXX: metadata */ |
919 | nvme_setup_prps(&c.common, sg, length); | 995 | prps = nvme_setup_prps(nvmeq, &c.common, sg, length); |
920 | 996 | ||
921 | nvmeq = get_nvmeq(ns); | ||
922 | /* Since nvme_submit_sync_cmd sleeps, we can't keep preemption | 997 | /* Since nvme_submit_sync_cmd sleeps, we can't keep preemption |
923 | * disabled. We may be preempted at any point, and be rescheduled | 998 | * disabled. We may be preempted at any point, and be rescheduled |
924 | * to a different CPU. That will cause cacheline bouncing, but no | 999 | * to a different CPU. That will cause cacheline bouncing, but no |
@@ -928,6 +1003,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) | |||
928 | status = nvme_submit_sync_cmd(nvmeq, &c, &result, IO_TIMEOUT); | 1003 | status = nvme_submit_sync_cmd(nvmeq, &c, &result, IO_TIMEOUT); |
929 | 1004 | ||
930 | nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); | 1005 | nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents); |
1006 | nvme_free_prps(nvmeq, prps); | ||
931 | put_user(result, &uio->result); | 1007 | put_user(result, &uio->result); |
932 | return status; | 1008 | return status; |
933 | } | 1009 | } |
@@ -940,6 +1016,7 @@ static int nvme_download_firmware(struct nvme_ns *ns, | |||
940 | struct nvme_command c; | 1016 | struct nvme_command c; |
941 | int nents, status; | 1017 | int nents, status; |
942 | struct scatterlist *sg; | 1018 | struct scatterlist *sg; |
1019 | struct nvme_prps *prps; | ||
943 | 1020 | ||
944 | if (copy_from_user(&dlfw, udlfw, sizeof(dlfw))) | 1021 | if (copy_from_user(&dlfw, udlfw, sizeof(dlfw))) |
945 | return -EFAULT; | 1022 | return -EFAULT; |
@@ -954,10 +1031,11 @@ static int nvme_download_firmware(struct nvme_ns *ns, | |||
954 | c.dlfw.opcode = nvme_admin_download_fw; | 1031 | c.dlfw.opcode = nvme_admin_download_fw; |
955 | c.dlfw.numd = cpu_to_le32(dlfw.length); | 1032 | c.dlfw.numd = cpu_to_le32(dlfw.length); |
956 | c.dlfw.offset = cpu_to_le32(dlfw.offset); | 1033 | c.dlfw.offset = cpu_to_le32(dlfw.offset); |
957 | nvme_setup_prps(&c.common, sg, dlfw.length * 4); | 1034 | prps = nvme_setup_prps(dev->queues[0], &c.common, sg, dlfw.length * 4); |
958 | 1035 | ||
959 | status = nvme_submit_admin_cmd(dev, &c, NULL); | 1036 | status = nvme_submit_admin_cmd(dev, &c, NULL); |
960 | nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents); | 1037 | nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents); |
1038 | nvme_free_prps(dev->queues[0], prps); | ||
961 | return status; | 1039 | return status; |
962 | } | 1040 | } |
963 | 1041 | ||