diff options
Diffstat (limited to 'drivers/gpu/nvgpu/os')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/ioctl_as.c | 13 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/swap.h | 41 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/vm.c | 1 |
3 files changed, 30 insertions, 25 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_as.c b/drivers/gpu/nvgpu/os/linux/ioctl_as.c index af6cdb5b..6348bb2a 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_as.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_as.c | |||
@@ -365,9 +365,9 @@ static int nvgpu_as_ioctl_write_swap_buffer( | |||
365 | // (Assuming that NVMe DRAM acceses are uncached) | 365 | // (Assuming that NVMe DRAM acceses are uncached) |
366 | gk20a_mm_l2_flush(g, false); | 366 | gk20a_mm_l2_flush(g, false); |
367 | 367 | ||
368 | // Copy out (blocking) TODO: non-blocking | 368 | // Copy out (non-blocking) |
369 | // Could fail on inaccessible swap device, etc | 369 | // Could fail on inaccessible swap device, etc |
370 | err = copy_out(m->os_priv.sgt); | 370 | err = copy_out(m->os_priv.sgt, m); |
371 | 371 | ||
372 | out: | 372 | out: |
373 | return err; | 373 | return err; |
@@ -393,7 +393,7 @@ static int nvgpu_as_ioctl_write_swap_buffer_finish( | |||
393 | nvgpu_log_fn(g, " "); | 393 | nvgpu_log_fn(g, " "); |
394 | 394 | ||
395 | // Wait for the pages to get written out | 395 | // Wait for the pages to get written out |
396 | //wait_for_completion_io(m->os_priv.swap_completion); | 396 | wait_for_completion_io(&m->os_priv.swap_io_done); |
397 | 397 | ||
398 | // Unpin needs to happen after copy out is done | 398 | // Unpin needs to happen after copy out is done |
399 | // (No return value check as it's a void function) | 399 | // (No return value check as it's a void function) |
@@ -448,9 +448,8 @@ static int nvgpu_as_ioctl_read_swap_buffer( | |||
448 | // Do any bookeeping not done by gk20a_mm_pin() | 448 | // Do any bookeeping not done by gk20a_mm_pin() |
449 | m->os_priv.sgt = sgt; | 449 | m->os_priv.sgt = sgt; |
450 | 450 | ||
451 | // Reload page contents from disk (blocking) | 451 | // Reload page contents from disk (non-blocking) |
452 | // TODO: non-blocking | 452 | err = copy_in(sgt, m); |
453 | err = copy_in(sgt); | ||
454 | if (err) { | 453 | if (err) { |
455 | int err2; | 454 | int err2; |
456 | // Rollback pinning and allocation | 455 | // Rollback pinning and allocation |
@@ -487,7 +486,7 @@ static int nvgpu_as_ioctl_read_swap_buffer_finish( | |||
487 | // Invalidate L2 so that TLB refill does not load stale PT | 486 | // Invalidate L2 so that TLB refill does not load stale PT |
488 | gk20a_mm_l2_flush(g, true); | 487 | gk20a_mm_l2_flush(g, true); |
489 | // Wait for read to complete if it hasn't yet | 488 | // Wait for read to complete if it hasn't yet |
490 | //wait_for_completion_io(m->os_priv.swap_completion); | 489 | wait_for_completion_io(&m->os_priv.swap_io_done); |
491 | 490 | ||
492 | return err; | 491 | return err; |
493 | } | 492 | } |
diff --git a/drivers/gpu/nvgpu/os/linux/swap.h b/drivers/gpu/nvgpu/os/linux/swap.h index f762ba81..1e986095 100644 --- a/drivers/gpu/nvgpu/os/linux/swap.h +++ b/drivers/gpu/nvgpu/os/linux/swap.h | |||
@@ -2,17 +2,23 @@ | |||
2 | #include <linux/bio.h> | 2 | #include <linux/bio.h> |
3 | //#include <nvgpu/bug.h> | 3 | //#include <nvgpu/bug.h> |
4 | 4 | ||
5 | // Callback for completion of the I/O chain | ||
6 | static void complete_swap_io(struct bio *bio) { | ||
7 | struct nvgpu_mapped_buf *m = bio->bi_private; | ||
8 | bio_put(bio); | ||
9 | complete(&m->os_priv.swap_io_done); | ||
10 | } | ||
11 | |||
5 | // Queue a command to copy out an SGT to disk | 12 | // Queue a command to copy out an SGT to disk |
6 | // TODO: Cache bdev | 13 | // TODO: Cache bdev |
7 | // TODO: Asynchronous I/O | ||
8 | // TODO: Don't hardcode sector 0 | 14 | // TODO: Don't hardcode sector 0 |
9 | int copy(struct sg_table *sgt, int op) { | 15 | // TODO: Figure out if submit_bio() can fail, and what to do then |
16 | int copy(struct sg_table *sgt, int op, struct nvgpu_mapped_buf *m) { | ||
10 | unsigned int i; | 17 | unsigned int i; |
11 | struct scatterlist *sg; | 18 | struct scatterlist *sg; |
12 | struct bio *bio; | 19 | struct bio *bio; |
13 | int err = 0; | 20 | int err = 0; |
14 | int sg_cnt = sgt->nents; | 21 | int sg_cnt = sgt->nents; |
15 | struct bio *bio_orig; | ||
16 | sector_t sector = 0; // XXX: For testing | 22 | sector_t sector = 0; // XXX: For testing |
17 | // Find and open the block device | 23 | // Find and open the block device |
18 | struct block_device *bdev = blkdev_get_by_path("/dev/nvme0n1", FMODE_READ | FMODE_WRITE, copy); | 24 | struct block_device *bdev = blkdev_get_by_path("/dev/nvme0n1", FMODE_READ | FMODE_WRITE, copy); |
@@ -20,12 +26,15 @@ int copy(struct sg_table *sgt, int op) { | |||
20 | printk(KERN_WARNING "Unabled to find `nvme0`, err %ld!\n", PTR_ERR(bdev)); | 26 | printk(KERN_WARNING "Unabled to find `nvme0`, err %ld!\n", PTR_ERR(bdev)); |
21 | return -ENODEV; | 27 | return -ENODEV; |
22 | } | 28 | } |
23 | // Will never fail when allocating <= BIO_MAX_PAGES | 29 | // Reset the .done variable in the completion |
30 | reinit_completion(&m->os_priv.swap_io_done); | ||
31 | // bio_alloc() will never fail when allocating <= BIO_MAX_PAGES | ||
24 | bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); | 32 | bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); |
25 | bio_orig = bio; | ||
26 | bio->bi_bdev = bdev; // Switch to bio_set_dev(bdev) in newer kernels | 33 | bio->bi_bdev = bdev; // Switch to bio_set_dev(bdev) in newer kernels |
27 | bio->bi_iter.bi_sector = sector; | 34 | bio->bi_iter.bi_sector = sector; |
28 | bio_set_op_attrs(bio, op, op == REQ_OP_WRITE ? WRITE_ODIRECT : 0);//REQ_SYNC); // XXX: Is REQ_SYNC necessary? | 35 | bio_set_op_attrs(bio, op, REQ_SYNC); // REQ_SYNC is identical to WRITE_ODIRECT |
36 | bio->bi_private = m; | ||
37 | bio->bi_end_io = complete_swap_io; | ||
29 | // Copy the scatter-gather table (sgt) into a block I/O vector (bio vec) | 38 | // Copy the scatter-gather table (sgt) into a block I/O vector (bio vec) |
30 | // bio_chain() approach borrowed from drivers/nvme/target/io-cmd.c:nvmet_execute_rw() | 39 | // bio_chain() approach borrowed from drivers/nvme/target/io-cmd.c:nvmet_execute_rw() |
31 | for_each_sg(sgt->sgl, sg, sgt->nents, i) { | 40 | for_each_sg(sgt->sgl, sg, sgt->nents, i) { |
@@ -47,16 +56,12 @@ int copy(struct sg_table *sgt, int op) { | |||
47 | sector += sg_dma_len(sg) >> 9; | 56 | sector += sg_dma_len(sg) >> 9; |
48 | sg_cnt--; | 57 | sg_cnt--; |
49 | } | 58 | } |
50 | // Use blocking submit for now | ||
51 | // TODO: Switch to async via submit_bio(bio) | ||
52 | err = submit_bio_wait(bio); | ||
53 | 59 | ||
54 | if (bio->bi_error && bio->bi_error != err) | 60 | // Async submit. Caller should wait_for_completion_io(&m->os_priv.swap_io_done); |
55 | printk(KERN_WARNING "nvgpu: bio->bi_error %d != return val from submit_bio_wait() %d\n", bio->bi_error, err); | 61 | submit_bio(bio); |
56 | 62 | ||
57 | //out: | 63 | // Release our block device handle |
58 | bio_put(bio_orig); // TODO: Move to completion handler | 64 | blkdev_put(bdev, FMODE_WRITE | FMODE_READ); // Is this safe? |
59 | blkdev_put(bdev, FMODE_WRITE|FMODE_READ); | ||
60 | return err; | 65 | return err; |
61 | } | 66 | } |
62 | 67 | ||
@@ -107,11 +112,11 @@ int copy_all(struct vm_gk20a *vm) { | |||
107 | return 0; | 112 | return 0; |
108 | } | 113 | } |
109 | 114 | ||
110 | int copy_out(struct sg_table *sgt) { | 115 | int copy_out(struct sg_table *sgt, struct nvgpu_mapped_buf *m) { |
111 | return copy(sgt, REQ_OP_WRITE); | 116 | return copy(sgt, REQ_OP_WRITE, m); |
112 | } | 117 | } |
113 | 118 | ||
114 | int copy_in(struct sg_table *sgt) { | 119 | int copy_in(struct sg_table *sgt, struct nvgpu_mapped_buf *m) { |
115 | return copy(sgt, REQ_OP_READ); | 120 | return copy(sgt, REQ_OP_READ, m); |
116 | } | 121 | } |
117 | 122 | ||
diff --git a/drivers/gpu/nvgpu/os/linux/vm.c b/drivers/gpu/nvgpu/os/linux/vm.c index fcb58ac4..9cd17981 100644 --- a/drivers/gpu/nvgpu/os/linux/vm.c +++ b/drivers/gpu/nvgpu/os/linux/vm.c | |||
@@ -262,6 +262,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm, | |||
262 | mapped_buffer->os_priv.dmabuf = dmabuf; | 262 | mapped_buffer->os_priv.dmabuf = dmabuf; |
263 | mapped_buffer->os_priv.attachment = attachment; | 263 | mapped_buffer->os_priv.attachment = attachment; |
264 | mapped_buffer->os_priv.sgt = sgt; | 264 | mapped_buffer->os_priv.sgt = sgt; |
265 | init_completion(&mapped_buffer->os_priv.swap_io_done); | ||
265 | nvmap_priv = nvmap_get_priv_list(dmabuf); | 266 | nvmap_priv = nvmap_get_priv_list(dmabuf); |
266 | if (!IS_ERR(nvmap_priv)) | 267 | if (!IS_ERR(nvmap_priv)) |
267 | list_add(&mapped_buffer->os_priv.nvmap_priv_entry, nvmap_priv); | 268 | list_add(&mapped_buffer->os_priv.nvmap_priv_entry, nvmap_priv); |