diff options
author | Joshua Bakita <jbakita@cs.unc.edu> | 2022-06-02 14:29:53 -0400 |
---|---|---|
committer | Joshua Bakita <jbakita@cs.unc.edu> | 2022-06-03 15:41:42 -0400 |
commit | 9296adcd450143f02faf32fbda5b77dba3f03bc7 (patch) | |
tree | 1db15efddd549e40f0d92869be6f0d499852d7d8 /drivers/gpu/nvgpu/os/linux | |
parent | 745b3ef2ac4d7afa99202e6afc441e3f0b97f5b4 (diff) |
gpu-paging: Allow for more than one buffer to be swapped at a timertss22-aegpu-paging
This uses a very primitive linear disk sector allocation scheme.
Sectors are only reused when userspace resets assignment to 0 with
an NVGPU_AS_IOCTL_SWAP_RESET ioctl (which invalidates all current
swap buffers).
This sector assignment scheme is sufficient for use in a TimeWall-
like system, where all allocations are assumed to be static after
after task system release. This is not suitable for a system with
dynamic allocations, unless userspace manually resets swap state
regularly (benchmarks run a reset at start for example).
Support for dynamic allocations is on the backlog.
No significant speed impact.
Benchmarks, 100 iters, after:
gpu_paging_speed, write: 186.0ms +/- 3.51
gpu_paging_speed, read: 162.7ms +/- 2.58
gpu_paging_overhead_speed, write start: 35.4ms +/- 4.47
gpu_paging_overhead_speed, write finish: 3.3ms +/- 0.18
gpu_paging_overhead_speed, read start: 69.8ms +/- 6.42
gpu_paging_overhead_speed, read finish: 43.2ms +/- 0.91
Diffstat (limited to 'drivers/gpu/nvgpu/os/linux')
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/ioctl_as.c | 8 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/swap.h | 31 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/vm.c | 1 |
3 files changed, 36 insertions, 4 deletions
diff --git a/drivers/gpu/nvgpu/os/linux/ioctl_as.c b/drivers/gpu/nvgpu/os/linux/ioctl_as.c index 6348bb2a..2bf8363a 100644 --- a/drivers/gpu/nvgpu/os/linux/ioctl_as.c +++ b/drivers/gpu/nvgpu/os/linux/ioctl_as.c | |||
@@ -672,6 +672,14 @@ long gk20a_as_dev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) | |||
672 | err = nvgpu_as_dev_ioctl_swap(cmd, as_share, | 672 | err = nvgpu_as_dev_ioctl_swap(cmd, as_share, |
673 | (struct nvgpu_as_swap_buffer_args *)buf); | 673 | (struct nvgpu_as_swap_buffer_args *)buf); |
674 | break; | 674 | break; |
675 | case NVGPU_AS_IOCTL_SWAP_RESET: | ||
676 | // On-disk sector assignment is linear currently, and needs to | ||
677 | // be reset to the start between task systems to avoid disk | ||
678 | // space exhaustion. | ||
679 | // TODO: Support garbage-collection- or callback-driven sector | ||
680 | // reclaiming rather than requiring manual reset. | ||
681 | atomic64_set(&nvgpu_swap_next_sector, 4); | ||
682 | break; | ||
675 | default: | 683 | default: |
676 | err = -ENOTTY; | 684 | err = -ENOTTY; |
677 | break; | 685 | break; |
diff --git a/drivers/gpu/nvgpu/os/linux/swap.h b/drivers/gpu/nvgpu/os/linux/swap.h index 1e986095..3a648b26 100644 --- a/drivers/gpu/nvgpu/os/linux/swap.h +++ b/drivers/gpu/nvgpu/os/linux/swap.h | |||
@@ -1,8 +1,12 @@ | |||
1 | #include <linux/scatterlist.h> | 1 | #include <linux/scatterlist.h> |
2 | #include <linux/bio.h> | 2 | #include <linux/bio.h> |
3 | //#include <nvgpu/bug.h> | 3 | #include <linux/blkdev.h> // For SECTOR_SHIFT |
4 | |||
5 | // Next sector to assign a mapped_buf to. Skip first disk block | ||
6 | atomic64_t nvgpu_swap_next_sector = {4}; | ||
4 | 7 | ||
5 | // Callback for completion of the I/O chain | 8 | // Callback for completion of the I/O chain |
9 | // TODO: Error checking and handling | ||
6 | static void complete_swap_io(struct bio *bio) { | 10 | static void complete_swap_io(struct bio *bio) { |
7 | struct nvgpu_mapped_buf *m = bio->bi_private; | 11 | struct nvgpu_mapped_buf *m = bio->bi_private; |
8 | bio_put(bio); | 12 | bio_put(bio); |
@@ -11,21 +15,38 @@ static void complete_swap_io(struct bio *bio) { | |||
11 | 15 | ||
12 | // Queue a command to copy out an SGT to disk | 16 | // Queue a command to copy out an SGT to disk |
13 | // TODO: Cache bdev | 17 | // TODO: Cache bdev |
14 | // TODO: Don't hardcode sector 0 | 18 | // TODO: Track, allocate, and recycle individual swap buffers on disk instead |
15 | // TODO: Figure out if submit_bio() can fail, and what to do then | 19 | // of only supporting a global reset |
16 | int copy(struct sg_table *sgt, int op, struct nvgpu_mapped_buf *m) { | 20 | int copy(struct sg_table *sgt, int op, struct nvgpu_mapped_buf *m) { |
17 | unsigned int i; | 21 | unsigned int i; |
18 | struct scatterlist *sg; | 22 | struct scatterlist *sg; |
19 | struct bio *bio; | 23 | struct bio *bio; |
20 | int err = 0; | 24 | int err = 0; |
21 | int sg_cnt = sgt->nents; | 25 | int sg_cnt = sgt->nents; |
22 | sector_t sector = 0; // XXX: For testing | 26 | sector_t sector = m->os_priv.swap_sector; |
23 | // Find and open the block device | 27 | // Find and open the block device |
24 | struct block_device *bdev = blkdev_get_by_path("/dev/nvme0n1", FMODE_READ | FMODE_WRITE, copy); | 28 | struct block_device *bdev = blkdev_get_by_path("/dev/nvme0n1", FMODE_READ | FMODE_WRITE, copy); |
25 | if (unlikely(IS_ERR(bdev))) { | 29 | if (unlikely(IS_ERR(bdev))) { |
26 | printk(KERN_WARNING "Unabled to find `nvme0`, err %ld!\n", PTR_ERR(bdev)); | 30 | printk(KERN_WARNING "Unabled to find `nvme0`, err %ld!\n", PTR_ERR(bdev)); |
27 | return -ENODEV; | 31 | return -ENODEV; |
28 | } | 32 | } |
33 | // Assign a sector on-disk (0 indicates unassigned, we start at 4) | ||
34 | if (sector == 0) { | ||
35 | // Read block device size in sectors, and fail if we'd use more than 1/3rd | ||
36 | // of the disk (to stay in SLC-emulation-mode). | ||
37 | // TODO: Issue NVMe DSM commands to try to manage this better? Read-only | ||
38 | // regions should be able to be moved to TLC safely, whereas other | ||
39 | // data should be kept in the SLC cache to reduce wear. | ||
40 | if (atomic64_read(&nvgpu_swap_next_sector) >= i_size_read(bdev->bd_inode)/3) { | ||
41 | err = -ENOMEM; | ||
42 | goto out_put; | ||
43 | } | ||
44 | // Hand out sectors sequentially, and statically | ||
45 | // TODO: Intelligent sector allocation | ||
46 | sector = atomic64_add_return(m->size >> SECTOR_SHIFT, &nvgpu_swap_next_sector); | ||
47 | sector -= (m->size >> SECTOR_SHIFT); | ||
48 | m->os_priv.swap_sector = sector; | ||
49 | } | ||
29 | // Reset the .done variable in the completion | 50 | // Reset the .done variable in the completion |
30 | reinit_completion(&m->os_priv.swap_io_done); | 51 | reinit_completion(&m->os_priv.swap_io_done); |
31 | // bio_alloc() will never fail when allocating <= BIO_MAX_PAGES | 52 | // bio_alloc() will never fail when allocating <= BIO_MAX_PAGES |
@@ -58,8 +79,10 @@ int copy(struct sg_table *sgt, int op, struct nvgpu_mapped_buf *m) { | |||
58 | } | 79 | } |
59 | 80 | ||
60 | // Async submit. Caller should wait_for_completion_io(&m->os_priv.swap_io_done); | 81 | // Async submit. Caller should wait_for_completion_io(&m->os_priv.swap_io_done); |
82 | // Does not fail. Error reported via completion handler. | ||
61 | submit_bio(bio); | 83 | submit_bio(bio); |
62 | 84 | ||
85 | out_put: | ||
63 | // Release our block device handle | 86 | // Release our block device handle |
64 | blkdev_put(bdev, FMODE_WRITE | FMODE_READ); // Is this safe? | 87 | blkdev_put(bdev, FMODE_WRITE | FMODE_READ); // Is this safe? |
65 | return err; | 88 | return err; |
diff --git a/drivers/gpu/nvgpu/os/linux/vm.c b/drivers/gpu/nvgpu/os/linux/vm.c index 9cd17981..a1c19a3a 100644 --- a/drivers/gpu/nvgpu/os/linux/vm.c +++ b/drivers/gpu/nvgpu/os/linux/vm.c | |||
@@ -269,6 +269,7 @@ int nvgpu_vm_map_linux(struct vm_gk20a *vm, | |||
269 | else | 269 | else |
270 | // So we can always safely call list_del() | 270 | // So we can always safely call list_del() |
271 | INIT_LIST_HEAD(&mapped_buffer->os_priv.nvmap_priv_entry); | 271 | INIT_LIST_HEAD(&mapped_buffer->os_priv.nvmap_priv_entry); |
272 | mapped_buffer->os_priv.swap_sector = 0; | ||
272 | 273 | ||
273 | *gpu_va = mapped_buffer->addr; | 274 | *gpu_va = mapped_buffer->addr; |
274 | return 0; | 275 | return 0; |