From ee26a2842ca891d3ae8b1de1b066d29234fc0115 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Tue, 24 May 2022 21:11:59 -0400 Subject: gpu-paging: Initial working implementation Supports synchronous page out or in of a specific buffer. Includes fast reverse struct mapped_buf lookup. Requires initial set of changes to nvmap as well. --- drivers/gpu/nvgpu/os/linux/swap.h | 117 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 drivers/gpu/nvgpu/os/linux/swap.h (limited to 'drivers/gpu/nvgpu/os/linux/swap.h') diff --git a/drivers/gpu/nvgpu/os/linux/swap.h b/drivers/gpu/nvgpu/os/linux/swap.h new file mode 100644 index 00000000..f762ba81 --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/swap.h @@ -0,0 +1,117 @@ +#include +#include +//#include + +// Queue a command to copy out an SGT to disk +// TODO: Cache bdev +// TODO: Asynchronous I/O +// TODO: Don't hardcode sector 0 +int copy(struct sg_table *sgt, int op) { + unsigned int i; + struct scatterlist *sg; + struct bio *bio; + int err = 0; + int sg_cnt = sgt->nents; + struct bio *bio_orig; + sector_t sector = 0; // XXX: For testing + // Find and open the block device + struct block_device *bdev = blkdev_get_by_path("/dev/nvme0n1", FMODE_READ | FMODE_WRITE, copy); + if (unlikely(IS_ERR(bdev))) { + printk(KERN_WARNING "Unabled to find `nvme0`, err %ld!\n", PTR_ERR(bdev)); + return -ENODEV; + } + // Will never fail when allocating <= BIO_MAX_PAGES + bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio_orig = bio; + bio->bi_bdev = bdev; // Switch to bio_set_dev(bdev) in newer kernels + bio->bi_iter.bi_sector = sector; + bio_set_op_attrs(bio, op, op == REQ_OP_WRITE ? WRITE_ODIRECT : 0);//REQ_SYNC); // XXX: Is REQ_SYNC necessary? + // Copy the scatter-gather table (sgt) into a block I/O vector (bio vec) + // bio_chain() approach borrowed from drivers/nvme/target/io-cmd.c:nvmet_execute_rw() + for_each_sg(sgt->sgl, sg, sgt->nents, i) { + // On most iterations, this inner loop shouldn't happen at all. This loop + // conditional only triggers if we fill up the bio and are unable to map + // the full length of an SGL entry. + while (bio_add_page(bio, sg_page(sg), sg_dma_len(sg), sg->offset) != sg_dma_len(sg)) { + // Uh oh! We ran out of space in the bio. Allocate a new one and chain it... + struct bio *prev = bio; + bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); + bio->bi_bdev = bdev; // Switch to bio_set_dev(bdev) in newer kernels + bio->bi_iter.bi_sector = sector; + bio_set_op_attrs(bio, op, op == REQ_OP_WRITE ? WRITE_ODIRECT : 0); + bio_chain(bio, prev); + // Get the I/O started + submit_bio(prev); + // No need to call bio_put() as that's automatically managed for chained bios + } + sector += sg_dma_len(sg) >> 9; + sg_cnt--; + } + // Use blocking submit for now + // TODO: Switch to async via submit_bio(bio) + err = submit_bio_wait(bio); + + if (bio->bi_error && bio->bi_error != err) + printk(KERN_WARNING "nvgpu: bio->bi_error %d != return val from submit_bio_wait() %d\n", bio->bi_error, err); + +//out: + bio_put(bio_orig); // TODO: Move to completion handler + blkdev_put(bdev, FMODE_WRITE|FMODE_READ); + return err; +} + +// Patterned off how __nvgpu_vm_find_mapped_buf_reverse() works in vm.c +// Needs struct nvgpu_rbtree_node *node, struct nvgpu_rbtree_node *root, +// and struct nvgpu_mapped_buf *m. +// Steps until end of rbtree OR !m +#define for_each_buffer(node, root, m) \ + for (nvgpu_rbtree_enum_start(0, &node, root); \ + node && (uintptr_t)(m = mapped_buffer_from_rbtree_node(node)); \ + nvgpu_rbtree_enum_next(&node, node)) + +// New, fast replacement to looking through with the above macro to match +struct nvgpu_mapped_buf* dmabuf_to_mapped_buf(struct dma_buf *dmabuf) { + struct list_head *nvmap_priv = nvmap_get_priv_list(dmabuf); + struct nvgpu_mapped_buf *mapped_buffer; + struct nvgpu_mapped_buf_priv *priv; + + if (IS_ERR(nvmap_priv)) + return ERR_PTR(-EOPNOTSUPP); + + priv = list_first_entry_or_null(nvmap_priv, struct nvgpu_mapped_buf_priv, nvmap_priv_entry); + if (unlikely(!priv)) { + printk(KERN_ERR "nvgpu: State tracking error for fast reverse lookups. Have unattached dmabuf!"); + return ERR_PTR(-ENOTRECOVERABLE); + } + + mapped_buffer = container_of(priv, struct nvgpu_mapped_buf, os_priv); + if (unlikely(mapped_buffer->os_priv.dmabuf != dmabuf)) { + printk(KERN_ERR "nvgpu: dmabuf_to_mapped_buf mapping inconsistent! BUG!\n"); + return ERR_PTR(-ENOTRECOVERABLE); + } + if (!list_is_singular(&priv->nvmap_priv_entry)) { + printk(KERN_WARNING "nvgpu: Requesting paging on memory with multiple mappings! Aborting...\n"); + return ERR_PTR(-EOPNOTSUPP); + } + return mapped_buffer; +} + +int copy_all(struct vm_gk20a *vm) { + struct nvgpu_rbtree_node *node; + struct nvgpu_mapped_buf *m; + + for_each_buffer(node, vm->mapped_buffers, m) { + // TODO + continue; + } + return 0; +} + +int copy_out(struct sg_table *sgt) { + return copy(sgt, REQ_OP_WRITE); +} + +int copy_in(struct sg_table *sgt) { + return copy(sgt, REQ_OP_READ); +} + -- cgit v1.2.2