2 files changed, 105 insertions, 50 deletions

diff --git a/drivers/block/nvme.c b/drivers/block/nvme.c
index 38a2d0631882..ad16c68c8645 100644
--- a/drivers/block/nvme.c
+++ b/drivers/block/nvme.c
@@ -79,6 +79,7 @@ struct nvme_dev {
         char serial[20];
         char model[40];
         char firmware_rev[8];
+        u32 max_hw_sectors;
 };
 
 /*
@@ -835,15 +836,15 @@ static int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
 }
 
 static int nvme_get_features(struct nvme_dev *dev, unsigned fid,
-                                unsigned dword11, dma_addr_t dma_addr)
+                                unsigned nsid, dma_addr_t dma_addr)
 {
         struct nvme_command c;
 
         memset(&c, 0, sizeof(c));
         c.features.opcode = nvme_admin_get_features;
+        c.features.nsid = cpu_to_le32(nsid);
         c.features.prp1 = cpu_to_le64(dma_addr);
         c.features.fid = cpu_to_le32(fid);
-        c.features.dword11 = cpu_to_le32(dword11);
 
         return nvme_submit_admin_cmd(dev, &c, NULL);
 }
@@ -862,11 +863,51 @@ static int nvme_set_features(struct nvme_dev *dev, unsigned fid,
         return nvme_submit_admin_cmd(dev, &c, result);
 }
 
+/**
+ * nvme_cancel_ios - Cancel outstanding I/Os
+ * @queue: The queue to cancel I/Os on
+ * @timeout: True to only cancel I/Os which have timed out
+ */
+static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
+{
+        int depth = nvmeq->q_depth - 1;
+        struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
+        unsigned long now = jiffies;
+        int cmdid;
+
+        for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
+                void *ctx;
+                nvme_completion_fn fn;
+                static struct nvme_completion cqe = {
+                        .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1,
+                };
+
+                if (timeout && !time_after(now, info[cmdid].timeout))
+                        continue;
+                dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
+                ctx = cancel_cmdid(nvmeq, cmdid, &fn);
+                fn(nvmeq->dev, ctx, &cqe);
+        }
+}
+
+static void nvme_free_queue_mem(struct nvme_queue *nvmeq)
+{
+        dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
+                                (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
+        dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
+                                        nvmeq->sq_cmds, nvmeq->sq_dma_addr);
+        kfree(nvmeq);
+}
+
 static void nvme_free_queue(struct nvme_dev *dev, int qid)
 {
         struct nvme_queue *nvmeq = dev->queues[qid];
         int vector = dev->entry[nvmeq->cq_vector].vector;
 
+        spin_lock_irq(&nvmeq->q_lock);
+        nvme_cancel_ios(nvmeq, false);
+        spin_unlock_irq(&nvmeq->q_lock);
+
         irq_set_affinity_hint(vector, NULL);
         free_irq(vector, nvmeq);
 
@@ -876,18 +917,15 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid)
                 adapter_delete_cq(dev, qid);
         }
 
-        dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
-                                (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
-        dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
-                                        nvmeq->sq_cmds, nvmeq->sq_dma_addr);
-        kfree(nvmeq);
+        nvme_free_queue_mem(nvmeq);
 }
 
 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
                                                         int depth, int vector)
 {
         struct device *dmadev = &dev->pci_dev->dev;
-        unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info));
+        unsigned extra = DIV_ROUND_UP(depth, 8) + (depth *
+                                                sizeof(struct nvme_cmd_info));
         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
         if (!nvmeq)
                 return NULL;
@@ -975,7 +1013,7 @@ static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev,
 
 static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)
 {
-        int result;
+        int result = 0;
         u32 aqa;
         u64 cap;
         unsigned long timeout;
@@ -1005,17 +1043,22 @@ static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)
         timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
         dev->db_stride = NVME_CAP_STRIDE(cap);
 
-        while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
+        while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
                 msleep(100);
                 if (fatal_signal_pending(current))
-                        return -EINTR;
+                        result = -EINTR;
                 if (time_after(jiffies, timeout)) {
                         dev_err(&dev->pci_dev->dev,
                                 "Device not ready; aborting initialisation\n");
-                        return -ENODEV;
+                        result = -ENODEV;
                 }
         }
 
+        if (result) {
+                nvme_free_queue_mem(nvmeq);
+                return result;
+        }
+
         result = queue_request_irq(dev, nvmeq, "nvme admin");
         dev->queues[0] = nvmeq;
         return result;
@@ -1037,6 +1080,8 @@ static struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
         offset = offset_in_page(addr);
         count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
         pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
+        if (!pages)
+                return ERR_PTR(-ENOMEM);
 
         err = get_user_pages_fast(addr, count, 1, pages);
         if (err < count) {
@@ -1146,14 +1191,13 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
         return status;
 }
 
-static int nvme_user_admin_cmd(struct nvme_ns *ns,
+static int nvme_user_admin_cmd(struct nvme_dev *dev,
                                         struct nvme_admin_cmd __user *ucmd)
 {
-        struct nvme_dev *dev = ns->dev;
         struct nvme_admin_cmd cmd;
         struct nvme_command c;
         int status, length;
-        struct nvme_iod *iod;
+        struct nvme_iod *uninitialized_var(iod);
 
         if (!capable(CAP_SYS_ADMIN))
                 return -EACCES;
@@ -1204,7 +1248,7 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
         case NVME_IOCTL_ID:
                 return ns->ns_id;
         case NVME_IOCTL_ADMIN_CMD:
-                return nvme_user_admin_cmd(ns, (void __user *)arg);
+                return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
         case NVME_IOCTL_SUBMIT_IO:
                 return nvme_submit_io(ns, (void __user *)arg);
         default:
@@ -1218,26 +1262,6 @@ static const struct block_device_operations nvme_fops = {
         .compat_ioctl   = nvme_ioctl,
 };
 
-static void nvme_timeout_ios(struct nvme_queue *nvmeq)
-{
-        int depth = nvmeq->q_depth - 1;
-        struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
-        unsigned long now = jiffies;
-        int cmdid;
-
-        for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
-                void *ctx;
-                nvme_completion_fn fn;
-                static struct nvme_completion cqe = { .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, };
-
-                if (!time_after(now, info[cmdid].timeout))
-                        continue;
-                dev_warn(nvmeq->q_dmadev, "Timing out I/O %d\n", cmdid);
-                ctx = cancel_cmdid(nvmeq, cmdid, &fn);
-                fn(nvmeq->dev, ctx, &cqe);
-        }
-}
-
 static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
 {
         while (bio_list_peek(&nvmeq->sq_cong)) {
@@ -1269,7 +1293,7 @@ static int nvme_kthread(void *data)
                                 spin_lock_irq(&nvmeq->q_lock);
                                 if (nvme_process_cq(nvmeq))
                                         printk("process_cq did something\n");
-                                nvme_timeout_ios(nvmeq);
+                                nvme_cancel_ios(nvmeq, true);
                                 nvme_resubmit_bios(nvmeq);
                                 spin_unlock_irq(&nvmeq->q_lock);
                         }
@@ -1339,6 +1363,9 @@ static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid,
         ns->disk = disk;
         lbaf = id->flbas & 0xf;
         ns->lba_shift = id->lbaf[lbaf].ds;
+        blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
+        if (dev->max_hw_sectors)
+                blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
 
         disk->major = nvme_major;
         disk->minors = NVME_MINORS;
@@ -1383,7 +1410,7 @@ static int set_queue_count(struct nvme_dev *dev, int count)
 
 static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)
 {
-        int result, cpu, i, nr_io_queues, db_bar_size;
+        int result, cpu, i, nr_io_queues, db_bar_size, q_depth;
 
         nr_io_queues = num_online_cpus();
         result = set_queue_count(dev, nr_io_queues);
@@ -1429,9 +1456,10 @@ static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)
                 cpu = cpumask_next(cpu, cpu_online_mask);
         }
 
+        q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
+                                                                NVME_Q_DEPTH);
         for (i = 0; i < nr_io_queues; i++) {
-                dev->queues[i + 1] = nvme_create_queue(dev, i + 1,
-                                                        NVME_Q_DEPTH, i);
+                dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i);
                 if (IS_ERR(dev->queues[i + 1]))
                         return PTR_ERR(dev->queues[i + 1]);
                 dev->queue_count++;
@@ -1480,6 +1508,10 @@ static int __devinit nvme_dev_add(struct nvme_dev *dev)
         memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
         memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
         memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
+        if (ctrl->mdts) {
+                int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
+                dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
+        }
 
         id_ns = mem;
         for (i = 1; i <= nn; i++) {
@@ -1523,8 +1555,6 @@ static int nvme_dev_remove(struct nvme_dev *dev)
         list_del(&dev->node);
         spin_unlock(&dev_list_lock);
 
-        /* TODO: wait all I/O finished or cancel them */
-
         list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
                 list_del(&ns->list);
                 del_gendisk(ns->disk);
@@ -1560,15 +1590,33 @@ static void nvme_release_prp_pools(struct nvme_dev *dev)
         dma_pool_destroy(dev->prp_small_pool);
 }
 
-/* XXX: Use an ida or something to let remove / add work correctly */
-static void nvme_set_instance(struct nvme_dev *dev)
+static DEFINE_IDA(nvme_instance_ida);
+
+static int nvme_set_instance(struct nvme_dev *dev)
 {
-        static int instance;
-        dev->instance = instance++;
+        int instance, error;
+
+        do {
+                if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
+                        return -ENODEV;
+
+                spin_lock(&dev_list_lock);
+                error = ida_get_new(&nvme_instance_ida, &instance);
+                spin_unlock(&dev_list_lock);
+        } while (error == -EAGAIN);
+
+        if (error)
+                return -ENODEV;
+
+        dev->instance = instance;
+        return 0;
 }
 
 static void nvme_release_instance(struct nvme_dev *dev)
 {
+        spin_lock(&dev_list_lock);
+        ida_remove(&nvme_instance_ida, dev->instance);
+        spin_unlock(&dev_list_lock);
 }
 
 static int __devinit nvme_probe(struct pci_dev *pdev,
@@ -1601,7 +1649,10 @@ static int __devinit nvme_probe(struct pci_dev *pdev,
         pci_set_drvdata(pdev, dev);
         dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
         dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
-        nvme_set_instance(dev);
+        result = nvme_set_instance(dev);
+        if (result)
+                goto disable;
+
         dev->entry[0].vector = pdev->irq;
 
         result = nvme_setup_prp_pools(dev);
@@ -1704,15 +1755,17 @@ static struct pci_driver nvme_driver = {
 
 static int __init nvme_init(void)
 {
-        int result = -EBUSY;
+        int result;
 
         nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
         if (IS_ERR(nvme_thread))
                 return PTR_ERR(nvme_thread);
 
-        nvme_major = register_blkdev(nvme_major, "nvme");
-        if (nvme_major <= 0)
+        result = register_blkdev(nvme_major, "nvme");
+        if (result < 0)
                 goto kill_kthread;
+        else if (result > 0)
+                nvme_major = result;
 
         result = pci_register_driver(&nvme_driver);
         if (result)
diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index 9490a00529f4..c25cccaa555a 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -35,8 +35,10 @@ struct nvme_bar {
         __u64                   acq;    /* Admin CQ Base Address */
 };
 
+#define NVME_CAP_MQES(cap)      ((cap) & 0xffff)
 #define NVME_CAP_TIMEOUT(cap)   (((cap) >> 24) & 0xff)
 #define NVME_CAP_STRIDE(cap)    (((cap) >> 32) & 0xf)
+#define NVME_CAP_MPSMIN(cap)    (((cap) >> 48) & 0xf)
 
 enum {
         NVME_CC_ENABLE          = 1 << 0,