Merge git://git.infradead.org/users/willy/linux-nvme

Pull NVMe driver updates from Matthew Wilcox:
 "Various updates to the NVMe driver.  The most user-visible change is
  that drive hotplugging now works and CPU hotplug while an NVMe drive
  is installed should also work better"

* git://git.infradead.org/users/willy/linux-nvme:
  NVMe: Retry failed commands with non-fatal errors
  NVMe: Add getgeo to block ops
  NVMe: Start-stop nvme_thread during device add-remove.
  NVMe: Make I/O timeout a module parameter
  NVMe: CPU hot plug notification
  NVMe: per-cpu io queues
  NVMe: Replace DEFINE_PCI_DEVICE_TABLE
  NVMe: Fix divide-by-zero in nvme_trans_io_get_num_cmds
  NVMe: IOCTL path RCU protect queue access
  NVMe: RCU protected access to io queues
  NVMe: Initialize device reference count earlier
  NVMe: Add CONFIG_PM_SLEEP to suspend/resume functions

4 files changed, 504 insertions, 245 deletions

diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c
index da085ff10d25..7c64fa756cce 100644
--- a/drivers/block/nvme-core.c
+++ b/drivers/block/nvme-core.c
@@ -1,6 +1,6 @@
 /*
  * NVM Express device driver
- * Copyright (c) 2011, Intel Corporation.
+ * Copyright (c) 2011-2014, Intel Corporation.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -20,10 +20,12 @@
 #include <linux/bio.h>
 #include <linux/bitops.h>
 #include <linux/blkdev.h>
+#include <linux/cpu.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/fs.h>
 #include <linux/genhd.h>
+#include <linux/hdreg.h>
 #include <linux/idr.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -35,6 +37,7 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/pci.h>
+#include <linux/percpu.h>
 #include <linux/poison.h>
 #include <linux/ptrace.h>
 #include <linux/sched.h>
@@ -47,6 +50,11 @@
 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
 #define ADMIN_TIMEOUT   (60 * HZ)
+#define IOD_TIMEOUT     (4 * NVME_IO_TIMEOUT)
+
+unsigned char io_timeout = 30;
+module_param(io_timeout, byte, 0644);
+MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
 
 static int nvme_major;
 module_param(nvme_major, int, 0);
@@ -58,6 +66,7 @@ static DEFINE_SPINLOCK(dev_list_lock);
 static LIST_HEAD(dev_list);
 static struct task_struct *nvme_thread;
 static struct workqueue_struct *nvme_workq;
+static wait_queue_head_t nvme_kthread_wait;
 
 static void nvme_reset_failed_dev(struct work_struct *ws);
 
@@ -74,6 +83,7 @@ struct async_cmd_info {
  * commands and one for I/O commands).
  */
 struct nvme_queue {
+        struct rcu_head r_head;
         struct device *q_dmadev;
         struct nvme_dev *dev;
         char irqname[24];       /* nvme4294967295-65535\0 */
@@ -85,6 +95,7 @@ struct nvme_queue {
         wait_queue_head_t sq_full;
         wait_queue_t sq_cong_wait;
         struct bio_list sq_cong;
+        struct list_head iod_bio;
         u32 __iomem *q_db;
         u16 q_depth;
         u16 cq_vector;
@@ -95,6 +106,7 @@ struct nvme_queue {
         u8 cq_phase;
         u8 cqe_seen;
         u8 q_suspended;
+        cpumask_var_t cpu_mask;
         struct async_cmd_info cmdinfo;
         unsigned long cmdid_data[];
 };
@@ -118,7 +130,7 @@ static inline void _nvme_check_size(void)
         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 }
 
-typedef void (*nvme_completion_fn)(struct nvme_dev *, void *,
+typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
                                                 struct nvme_completion *);
 
 struct nvme_cmd_info {
@@ -190,7 +202,7 @@ static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
 #define CMD_CTX_FLUSH           (0x318 + CMD_CTX_BASE)
 #define CMD_CTX_ABORT           (0x31C + CMD_CTX_BASE)
 
-static void special_completion(struct nvme_dev *dev, void *ctx,
+static void special_completion(struct nvme_queue *nvmeq, void *ctx,
                                                 struct nvme_completion *cqe)
 {
         if (ctx == CMD_CTX_CANCELLED)
@@ -198,26 +210,26 @@ static void special_completion(struct nvme_dev *dev, void *ctx,
         if (ctx == CMD_CTX_FLUSH)
                 return;
         if (ctx == CMD_CTX_ABORT) {
-                ++dev->abort_limit;
+                ++nvmeq->dev->abort_limit;
                 return;
         }
         if (ctx == CMD_CTX_COMPLETED) {
-                dev_warn(&dev->pci_dev->dev,
+                dev_warn(nvmeq->q_dmadev,
                                 "completed id %d twice on queue %d\n",
                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
                 return;
         }
         if (ctx == CMD_CTX_INVALID) {
-                dev_warn(&dev->pci_dev->dev,
+                dev_warn(nvmeq->q_dmadev,
                                 "invalid id %d completed on queue %d\n",
                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
                 return;
         }
 
-        dev_warn(&dev->pci_dev->dev, "Unknown special completion %p\n", ctx);
+        dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
 }
 
-static void async_completion(struct nvme_dev *dev, void *ctx,
+static void async_completion(struct nvme_queue *nvmeq, void *ctx,
                                                 struct nvme_completion *cqe)
 {
         struct async_cmd_info *cmdinfo = ctx;
@@ -262,14 +274,34 @@ static void *cancel_cmdid(struct nvme_queue *nvmeq, int cmdid,
         return ctx;
 }
 
-struct nvme_queue *get_nvmeq(struct nvme_dev *dev)
+static struct nvme_queue *raw_nvmeq(struct nvme_dev *dev, int qid)
+{
+        return rcu_dereference_raw(dev->queues[qid]);
+}
+
+static struct nvme_queue *get_nvmeq(struct nvme_dev *dev) __acquires(RCU)
+{
+        unsigned queue_id = get_cpu_var(*dev->io_queue);
+        rcu_read_lock();
+        return rcu_dereference(dev->queues[queue_id]);
+}
+
+static void put_nvmeq(struct nvme_queue *nvmeq) __releases(RCU)
 {
-        return dev->queues[get_cpu() + 1];
+        rcu_read_unlock();
+        put_cpu_var(nvmeq->dev->io_queue);
 }