nvme-rdma: centralize controller setup sequence

Centralize controller sequence to a single routine that correctly cleans
up after failures instead of having multiple apperances in several flows
(create, reset, reconnect).

One thing that we also gain here are the sanity/boundary checks also
when connecting back to a dynamic controller.

Signed-off-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Christoph Hellwig <hch@lst.de>

1 files changed, 53 insertions, 77 deletions

diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c
index 2b683b8d4763..c22125c5661b 100644
--- a/drivers/nvme/host/rdma.c
+++ b/drivers/nvme/host/rdma.c
@@ -917,24 +917,44 @@ static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
         }
 }
 
-static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
+static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
 {
-        struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
-                        struct nvme_rdma_ctrl, reconnect_work);
+        int ret = -EINVAL;
         bool changed;
-        int ret;
-
-        ++ctrl->ctrl.nr_reconnects;
 
-        ret = nvme_rdma_configure_admin_queue(ctrl, false);
+        ret = nvme_rdma_configure_admin_queue(ctrl, new);
         if (ret)
-                goto requeue;
+                return ret;
+
+        if (ctrl->ctrl.icdoff) {
+                dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
+                goto destroy_admin;
+        }
+
+        if (!(ctrl->ctrl.sgls & (1 << 2))) {
+                dev_err(ctrl->ctrl.device,
+                        "Mandatory keyed sgls are not supported!\n");
+                goto destroy_admin;
+        }
+
+        if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize + 1) {
+                dev_warn(ctrl->ctrl.device,
+                        "queue_size %zu > ctrl sqsize %u, clamping down\n",
+                        ctrl->ctrl.opts->queue_size, ctrl->ctrl.sqsize + 1);
+        }
+
+        if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) {
+                dev_warn(ctrl->ctrl.device,
+                        "sqsize %u > ctrl maxcmd %u, clamping down\n",
+                        ctrl->ctrl.sqsize + 1, ctrl->ctrl.maxcmd);
+                ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd - 1;
+        }
 
         if (ctrl->ctrl.sgls & (1 << 20))
                 ctrl->use_inline_data = true;
 
         if (ctrl->ctrl.queue_count > 1) {
-                ret = nvme_rdma_configure_io_queues(ctrl, false);
+                ret = nvme_rdma_configure_io_queues(ctrl, new);
                 if (ret)
                         goto destroy_admin;
         }
@@ -943,10 +963,31 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
         if (!changed) {
                 /* state change failure is ok if we're in DELETING state */
                 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
-                return;
+                ret = -EINVAL;
+                goto destroy_io;
         }
 
         nvme_start_ctrl(&ctrl->ctrl);
+        return 0;
+
+destroy_io:
+        if (ctrl->ctrl.queue_count > 1)
+                nvme_rdma_destroy_io_queues(ctrl, new);
+destroy_admin:
+        nvme_rdma_stop_queue(&ctrl->queues[0]);
+        nvme_rdma_destroy_admin_queue(ctrl, new);
+        return ret;
+}
+
+static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
+{
+        struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
+                        struct nvme_rdma_ctrl, reconnect_work);
+
+        ++ctrl->ctrl.nr_reconnects;
+
+        if (nvme_rdma_setup_ctrl(ctrl, false))
+                goto requeue;
 
         dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
                         ctrl->ctrl.nr_reconnects);
@@ -955,9 +996,6 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
 
         return;
 
-destroy_admin:
-        nvme_rdma_stop_queue(&ctrl->queues[0]);
-        nvme_rdma_destroy_admin_queue(ctrl, false);
 requeue:
         dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
                         ctrl->ctrl.nr_reconnects);
@@ -1786,8 +1824,6 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
 {
         struct nvme_rdma_ctrl *ctrl =
                 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
-        int ret;
-        bool changed;
 
         nvme_stop_ctrl(&ctrl->ctrl);
         nvme_rdma_shutdown_ctrl(ctrl, false);
@@ -1798,25 +1834,9 @@ static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
                 return;
         }
 
-        ret = nvme_rdma_configure_admin_queue(ctrl, false);
-        if (ret)
+        if (nvme_rdma_setup_ctrl(ctrl, false))
                 goto out_fail;
 
-        if (ctrl->ctrl.queue_count > 1) {
-                ret = nvme_rdma_configure_io_queues(ctrl, false);
-                if (ret)
-                        goto out_fail;
-        }
-
-        changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
-        if (!changed) {
-                /* state change failure is ok if we're in DELETING state */
-                WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING);
-                return;
-        }
-
-        nvme_start_ctrl(&ctrl->ctrl);
-
         return;
 
 out_fail:
@@ -1979,49 +1999,10 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING);
         WARN_ON_ONCE(!changed);
 
-        ret = nvme_rdma_configure_admin_queue(ctrl, true);
+        ret = nvme_rdma_setup_ctrl(ctrl, true);
         if (ret)
                 goto out_uninit_ctrl;
 
-        /* sanity check icdoff */
-        if (ctrl->ctrl.icdoff) {
-                dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
-                ret = -EINVAL;
-                goto out_remove_admin_queue;
-        }
-
-        /* sanity check keyed sgls */
-        if (!(ctrl->ctrl.sgls & (1 << 2))) {
-                dev_err(ctrl->ctrl.device,
-                        "Mandatory keyed sgls are not supported!\n");
-                ret = -EINVAL;
-                goto out_remove_admin_queue;
-        }
-
-        /* only warn if argument is too large here, will clamp later */
-        if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
-                dev_warn(ctrl->ctrl.device,
-                        "queue_size %zu > ctrl sqsize %u, clamping down\n",
-                        opts->queue_size, ctrl->ctrl.sqsize + 1);
-        }
-
-        /* warn if maxcmd is lower than sqsize+1 */
-        if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) {
-                dev_warn(ctrl->ctrl.device,
-                        "sqsize %u > ctrl maxcmd %u, clamping down\n",
-                        ctrl->ctrl.sqsize + 1, ctrl->ctrl.maxcmd);
-                ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd - 1;
-        }
-
-        if (opts->nr_io_queues) {
-                ret = nvme_rdma_configure_io_queues(ctrl, true);
-                if (ret)
-                        goto out_remove_admin_queue;
-        }
-
-        changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
-        WARN_ON_ONCE(!changed);
-
         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
 
@@ -2031,13 +2012,8 @@ static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
         list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
         mutex_unlock(&nvme_rdma_ctrl_mutex);
 
-        nvme_start_ctrl(&ctrl->ctrl);
-
         return &ctrl->ctrl;
 
-out_remove_admin_queue:
-        nvme_rdma_stop_queue(&ctrl->queues[0]);
-        nvme_rdma_destroy_admin_queue(ctrl, true);
 out_uninit_ctrl:
         nvme_uninit_ctrl(&ctrl->ctrl);
         nvme_put_ctrl(&ctrl->ctrl);