IB/qib: Optimize CQ callbacks

The current workqueue implemention has the following performance
deficiencies on QDR HCAs:

- The CQ call backs tend to run on the CPUs processing the
  receive queues
- The single thread queue isn't optimal for multiple HCAs

This patch adds a dedicated per HCA bound thread to process CQ callbacks.

Reviewed-by: Ramkrishna Vepa <ramkrishna.vepa@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>

1 files changed, 4 insertions, 14 deletions

diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index e02217b5c46d..ff36903474ea 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -97,8 +97,6 @@ unsigned qib_wc_pat = 1; /* default (1) is to use PAT, not MTRR */
 module_param_named(wc_pat, qib_wc_pat, uint, S_IRUGO);
 MODULE_PARM_DESC(wc_pat, "enable write-combining via PAT mechanism");
 
-struct workqueue_struct *qib_cq_wq;
-
 static void verify_interrupt(unsigned long);
 
 static struct idr qib_unit_table;
@@ -445,6 +443,7 @@ static int loadtime_init(struct qib_devdata *dd)
         dd->intrchk_timer.function = verify_interrupt;
         dd->intrchk_timer.data = (unsigned long) dd;
 
+        ret = qib_cq_init(dd);
 done:
         return ret;
 }
@@ -1215,12 +1214,6 @@ static int __init qlogic_ib_init(void)
         if (ret)
                 goto bail;
 
-        qib_cq_wq = create_singlethread_workqueue("qib_cq");
-        if (!qib_cq_wq) {
-                ret = -ENOMEM;
-                goto bail_dev;
-        }
-
         /*
          * These must be called before the driver is registered with
          * the PCI subsystem.
@@ -1233,7 +1226,7 @@ static int __init qlogic_ib_init(void)
         ret = pci_register_driver(&qib_driver);
         if (ret < 0) {
                 pr_err("Unable to register driver: error %d\n", -ret);
-                goto bail_unit;
+                goto bail_dev;
         }
 
         /* not fatal if it doesn't work */
@@ -1241,13 +1234,11 @@ static int __init qlogic_ib_init(void)
                 pr_err("Unable to register ipathfs\n");
         goto bail; /* all OK */
 
-bail_unit:
+bail_dev:
 #ifdef CONFIG_INFINIBAND_QIB_DCA
         dca_unregister_notify(&dca_notifier);
 #endif
         idr_destroy(&qib_unit_table);
-        destroy_workqueue(qib_cq_wq);
-bail_dev:
         qib_dev_cleanup();
 bail:
         return ret;
@@ -1273,8 +1264,6 @@ static void __exit qlogic_ib_cleanup(void)
 #endif
         pci_unregister_driver(&qib_driver);
 
-        destroy_workqueue(qib_cq_wq);
-
         qib_cpulist_count = 0;
         kfree(qib_cpulist);
 
@@ -1365,6 +1354,7 @@ static void cleanup_device_data(struct qib_devdata *dd)
         }
         kfree(tmp);
         kfree(dd->boardname);
+        qib_cq_exit(dd);
 }
 
 /*