4 files changed, 156 insertions, 65 deletions

diff --git a/drivers/ntb/ntb_hw.c b/drivers/ntb/ntb_hw.c
index 1cb6e51e6bda..170e8e60cdb7 100644
--- a/drivers/ntb/ntb_hw.c
+++ b/drivers/ntb/ntb_hw.c
@@ -141,6 +141,24 @@ void ntb_unregister_event_callback(struct ntb_device *ndev)
         ndev->event_cb = NULL;
 }
 
+static void ntb_irq_work(unsigned long data)
+{
+        struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;
+        int rc;
+
+        rc = db_cb->callback(db_cb->data, db_cb->db_num);
+        if (rc)
+                tasklet_schedule(&db_cb->irq_work);
+        else {
+                struct ntb_device *ndev = db_cb->ndev;
+                unsigned long mask;
+
+                mask = readw(ndev->reg_ofs.ldb_mask);
+                clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
+                writew(mask, ndev->reg_ofs.ldb_mask);
+        }
+}
+
 /**
  * ntb_register_db_callback() - register a callback for doorbell interrupt
  * @ndev: pointer to ntb_device instance
@@ -155,7 +173,7 @@ void ntb_unregister_event_callback(struct ntb_device *ndev)
  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  */
 int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
-                             void *data, void (*func)(void *data, int db_num))
+                             void *data, int (*func)(void *data, int db_num))
 {
         unsigned long mask;
 
@@ -166,6 +184,10 @@ int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
 
         ndev->db_cb[idx].callback = func;
         ndev->db_cb[idx].data = data;
+        ndev->db_cb[idx].ndev = ndev;
+
+        tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,
+                     (unsigned long) &ndev->db_cb[idx]);
 
         /* unmask interrupt */
         mask = readw(ndev->reg_ofs.ldb_mask);
@@ -194,6 +216,8 @@ void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
         set_bit(idx * ndev->bits_per_vector, &mask);
         writew(mask, ndev->reg_ofs.ldb_mask);
 
+        tasklet_disable(&ndev->db_cb[idx].irq_work);
+
         ndev->db_cb[idx].callback = NULL;
 }
 
@@ -678,6 +702,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
                                 return -EINVAL;
 
                         ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
+                        ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
                         ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
                                                    SNB_SPAD_OFFSET;
                         ndev->reg_ofs.rdb = ndev->mw[1].vbase +
@@ -688,8 +713,21 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
                          */
                         writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
                                SNB_PBAR4LMT_OFFSET);
+                        /* HW errata on the Limit registers.  They can only be
+                         * written when the base register is 4GB aligned and
+                         * < 32bit.  This should already be the case based on the
+                         * driver defaults, but write the Limit registers first
+                         * just in case.
+                         */
                 } else {
                         ndev->limits.max_mw = SNB_MAX_MW;
+
+                        /* HW Errata on bit 14 of b2bdoorbell register.  Writes
+                         * will not be mirrored to the remote system.  Shrink
+                         * the number of bits by one, since bit 14 is the last
+                         * bit.
+                         */
+                        ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
                         ndev->reg_ofs.spad_write = ndev->reg_base +
                                                    SNB_B2B_SPAD_OFFSET;
                         ndev->reg_ofs.rdb = ndev->reg_base +
@@ -699,6 +737,12 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
                          * something silly
                          */
                         writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
+                        /* HW errata on the Limit registers.  They can only be
+                         * written when the base register is 4GB aligned and
+                         * < 32bit.  This should already be the case based on the
+                         * driver defaults, but write the Limit registers first
+                         * just in case.
+                         */
                 }
 
                 /* The Xeon errata workaround requires setting SBAR Base
@@ -769,6 +813,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
                  * have an equal amount.
                  */
                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
+                ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
                 /* Note: The SDOORBELL is the cause of the errata.  You REALLY
                  * don't want to touch it.
                  */
@@ -793,6 +838,7 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
                  * have an equal amount.
                  */
                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
+                ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
                 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
@@ -819,7 +865,6 @@ static int ntb_xeon_setup(struct ntb_device *ndev)
         ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
         ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
 
-        ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
         ndev->limits.msix_cnt = SNB_MSIX_CNT;
         ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
 
@@ -934,12 +979,16 @@ static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
 {
         struct ntb_db_cb *db_cb = data;
         struct ntb_device *ndev = db_cb->ndev;
+        unsigned long mask;
 
         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
                 db_cb->db_num);
 
-        if (db_cb->callback)
-                db_cb->callback(db_cb->data, db_cb->db_num);
+        mask = readw(ndev->reg_ofs.ldb_mask);
+        set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
+        writew(mask, ndev->reg_ofs.ldb_mask);
+
+        tasklet_schedule(&db_cb->irq_work);
 
         /* No need to check for the specific HB irq, any interrupt means
          * we're connected.
@@ -955,12 +1004,16 @@ static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
 {
         struct ntb_db_cb *db_cb = data;
         struct ntb_device *ndev = db_cb->ndev;
+        unsigned long mask;
 
         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
                 db_cb->db_num);
 
-        if (db_cb->callback)
-                db_cb->callback(db_cb->data, db_cb->db_num);
+        mask = readw(ndev->reg_ofs.ldb_mask);
+        set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
+        writew(mask, ndev->reg_ofs.ldb_mask);
+
+        tasklet_schedule(&db_cb->irq_work);
 
         /* On Sandybridge, there are 16 bits in the interrupt register
          * but only 4 vectors.  So, 5 bits are assigned to the first 3
@@ -986,7 +1039,7 @@ static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
                 dev_err(&ndev->pdev->dev, "Error determining link status\n");
 
         /* bit 15 is always the link bit */
-        writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);
+        writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);
 
         return IRQ_HANDLED;
 }
@@ -1075,6 +1128,10 @@ static int ntb_setup_msix(struct ntb_device *ndev)
                          "Only %d MSI-X vectors.  Limiting the number of queues to that number.\n",
                          rc);
                 msix_entries = rc;
+
+                rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
+                if (rc)
+                        goto err1;
         }
 
         for (i = 0; i < msix_entries; i++) {
@@ -1176,9 +1233,10 @@ static int ntb_setup_interrupts(struct ntb_device *ndev)
          */
         if (ndev->hw_type == BWD_HW)
                 writeq(~0, ndev->reg_ofs.ldb_mask);
-        else
-                writew(~(1 << ndev->limits.max_db_bits),
-                       ndev->reg_ofs.ldb_mask);
+        else {
+                u16 var = 1 << SNB_LINK_DB;
+                writew(~var, ndev->reg_ofs.ldb_mask);
+        }
 
         rc = ntb_setup_msix(ndev);
         if (!rc)
@@ -1286,6 +1344,39 @@ static void ntb_free_debugfs(struct ntb_device *ndev)
         }
 }
 
+static void ntb_hw_link_up(struct ntb_device *ndev)
+{
+        if (ndev->conn_type == NTB_CONN_TRANSPARENT)
+                ntb_link_event(ndev, NTB_LINK_UP);
+        else {
+                u32 ntb_cntl;
+
+                /* Let's bring the NTB link up */
+                ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
+                ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
+                ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
+                ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP;
+                writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
+        }
+}
+
+static void ntb_hw_link_down(struct ntb_device *ndev)
+{
+        u32 ntb_cntl;
+
+        if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
+                ntb_link_event(ndev, NTB_LINK_DOWN);
+                return;
+        }
+
+        /* Bring NTB link down */
+        ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
+        ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
+        ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP);
+        ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
+        writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
+}
+
 static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
         struct ntb_device *ndev;
@@ -1374,9 +1465,7 @@ static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
         if (rc)
                 goto err6;
 
-        /* Let's bring the NTB link up */
-        writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
-               ndev->reg_ofs.lnk_cntl);
+        ntb_hw_link_up(ndev);
 
         return 0;
 
@@ -1406,12 +1495,8 @@ static void ntb_pci_remove(struct pci_dev *pdev)
 {
         struct ntb_device *ndev = pci_get_drvdata(pdev);
         int i;
-        u32 ntb_cntl;
 
-        /* Bring NTB link down */
-        ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
-        ntb_cntl |= NTB_CNTL_LINK_DISABLE;
-        writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
+        ntb_hw_link_down(ndev);
 
         ntb_transport_free(ndev->ntb_transport);
 
diff --git a/drivers/ntb/ntb_hw.h b/drivers/ntb/ntb_hw.h
index 0a31cedae7d4..bbdb7edca10c 100644
--- a/drivers/ntb/ntb_hw.h
+++ b/drivers/ntb/ntb_hw.h
@@ -106,10 +106,11 @@ struct ntb_mw {
 };
 
 struct ntb_db_cb {
-        void (*callback) (void *data, int db_num);
+        int (*callback)(void *data, int db_num);
         unsigned int db_num;
         void *data;
         struct ntb_device *ndev;
+        struct tasklet_struct irq_work;
 };
 
 struct ntb_device {
@@ -228,8 +229,8 @@ struct ntb_device *ntb_register_transport(struct pci_dev *pdev,
 void ntb_unregister_transport(struct ntb_device *ndev);
 void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr);
 int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
-                             void *data, void (*db_cb_func) (void *data,
-                                                             int db_num));
+                             void *data, int (*db_cb_func)(void *data,
+                                                           int db_num));
 void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx);
 int ntb_register_event_callback(struct ntb_device *ndev,
                                 void (*event_cb_func) (void *handle,
diff --git a/drivers/ntb/ntb_regs.h b/drivers/ntb/ntb_regs.h
index aa4bdd393c58..9774506419d7 100644
--- a/drivers/ntb/ntb_regs.h
+++ b/drivers/ntb/ntb_regs.h
@@ -55,6 +55,7 @@
 #define SNB_MAX_COMPAT_SPADS    16
 /* Reserve the uppermost bit for link interrupt */
 #define SNB_MAX_DB_BITS         15
+#define SNB_LINK_DB             15
 #define SNB_DB_BITS_PER_VEC     5
 #define SNB_MAX_MW              2
 #define SNB_ERRATA_MAX_MW       1
@@ -75,9 +76,6 @@
 #define SNB_SBAR2XLAT_OFFSET    0x0030
 #define SNB_SBAR4XLAT_OFFSET    0x0038
 #define SNB_SBAR0BASE_OFFSET    0x0040
-#define SNB_SBAR0BASE_OFFSET    0x0040
-#define SNB_SBAR2BASE_OFFSET    0x0048
-#define SNB_SBAR4BASE_OFFSET    0x0050
 #define SNB_SBAR2BASE_OFFSET    0x0048
 #define SNB_SBAR4BASE_OFFSET    0x0050
 #define SNB_NTBCNTL_OFFSET      0x0058
@@ -145,11 +143,13 @@
 #define BWD_LTSSMSTATEJMP_FORCEDETECT   (1 << 2)
 #define BWD_IBIST_ERR_OFLOW     0x7FFF7FFF
 
-#define NTB_CNTL_CFG_LOCK       (1 << 0)
-#define NTB_CNTL_LINK_DISABLE   (1 << 1)
-#define NTB_CNTL_BAR23_SNOOP    (1 << 2)
-#define NTB_CNTL_BAR45_SNOOP    (1 << 6)
-#define BWD_CNTL_LINK_DOWN      (1 << 16)
+#define NTB_CNTL_CFG_LOCK               (1 << 0)
+#define NTB_CNTL_LINK_DISABLE           (1 << 1)
+#define NTB_CNTL_S2P_BAR23_SNOOP        (1 << 2)
+#define NTB_CNTL_P2S_BAR23_SNOOP        (1 << 4)
+#define NTB_CNTL_S2P_BAR45_SNOOP        (1 << 6)
+#define NTB_CNTL_P2S_BAR45_SNOOP        (1 << 8)
+#define BWD_CNTL_LINK_DOWN              (1 << 16)
 
 #define NTB_PPD_OFFSET          0x00D4
 #define SNB_PPD_CONN_TYPE       0x0003
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index d0222f13d154..3217f394d45b 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -119,7 +119,6 @@ struct ntb_transport_qp {
 
         void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data,
                             void *data, int len);
-        struct tasklet_struct rx_work;
         struct list_head rx_pend_q;
         struct list_head rx_free_q;
         spinlock_t ntb_rx_pend_q_lock;
@@ -584,11 +583,8 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)
         return 0;
 }
 
-static void ntb_qp_link_cleanup(struct work_struct *work)
+static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
 {
-        struct ntb_transport_qp *qp = container_of(work,
-                                                   struct ntb_transport_qp,
-                                                   link_cleanup);
         struct ntb_transport *nt = qp->transport;
         struct pci_dev *pdev = ntb_query_pdev(nt->ndev);
 
@@ -602,6 +598,16 @@ static void ntb_qp_link_cleanup(struct work_struct *work)
 
         dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);
         qp->qp_link = NTB_LINK_DOWN;
+}
+
+static void ntb_qp_link_cleanup_work(struct work_struct *work)
+{
+        struct ntb_transport_qp *qp = container_of(work,
+                                                   struct ntb_transport_qp,
+                                                   link_cleanup);
+        struct ntb_transport *nt = qp->transport;
+
+        ntb_qp_link_cleanup(qp);
 
         if (nt->transport_link == NTB_LINK_UP)
                 schedule_delayed_work(&qp->link_work,
@@ -613,22 +619,20 @@ static void ntb_qp_link_down(struct ntb_transport_qp *qp)
         schedule_work(&qp->link_cleanup);
 }
 
-static void ntb_transport_link_cleanup(struct work_struct *work)
+static void ntb_transport_link_cleanup(struct ntb_transport *nt)
 {
-        struct ntb_transport *nt = container_of(work, struct ntb_transport,
-                                                link_cleanup);
         int i;
 
+        /* Pass along the info to any clients */
+        for (i = 0; i < nt->max_qps; i++)
+                if (!test_bit(i, &nt->qp_bitmap))
+                        ntb_qp_link_cleanup(&nt->qps[i]);
+
         if (nt->transport_link == NTB_LINK_DOWN)
                 cancel_delayed_work_sync(&nt->link_work);
         else
                 nt->transport_link = NTB_LINK_DOWN;
 
-        /* Pass along the info to any clients */
-        for (i = 0; i < nt->max_qps; i++)
-                if (!test_bit(i, &nt->qp_bitmap))
-                        ntb_qp_link_down(&nt->qps[i]);
-
         /* The scratchpad registers keep the values if the remote side
          * goes down, blast them now to give them a sane value the next
          * time they are accessed
@@ -637,6 +641,14 @@ static void ntb_transport_link_cleanup(struct work_struct *work)
                 ntb_write_local_spad(nt->ndev, i, 0);
 }
 
+static void ntb_transport_link_cleanup_work(struct work_struct *work)
+{
+        struct ntb_transport *nt = container_of(work, struct ntb_transport,
+                                                link_cleanup);
+
+        ntb_transport_link_cleanup(nt);
+}
+
 static void ntb_transport_event_callback(void *data, enum ntb_hw_event event)
 {
         struct ntb_transport *nt = data;
@@ -880,7 +892,7 @@ static int ntb_transport_init_queue(struct ntb_transport *nt,
         }
 
         INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
-        INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup);
+        INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
 
         spin_lock_init(&qp->ntb_rx_pend_q_lock);
         spin_lock_init(&qp->ntb_rx_free_q_lock);
@@ -936,7 +948,7 @@ int ntb_transport_init(struct pci_dev *pdev)
         }
 
         INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
-        INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup);
+        INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);
 
         rc = ntb_register_event_callback(nt->ndev,
                                          ntb_transport_event_callback);
@@ -972,7 +984,7 @@ void ntb_transport_free(void *transport)
         struct ntb_device *ndev = nt->ndev;
         int i;
 
-        nt->transport_link = NTB_LINK_DOWN;
+        ntb_transport_link_cleanup(nt);
 
         /* verify that all the qp's are freed */
         for (i = 0; i < nt->max_qps; i++) {
@@ -1188,11 +1200,14 @@ err:
         goto out;
 }
 
-static void ntb_transport_rx(unsigned long data)
+static int ntb_transport_rxc_db(void *data, int db_num)
 {
-        struct ntb_transport_qp *qp = (struct ntb_transport_qp *)data;
+        struct ntb_transport_qp *qp = data;
         int rc, i;
 
+        dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",
+                __func__, db_num);
+
         /* Limit the number of packets processed in a single interrupt to
          * provide fairness to others
          */
@@ -1204,16 +1219,8 @@ static void ntb_transport_rx(unsigned long data)
 
         if (qp->dma_chan)
                 dma_async_issue_pending(qp->dma_chan);
-}
-
-static void ntb_transport_rxc_db(void *data, int db_num)
-{
-        struct ntb_transport_qp *qp = data;
-
-        dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n",
-                __func__, db_num);
 
-        tasklet_schedule(&qp->rx_work);
+        return i;
 }
 
 static void ntb_tx_copy_callback(void *data)
@@ -1432,11 +1439,12 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
         qp->tx_handler = handlers->tx_handler;
         qp->event_handler = handlers->event_handler;
 
+        dmaengine_get();
         qp->dma_chan = dma_find_channel(DMA_MEMCPY);
-        if (!qp->dma_chan)
+        if (!qp->dma_chan) {
+                dmaengine_put();
                 dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n");
-        else
-                dmaengine_get();
+        }
 
         for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
                 entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC);
@@ -1458,25 +1466,23 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
                              &qp->tx_free_q);
         }
 
-        tasklet_init(&qp->rx_work, ntb_transport_rx, (unsigned long) qp);
-
         rc = ntb_register_db_callback(qp->ndev, free_queue, qp,
                                       ntb_transport_rxc_db);
         if (rc)
-                goto err3;
+                goto err2;
 
         dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);
 
         return qp;
 
-err3:
-        tasklet_disable(&qp->rx_work);
 err2:
         while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
                 kfree(entry);
 err1:
         while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
                 kfree(entry);
+        if (qp->dma_chan)
+                dmaengine_put();
         set_bit(free_queue, &nt->qp_bitmap);
 err:
         return NULL;
@@ -1515,7 +1521,6 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
         }
 
         ntb_unregister_db_callback(qp->ndev, qp->qp_num);
-        tasklet_disable(&qp->rx_work);
 
         cancel_delayed_work_sync(&qp->link_work);