mlx4_core: Move kernel doorbell management into core

In addition to mlx4_ib, there will be ethernet and FC consumers of
mlx4_core, so move the code for managing kernel doorbells into the
core module to avoid having to duplicate this multiple times.

Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

10 files changed, 162 insertions, 160 deletions

diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 3557e7edc9b6..5e570bb0bb6f 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -204,7 +204,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
 
                 uar = &to_mucontext(context)->uar;
         } else {
-                err = mlx4_ib_db_alloc(dev, &cq->db, 1);
+                err = mlx4_db_alloc(dev->dev, &cq->db, 1);
                 if (err)
                         goto err_cq;
 
@@ -250,7 +250,7 @@ err_mtt:
 
 err_db:
         if (!context)
-                mlx4_ib_db_free(dev, &cq->db);
+                mlx4_db_free(dev->dev, &cq->db);
 
 err_cq:
         kfree(cq);
@@ -435,7 +435,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq)
                 ib_umem_release(mcq->umem);
         } else {
                 mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe + 1);
-                mlx4_ib_db_free(dev, &mcq->db);
+                mlx4_db_free(dev->dev, &mcq->db);
         }
 
         kfree(mcq);
diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c
index 1c36087aef14..8e342cc9baec 100644
--- a/drivers/infiniband/hw/mlx4/doorbell.c
+++ b/drivers/infiniband/hw/mlx4/doorbell.c
@@ -34,124 +34,6 @@
 
 #include "mlx4_ib.h"
 
-struct mlx4_ib_db_pgdir {
-        struct list_head        list;
-        DECLARE_BITMAP(order0, MLX4_IB_DB_PER_PAGE);
-        DECLARE_BITMAP(order1, MLX4_IB_DB_PER_PAGE / 2);
-        unsigned long          *bits[2];
-        __be32                 *db_page;
-        dma_addr_t              db_dma;
-};
-
-static struct mlx4_ib_db_pgdir *mlx4_ib_alloc_db_pgdir(struct mlx4_ib_dev *dev)
-{
-        struct mlx4_ib_db_pgdir *pgdir;
-
-        pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL);
-        if (!pgdir)
-                return NULL;
-
-        bitmap_fill(pgdir->order1, MLX4_IB_DB_PER_PAGE / 2);
-        pgdir->bits[0] = pgdir->order0;
-        pgdir->bits[1] = pgdir->order1;
-        pgdir->db_page = dma_alloc_coherent(dev->ib_dev.dma_device,
-                                            PAGE_SIZE, &pgdir->db_dma,
-                                            GFP_KERNEL);
-        if (!pgdir->db_page) {
-                kfree(pgdir);
-                return NULL;
-        }
-
-        return pgdir;
-}
-
-static int mlx4_ib_alloc_db_from_pgdir(struct mlx4_ib_db_pgdir *pgdir,
-                                       struct mlx4_ib_db *db, int order)
-{
-        int o;
-        int i;
-
-        for (o = order; o <= 1; ++o) {
-                i = find_first_bit(pgdir->bits[o], MLX4_IB_DB_PER_PAGE >> o);
-                if (i < MLX4_IB_DB_PER_PAGE >> o)
-                        goto found;
-        }
-
-        return -ENOMEM;
-
-found:
-        clear_bit(i, pgdir->bits[o]);
-
-        i <<= o;
-
-        if (o > order)
-                set_bit(i ^ 1, pgdir->bits[order]);
-
-        db->u.pgdir = pgdir;
-        db->index   = i;
-        db->db      = pgdir->db_page + db->index;
-        db->dma     = pgdir->db_dma  + db->index * 4;
-        db->order   = order;
-
-        return 0;
-}
-
-int mlx4_ib_db_alloc(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db, int order)
-{
-        struct mlx4_ib_db_pgdir *pgdir;
-        int ret = 0;
-
-        mutex_lock(&dev->pgdir_mutex);
-
-        list_for_each_entry(pgdir, &dev->pgdir_list, list)
-                if (!mlx4_ib_alloc_db_from_pgdir(pgdir, db, order))
-                        goto out;
-
-        pgdir = mlx4_ib_alloc_db_pgdir(dev);
-        if (!pgdir) {
-                ret = -ENOMEM;
-                goto out;
-        }
-
-        list_add(&pgdir->list, &dev->pgdir_list);
-
-        /* This should never fail -- we just allocated an empty page: */
-        WARN_ON(mlx4_ib_alloc_db_from_pgdir(pgdir, db, order));
-
-out:
-        mutex_unlock(&dev->pgdir_mutex);
-
-        return ret;
-}
-
-void mlx4_ib_db_free(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db)
-{
-        int o;
-        int i;
-
-        mutex_lock(&dev->pgdir_mutex);
-
-        o = db->order;
-        i = db->index;
-
-        if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
-                clear_bit(i ^ 1, db->u.pgdir->order0);
-                ++o;
-        }
-
-        i >>= o;
-        set_bit(i, db->u.pgdir->bits[o]);
-
-        if (bitmap_full(db->u.pgdir->order1, MLX4_IB_DB_PER_PAGE / 2)) {
-                dma_free_coherent(dev->ib_dev.dma_device, PAGE_SIZE,
-                                  db->u.pgdir->db_page, db->u.pgdir->db_dma);
-                list_del(&db->u.pgdir->list);
-                kfree(db->u.pgdir);
-        }
-
-        mutex_unlock(&dev->pgdir_mutex);
-}
-
 struct mlx4_ib_user_db_page {
         struct list_head        list;
         struct ib_umem         *umem;
@@ -160,7 +42,7 @@ struct mlx4_ib_user_db_page {
 };
 
 int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
-                        struct mlx4_ib_db *db)
+                        struct mlx4_db *db)
 {
         struct mlx4_ib_user_db_page *page;
         struct ib_umem_chunk *chunk;
@@ -202,7 +84,7 @@ out:
         return err;
 }
 
-void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_ib_db *db)
+void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db)
 {
         mutex_lock(&context->db_page_mutex);
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 4d9b5ac42202..4d61e32866c6 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -557,9 +557,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
                 goto err_uar;
         MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);
 
-        INIT_LIST_HEAD(&ibdev->pgdir_list);
-        mutex_init(&ibdev->pgdir_mutex);
-
         ibdev->dev = dev;
 
         strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 9e637323c155..5cf994794d25 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -43,24 +43,6 @@
 #include <linux/mlx4/device.h>
 #include <linux/mlx4/doorbell.h>
 
-enum {
-        MLX4_IB_DB_PER_PAGE     = PAGE_SIZE / 4
-};
-
-struct mlx4_ib_db_pgdir;
-struct mlx4_ib_user_db_page;
-
-struct mlx4_ib_db {
-        __be32                 *db;
-        union {
-                struct mlx4_ib_db_pgdir        *pgdir;
-                struct mlx4_ib_user_db_page    *user_page;
-        }                       u;
-        dma_addr_t              dma;
-        int                     index;
-        int                     order;
-};
-
 struct mlx4_ib_ucontext {
         struct ib_ucontext      ibucontext;
         struct mlx4_uar         uar;
@@ -88,7 +70,7 @@ struct mlx4_ib_cq {
         struct mlx4_cq          mcq;
         struct mlx4_ib_cq_buf   buf;
         struct mlx4_ib_cq_resize *resize_buf;
-        struct mlx4_ib_db       db;
+        struct mlx4_db          db;
         spinlock_t              lock;
         struct mutex            resize_mutex;
         struct ib_umem         *umem;
@@ -127,7 +109,7 @@ struct mlx4_ib_qp {
         struct mlx4_qp          mqp;
         struct mlx4_buf         buf;
 
-        struct mlx4_ib_db       db;
+        struct mlx4_db          db;
         struct mlx4_ib_wq       rq;
 
         u32                     doorbell_qpn;
@@ -154,7 +136,7 @@ struct mlx4_ib_srq {
         struct ib_srq           ibsrq;
         struct mlx4_srq         msrq;
         struct mlx4_buf         buf;
-        struct mlx4_ib_db       db;
+        struct mlx4_db          db;
         u64                    *wrid;
         spinlock_t              lock;
         int                     head;
@@ -175,9 +157,6 @@ struct mlx4_ib_dev {
         struct mlx4_dev        *dev;
         void __iomem           *uar_map;
 
-        struct list_head        pgdir_list;
-        struct mutex            pgdir_mutex;
-
         struct mlx4_uar         priv_uar;
         u32                     priv_pdn;
         MLX4_DECLARE_DOORBELL_LOCK(uar_lock);
@@ -248,11 +227,9 @@ static inline struct mlx4_ib_ah *to_mah(struct ib_ah *ibah)
         return container_of(ibah, struct mlx4_ib_ah, ibah);
 }
 
-int mlx4_ib_db_alloc(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db, int order);
-void mlx4_ib_db_free(struct mlx4_ib_dev *dev, struct mlx4_ib_db *db);
 int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context, unsigned long virt,
-                        struct mlx4_ib_db *db);
-void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_ib_db *db);
+                        struct mlx4_db *db);
+void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db);
 
 struct ib_mr *mlx4_ib_get_dma_mr(struct ib_pd *pd, int acc);
 int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index b75efae7e449..80ea8b9e7761 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -514,7 +514,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
                         goto err;
 
                 if (!init_attr->srq) {
-                        err = mlx4_ib_db_alloc(dev, &qp->db, 0);
+                        err = mlx4_db_alloc(dev->dev, &qp->db, 0);
                         if (err)
                                 goto err;
 
@@ -580,7 +580,7 @@ err_buf:
 
 err_db:
         if (!pd->uobject && !init_attr->srq)
-                mlx4_ib_db_free(dev, &qp->db);
+                mlx4_db_free(dev->dev, &qp->db);
 
 err:
         return err;
@@ -666,7 +666,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
                 kfree(qp->rq.wrid);
                 mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
                 if (!qp->ibqp.srq)
-                        mlx4_ib_db_free(dev, &qp->db);
+                        mlx4_db_free(dev->dev, &qp->db);
         }
 }
 
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index beaa3b06cf58..204619702f9d 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -129,7 +129,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
                 if (err)
                         goto err_mtt;
         } else {
-                err = mlx4_ib_db_alloc(dev, &srq->db, 0);
+                err = mlx4_db_alloc(dev->dev, &srq->db, 0);
                 if (err)
                         goto err_srq;
 
@@ -200,7 +200,7 @@ err_buf:
 
 err_db:
         if (!pd->uobject)
-                mlx4_ib_db_free(dev, &srq->db);
+                mlx4_db_free(dev->dev, &srq->db);
 
 err_srq:
         kfree(srq);
@@ -267,7 +267,7 @@ int mlx4_ib_destroy_srq(struct ib_srq *srq)
                 kfree(msrq->wrid);
                 mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
                               &msrq->buf);
-                mlx4_ib_db_free(dev, &msrq->db);
+                mlx4_db_free(dev->dev, &msrq->db);
         }
 
         kfree(msrq);
diff --git a/drivers/net/mlx4/alloc.c b/drivers/net/mlx4/alloc.c
index 75ef9d0d974d..43c6d04bb880 100644
--- a/drivers/net/mlx4/alloc.c
+++ b/drivers/net/mlx4/alloc.c
@@ -196,3 +196,114 @@ void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
         }
 }
 EXPORT_SYMBOL_GPL(mlx4_buf_free);
+
+static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device)
+{
+        struct mlx4_db_pgdir *pgdir;
+
+        pgdir = kzalloc(sizeof *pgdir, GFP_KERNEL);
+        if (!pgdir)
+                return NULL;
+
+        bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
+        pgdir->bits[0] = pgdir->order0;
+        pgdir->bits[1] = pgdir->order1;
+        pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
+                                            &pgdir->db_dma, GFP_KERNEL);
+        if (!pgdir->db_page) {
+                kfree(pgdir);
+                return NULL;
+        }
+
+        return pgdir;
+}
+
+static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
+                                    struct mlx4_db *db, int order)
+{
+        int o;
+        int i;
+
+        for (o = order; o <= 1; ++o) {
+                i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
+                if (i < MLX4_DB_PER_PAGE >> o)
+                        goto found;
+        }
+
+        return -ENOMEM;
+
+found:
+        clear_bit(i, pgdir->bits[o]);
+
+        i <<= o;
+
+        if (o > order)
+                set_bit(i ^ 1, pgdir->bits[order]);
+
+        db->u.pgdir = pgdir;
+        db->index   = i;
+        db->db      = pgdir->db_page + db->index;
+        db->dma     = pgdir->db_dma  + db->index * 4;
+        db->order   = order;
+
+        return 0;
+}
+
+int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct mlx4_db_pgdir *pgdir;
+        int ret = 0;
+
+        mutex_lock(&priv->pgdir_mutex);
+
+        list_for_each_entry(pgdir, &priv->pgdir_list, list)
+                if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
+                        goto out;
+
+        pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev));
+        if (!pgdir) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        list_add(&pgdir->list, &priv->pgdir_list);
+
+        /* This should never fail -- we just allocated an empty page: */
+        WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));
+
+out:
+        mutex_unlock(&priv->pgdir_mutex);
+
+        return ret;
+}
+EXPORT_SYMBOL_GPL(mlx4_db_alloc);
+
+void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        int o;
+        int i;
+
+        mutex_lock(&priv->pgdir_mutex);
+
+        o = db->order;
+        i = db->index;
+
+        if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
+                clear_bit(i ^ 1, db->u.pgdir->order0);
+                ++o;
+        }
+        i >>= o;
+        set_bit(i, db->u.pgdir->bits[o]);
+
+        if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
+                dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
+                                  db->u.pgdir->db_page, db->u.pgdir->db_dma);
+                list_del(&db->u.pgdir->list);
+                kfree(db->u.pgdir);
+        }
+
+        mutex_unlock(&priv->pgdir_mutex);
+}
+EXPORT_SYMBOL_GPL(mlx4_db_free);
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 49a4acab5e82..a6aa49fc1d68 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -798,6 +798,9 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
         INIT_LIST_HEAD(&priv->ctx_list);
         spin_lock_init(&priv->ctx_lock);
 
+        INIT_LIST_HEAD(&priv->pgdir_list);
+        mutex_init(&priv->pgdir_mutex);
+
         /*
          * Now reset the HCA before we touch the PCI capabilities or
          * attempt a firmware command, since a boot ROM may have left
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 73336810e652..a4023c2dd050 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -257,6 +257,9 @@ struct mlx4_priv {
         struct list_head        ctx_list;
         spinlock_t              ctx_lock;
 
+        struct list_head        pgdir_list;
+        struct mutex            pgdir_mutex;
+
         struct mlx4_fw          fw;
         struct mlx4_cmd         cmd;
 
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index ff7df1a2222f..0a47457931ac 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -208,6 +208,32 @@ struct mlx4_mtt {
         int                     page_shift;
 };
 
+enum {
+        MLX4_DB_PER_PAGE = PAGE_SIZE / 4
+};
+
+struct mlx4_db_pgdir {
+        struct list_head        list;
+        DECLARE_BITMAP(order0, MLX4_DB_PER_PAGE);
+        DECLARE_BITMAP(order1, MLX4_DB_PER_PAGE / 2);
+        unsigned long          *bits[2];
+        __be32                 *db_page;
+        dma_addr_t              db_dma;
+};
+
+struct mlx4_ib_user_db_page;
+
+struct mlx4_db {
+        __be32                  *db;
+        union {
+                struct mlx4_db_pgdir            *pgdir;
+                struct mlx4_ib_user_db_page     *user_page;
+        }                       u;
+        dma_addr_t              dma;
+        int                     index;
+        int                     order;
+};
+
 struct mlx4_mr {
         struct mlx4_mtt         mtt;
         u64                     iova;
@@ -341,6 +367,9 @@ int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
                        struct mlx4_buf *buf);
 
+int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order);
+void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db);
+
 int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
                   struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq);
 void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq);