Auto merge with /home/aegl/GIT/linus

10 files changed, 762 insertions, 143 deletions

diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 766e9031ec45..b5aea7b869f6 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -742,6 +743,7 @@ err_out:
 }
 
 int mthca_init_cq(struct mthca_dev *dev, int nent,
+                  struct mthca_ucontext *ctx, u32 pdn,
                   struct mthca_cq *cq)
 {
         int size = nent * MTHCA_CQ_ENTRY_SIZE;
@@ -753,30 +755,33 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
 
         might_sleep();
 
-        cq->ibcq.cqe = nent - 1;
+        cq->ibcq.cqe  = nent - 1;
+        cq->is_kernel = !ctx;
 
         cq->cqn = mthca_alloc(&dev->cq_table.alloc);
         if (cq->cqn == -1)
                 return -ENOMEM;
 
         if (mthca_is_memfree(dev)) {
-                cq->arm_sn = 1;
-
                 err = mthca_table_get(dev, dev->cq_table.table, cq->cqn);
                 if (err)
                         goto err_out;
 
-                err = -ENOMEM;
+                if (cq->is_kernel) {
+                        cq->arm_sn = 1;
+
+                        err = -ENOMEM;
 
-                cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
-                                                     cq->cqn, &cq->set_ci_db);
-                if (cq->set_ci_db_index < 0)
-                        goto err_out_icm;
+                        cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
+                                                             cq->cqn, &cq->set_ci_db);
+                        if (cq->set_ci_db_index < 0)
+                                goto err_out_icm;
 
-                cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
-                                                  cq->cqn, &cq->arm_db);
-                if (cq->arm_db_index < 0)
-                        goto err_out_ci;
+                        cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
+                                                          cq->cqn, &cq->arm_db);
+                        if (cq->arm_db_index < 0)
+                                goto err_out_ci;
+                }
         }
 
         mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
@@ -785,12 +790,14 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
 
         cq_context = mailbox->buf;
 
-        err = mthca_alloc_cq_buf(dev, size, cq);
-        if (err)
-                goto err_out_mailbox;
+        if (cq->is_kernel) {
+                err = mthca_alloc_cq_buf(dev, size, cq);
+                if (err)
+                        goto err_out_mailbox;
 
-        for (i = 0; i < nent; ++i)
-                set_cqe_hw(get_cqe(cq, i));
+                for (i = 0; i < nent; ++i)
+                        set_cqe_hw(get_cqe(cq, i));
+        }
 
         spin_lock_init(&cq->lock);
         atomic_set(&cq->refcount, 1);
@@ -801,11 +808,14 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
                                                   MTHCA_CQ_STATE_DISARMED |
                                                   MTHCA_CQ_FLAG_TR);
         cq_context->start           = cpu_to_be64(0);
-        cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
-                                                  dev->driver_uar.index);
+        cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
+        if (ctx)
+                cq_context->logsize_usrpage |= cpu_to_be32(ctx->uar.index);
+        else
+                cq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
         cq_context->error_eqn       = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);
         cq_context->comp_eqn        = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn);
-        cq_context->pd              = cpu_to_be32(dev->driver_pd.pd_num);
+        cq_context->pd              = cpu_to_be32(pdn);
         cq_context->lkey            = cpu_to_be32(cq->mr.ibmr.lkey);
         cq_context->cqn             = cpu_to_be32(cq->cqn);
 
@@ -843,18 +853,20 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
         return 0;
 
 err_out_free_mr:
-        mthca_free_mr(dev, &cq->mr);
-        mthca_free_cq_buf(dev, cq);
+        if (cq->is_kernel) {
+                mthca_free_mr(dev, &cq->mr);
+                mthca_free_cq_buf(dev, cq);
+        }
 
 err_out_mailbox:
         mthca_free_mailbox(dev, mailbox);
 
 err_out_arm:
-        if (mthca_is_memfree(dev))
+        if (cq->is_kernel && mthca_is_memfree(dev))
                 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
 
 err_out_ci:
-        if (mthca_is_memfree(dev))
+        if (cq->is_kernel && mthca_is_memfree(dev))
                 mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
 
 err_out_icm:
@@ -892,7 +904,8 @@ void mthca_free_cq(struct mthca_dev *dev,
                 int j;
 
                 printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
-                       cq->cqn, cq->cons_index, !!next_cqe_sw(cq));
+                       cq->cqn, cq->cons_index,
+                       cq->is_kernel ? !!next_cqe_sw(cq) : 0);
                 for (j = 0; j < 16; ++j)
                         printk(KERN_ERR "[%2x] %08x\n", j * 4, be32_to_cpu(ctx[j]));
         }
@@ -910,12 +923,13 @@ void mthca_free_cq(struct mthca_dev *dev,
         atomic_dec(&cq->refcount);
         wait_event(cq->wait, !atomic_read(&cq->refcount));
 
-        mthca_free_mr(dev, &cq->mr);
-        mthca_free_cq_buf(dev, cq);
-
-        if (mthca_is_memfree(dev)) {
-                mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);
-                mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
+        if (cq->is_kernel) {
+                mthca_free_mr(dev, &cq->mr);
+                mthca_free_cq_buf(dev, cq);
+                if (mthca_is_memfree(dev)) {
+                        mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);
+                        mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
+                }
         }
 
         mthca_table_put(dev, dev->cq_table.table, cq->cqn);
diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
index 4127f09dc5ec..5ecdd2eeeb0f 100644
--- a/drivers/infiniband/hw/mthca/mthca_dev.h
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -378,7 +379,7 @@ void mthca_unregister_device(struct mthca_dev *dev);
 int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar);
 void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar);
 
-int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd);
+int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd);
 void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);
 
 struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size);
@@ -413,6 +414,7 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
 int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
 int mthca_init_cq(struct mthca_dev *dev, int nent,
+                  struct mthca_ucontext *ctx, u32 pdn,
                   struct mthca_cq *cq);
 void mthca_free_cq(struct mthca_dev *dev,
                    struct mthca_cq *cq);
@@ -438,12 +440,14 @@ int mthca_alloc_qp(struct mthca_dev *dev,
                    struct mthca_cq *recv_cq,
                    enum ib_qp_type type,
                    enum ib_sig_type send_policy,
+                   struct ib_qp_cap *cap,
                    struct mthca_qp *qp);
 int mthca_alloc_sqp(struct mthca_dev *dev,
                     struct mthca_pd *pd,
                     struct mthca_cq *send_cq,
                     struct mthca_cq *recv_cq,
                     enum ib_sig_type send_policy,
+                    struct ib_qp_cap *cap,
                     int qpn,
                     int port,
                     struct mthca_sqp *sqp);
diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
index 09519b604c08..2ef916859e17 100644
--- a/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -665,7 +665,7 @@ static int __devinit mthca_setup_hca(struct mthca_dev *dev)
                 goto err_pd_table_free;
         }
 
-        err = mthca_pd_alloc(dev, &dev->driver_pd);
+        err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
         if (err) {
                 mthca_err(dev, "Failed to create driver PD, "
                           "aborting.\n");
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c
index 6d3b05dd9e3f..2a8646150355 100644
--- a/drivers/infiniband/hw/mthca/mthca_memfree.c
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -47,6 +48,15 @@ enum {
         MTHCA_TABLE_CHUNK_SIZE = 1 << 18
 };
 
+struct mthca_user_db_table {
+        struct semaphore mutex;
+        struct {
+                u64                uvirt;
+                struct scatterlist mem;
+                int                refcount;
+        }                page[0];
+};
+
 void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm)
 {
         struct mthca_icm_chunk *chunk, *tmp;
@@ -344,13 +354,133 @@ void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table)
         kfree(table);
 }
 
-static u64 mthca_uarc_virt(struct mthca_dev *dev, int page)
+static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page)
 {
         return dev->uar_table.uarc_base +
-                dev->driver_uar.index * dev->uar_table.uarc_size +
+                uar->index * dev->uar_table.uarc_size +
                 page * 4096;
 }
 
+int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+                      struct mthca_user_db_table *db_tab, int index, u64 uaddr)
+{
+        int ret = 0;
+        u8 status;
+        int i;
+
+        if (!mthca_is_memfree(dev))
+                return 0;
+
+        if (index < 0 || index > dev->uar_table.uarc_size / 8)
+                return -EINVAL;
+
+        down(&db_tab->mutex);
+
+        i = index / MTHCA_DB_REC_PER_PAGE;
+
+        if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE)       ||
+            (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) ||
+            (uaddr & 4095)) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        if (db_tab->page[i].refcount) {
+                ++db_tab->page[i].refcount;
+                goto out;
+        }
+
+        ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0,
+                             &db_tab->page[i].mem.page, NULL);
+        if (ret < 0)
+                goto out;
+
+        db_tab->page[i].mem.length = 4096;
+        db_tab->page[i].mem.offset = uaddr & ~PAGE_MASK;
+
+        ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
+        if (ret < 0) {
+                put_page(db_tab->page[i].mem.page);
+                goto out;
+        }
+
+        ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem),
+                                 mthca_uarc_virt(dev, uar, i), &status);
+        if (!ret && status)
+                ret = -EINVAL;
+        if (ret) {
+                pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
+                put_page(db_tab->page[i].mem.page);
+                goto out;
+        }
+
+        db_tab->page[i].uvirt    = uaddr;
+        db_tab->page[i].refcount = 1;
+
+out:
+        up(&db_tab->mutex);
+        return ret;
+}
+
+void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+                         struct mthca_user_db_table *db_tab, int index)
+{
+        if (!mthca_is_memfree(dev))
+                return;
+
+        /*
+         * To make our bookkeeping simpler, we don't unmap DB
+         * pages until we clean up the whole db table.
+         */
+
+        down(&db_tab->mutex);
+
+        --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount;
+
+        up(&db_tab->mutex);
+}
+
+struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev)
+{
+        struct mthca_user_db_table *db_tab;
+        int npages;
+        int i;
+
+        if (!mthca_is_memfree(dev))
+                return NULL;
+
+        npages = dev->uar_table.uarc_size / 4096;
+        db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL);
+        if (!db_tab)
+                return ERR_PTR(-ENOMEM);
+
+        init_MUTEX(&db_tab->mutex);
+        for (i = 0; i < npages; ++i) {
+                db_tab->page[i].refcount = 0;
+                db_tab->page[i].uvirt    = 0;
+        }
+
+        return db_tab;
+}
+
+void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
+                               struct mthca_user_db_table *db_tab)
+{
+        int i;
+        u8 status;
+
+        if (!mthca_is_memfree(dev))
+                return;
+
+        for (i = 0; i < dev->uar_table.uarc_size / 4096; ++i) {
+                if (db_tab->page[i].uvirt) {
+                        mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status);
+                        pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE);
+                        put_page(db_tab->page[i].mem.page);
+                }
+        }
+}
+
 int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db)
 {
         int group;
@@ -407,7 +537,8 @@ int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db)
         }
         memset(page->db_rec, 0, 4096);
 
-        ret = mthca_MAP_ICM_page(dev, page->mapping, mthca_uarc_virt(dev, i), &status);
+        ret = mthca_MAP_ICM_page(dev, page->mapping,
+                                 mthca_uarc_virt(dev, &dev->driver_uar, i), &status);
         if (!ret && status)
                 ret = -EINVAL;
         if (ret) {
@@ -461,7 +592,7 @@ void mthca_free_db(struct mthca_dev *dev, int type, int db_index)
 
         if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) &&
             i >= dev->db_tab->max_group1 - 1) {
-                mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status);
+                mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status);
 
                 dma_free_coherent(&dev->pdev->dev, 4096,
                                   page->db_rec, page->mapping);
@@ -530,7 +661,7 @@ void mthca_cleanup_db_tab(struct mthca_dev *dev)
                 if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE))
                         mthca_warn(dev, "Kernel UARC page %d not empty\n", i);
 
-                mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, i), 1, &status);
+                mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status);
 
                 dma_free_coherent(&dev->pdev->dev, 4096,
                                   dev->db_tab->page[i].db_rec,
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.h b/drivers/infiniband/hw/mthca/mthca_memfree.h
index fe7be2a6bc4a..4761d844cb5f 100644
--- a/drivers/infiniband/hw/mthca/mthca_memfree.h
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -148,7 +149,7 @@ struct mthca_db_table {
         struct semaphore      mutex;
 };
 
-enum {
+enum mthca_db_type {
         MTHCA_DB_TYPE_INVALID   = 0x0,
         MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
         MTHCA_DB_TYPE_CQ_ARM    = 0x2,
@@ -158,6 +159,17 @@ enum {
         MTHCA_DB_TYPE_GROUP_SEP = 0x7
 };
 
+struct mthca_user_db_table;
+struct mthca_uar;
+
+int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+                      struct mthca_user_db_table *db_tab, int index, u64 uaddr);
+void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
+                         struct mthca_user_db_table *db_tab, int index);
+struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev);
+void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar,
+                               struct mthca_user_db_table *db_tab);
+
 int mthca_init_db_tab(struct mthca_dev *dev);
 void mthca_cleanup_db_tab(struct mthca_dev *dev);
 int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db);
diff --git a/drivers/infiniband/hw/mthca/mthca_pd.c b/drivers/infiniband/hw/mthca/mthca_pd.c
index ea66847e4ea3..c2c899844e98 100644
--- a/drivers/infiniband/hw/mthca/mthca_pd.c
+++ b/drivers/infiniband/hw/mthca/mthca_pd.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -37,23 +38,27 @@
 
 #include "mthca_dev.h"
 
-int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd)
+int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd)
 {
-        int err;
+        int err = 0;
 
         might_sleep();
 
+        pd->privileged = privileged;
+
         atomic_set(&pd->sqp_count, 0);
         pd->pd_num = mthca_alloc(&dev->pd_table.alloc);
         if (pd->pd_num == -1)
                 return -ENOMEM;
 
-        err = mthca_mr_alloc_notrans(dev, pd->pd_num,
-                                     MTHCA_MPT_FLAG_LOCAL_READ |
-                                     MTHCA_MPT_FLAG_LOCAL_WRITE,
-                                     &pd->ntmr);
-        if (err)
-                mthca_free(&dev->pd_table.alloc, pd->pd_num);
+        if (privileged) {
+                err = mthca_mr_alloc_notrans(dev, pd->pd_num,
+                                             MTHCA_MPT_FLAG_LOCAL_READ |
+                                             MTHCA_MPT_FLAG_LOCAL_WRITE,
+                                             &pd->ntmr);
+                if (err)
+                        mthca_free(&dev->pd_table.alloc, pd->pd_num);
+        }
 
         return err;
 }
@@ -61,7 +66,8 @@ int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd)
 void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd)
 {
         might_sleep();
-        mthca_free_mr(dev, &pd->ntmr);
+        if (pd->privileged)
+                mthca_free_mr(dev, &pd->ntmr);
         mthca_free(&dev->pd_table.alloc, pd->pd_num);
 }
 
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 0b5adfd91597..7a58ce90e179 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -34,9 +35,12 @@
  */
 
 #include <ib_smi.h>
+#include <linux/mm.h>
 
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
+#include "mthca_user.h"
+#include "mthca_memfree.h"
 
 static int mthca_query_device(struct ib_device *ibdev,
                               struct ib_device_attr *props)
@@ -284,7 +288,78 @@ static int mthca_query_gid(struct ib_device *ibdev, u8 port,
         return err;
 }
 
-static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev)
+static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
+                                                struct ib_udata *udata)
+{
+        struct mthca_alloc_ucontext_resp uresp;
+        struct mthca_ucontext           *context;
+        int                              err;
+
+        memset(&uresp, 0, sizeof uresp);
+
+        uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
+        if (mthca_is_memfree(to_mdev(ibdev)))
+                uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
+        else
+                uresp.uarc_size = 0;
+
+        context = kmalloc(sizeof *context, GFP_KERNEL);
+        if (!context)
+                return ERR_PTR(-ENOMEM);
+
+        err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
+        if (err) {
+                kfree(context);
+                return ERR_PTR(err);
+        }
+
+        context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
+        if (IS_ERR(context->db_tab)) {
+                err = PTR_ERR(context->db_tab);
+                mthca_uar_free(to_mdev(ibdev), &context->uar);
+                kfree(context);
+                return ERR_PTR(err);
+        }
+
+        if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
+                mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
+                mthca_uar_free(to_mdev(ibdev), &context->uar);
+                kfree(context);
+                return ERR_PTR(-EFAULT);
+        }
+
+        return &context->ibucontext;
+}
+
+static int mthca_dealloc_ucontext(struct ib_ucontext *context)
+{
+        mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar,
+                                  to_mucontext(context)->db_tab);
+        mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar);
+        kfree(to_mucontext(context));
+
+        return 0;
+}
+
+static int mthca_mmap_uar(struct ib_ucontext *context,
+                          struct vm_area_struct *vma)
+{
+        if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+                return -EINVAL;
+
+        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+        if (remap_pfn_range(vma, vma->vm_start,
+                            to_mucontext(context)->uar.pfn,
+                            PAGE_SIZE, vma->vm_page_prot))
+                return -EAGAIN;
+
+        return 0;
+}
+
+static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
+                                    struct ib_ucontext *context,
+                                    struct ib_udata *udata)
 {
         struct mthca_pd *pd;
         int err;
@@ -293,12 +368,20 @@ static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev)
         if (!pd)
                 return ERR_PTR(-ENOMEM);
 
-        err = mthca_pd_alloc(to_mdev(ibdev), pd);
+        err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
         if (err) {
                 kfree(pd);
                 return ERR_PTR(err);
         }
 
+        if (context) {
+                if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
+                        mthca_pd_free(to_mdev(ibdev), pd);
+                        kfree(pd);
+                        return ERR_PTR(-EFAULT);
+                }
+        }
+
         return &pd->ibpd;
 }
 
@@ -338,8 +421,10 @@ static int mthca_ah_destroy(struct ib_ah *ah)
 }
 
 static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
-                                     struct ib_qp_init_attr *init_attr)
+                                     struct ib_qp_init_attr *init_attr,
+                                     struct ib_udata *udata)
 {
+        struct mthca_create_qp ucmd;
         struct mthca_qp *qp;
         int err;
 
@@ -348,41 +433,82 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
         case IB_QPT_UC:
         case IB_QPT_UD:
         {
+                struct mthca_ucontext *context;
+
                 qp = kmalloc(sizeof *qp, GFP_KERNEL);
                 if (!qp)
                         return ERR_PTR(-ENOMEM);
 
-                qp->sq.max    = init_attr->cap.max_send_wr;
-                qp->rq.max    = init_attr->cap.max_recv_wr;
-                qp->sq.max_gs = init_attr->cap.max_send_sge;
-                qp->rq.max_gs = init_attr->cap.max_recv_sge;
+                if (pd->uobject) {
+                        context = to_mucontext(pd->uobject->context);
+
+                        if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
+                                return ERR_PTR(-EFAULT);
+
+                        err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
+                                                context->db_tab,
+                                                ucmd.sq_db_index, ucmd.sq_db_page);
+                        if (err) {
+                                kfree(qp);
+                                return ERR_PTR(err);
+                        }
+
+                        err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
+                                                context->db_tab,
+                                                ucmd.rq_db_index, ucmd.rq_db_page);
+                        if (err) {
+                                mthca_unmap_user_db(to_mdev(pd->device),
+                                                    &context->uar,
+                                                    context->db_tab,
+                                                    ucmd.sq_db_index);
+                                kfree(qp);
+                                return ERR_PTR(err);
+                        }
+
+                        qp->mr.ibmr.lkey = ucmd.lkey;
+                        qp->sq.db_index  = ucmd.sq_db_index;
+                        qp->rq.db_index  = ucmd.rq_db_index;
+                }
 
                 err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd),
                                      to_mcq(init_attr->send_cq),
                                      to_mcq(init_attr->recv_cq),
                                      init_attr->qp_type, init_attr->sq_sig_type,
-                                     qp);
+                                     &init_attr->cap, qp);
+
+                if (err && pd->uobject) {
+                        context = to_mucontext(pd->uobject->context);
+
+                        mthca_unmap_user_db(to_mdev(pd->device),
+                                            &context->uar,
+                                            context->db_tab,
+                                            ucmd.sq_db_index);
+                        mthca_unmap_user_db(to_mdev(pd->device),
+                                            &context->uar,
+                                            context->db_tab,
+                                            ucmd.rq_db_index);
+                }
+
                 qp->ibqp.qp_num = qp->qpn;
                 break;
         }
         case IB_QPT_SMI:
         case IB_QPT_GSI:
         {
+                /* Don't allow userspace to create special QPs */
+                if (pd->uobject)
+                        return ERR_PTR(-EINVAL);
+
                 qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
                 if (!qp)
                         return ERR_PTR(-ENOMEM);
 
-                qp->sq.max    = init_attr->cap.max_send_wr;
-                qp->rq.max    = init_attr->cap.max_recv_wr;
-                qp->sq.max_gs = init_attr->cap.max_send_sge;
-                qp->rq.max_gs = init_attr->cap.max_recv_sge;
-
                 qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
 
                 err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
                                       to_mcq(init_attr->send_cq),
                                       to_mcq(init_attr->recv_cq),
-                                      init_attr->sq_sig_type,
+                                      init_attr->sq_sig_type, &init_attr->cap,
                                       qp->ibqp.qp_num, init_attr->port_num,
                                       to_msqp(qp));
                 break;
@@ -397,42 +523,115 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
                 return ERR_PTR(err);
         }
 
-        init_attr->cap.max_inline_data = 0;
+        init_attr->cap.max_inline_data = 0;
+        init_attr->cap.max_send_wr     = qp->sq.max;
+        init_attr->cap.max_recv_wr     = qp->rq.max;
+        init_attr->cap.max_send_sge    = qp->sq.max_gs;
+        init_attr->cap.max_recv_sge    = qp->rq.max_gs;
 
         return &qp->ibqp;
 }
 
 static int mthca_destroy_qp(struct ib_qp *qp)
 {
+        if (qp->uobject) {
+                mthca_unmap_user_db(to_mdev(qp->device),
+                                    &to_mucontext(qp->uobject->context)->uar,
+                                    to_mucontext(qp->uobject->context)->db_tab,
+                                    to_mqp(qp)->sq.db_index);
+                mthca_unmap_user_db(to_mdev(qp->device),
+                                    &to_mucontext(qp->uobject->context)->uar,
+                                    to_mucontext(qp->uobject->context)->db_tab,
+                                    to_mqp(qp)->rq.db_index);
+        }
         mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
         kfree(qp);
         return 0;
 }
 
-static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries)
+static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries,
+                                     struct ib_ucontext *context,
+                                     struct ib_udata *udata)
 {
+        struct mthca_create_cq ucmd;
         struct mthca_cq *cq;
         int nent;
         int err;
 
+        if (context) {
+                if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
+                        return ERR_PTR(-EFAULT);
+
+                err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
+                                        to_mucontext(context)->db_tab,
+                                        ucmd.set_db_index, ucmd.set_db_page);
+                if (err)
+                        return ERR_PTR(err);
+
+                err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
+                                        to_mucontext(context)->db_tab,
+                                        ucmd.arm_db_index, ucmd.arm_db_page);
+                if (err)
+                        goto err_unmap_set;
+        }
+
         cq = kmalloc(sizeof *cq, GFP_KERNEL);
-        if (!cq)
-                return ERR_PTR(-ENOMEM);
+        if (!cq) {
+                err = -ENOMEM;
+                goto err_unmap_arm;
+        }
+
+        if (context) {
+                cq->mr.ibmr.lkey    = ucmd.lkey;
+                cq->set_ci_db_index = ucmd.set_db_index;
+                cq->arm_db_index    = ucmd.arm_db_index;
+        }
 
         for (nent = 1; nent <= entries; nent <<= 1)
                 ; /* nothing */
 
-        err = mthca_init_cq(to_mdev(ibdev), nent, cq);
-        if (err) {
-                kfree(cq);
-                cq = ERR_PTR(err);
+        err = mthca_init_cq(to_mdev(ibdev), nent,
+                            context ? to_mucontext(context) : NULL,
+                            context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
+                            cq);
+        if (err)
+                goto err_free;
+
+        if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) {
+                mthca_free_cq(to_mdev(ibdev), cq);
+                goto err_free;
         }
 
         return &cq->ibcq;
+
+err_free:
+        kfree(cq);
+
+err_unmap_arm:
+        if (context)
+                mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
+                                    to_mucontext(context)->db_tab, ucmd.arm_db_index);
+
+err_unmap_set:
+        if (context)
+                mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
+                                    to_mucontext(context)->db_tab, ucmd.set_db_index);
+
+        return ERR_PTR(err);
 }
 
 static int mthca_destroy_cq(struct ib_cq *cq)
 {
+        if (cq->uobject) {
+                mthca_unmap_user_db(to_mdev(cq->device),
+                                    &to_mucontext(cq->uobject->context)->uar,
+                                    to_mucontext(cq->uobject->context)->db_tab,
+                                    to_mcq(cq)->arm_db_index);
+                mthca_unmap_user_db(to_mdev(cq->device),
+                                    &to_mucontext(cq->uobject->context)->uar,
+                                    to_mucontext(cq->uobject->context)->db_tab,
+                                    to_mcq(cq)->set_ci_db_index);
+        }
         mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
         kfree(cq);
 
@@ -568,6 +767,87 @@ static struct ib_mr *mthca_reg_phys_mr(struct ib_pd       *pd,
         return &mr->ibmr;
 }
 
+static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region,
+                                       int acc, struct ib_udata *udata)
+{
+        struct mthca_dev *dev = to_mdev(pd->device);
+        struct ib_umem_chunk *chunk;
+        struct mthca_mr *mr;
+        u64 *pages;
+        int shift, n, len;
+        int i, j, k;
+        int err = 0;
+
+        shift = ffs(region->page_size) - 1;
+
+        mr = kmalloc(sizeof *mr, GFP_KERNEL);
+        if (!mr)
+                return ERR_PTR(-ENOMEM);
+
+        n = 0;
+        list_for_each_entry(chunk, &region->chunk_list, list)
+                n += chunk->nents;
+
+        mr->mtt = mthca_alloc_mtt(dev, n);
+        if (IS_ERR(mr->mtt)) {
+                err = PTR_ERR(mr->mtt);
+                goto err;
+        }
+
+        pages = (u64 *) __get_free_page(GFP_KERNEL);
+        if (!pages) {
+                err = -ENOMEM;
+                goto err_mtt;
+        }
+
+        i = n = 0;
+
+        list_for_each_entry(chunk, &region->chunk_list, list)
+                for (j = 0; j < chunk->nmap; ++j) {
+                        len = sg_dma_len(&chunk->page_list[j]) >> shift;
+                        for (k = 0; k < len; ++k) {
+                                pages[i++] = sg_dma_address(&chunk->page_list[j]) +
+                                        region->page_size * k;
+                                /*
+                                 * Be friendly to WRITE_MTT command
+                                 * and leave two empty slots for the
+                                 * index and reserved fields of the
+                                 * mailbox.
+                                 */
+                                if (i == PAGE_SIZE / sizeof (u64) - 2) {
+                                        err = mthca_write_mtt(dev, mr->mtt,
+                                                              n, pages, i);
+                                        if (err)
+                                                goto mtt_done;
+                                        n += i;
+                                        i = 0;
+                                }
+                        }
+                }
+
+        if (i)
+                err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
+mtt_done:
+        free_page((unsigned long) pages);
+        if (err)
+                goto err_mtt;
+
+        err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, region->virt_base,
+                             region->length, convert_access(acc), mr);
+
+        if (err)
+                goto err_mtt;
+
+        return &mr->ibmr;
+
+err_mtt:
+        mthca_free_mtt(dev, mr->mtt);
+
+err:
+        kfree(mr);
+        return ERR_PTR(err);
+}
+
 static int mthca_dereg_mr(struct ib_mr *mr)
 {
         struct mthca_mr *mmr = to_mmr(mr);
@@ -692,6 +972,8 @@ int mthca_register_device(struct mthca_dev *dev)
         int i;
 
         strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
+        dev->ib_dev.owner                = THIS_MODULE;
+
         dev->ib_dev.node_type            = IB_NODE_CA;
         dev->ib_dev.phys_port_cnt        = dev->limits.num_ports;
         dev->ib_dev.dma_device           = &dev->pdev->dev;
@@ -701,6 +983,9 @@ int mthca_register_device(struct mthca_dev *dev)
         dev->ib_dev.modify_port          = mthca_modify_port;
         dev->ib_dev.query_pkey           = mthca_query_pkey;
         dev->ib_dev.query_gid            = mthca_query_gid;
+        dev->ib_dev.alloc_ucontext       = mthca_alloc_ucontext;
+        dev->ib_dev.dealloc_ucontext     = mthca_dealloc_ucontext;
+        dev->ib_dev.mmap                 = mthca_mmap_uar;
         dev->ib_dev.alloc_pd             = mthca_alloc_pd;
         dev->ib_dev.dealloc_pd           = mthca_dealloc_pd;
         dev->ib_dev.create_ah            = mthca_ah_create;
@@ -713,6 +998,7 @@ int mthca_register_device(struct mthca_dev *dev)
         dev->ib_dev.poll_cq              = mthca_poll_cq;
         dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
         dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
+        dev->ib_dev.reg_user_mr          = mthca_reg_user_mr;
         dev->ib_dev.dereg_mr             = mthca_dereg_mr;
 
         if (dev->mthca_flags & MTHCA_FLAG_FMR) {
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.h b/drivers/infiniband/hw/mthca/mthca_provider.h
index 4d976cccb1a8..1d032791cc8b 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.h
+++ b/drivers/infiniband/hw/mthca/mthca_provider.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -54,6 +55,14 @@ struct mthca_uar {
         int           index;
 };
 
+struct mthca_user_db_table;
+
+struct mthca_ucontext {
+        struct ib_ucontext          ibucontext;
+        struct mthca_uar            uar;
+        struct mthca_user_db_table *db_tab;
+};
+
 struct mthca_mtt;
 
 struct mthca_mr {
@@ -83,6 +92,7 @@ struct mthca_pd {
         u32             pd_num;
         atomic_t        sqp_count;
         struct mthca_mr ntmr;
+        int             privileged;
 };
 
 struct mthca_eq {
@@ -167,6 +177,7 @@ struct mthca_cq {
         int                    cqn;
         u32                    cons_index;
         int                    is_direct;
+        int                    is_kernel;
 
         /* Next fields are Arbel only */
         int                    set_ci_db_index;
@@ -236,6 +247,11 @@ struct mthca_sqp {
         dma_addr_t      header_dma;
 };
 
+static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
+{
+        return container_of(ibucontext, struct mthca_ucontext, ibucontext);
+}
+
 static inline struct mthca_fmr *to_mfmr(struct ib_fmr *ibmr)
 {
         return container_of(ibmr, struct mthca_fmr, ibmr);
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 163a8ef4186f..f7126b14d5ae 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -46,7 +47,9 @@ enum {
         MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE,
         MTHCA_ACK_REQ_FREQ       = 10,
         MTHCA_FLIGHT_LIMIT       = 9,
-        MTHCA_UD_HEADER_SIZE     = 72 /* largest UD header possible */
+        MTHCA_UD_HEADER_SIZE     = 72, /* largest UD header possible */
+        MTHCA_INLINE_HEADER_SIZE = 4,  /* data segment overhead for inline */
+        MTHCA_INLINE_CHUNK_SIZE  = 16  /* inline data segment chunk */
 };
 
 enum {
@@ -689,7 +692,11 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
 
         /* leave arbel_sched_queue as 0 */
 
-        qp_context->usr_page   = cpu_to_be32(dev->driver_uar.index);
+        if (qp->ibqp.uobject)
+                qp_context->usr_page =
+                        cpu_to_be32(to_mucontext(qp->ibqp.uobject->context)->uar.index);
+        else
+                qp_context->usr_page = cpu_to_be32(dev->driver_uar.index);
         qp_context->local_qpn  = cpu_to_be32(qp->qpn);
         if (attr_mask & IB_QP_DEST_QPN) {
                 qp_context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
@@ -954,6 +961,15 @@ static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
 
         qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift,
                                     1 << qp->sq.wqe_shift);
+
+        /*
+         * If this is a userspace QP, we don't actually have to
+         * allocate anything.  All we need is to calculate the WQE
+         * sizes and the send_wqe_offset, so we're done now.
+         */
+        if (pd->ibpd.uobject)
+                return 0;
+
         size = PAGE_ALIGN(qp->send_wqe_offset +
                           (qp->sq.max << qp->sq.wqe_shift));
 
@@ -1053,10 +1069,32 @@ static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
         return err;
 }
 
-static int mthca_alloc_memfree(struct mthca_dev *dev,
+static void mthca_free_wqe_buf(struct mthca_dev *dev,
                                struct mthca_qp *qp)
 {
-        int ret = 0;
+        int i;
+        int size = PAGE_ALIGN(qp->send_wqe_offset +
+                              (qp->sq.max << qp->sq.wqe_shift));
+
+        if (qp->is_direct) {
+                dma_free_coherent(&dev->pdev->dev, size, qp->queue.direct.buf,
+                                  pci_unmap_addr(&qp->queue.direct, mapping));
+        } else {
+                for (i = 0; i < size / PAGE_SIZE; ++i) {
+                        dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
+                                          qp->queue.page_list[i].buf,
+                                          pci_unmap_addr(&qp->queue.page_list[i],
+                                                         mapping));
+                }
+        }
+
+        kfree(qp->wrid);
+}
+
+static int mthca_map_memfree(struct mthca_dev *dev,
+                             struct mthca_qp *qp)
+{
+        int ret;
 
         if (mthca_is_memfree(dev)) {
                 ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn);
@@ -1067,35 +1105,15 @@ static int mthca_alloc_memfree(struct mthca_dev *dev,
                 if (ret)
                         goto err_qpc;
 
-                ret = mthca_table_get(dev, dev->qp_table.rdb_table,
-                                      qp->qpn << dev->qp_table.rdb_shift);
-                if (ret)
-                        goto err_eqpc;
-
-                qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,
-                                                 qp->qpn, &qp->rq.db);
-                if (qp->rq.db_index < 0) {
-                        ret = -ENOMEM;
-                        goto err_rdb;
-                }
+                ret = mthca_table_get(dev, dev->qp_table.rdb_table,
+                                      qp->qpn << dev->qp_table.rdb_shift);
+                if (ret)
+                        goto err_eqpc;
 
-                qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,
-                                                 qp->qpn, &qp->sq.db);
-                if (qp->sq.db_index < 0) {
-                        ret = -ENOMEM;
-                        goto err_rq_db;
-                }
         }
 
         return 0;
 
-err_rq_db:
-        mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
-
-err_rdb:
-        mthca_table_put(dev, dev->qp_table.rdb_table,
-                        qp->qpn << dev->qp_table.rdb_shift);
-
 err_eqpc:
         mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
 
@@ -1105,6 +1123,35 @@ err_qpc:
         return ret;
 }
 
+static void mthca_unmap_memfree(struct mthca_dev *dev,
+                                struct mthca_qp *qp)
+{
+        mthca_table_put(dev, dev->qp_table.rdb_table,
+                        qp->qpn << dev->qp_table.rdb_shift);
+        mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
+        mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
+}
+
+static int mthca_alloc_memfree(struct mthca_dev *dev,
+                               struct mthca_qp *qp)
+{
+        int ret = 0;
+
+        if (mthca_is_memfree(dev)) {
+                qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ,
+                                                 qp->qpn, &qp->rq.db);
+                if (qp->rq.db_index < 0)
+                        return ret;
+
+                qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ,
+                                                 qp->qpn, &qp->sq.db);
+                if (qp->sq.db_index < 0)
+                        mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
+        }
+
+        return ret;
+}
+
 static void mthca_free_memfree(struct mthca_dev *dev,
                                struct mthca_qp *qp)
 {
@@ -1112,11 +1159,6 @@ static void mthca_free_memfree(struct mthca_dev *dev,
                 mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index);
                 mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index);
         }
-
-        mthca_table_put(dev, dev->qp_table.rdb_table,
-                        qp->qpn << dev->qp_table.rdb_shift);
-        mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn);
-        mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn);
 }
 
 static void mthca_wq_init(struct mthca_wq* wq)
@@ -1147,13 +1189,28 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
         mthca_wq_init(&qp->sq);
         mthca_wq_init(&qp->rq);
 
-        ret = mthca_alloc_memfree(dev, qp);
+        ret = mthca_map_memfree(dev, qp);
         if (ret)
                 return ret;
 
         ret = mthca_alloc_wqe_buf(dev, pd, qp);
         if (ret) {
-                mthca_free_memfree(dev, qp);
+                mthca_unmap_memfree(dev, qp);
+                return ret;
+        }
+
+        /*
+         * If this is a userspace QP, we're done now.  The doorbells
+         * will be allocated and buffers will be initialized in
+         * userspace.
+         */
+        if (pd->ibpd.uobject)
+                return 0;
+
+        ret = mthca_alloc_memfree(dev, qp);
+        if (ret) {
+                mthca_free_wqe_buf(dev, qp);
+                mthca_unmap_memfree(dev, qp);
                 return ret;
         }
 
@@ -1186,22 +1243,39 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
         return 0;
 }
 
-static void mthca_align_qp_size(struct mthca_dev *dev, struct mthca_qp *qp)
+static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap,
+                             struct mthca_qp *qp)
 {
-        int i;
-
-        if (!mthca_is_memfree(dev))
-                return;
+        /* Sanity check QP size before proceeding */
+        if (cap->max_send_wr  > 65536 || cap->max_recv_wr  > 65536 ||
+            cap->max_send_sge > 64    || cap->max_recv_sge > 64)
+                return -EINVAL;
 
-        for (i = 0; 1 << i < qp->rq.max; ++i)
-                ; /* nothing */
+        if (mthca_is_memfree(dev)) {
+                qp->rq.max = cap->max_recv_wr ?
+                        roundup_pow_of_two(cap->max_recv_wr) : 0;
+                qp->sq.max = cap->max_send_wr ?
+                        roundup_pow_of_two(cap->max_send_wr) : 0;
+        } else {
+                qp->rq.max = cap->max_recv_wr;
+                qp->sq.max = cap->max_send_wr;
+        }
 
-        qp->rq.max = 1 << i;
+        qp->rq.max_gs = cap->max_recv_sge;
+        qp->sq.max_gs = max_t(int, cap->max_send_sge,
+                              ALIGN(cap->max_inline_data + MTHCA_INLINE_HEADER_SIZE,
+                                    MTHCA_INLINE_CHUNK_SIZE) /
+                              sizeof (struct mthca_data_seg));
 
-        for (i = 0; 1 << i < qp->sq.max; ++i)
-                ; /* nothing */
+        /*
+         * For MLX transport we need 2 extra S/G entries:
+         * one for the header and one for the checksum at the end
+         */
+        if ((qp->transport == MLX && qp->sq.max_gs + 2 > dev->limits.max_sg) ||
+            qp->sq.max_gs > dev->limits.max_sg || qp->rq.max_gs > dev->limits.max_sg)
+                return -EINVAL;
 
-        qp->sq.max = 1 << i;
+        return 0;
 }
 
 int mthca_alloc_qp(struct mthca_dev *dev,
@@ -1210,11 +1284,14 @@ int mthca_alloc_qp(struct mthca_dev *dev,
                    struct mthca_cq *recv_cq,
                    enum ib_qp_type type,
                    enum ib_sig_type send_policy,
+                   struct ib_qp_cap *cap,
                    struct mthca_qp *qp)
 {
         int err;
 
-        mthca_align_qp_size(dev, qp);
+        err = mthca_set_qp_size(dev, cap, qp);
+        if (err)
+                return err;
 
         switch (type) {
         case IB_QPT_RC: qp->transport = RC; break;
@@ -1247,14 +1324,17 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
                     struct mthca_cq *send_cq,
                     struct mthca_cq *recv_cq,
                     enum ib_sig_type send_policy,
+                    struct ib_qp_cap *cap,
                     int qpn,
                     int port,
                     struct mthca_sqp *sqp)
 {
-        int err = 0;
         u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;
+        int err;
 
-        mthca_align_qp_size(dev, &sqp->qp);
+        err = mthca_set_qp_size(dev, cap, &sqp->qp);
+        if (err)
+                return err;
 
         sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE;
         sqp->header_buf = dma_alloc_coherent(&dev->pdev->dev, sqp->header_buf_size,
@@ -1313,8 +1393,6 @@ void mthca_free_qp(struct mthca_dev *dev,
                    struct mthca_qp *qp)
 {
         u8 status;
-        int size;
-        int i;
         struct mthca_cq *send_cq;
         struct mthca_cq *recv_cq;
 
@@ -1344,31 +1422,22 @@ void mthca_free_qp(struct mthca_dev *dev,
         if (qp->state != IB_QPS_RESET)
                 mthca_MODIFY_QP(dev, MTHCA_TRANS_ANY2RST, qp->qpn, 0, NULL, 0, &status);
 
-        mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn);
-        if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
-                mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn);
-
-        mthca_free_mr(dev, &qp->mr);
-
-        size = PAGE_ALIGN(qp->send_wqe_offset +
-                          (qp->sq.max << qp->sq.wqe_shift));
+        /*
+         * If this is a userspace QP, the buffers, MR, CQs and so on
+         * will be cleaned up in userspace, so all we have to do is
+         * unref the mem-free tables and free the QPN in our table.
+         */
+        if (!qp->ibqp.uobject) {
+                mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq)->cqn, qp->qpn);
+                if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
+                        mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq)->cqn, qp->qpn);
 
-        if (qp->is_direct) {
-                pci_free_consistent(dev->pdev, size,
-                                    qp->queue.direct.buf,
-                                    pci_unmap_addr(&qp->queue.direct, mapping));
-        } else {
-                for (i = 0; i < size / PAGE_SIZE; ++i) {
-                        pci_free_consistent(dev->pdev, PAGE_SIZE,
-                                            qp->queue.page_list[i].buf,
-                                            pci_unmap_addr(&qp->queue.page_list[i],
-                                                           mapping));
-                }
+                mthca_free_mr(dev, &qp->mr);
+                mthca_free_memfree(dev, qp);
+                mthca_free_wqe_buf(dev, qp);
         }
 
-        kfree(qp->wrid);
-
-        mthca_free_memfree(dev, qp);
+        mthca_unmap_memfree(dev, qp);
 
         if (is_sqp(dev, qp)) {
                 atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count));
diff --git a/drivers/infiniband/hw/mthca/mthca_user.h b/drivers/infiniband/hw/mthca/mthca_user.h
new file mode 100644
index 000000000000..3024c1b4547d
--- /dev/null
+++ b/drivers/infiniband/hw/mthca/mthca_user.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef MTHCA_USER_H
+#define MTHCA_USER_H
+
+#include <linux/types.h>
+
+/*
+ * Make sure that all structs defined in this file remain laid out so
+ * that they pack the same way on 32-bit and 64-bit architectures (to
+ * avoid incompatibility between 32-bit userspace and 64-bit kernels).
+ * In particular do not use pointer types -- pass pointers in __u64
+ * instead.
+ */
+
+struct mthca_alloc_ucontext_resp {
+        __u32 qp_tab_size;
+        __u32 uarc_size;
+};
+
+struct mthca_alloc_pd_resp {
+        __u32 pdn;
+        __u32 reserved;
+};
+
+struct mthca_create_cq {
+        __u32 lkey;
+        __u32 pdn;
+        __u64 arm_db_page;
+        __u64 set_db_page;
+        __u32 arm_db_index;
+        __u32 set_db_index;
+};
+
+struct mthca_create_cq_resp {
+        __u32 cqn;
+        __u32 reserved;
+};
+
+struct mthca_create_qp {
+        __u32 lkey;
+        __u32 reserved;
+        __u64 sq_db_page;
+        __u64 rq_db_page;
+        __u32 sq_db_index;
+        __u32 rq_db_index;
+};
+
+#endif /* MTHCA_USER_H */