Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband: (87 commits)
  mlx4_core: Fix section mismatches
  IPoIB: Allow setting policy to ignore multicast groups
  IB/mthca: Mark error paths as unlikely() in post_srq_recv functions
  IB/ipath: Minor fix to ordering of freeing and zeroing of tid pages.
  IB/ipath: Remove redundant link state checks
  IB/ipath: Fix IB_EVENT_PORT_ERR event
  IB/ipath: Better handling of unexpected GPIO interrupts
  IB/ipath: Maintain active time on all chips
  IB/ipath: Fix QHT7040 serial number check
  IB/ipath: Indicate a couple of chip bugs to userspace
  IB/ipath: iba6110 rev4 no longer needs recv header overrun workaround
  IB/ipath: Use counters in ipath_poll and cleanup interrupts in ipath_close
  IB/ipath: Remove duplicate copy of LMC
  IB/ipath: Add ability to set the LMC via the sysfs debugging interface
  IB/ipath: Optimize completion queue entry insertion and polling
  IB/ipath: Implement IB_EVENT_QP_LAST_WQE_REACHED
  IB/ipath: Generate flush CQE when QP is in error state
  IB/ipath: Remove redundant code
  IB/ipath: Future proof eeprom checksum code (contents reading)
  IB/ipath: UC RDMA WRITE with IMMEDIATE doesn't send the immediate
  ...

13 files changed, 415 insertions, 151 deletions

diff --git a/drivers/net/mlx4/cmd.c b/drivers/net/mlx4/cmd.c
index a9f31753661..db49051b97b 100644
--- a/drivers/net/mlx4/cmd.c
+++ b/drivers/net/mlx4/cmd.c
@@ -95,7 +95,7 @@ enum {
 };
 
 enum {
-        GO_BIT_TIMEOUT          = 10000
+        GO_BIT_TIMEOUT_MSECS    = 10000
 };
 
 struct mlx4_cmd_context {
@@ -155,7 +155,7 @@ static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param,
 
         end = jiffies;
         if (event)
-                end += HZ * 10;
+                end += msecs_to_jiffies(GO_BIT_TIMEOUT_MSECS);
 
         while (cmd_pending(dev)) {
                 if (time_after_eq(jiffies, end))
@@ -184,6 +184,13 @@ static int mlx4_cmd_post(struct mlx4_dev *dev, u64 in_param, u64 out_param,
                                                (event ? (1 << HCR_E_BIT) : 0)   |
                                                (op_modifier << HCR_OPMOD_SHIFT) |
                                                op),                       hcr + 6);
+
+        /*
+         * Make sure that our HCR writes don't get mixed in with
+         * writes from another CPU starting a FW command.
+         */
+        mmiowb();
+
         cmd->toggle = cmd->toggle ^ 1;
 
         ret = 0;
diff --git a/drivers/net/mlx4/cq.c b/drivers/net/mlx4/cq.c
index 39253d0c159..d4441fee3d8 100644
--- a/drivers/net/mlx4/cq.c
+++ b/drivers/net/mlx4/cq.c
@@ -231,7 +231,7 @@ void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq)
 }
 EXPORT_SYMBOL_GPL(mlx4_cq_free);
 
-int __devinit mlx4_init_cq_table(struct mlx4_dev *dev)
+int mlx4_init_cq_table(struct mlx4_dev *dev)
 {
         struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table;
         int err;
diff --git a/drivers/net/mlx4/eq.c b/drivers/net/mlx4/eq.c
index 2095c843fa1..9c36c203403 100644
--- a/drivers/net/mlx4/eq.c
+++ b/drivers/net/mlx4/eq.c
@@ -300,8 +300,7 @@ static int mlx4_HW2SW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
                             MLX4_CMD_TIME_CLASS_A);
 }
 
-static void __devinit __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev,
-                                               struct mlx4_eq *eq)
+static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int index;
@@ -323,8 +322,8 @@ static void __devinit __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev,
         return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4);
 }
 
-static int __devinit mlx4_create_eq(struct mlx4_dev *dev, int nent,
-                                    u8 intr, struct mlx4_eq *eq)
+static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
+                          u8 intr, struct mlx4_eq *eq)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         struct mlx4_cmd_mailbox *mailbox;
@@ -485,7 +484,7 @@ static void mlx4_free_irqs(struct mlx4_dev *dev)
                         free_irq(eq_table->eq[i].irq, eq_table->eq + i);
 }
 
-static int __devinit mlx4_map_clr_int(struct mlx4_dev *dev)
+static int mlx4_map_clr_int(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
 
@@ -506,7 +505,7 @@ static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
         iounmap(priv->clr_base);
 }
 
-int __devinit mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt)
+int mlx4_map_eq_icm(struct mlx4_dev *dev, u64 icm_virt)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int ret;
@@ -548,7 +547,7 @@ void mlx4_unmap_eq_icm(struct mlx4_dev *dev)
         __free_page(priv->eq_table.icm_page);
 }
 
-int __devinit mlx4_init_eq_table(struct mlx4_dev *dev)
+int mlx4_init_eq_table(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int err;
diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index c45cbe43a0c..6471d33afb7 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -76,7 +76,7 @@ static void dump_dev_cap_flags(struct mlx4_dev *dev, u32 flags)
                 [ 0] = "RC transport",
                 [ 1] = "UC transport",
                 [ 2] = "UD transport",
-                [ 3] = "SRC transport",
+                [ 3] = "XRC transport",
                 [ 4] = "reliable multicast",
                 [ 5] = "FCoIB support",
                 [ 6] = "SRQ support",
diff --git a/drivers/net/mlx4/icm.c b/drivers/net/mlx4/icm.c
index b7a4aa8476f..4b3c109d5ea 100644
--- a/drivers/net/mlx4/icm.c
+++ b/drivers/net/mlx4/icm.c
@@ -34,6 +34,7 @@
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/mm.h>
+#include <linux/scatterlist.h>
 
 #include <linux/mlx4/cmd.h>
 
@@ -50,19 +51,41 @@ enum {
         MLX4_TABLE_CHUNK_SIZE   = 1 << 18
 };
 
-void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm)
+static void mlx4_free_icm_pages(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
 {
-        struct mlx4_icm_chunk *chunk, *tmp;
         int i;
 
-        list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
-                if (chunk->nsg > 0)
-                        pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
-                                     PCI_DMA_BIDIRECTIONAL);
+        if (chunk->nsg > 0)
+                pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages,
+                             PCI_DMA_BIDIRECTIONAL);
+
+        for (i = 0; i < chunk->npages; ++i)
+                __free_pages(chunk->mem[i].page,
+                             get_order(chunk->mem[i].length));
+}
+
+static void mlx4_free_icm_coherent(struct mlx4_dev *dev, struct mlx4_icm_chunk *chunk)
+{
+        int i;
+
+        for (i = 0; i < chunk->npages; ++i)
+                dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length,
+                                  lowmem_page_address(chunk->mem[i].page),
+                                  sg_dma_address(&chunk->mem[i]));
+}
+
+void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent)
+{
+        struct mlx4_icm_chunk *chunk, *tmp;
 
-                for (i = 0; i < chunk->npages; ++i)
-                        __free_pages(chunk->mem[i].page,
-                                     get_order(chunk->mem[i].length));
+        if (!icm)
+                return;
+
+        list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) {
+                if (coherent)
+                        mlx4_free_icm_coherent(dev, chunk);
+                else
+                        mlx4_free_icm_pages(dev, chunk);
 
                 kfree(chunk);
         }
@@ -70,16 +93,45 @@ void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm)
         kfree(icm);
 }
 
+static int mlx4_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask)
+{
+        mem->page = alloc_pages(gfp_mask, order);
+        if (!mem->page)
+                return -ENOMEM;
+
+        mem->length = PAGE_SIZE << order;
+        mem->offset = 0;
+        return 0;
+}
+
+static int mlx4_alloc_icm_coherent(struct device *dev, struct scatterlist *mem,
+                                    int order, gfp_t gfp_mask)
+{
+        void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order,
+                                       &sg_dma_address(mem), gfp_mask);
+        if (!buf)
+                return -ENOMEM;
+
+        sg_set_buf(mem, buf, PAGE_SIZE << order);
+        BUG_ON(mem->offset);
+        sg_dma_len(mem) = PAGE_SIZE << order;
+        return 0;
+}
+
 struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
-                                gfp_t gfp_mask)
+                                gfp_t gfp_mask, int coherent)
 {
         struct mlx4_icm *icm;
         struct mlx4_icm_chunk *chunk = NULL;
         int cur_order;
+        int ret;
+
+        /* We use sg_set_buf for coherent allocs, which assumes low memory */
+        BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM));
 
         icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
         if (!icm)
-                return icm;
+                return NULL;
 
         icm->refcount = 0;
         INIT_LIST_HEAD(&icm->chunk_list);
@@ -101,12 +153,20 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
                 while (1 << cur_order > npages)
                         --cur_order;
 
-                chunk->mem[chunk->npages].page = alloc_pages(gfp_mask, cur_order);
-                if (chunk->mem[chunk->npages].page) {
-                        chunk->mem[chunk->npages].length = PAGE_SIZE << cur_order;
-                        chunk->mem[chunk->npages].offset = 0;
+                if (coherent)
+                        ret = mlx4_alloc_icm_coherent(&dev->pdev->dev,
+                                                      &chunk->mem[chunk->npages],
+                                                      cur_order, gfp_mask);
+                else
+                        ret = mlx4_alloc_icm_pages(&chunk->mem[chunk->npages],
+                                                   cur_order, gfp_mask);
+
+                if (!ret) {
+                        ++chunk->npages;
 
-                        if (++chunk->npages == MLX4_ICM_CHUNK_LEN) {
+                        if (coherent)
+                                ++chunk->nsg;
+                        else if (chunk->npages == MLX4_ICM_CHUNK_LEN) {
                                 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
                                                         chunk->npages,
                                                         PCI_DMA_BIDIRECTIONAL);
@@ -125,7 +185,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
                 }
         }
 
-        if (chunk) {
+        if (!coherent && chunk) {
                 chunk->nsg = pci_map_sg(dev->pdev, chunk->mem,
                                         chunk->npages,
                                         PCI_DMA_BIDIRECTIONAL);
@@ -137,7 +197,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
         return icm;
 
 fail:
-        mlx4_free_icm(dev, icm);
+        mlx4_free_icm(dev, icm, coherent);
         return NULL;
 }
 
@@ -202,7 +262,7 @@ int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj)
 
         table->icm[i] = mlx4_alloc_icm(dev, MLX4_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
                                        (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
-                                       __GFP_NOWARN);
+                                       __GFP_NOWARN, table->coherent);
         if (!table->icm[i]) {
                 ret = -ENOMEM;
                 goto out;
@@ -210,7 +270,7 @@ int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj)
 
         if (mlx4_MAP_ICM(dev, table->icm[i], table->virt +
                          (u64) i * MLX4_TABLE_CHUNK_SIZE)) {
-                mlx4_free_icm(dev, table->icm[i]);
+                mlx4_free_icm(dev, table->icm[i], table->coherent);
                 table->icm[i] = NULL;
                 ret = -ENOMEM;
                 goto out;
@@ -234,16 +294,16 @@ void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj)
         if (--table->icm[i]->refcount == 0) {
                 mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,
                                MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
-                mlx4_free_icm(dev, table->icm[i]);
+                mlx4_free_icm(dev, table->icm[i], table->coherent);
                 table->icm[i] = NULL;
         }
 
         mutex_unlock(&table->mutex);
 }
 
-void *mlx4_table_find(struct mlx4_icm_table *table, int obj)
+void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle)
 {
-        int idx, offset, i;
+        int idx, offset, dma_offset, i;
         struct mlx4_icm_chunk *chunk;
         struct mlx4_icm *icm;
         struct page *page = NULL;
@@ -253,15 +313,26 @@ void *mlx4_table_find(struct mlx4_icm_table *table, int obj)
 
         mutex_lock(&table->mutex);
 
-        idx = obj & (table->num_obj - 1);
-        icm = table->icm[idx / (MLX4_TABLE_CHUNK_SIZE / table->obj_size)];
-        offset = idx % (MLX4_TABLE_CHUNK_SIZE / table->obj_size);
+        idx = (obj & (table->num_obj - 1)) * table->obj_size;
+        icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE];
+        dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE;
 
         if (!icm)
                 goto out;
 
         list_for_each_entry(chunk, &icm->chunk_list, list) {
                 for (i = 0; i < chunk->npages; ++i) {
+                        if (dma_handle && dma_offset >= 0) {
+                                if (sg_dma_len(&chunk->mem[i]) > dma_offset)
+                                        *dma_handle = sg_dma_address(&chunk->mem[i]) +
+                                                dma_offset;
+                                dma_offset -= sg_dma_len(&chunk->mem[i]);
+                        }
+                        /*
+                         * DMA mapping can merge pages but not split them,
+                         * so if we found the page, dma_handle has already
+                         * been assigned to.
+                         */
                         if (chunk->mem[i].length > offset) {
                                 page = chunk->mem[i].page;
                                 goto out;
@@ -309,7 +380,7 @@ void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
 
 int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
                         u64 virt, int obj_size, int nobj, int reserved,
-                        int use_lowmem)
+                        int use_lowmem, int use_coherent)
 {
         int obj_per_chunk;
         int num_icm;
@@ -327,6 +398,7 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
         table->num_obj  = nobj;
         table->obj_size = obj_size;
         table->lowmem   = use_lowmem;
+        table->coherent = use_coherent;
         mutex_init(&table->mutex);
 
         for (i = 0; i * MLX4_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {
@@ -336,11 +408,11 @@ int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
 
                 table->icm[i] = mlx4_alloc_icm(dev, chunk_size >> PAGE_SHIFT,
                                                (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
-                                               __GFP_NOWARN);
+                                               __GFP_NOWARN, use_coherent);
                 if (!table->icm[i])
                         goto err;
                 if (mlx4_MAP_ICM(dev, table->icm[i], virt + i * MLX4_TABLE_CHUNK_SIZE)) {
-                        mlx4_free_icm(dev, table->icm[i]);
+                        mlx4_free_icm(dev, table->icm[i], use_coherent);
                         table->icm[i] = NULL;
                         goto err;
                 }
@@ -359,7 +431,7 @@ err:
                 if (table->icm[i]) {
                         mlx4_UNMAP_ICM(dev, virt + i * MLX4_TABLE_CHUNK_SIZE,
                                        MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
-                        mlx4_free_icm(dev, table->icm[i]);
+                        mlx4_free_icm(dev, table->icm[i], use_coherent);
                 }
 
         return -ENOMEM;
@@ -373,7 +445,7 @@ void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table)
                 if (table->icm[i]) {
                         mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE,
                                        MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE);
-                        mlx4_free_icm(dev, table->icm[i]);
+                        mlx4_free_icm(dev, table->icm[i], table->coherent);
                 }
 
         kfree(table->icm);
diff --git a/drivers/net/mlx4/icm.h b/drivers/net/mlx4/icm.h
index bea223d879a..6c44edf3584 100644
--- a/drivers/net/mlx4/icm.h
+++ b/drivers/net/mlx4/icm.h
@@ -67,8 +67,9 @@ struct mlx4_icm_iter {
 
 struct mlx4_dev;
 
-struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, gfp_t gfp_mask);
-void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm);
+struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
+                                gfp_t gfp_mask, int coherent);
+void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent);
 
 int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
 void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
@@ -78,11 +79,11 @@ void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
                           int start, int end);
 int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table,
                         u64 virt, int obj_size, int nobj, int reserved,
-                        int use_lowmem);
+                        int use_lowmem, int use_coherent);
 void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table);
 int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
 void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj);
-void *mlx4_table_find(struct mlx4_icm_table *table, int obj);
+void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle);
 int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
                          int start, int end);
 void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table,
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 4dc9dc19b71..e029b8afbd3 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -61,7 +61,7 @@ MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
 
 #ifdef CONFIG_PCI_MSI
 
-static int msi_x;
+static int msi_x = 1;
 module_param(msi_x, int, 0444);
 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
 
@@ -85,7 +85,7 @@ static struct mlx4_profile default_profile = {
         .num_mtt        = 1 << 20,
 };
 
-static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
+static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 {
         int err;
         int i;
@@ -149,7 +149,8 @@ static int __devinit mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev
         dev->caps.max_cqes           = dev_cap->max_cq_sz - 1;
         dev->caps.reserved_cqs       = dev_cap->reserved_cqs;
         dev->caps.reserved_eqs       = dev_cap->reserved_eqs;
-        dev->caps.reserved_mtts      = dev_cap->reserved_mtts;
+        dev->caps.reserved_mtts      = DIV_ROUND_UP(dev_cap->reserved_mtts,
+                                                    MLX4_MTT_ENTRY_PER_SEG);
         dev->caps.reserved_mrws      = dev_cap->reserved_mrws;
         dev->caps.reserved_uars      = dev_cap->reserved_uars;
         dev->caps.reserved_pds       = dev_cap->reserved_pds;
@@ -168,7 +169,7 @@ static int __devinit mlx4_load_fw(struct mlx4_dev *dev)
         int err;
 
         priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,
-                                         GFP_HIGHUSER | __GFP_NOWARN);
+                                         GFP_HIGHUSER | __GFP_NOWARN, 0);
         if (!priv->fw.fw_icm) {
                 mlx4_err(dev, "Couldn't allocate FW area, aborting.\n");
                 return -ENOMEM;
@@ -192,7 +193,7 @@ err_unmap_fa:
         mlx4_UNMAP_FA(dev);
 
 err_free:
-        mlx4_free_icm(dev, priv->fw.fw_icm);
+        mlx4_free_icm(dev, priv->fw.fw_icm, 0);
         return err;
 }
 
@@ -207,7 +208,7 @@ static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
                                   ((u64) (MLX4_CMPT_TYPE_QP *
                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
                                   cmpt_entry_sz, dev->caps.num_qps,
-                                  dev->caps.reserved_qps, 0);
+                                  dev->caps.reserved_qps, 0, 0);
         if (err)
                 goto err;
 
@@ -216,7 +217,7 @@ static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
                                   ((u64) (MLX4_CMPT_TYPE_SRQ *
                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
                                   cmpt_entry_sz, dev->caps.num_srqs,
-                                  dev->caps.reserved_srqs, 0);
+                                  dev->caps.reserved_srqs, 0, 0);
         if (err)
                 goto err_qp;
 
@@ -225,7 +226,7 @@ static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
                                   ((u64) (MLX4_CMPT_TYPE_CQ *
                                           cmpt_entry_sz) << MLX4_CMPT_SHIFT),
                                   cmpt_entry_sz, dev->caps.num_cqs,
-                                  dev->caps.reserved_cqs, 0);
+                                  dev->caps.reserved_cqs, 0, 0);
         if (err)
                 goto err_srq;
 
@@ -236,7 +237,7 @@ static int __devinit mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
                                   cmpt_entry_sz,
                                   roundup_pow_of_two(MLX4_NUM_EQ +
                                                      dev->caps.reserved_eqs),
-                                  MLX4_NUM_EQ + dev->caps.reserved_eqs, 0);
+                                  MLX4_NUM_EQ + dev->caps.reserved_eqs, 0, 0);
         if (err)
                 goto err_cq;
 
@@ -255,10 +256,8 @@ err:
         return err;
 }
 
-static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
-                                   struct mlx4_dev_cap *dev_cap,
-                                   struct mlx4_init_hca_param *init_hca,
-                                   u64 icm_size)
+static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
+                         struct mlx4_init_hca_param *init_hca, u64 icm_size)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         u64 aux_pages;
@@ -275,7 +274,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                  (unsigned long long) aux_pages << 2);
 
         priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
-                                          GFP_HIGHUSER | __GFP_NOWARN);
+                                          GFP_HIGHUSER | __GFP_NOWARN, 0);
         if (!priv->fw.aux_icm) {
                 mlx4_err(dev, "Couldn't allocate aux memory, aborting.\n");
                 return -ENOMEM;
@@ -299,11 +298,22 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                 goto err_unmap_cmpt;
         }
 
+        /*
+         * Reserved MTT entries must be aligned up to a cacheline
+         * boundary, since the FW will write to them, while the driver
+         * writes to all other MTT entries. (The variable
+         * dev->caps.mtt_entry_sz below is really the MTT segment
+         * size, not the raw entry size)
+         */
+        dev->caps.reserved_mtts =
+                ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
+                      dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
+
         err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
                                   init_hca->mtt_base,
                                   dev->caps.mtt_entry_sz,
                                   dev->caps.num_mtt_segs,
-                                  dev->caps.reserved_mtts, 1);
+                                  dev->caps.reserved_mtts, 1, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map MTT context memory, aborting.\n");
                 goto err_unmap_eq;
@@ -313,7 +323,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->dmpt_base,
                                   dev_cap->dmpt_entry_sz,
                                   dev->caps.num_mpts,
-                                  dev->caps.reserved_mrws, 1);
+                                  dev->caps.reserved_mrws, 1, 1);
         if (err) {
                 mlx4_err(dev, "Failed to map dMPT context memory, aborting.\n");
                 goto err_unmap_mtt;
@@ -323,7 +333,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->qpc_base,
                                   dev_cap->qpc_entry_sz,
                                   dev->caps.num_qps,
-                                  dev->caps.reserved_qps, 0);
+                                  dev->caps.reserved_qps, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map QP context memory, aborting.\n");
                 goto err_unmap_dmpt;
@@ -333,7 +343,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->auxc_base,
                                   dev_cap->aux_entry_sz,
                                   dev->caps.num_qps,
-                                  dev->caps.reserved_qps, 0);
+                                  dev->caps.reserved_qps, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map AUXC context memory, aborting.\n");
                 goto err_unmap_qp;
@@ -343,7 +353,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->altc_base,
                                   dev_cap->altc_entry_sz,
                                   dev->caps.num_qps,
-                                  dev->caps.reserved_qps, 0);
+                                  dev->caps.reserved_qps, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map ALTC context memory, aborting.\n");
                 goto err_unmap_auxc;
@@ -353,7 +363,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->rdmarc_base,
                                   dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
                                   dev->caps.num_qps,
-                                  dev->caps.reserved_qps, 0);
+                                  dev->caps.reserved_qps, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
                 goto err_unmap_altc;
@@ -363,7 +373,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->cqc_base,
                                   dev_cap->cqc_entry_sz,
                                   dev->caps.num_cqs,
-                                  dev->caps.reserved_cqs, 0);
+                                  dev->caps.reserved_cqs, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map CQ context memory, aborting.\n");
                 goto err_unmap_rdmarc;
@@ -373,7 +383,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->srqc_base,
                                   dev_cap->srq_entry_sz,
                                   dev->caps.num_srqs,
-                                  dev->caps.reserved_srqs, 0);
+                                  dev->caps.reserved_srqs, 0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map SRQ context memory, aborting.\n");
                 goto err_unmap_cq;
@@ -388,7 +398,7 @@ static int __devinit mlx4_init_icm(struct mlx4_dev *dev,
                                   init_hca->mc_base, MLX4_MGM_ENTRY_SIZE,
                                   dev->caps.num_mgms + dev->caps.num_amgms,
                                   dev->caps.num_mgms + dev->caps.num_amgms,
-                                  0);
+                                  0, 0);
         if (err) {
                 mlx4_err(dev, "Failed to map MCG context memory, aborting.\n");
                 goto err_unmap_srq;
@@ -433,7 +443,7 @@ err_unmap_aux:
         mlx4_UNMAP_ICM_AUX(dev);
 
 err_free_aux:
-        mlx4_free_icm(dev, priv->fw.aux_icm);
+        mlx4_free_icm(dev, priv->fw.aux_icm, 0);
 
         return err;
 }
@@ -458,7 +468,7 @@ static void mlx4_free_icms(struct mlx4_dev *dev)
         mlx4_unmap_eq_icm(dev);
 
         mlx4_UNMAP_ICM_AUX(dev);
-        mlx4_free_icm(dev, priv->fw.aux_icm);
+        mlx4_free_icm(dev, priv->fw.aux_icm, 0);
 }
 
 static void mlx4_close_hca(struct mlx4_dev *dev)
@@ -466,10 +476,10 @@ static void mlx4_close_hca(struct mlx4_dev *dev)
         mlx4_CLOSE_HCA(dev, 0);
         mlx4_free_icms(dev);
         mlx4_UNMAP_FA(dev);
-        mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm);
+        mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
 }
 
-static int __devinit mlx4_init_hca(struct mlx4_dev *dev)
+static int mlx4_init_hca(struct mlx4_dev *dev)
 {
         struct mlx4_priv          *priv = mlx4_priv(dev);
         struct mlx4_adapter        adapter;
@@ -524,8 +534,8 @@ static int __devinit mlx4_init_hca(struct mlx4_dev *dev)
         }
 
         priv->eq_table.inta_pin = adapter.inta_pin;
-        priv->rev_id            = adapter.revision_id;
-        memcpy(priv->board_id, adapter.board_id, sizeof priv->board_id);
+        dev->rev_id             = adapter.revision_id;
+        memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id);
 
         return 0;
 
@@ -537,12 +547,12 @@ err_free_icm:
 
 err_stop_fw:
         mlx4_UNMAP_FA(dev);
-        mlx4_free_icm(dev, priv->fw.fw_icm);
+        mlx4_free_icm(dev, priv->fw.fw_icm, 0);
 
         return err;
 }
 
-static int __devinit mlx4_setup_hca(struct mlx4_dev *dev)
+static int mlx4_setup_hca(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int err;
@@ -599,13 +609,17 @@ static int __devinit mlx4_setup_hca(struct mlx4_dev *dev)
 
         err = mlx4_NOP(dev);
         if (err) {
-                mlx4_err(dev, "NOP command failed to generate interrupt "
-                         "(IRQ %d), aborting.\n",
-                         priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
-                if (dev->flags & MLX4_FLAG_MSI_X)
-                        mlx4_err(dev, "Try again with MSI-X disabled.\n");
-                else
+                if (dev->flags & MLX4_FLAG_MSI_X) {
+                        mlx4_warn(dev, "NOP command failed to generate MSI-X "
+                                  "interrupt IRQ %d).\n",
+                                  priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
+                        mlx4_warn(dev, "Trying again without MSI-X.\n");
+                } else {
+                        mlx4_err(dev, "NOP command failed to generate interrupt "
+                                 "(IRQ %d), aborting.\n",
+                                 priv->eq_table.eq[MLX4_EQ_ASYNC].irq);
                         mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
+                }
 
                 goto err_cmd_poll;
         }
@@ -705,19 +719,12 @@ no_msi:
                 priv->eq_table.eq[i].irq = dev->pdev->irq;
 }
 
-static int __devinit mlx4_init_one(struct pci_dev *pdev,
-                                   const struct pci_device_id *id)
+static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-        static int mlx4_version_printed;
         struct mlx4_priv *priv;
         struct mlx4_dev *dev;
         int err;
 
-        if (!mlx4_version_printed) {
-                printk(KERN_INFO "%s", mlx4_version);
-                ++mlx4_version_printed;
-        }
-
         printk(KERN_INFO PFX "Initializing %s\n",
                pci_name(pdev));
 
@@ -803,8 +810,6 @@ static int __devinit mlx4_init_one(struct pci_dev *pdev,
                 goto err_free_dev;
         }
 
-        mlx4_enable_msi_x(dev);
-
         if (mlx4_cmd_init(dev)) {
                 mlx4_err(dev, "Failed to init command interface, aborting.\n");
                 goto err_free_dev;
@@ -814,7 +819,15 @@ static int __devinit mlx4_init_one(struct pci_dev *pdev,
         if (err)
                 goto err_cmd;
 
+        mlx4_enable_msi_x(dev);
+
         err = mlx4_setup_hca(dev);
+        if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X)) {
+                dev->flags &= ~MLX4_FLAG_MSI_X;
+                pci_disable_msix(pdev);
+                err = mlx4_setup_hca(dev);
+        }
+
         if (err)
                 goto err_close;
 
@@ -838,15 +851,15 @@ err_cleanup:
         mlx4_cleanup_uar_table(dev);
 
 err_close:
+        if (dev->flags & MLX4_FLAG_MSI_X)
+                pci_disable_msix(pdev);
+
         mlx4_close_hca(dev);
 
 err_cmd:
         mlx4_cmd_cleanup(dev);
 
 err_free_dev:
-        if (dev->flags & MLX4_FLAG_MSI_X)
-                pci_disable_msix(pdev);
-
         kfree(priv);
 
 err_release_bar2:
@@ -861,7 +874,20 @@ err_disable_pdev:
         return err;
 }
 
-static void __devexit mlx4_remove_one(struct pci_dev *pdev)
+static int __devinit mlx4_init_one(struct pci_dev *pdev,
+                                   const struct pci_device_id *id)
+{
+        static int mlx4_version_printed;
+
+        if (!mlx4_version_printed) {
+                printk(KERN_INFO "%s", mlx4_version);
+                ++mlx4_version_printed;
+        }
+
+        return mlx4_init_one(pdev, id);
+}
+
+static void mlx4_remove_one(struct pci_dev *pdev)
 {
         struct mlx4_dev  *dev  = pci_get_drvdata(pdev);
         struct mlx4_priv *priv = mlx4_priv(dev);
@@ -902,7 +928,7 @@ static void __devexit mlx4_remove_one(struct pci_dev *pdev)
 int mlx4_restart_one(struct pci_dev *pdev)
 {
         mlx4_remove_one(pdev);
-        return mlx4_init_one(pdev, NULL);
+        return __mlx4_init_one(pdev, NULL);
 }
 
 static struct pci_device_id mlx4_pci_table[] = {
diff --git a/drivers/net/mlx4/mcg.c b/drivers/net/mlx4/mcg.c
index 672024a0ee7..a99e7729d33 100644
--- a/drivers/net/mlx4/mcg.c
+++ b/drivers/net/mlx4/mcg.c
@@ -359,7 +359,7 @@ out:
 }
 EXPORT_SYMBOL_GPL(mlx4_multicast_detach);
 
-int __devinit mlx4_init_mcg_table(struct mlx4_dev *dev)
+int mlx4_init_mcg_table(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int err;
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index be304a7c2c9..53a1cdddfc1 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -56,11 +56,7 @@ enum {
 };
 
 enum {
-        MLX4_BOARD_ID_LEN       = 64
-};
-
-enum {
-        MLX4_MGM_ENTRY_SIZE     =  0x40,
+        MLX4_MGM_ENTRY_SIZE     =  0x100,
         MLX4_QP_PER_MGM         = 4 * (MLX4_MGM_ENTRY_SIZE / 16 - 2),
         MLX4_MTT_ENTRY_PER_SEG  = 8
 };
@@ -133,6 +129,7 @@ struct mlx4_icm_table {
         int                     num_obj;
         int                     obj_size;
         int                     lowmem;
+        int                     coherent;
         struct mutex            mutex;
         struct mlx4_icm       **icm;
 };
@@ -277,9 +274,6 @@ struct mlx4_priv {
 
         struct mlx4_uar         driver_uar;
         void __iomem           *kar;
-
-        u32                     rev_id;
-        char                    board_id[MLX4_BOARD_ID_LEN];
 };
 
 static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
diff --git a/drivers/net/mlx4/mr.c b/drivers/net/mlx4/mr.c
index 5b87183e62c..0c05a10bae3 100644
--- a/drivers/net/mlx4/mr.c
+++ b/drivers/net/mlx4/mr.c
@@ -68,6 +68,9 @@ struct mlx4_mpt_entry {
 
 #define MLX4_MTT_FLAG_PRESENT           1
 
+#define MLX4_MPT_STATUS_SW              0xF0
+#define MLX4_MPT_STATUS_HW              0x00
+
 static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order)
 {
         int o;
@@ -349,58 +352,57 @@ err_table:
 }
 EXPORT_SYMBOL_GPL(mlx4_mr_enable);
 
-static int mlx4_WRITE_MTT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
-                          int num_mtt)
+static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
+                                int start_index, int npages, u64 *page_list)
 {
-        return mlx4_cmd(dev, mailbox->dma, num_mtt, 0, MLX4_CMD_WRITE_MTT,
-                        MLX4_CMD_TIME_CLASS_B);
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        __be64 *mtts;
+        dma_addr_t dma_handle;
+        int i;
+        int s = start_index * sizeof (u64);
+
+        /* All MTTs must fit in the same page */
+        if (start_index / (PAGE_SIZE / sizeof (u64)) !=
+            (start_index + npages - 1) / (PAGE_SIZE / sizeof (u64)))
+                return -EINVAL;
+
+        if (start_index & (MLX4_MTT_ENTRY_PER_SEG - 1))
+                return -EINVAL;
+
+        mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->first_seg +
+                                s / dev->caps.mtt_entry_sz, &dma_handle);
+        if (!mtts)
+                return -ENOMEM;
+
+        for (i = 0; i < npages; ++i)
+                mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
+
+        dma_sync_single(&dev->pdev->dev, dma_handle, npages * sizeof (u64), DMA_TO_DEVICE);
+
+        return 0;
 }
 
 int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
                    int start_index, int npages, u64 *page_list)
 {
-        struct mlx4_cmd_mailbox *mailbox;
-        __be64 *mtt_entry;
-        int i;
-        int err = 0;
+        int chunk;
+        int err;
 
         if (mtt->order < 0)
                 return -EINVAL;
 
-        mailbox = mlx4_alloc_cmd_mailbox(dev);
-        if (IS_ERR(mailbox))
-                return PTR_ERR(mailbox);
-
-        mtt_entry = mailbox->buf;
-
         while (npages > 0) {
-                mtt_entry[0] = cpu_to_be64(mlx4_mtt_addr(dev, mtt) + start_index * 8);
-                mtt_entry[1] = 0;
-
-                for (i = 0; i < npages && i < MLX4_MAILBOX_SIZE / 8 - 2; ++i)
-                        mtt_entry[i + 2] = cpu_to_be64(page_list[i] |
-                                                       MLX4_MTT_FLAG_PRESENT);
-
-                /*
-                 * If we have an odd number of entries to write, add
-                 * one more dummy entry for firmware efficiency.
-                 */
-                if (i & 1)
-                        mtt_entry[i + 2] = 0;
-
-                err = mlx4_WRITE_MTT(dev, mailbox, (i + 1) & ~1);
+                chunk = min_t(int, PAGE_SIZE / sizeof(u64), npages);
+                err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list);
                 if (err)
-                        goto out;
+                        return err;
 
-                npages      -= i;
-                start_index += i;
-                page_list   += i;
+                npages      -= chunk;
+                start_index += chunk;
+                page_list   += chunk;
         }
 
-out:
-        mlx4_free_cmd_mailbox(dev, mailbox);
-
-        return err;
+        return 0;
 }
 EXPORT_SYMBOL_GPL(mlx4_write_mtt);
 
@@ -428,7 +430,7 @@ int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
 }
 EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt);
 
-int __devinit mlx4_init_mr_table(struct mlx4_dev *dev)
+int mlx4_init_mr_table(struct mlx4_dev *dev)
 {
         struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table;
         int err;
@@ -444,7 +446,7 @@ int __devinit mlx4_init_mr_table(struct mlx4_dev *dev)
                 goto err_buddy;
 
         if (dev->caps.reserved_mtts) {
-                if (mlx4_alloc_mtt_range(dev, ilog2(dev->caps.reserved_mtts)) == -1) {
+                if (mlx4_alloc_mtt_range(dev, fls(dev->caps.reserved_mtts - 1)) == -1) {
                         mlx4_warn(dev, "MTT table of order %d is too small.\n",
                                   mr_table->mtt_buddy.max_order);
                         err = -ENOMEM;
@@ -470,3 +472,165 @@ void mlx4_cleanup_mr_table(struct mlx4_dev *dev)
         mlx4_buddy_cleanup(&mr_table->mtt_buddy);
         mlx4_bitmap_cleanup(&mr_table->mpt_bitmap);
 }
+
+static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list,
+                                  int npages, u64 iova)
+{
+        int i, page_mask;
+
+        if (npages > fmr->max_pages)
+                return -EINVAL;
+
+        page_mask = (1 << fmr->page_shift) - 1;
+
+        /* We are getting page lists, so va must be page aligned. */
+        if (iova & page_mask)
+                return -EINVAL;
+
+        /* Trust the user not to pass misaligned data in page_list */
+        if (0)
+                for (i = 0; i < npages; ++i) {
+                        if (page_list[i] & ~page_mask)
+                                return -EINVAL;
+                }
+
+        if (fmr->maps >= fmr->max_maps)
+                return -EINVAL;
+
+        return 0;
+}
+
+int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,
+                      int npages, u64 iova, u32 *lkey, u32 *rkey)
+{
+        u32 key;
+        int i, err;
+
+        err = mlx4_check_fmr(fmr, page_list, npages, iova);
+        if (err)
+                return err;
+
+        ++fmr->maps;
+
+        key = key_to_hw_index(fmr->mr.key);
+        key += dev->caps.num_mpts;
+        *lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
+
+        *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
+
+        /* Make sure MPT status is visible before writing MTT entries */
+        wmb();
+
+        for (i = 0; i < npages; ++i)
+                fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);
+
+        dma_sync_single(&dev->pdev->dev, fmr->dma_handle,
+                        npages * sizeof(u64), DMA_TO_DEVICE);
+
+        fmr->mpt->key    = cpu_to_be32(key);
+        fmr->mpt->lkey   = cpu_to_be32(key);
+        fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift));
+        fmr->mpt->start  = cpu_to_be64(iova);
+
+        /* Make MTT entries are visible before setting MPT status */
+        wmb();
+
+        *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW;
+
+        /* Make sure MPT status is visible before consumer can use FMR */
+        wmb();
+
+        return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr);
+
+int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,
+                   int max_maps, u8 page_shift, struct mlx4_fmr *fmr)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        u64 mtt_seg;
+        int err = -ENOMEM;
+
+        if (page_shift < 12 || page_shift >= 32)
+                return -EINVAL;
+
+        /* All MTTs must fit in the same page */
+        if (max_pages * sizeof *fmr->mtts > PAGE_SIZE)
+                return -EINVAL;
+
+        fmr->page_shift = page_shift;
+        fmr->max_pages  = max_pages;
+        fmr->max_maps   = max_maps;
+        fmr->maps = 0;
+
+        err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages,
+                            page_shift, &fmr->mr);
+        if (err)
+                return err;
+
+        mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz;
+
+        fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table,
+                                    fmr->mr.mtt.first_seg,
+                                    &fmr->dma_handle);
+        if (!fmr->mtts) {
+                err = -ENOMEM;
+                goto err_free;
+        }
+
+        fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table,
+                                    key_to_hw_index(fmr->mr.key), NULL);
+        if (!fmr->mpt) {
+                err = -ENOMEM;
+                goto err_free;
+        }
+
+        return 0;
+
+err_free:
+        mlx4_mr_free(dev, &fmr->mr);
+        return err;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_alloc);
+
+int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
+{
+        return mlx4_mr_enable(dev, &fmr->mr);
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
+
+void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
+                    u32 *lkey, u32 *rkey)
+{
+        u32 key;
+
+        if (!fmr->maps)
+                return;
+
+        key = key_to_hw_index(fmr->mr.key);
+        key &= dev->caps.num_mpts - 1;
+        *lkey = *rkey = fmr->mr.key = hw_index_to_key(key);
+
+        fmr->maps = 0;
+
+        *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);
+
+int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
+{
+        if (fmr->maps)
+                return -EBUSY;
+
+        fmr->mr.enabled = 0;
+        mlx4_mr_free(dev, &fmr->mr);
+
+        return 0;
+}
+EXPORT_SYMBOL_GPL(mlx4_fmr_free);
+
+int mlx4_SYNC_TPT(struct mlx4_dev *dev)
+{
+        return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000);
+}
+EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT);
diff --git a/drivers/net/mlx4/pd.c b/drivers/net/mlx4/pd.c
index 23dea1ee775..3a93c5f0f7a 100644
--- a/drivers/net/mlx4/pd.c
+++ b/drivers/net/mlx4/pd.c
@@ -57,7 +57,7 @@ void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn)
 }
 EXPORT_SYMBOL_GPL(mlx4_pd_free);
 
-int __devinit mlx4_init_pd_table(struct mlx4_dev *dev)
+int mlx4_init_pd_table(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
 
diff --git a/drivers/net/mlx4/qp.c b/drivers/net/mlx4/qp.c
index 19b48c71cf7..cc4b1be1821 100644
--- a/drivers/net/mlx4/qp.c
+++ b/drivers/net/mlx4/qp.c
@@ -240,7 +240,8 @@ void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
         mlx4_table_put(dev, &qp_table->auxc_table, qp->qpn);
         mlx4_table_put(dev, &qp_table->qp_table, qp->qpn);
 
-        mlx4_bitmap_free(&qp_table->bitmap, qp->qpn);
+        if (qp->qpn < dev->caps.sqp_start + 8)
+                mlx4_bitmap_free(&qp_table->bitmap, qp->qpn);
 }
 EXPORT_SYMBOL_GPL(mlx4_qp_free);
 
@@ -250,7 +251,7 @@ static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, u32 base_qpn)
                         MLX4_CMD_TIME_CLASS_B);
 }
 
-int __devinit mlx4_init_qp_table(struct mlx4_dev *dev)
+int mlx4_init_qp_table(struct mlx4_dev *dev)
 {
         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
         int err;
diff --git a/drivers/net/mlx4/srq.c b/drivers/net/mlx4/srq.c
index b061c86d683..d23f46d692e 100644
--- a/drivers/net/mlx4/srq.c
+++ b/drivers/net/mlx4/srq.c
@@ -227,7 +227,7 @@ int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_waterm
         err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn);
         if (err)
                 goto err_out;
-        *limit_watermark = srq_context->limit_watermark;
+        *limit_watermark = be16_to_cpu(srq_context->limit_watermark);
 
 err_out:
         mlx4_free_cmd_mailbox(dev, mailbox);
@@ -235,7 +235,7 @@ err_out:
 }
 EXPORT_SYMBOL_GPL(mlx4_srq_query);
 
-int __devinit mlx4_init_srq_table(struct mlx4_dev *dev)
+int mlx4_init_srq_table(struct mlx4_dev *dev)
 {
         struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
         int err;