10 files changed, 105 insertions, 25 deletions

diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c
index 3d1899ff1076..fdc5f23d8e9f 100644
--- a/drivers/net/ethernet/mellanox/mlx4/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c
@@ -1498,6 +1498,7 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
         u32 reply;
         u8 is_going_down = 0;
         int i;
+        unsigned long flags;
 
         slave_state[slave].comm_toggle ^= 1;
         reply = (u32) slave_state[slave].comm_toggle << 31;
@@ -1576,12 +1577,12 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
                 mlx4_warn(dev, "Bad comm cmd:%d from slave:%d\n", cmd, slave);
                 goto reset_slave;
         }
-        spin_lock(&priv->mfunc.master.slave_state_lock);
+        spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
         if (!slave_state[slave].is_slave_going_down)
                 slave_state[slave].last_cmd = cmd;
         else
                 is_going_down = 1;
-        spin_unlock(&priv->mfunc.master.slave_state_lock);
+        spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
         if (is_going_down) {
                 mlx4_warn(dev, "Slave is going down aborting command(%d)"
                           " executing from slave:%d\n",
@@ -1597,10 +1598,10 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
 reset_slave:
         /* cleanup any slave resources */
         mlx4_delete_all_resources_for_slave(dev, slave);
-        spin_lock(&priv->mfunc.master.slave_state_lock);
+        spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
         if (!slave_state[slave].is_slave_going_down)
                 slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET;
-        spin_unlock(&priv->mfunc.master.slave_state_lock);
+        spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
         /*with slave in the middle of flr, no need to clean resources again.*/
 inform_slave_state:
         memset(&slave_state[slave].event_eq, 0,
@@ -1755,7 +1756,7 @@ int mlx4_multi_func_init(struct mlx4_dev *dev)
                         spin_lock_init(&s_state->lock);
                 }
 
-                memset(&priv->mfunc.master.cmd_eqe, 0, sizeof(struct mlx4_eqe));
+                memset(&priv->mfunc.master.cmd_eqe, 0, dev->caps.eqe_size);
                 priv->mfunc.master.cmd_eqe.type = MLX4_EVENT_TYPE_CMD;
                 INIT_WORK(&priv->mfunc.master.comm_work,
                           mlx4_master_comm_channel);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index aa9c2f6cf3c0..b8d0854a7ad1 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -51,7 +51,7 @@ int mlx4_en_create_cq(struct mlx4_en_priv *priv,
         int err;
 
         cq->size = entries;
-        cq->buf_size = cq->size * sizeof(struct mlx4_cqe);
+        cq->buf_size = cq->size * mdev->dev->caps.cqe_size;
 
         cq->ring = ring;
         cq->is_tx = mode;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 7d1287f81a31..75a3f467bb5b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -1604,6 +1604,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
                 goto out;
         }
         priv->rx_ring_num = prof->rx_ring_num;
+        priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0;
         priv->mac_index = -1;
         priv->msg_enable = MLX4_EN_MSG_LEVEL;
         spin_lock_init(&priv->stats_lock);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index f76c9671f362..fed26d867f4e 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -566,6 +566,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
         struct ethhdr *ethh;
         dma_addr_t dma;
         u64 s_mac;
+        int factor = priv->cqe_factor;
 
         if (!priv->port_up)
                 return 0;
@@ -574,7 +575,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
          * descriptor offset can be deduced from the CQE index instead of
          * reading 'cqe->index' */
         index = cq->mcq.cons_index & ring->size_mask;
-        cqe = &cq->buf[index];
+        cqe = &cq->buf[(index << factor) + factor];
 
         /* Process all completed CQEs */
         while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
@@ -709,7 +710,7 @@ next:
 
                 ++cq->mcq.cons_index;
                 index = (cq->mcq.cons_index) & ring->size_mask;
-                cqe = &cq->buf[index];
+                cqe = &cq->buf[(index << factor) + factor];
                 if (++polled == budget)
                         goto out;
         }
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 1f571d009155..2b799f4f1c37 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -315,12 +315,13 @@ static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq)
         struct mlx4_cqe *buf = cq->buf;
         u32 packets = 0;
         u32 bytes = 0;
+        int factor = priv->cqe_factor;
 
         if (!priv->port_up)
                 return;
 
         index = cons_index & size_mask;
-        cqe = &buf[index];
+        cqe = &buf[(index << factor) + factor];
         ring_index = ring->cons & size_mask;
 
         /* Process all completed CQEs */
@@ -349,7 +350,7 @@ static void mlx4_en_process_tx_cq(struct net_device *dev, struct mlx4_en_cq *cq)
 
                 ++cons_index;
                 index = cons_index & size_mask;
-                cqe = &buf[index];
+                cqe = &buf[(index << factor) + factor];
         }
 
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c
index c48cf6f6529c..251ae2f93116 100644
--- a/drivers/net/ethernet/mellanox/mlx4/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/eq.c
@@ -101,15 +101,21 @@ static void eq_set_ci(struct mlx4_eq *eq, int req_not)
         mb();
 }
 
-static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry)
+static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, u32 entry, u8 eqe_factor)
 {
-        unsigned long off = (entry & (eq->nent - 1)) * MLX4_EQ_ENTRY_SIZE;
-        return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;
+        /* (entry & (eq->nent - 1)) gives us a cyclic array */
+        unsigned long offset = (entry & (eq->nent - 1)) * (MLX4_EQ_ENTRY_SIZE << eqe_factor);
+        /* CX3 is capable of extending the EQE from 32 to 64 bytes.
+         * When this feature is enabled, the first (in the lower addresses)
+         * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes
+         * contain the legacy EQE information.
+         */
+        return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE;
 }
 
-static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq)
+static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, u8 eqe_factor)
 {
-        struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index);
+        struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor);
         return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
 }
 
@@ -177,7 +183,7 @@ static void slave_event(struct mlx4_dev *dev, u8 slave, struct mlx4_eqe *eqe)
                 return;
         }
 
-        memcpy(s_eqe, eqe, sizeof(struct mlx4_eqe) - 1);
+        memcpy(s_eqe, eqe, dev->caps.eqe_size - 1);
         s_eqe->slave_id = slave;
         /* ensure all information is written before setting the ownersip bit */
         wmb();
@@ -401,6 +407,7 @@ void mlx4_master_handle_slave_flr(struct work_struct *work)
         struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
         int i;
         int err;
+        unsigned long flags;
 
         mlx4_dbg(dev, "mlx4_handle_slave_flr\n");
 
@@ -412,10 +419,10 @@ void mlx4_master_handle_slave_flr(struct work_struct *work)
 
                         mlx4_delete_all_resources_for_slave(dev, i);
                         /*return the slave to running mode*/
-                        spin_lock(&priv->mfunc.master.slave_state_lock);
+                        spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
                         slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
                         slave_state[i].is_slave_going_down = 0;
-                        spin_unlock(&priv->mfunc.master.slave_state_lock);
+                        spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
                         /*notify the FW:*/
                         err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE,
                                        MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
@@ -440,8 +447,9 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
         u8 update_slave_state;
         int i;
         enum slave_port_gen_event gen_event;
+        unsigned long flags;
 
-        while ((eqe = next_eqe_sw(eq))) {
+        while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor))) {
                 /*
                  * Make sure we read EQ entry contents after we've
                  * checked the ownership bit.
@@ -647,13 +655,13 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
                         } else
                                 update_slave_state = 1;
 
-                        spin_lock(&priv->mfunc.master.slave_state_lock);
+                        spin_lock_irqsave(&priv->mfunc.master.slave_state_lock, flags);
                         if (update_slave_state) {
                                 priv->mfunc.master.slave_state[flr_slave].active = false;
                                 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR;
                                 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1;
                         }
-                        spin_unlock(&priv->mfunc.master.slave_state_lock);
+                        spin_unlock_irqrestore(&priv->mfunc.master.slave_state_lock, flags);
                         queue_work(priv->mfunc.master.comm_wq,
                                    &priv->mfunc.master.slave_flr_event_work);
                         break;
@@ -864,7 +872,8 @@ static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
 
         eq->dev   = dev;
         eq->nent  = roundup_pow_of_two(max(nent, 2));
-        npages = PAGE_ALIGN(eq->nent * MLX4_EQ_ENTRY_SIZE) / PAGE_SIZE;
+        /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes */
+        npages = PAGE_ALIGN(eq->nent * (MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor)) / PAGE_SIZE;
 
         eq->page_list = kmalloc(npages * sizeof *eq->page_list,
                                 GFP_KERNEL);
@@ -966,8 +975,9 @@ static void mlx4_free_eq(struct mlx4_dev *dev,
         struct mlx4_priv *priv = mlx4_priv(dev);
         struct mlx4_cmd_mailbox *mailbox;
         int err;
-        int npages = PAGE_ALIGN(MLX4_EQ_ENTRY_SIZE * eq->nent) / PAGE_SIZE;
         int i;
+        /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes */
+        int npages = PAGE_ALIGN((MLX4_EQ_ENTRY_SIZE << dev->caps.eqe_factor) * eq->nent) / PAGE_SIZE;
 
         mailbox = mlx4_alloc_cmd_mailbox(dev);
         if (IS_ERR(mailbox))
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index 4f30b99324cf..9a9de51ecc91 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -110,6 +110,8 @@ static void dump_dev_cap_flags(struct mlx4_dev *dev, u64 flags)
                 [42] = "Multicast VEP steering support",
                 [48] = "Counters support",
                 [59] = "Port management change event support",
+                [61] = "64 byte EQE support",
+                [62] = "64 byte CQE support",
         };
         int i;
 
@@ -235,7 +237,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
                 field = dev->caps.num_ports;
                 MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);
 
-                size = 0; /* no PF behaviour is set for now */
+                size = dev->caps.function_caps; /* set PF behaviours */
                 MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_PF_BHVR_OFFSET);
 
                 field = 0; /* protected FMR support not available as yet */
@@ -1237,6 +1239,24 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS)
                 *(inbox + INIT_HCA_FLAGS_OFFSET / 4) |= cpu_to_be32(1 << 4);
 
+        /* CX3 is capable of extending CQEs/EQEs from 32 to 64 bytes */
+        if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_EQE) {
+                *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 29);
+                dev->caps.eqe_size   = 64;
+                dev->caps.eqe_factor = 1;
+        } else {
+                dev->caps.eqe_size   = 32;
+                dev->caps.eqe_factor = 0;
+        }
+
+        if (dev->caps.flags & MLX4_DEV_CAP_FLAG_64B_CQE) {
+                *(inbox + INIT_HCA_EQE_CQE_OFFSETS / 4) |= cpu_to_be32(1 << 30);
+                dev->caps.cqe_size   = 64;
+                dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE;
+        } else {
+                dev->caps.cqe_size   = 32;
+        }
+
         /* QPC/EEC/CQC/EQC/RDMARC attributes */
 
         MLX4_PUT(inbox, param->qpc_base,      INIT_HCA_QPC_BASE_OFFSET);
@@ -1319,6 +1339,7 @@ int mlx4_QUERY_HCA(struct mlx4_dev *dev,
         struct mlx4_cmd_mailbox *mailbox;
         __be32 *outbox;
         int err;
+        u8 byte_field;
 
 #define QUERY_HCA_GLOBAL_CAPS_OFFSET    0x04
 
@@ -1370,6 +1391,13 @@ int mlx4_QUERY_HCA(struct mlx4_dev *dev,
                          INIT_HCA_LOG_MC_TABLE_SZ_OFFSET);
         }
 
+        /* CX3 is capable of extending CQEs/EQEs from 32 to 64 bytes */
+        MLX4_GET(byte_field, outbox, INIT_HCA_EQE_CQE_OFFSETS);
+        if (byte_field & 0x20) /* 64-bytes eqe enabled */
+                param->dev_cap_enabled |= MLX4_DEV_CAP_64B_EQE_ENABLED;
+        if (byte_field & 0x40) /* 64-bytes cqe enabled */
+                param->dev_cap_enabled |= MLX4_DEV_CAP_64B_CQE_ENABLED;
+
         /* TPT attributes */
 
         MLX4_GET(param->dmpt_base,  outbox, INIT_HCA_DMPT_BASE_OFFSET);
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.h b/drivers/net/ethernet/mellanox/mlx4/fw.h
index 85abe9c11a22..2c2e7ade2a34 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.h
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.h
@@ -172,6 +172,7 @@ struct mlx4_init_hca_param {
         u8  log_uar_sz;
         u8  uar_page_sz; /* log pg sz in 4k chunks */
         u8  fs_hash_enable_bits;
+        u64 dev_cap_enabled;
 };
 
 struct mlx4_init_ib_param {
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 200cc0ec8052..b2acbe7706a3 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -95,8 +95,14 @@ MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num"
                                          " Not in use with device managed"
                                          " flow steering");
 
+static bool enable_64b_cqe_eqe;
+module_param(enable_64b_cqe_eqe, bool, 0444);
+MODULE_PARM_DESC(enable_64b_cqe_eqe,
+                 "Enable 64 byte CQEs/EQEs when the the FW supports this");
+
 #define HCA_GLOBAL_CAP_MASK            0
-#define PF_CONTEXT_BEHAVIOUR_MASK      0
+
+#define PF_CONTEXT_BEHAVIOUR_MASK       MLX4_FUNC_CAP_64B_EQE_CQE
 
 static char mlx4_version[] =
         DRV_NAME ": Mellanox ConnectX core driver v"
@@ -386,6 +392,21 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
 
         dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0;
+
+        if (!enable_64b_cqe_eqe) {
+                if (dev_cap->flags &
+                    (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) {
+                        mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n");
+                        dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
+                        dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
+                }
+        }
+
+        if ((dev_cap->flags &
+            (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) &&
+            mlx4_is_master(dev))
+                dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE;
+
         return 0;
 }
 /*The function checks if there are live vf, return the num of them*/
@@ -599,6 +620,21 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
                 goto err_mem;
         }
 
+        if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
+                dev->caps.eqe_size   = 64;
+                dev->caps.eqe_factor = 1;
+        } else {
+                dev->caps.eqe_size   = 32;
+                dev->caps.eqe_factor = 0;
+        }
+
+        if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
+                dev->caps.cqe_size   = 64;
+                dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_64B_CQE;
+        } else {
+                dev->caps.cqe_size   = 32;
+        }
+
         return 0;
 
 err_mem:
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 334ec483480b..8d54412ada63 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -473,6 +473,7 @@ struct mlx4_en_priv {
         int mac_index;
         unsigned max_mtu;
         int base_qpn;
+        int cqe_factor;
 
         struct mlx4_en_rss_map rss_map;
         __be32 ctrl_flags;