net/mlx4: Add A0 hybrid steering

A0 hybrid steering is a form of high performance flow steering.
By using this mode, mlx4 cards use a fast limited table based steering,
in order to enable fast steering of unicast packets to a QP.

In order to implement A0 hybrid steering we allocate resources
from different zones:
(1) General range
(2) Special MAC-assigned QPs [RSS, Raw-Ethernet] each has its own region.

When we create a rss QP or a raw ethernet (A0 steerable and BF ready) QP,
we try hard to allocate the QP from range (2). Otherwise, we try hard not
to allocate from this  range. However, when the system is pushed to its
limits and one needs every resource, the allocator uses every region it can.

Meaning, when we run out of raw-eth qps, the allocator allocates from the
general range (and the special-A0 area is no longer active). If we run out
of RSS qps, the mechanism tries to allocate from the raw-eth QP zone. If that
is also exhausted, the allocator will allocate from the general range
(and the A0 region is no longer active).

Note that if a raw-eth qp is allocated from the general range, it attempts
to allocate the range such that bits 6 and 7 (blueflame bits) in the
QP number are not set.

When the feature is used in SRIOV, the VF has to notify the PF what
kind of QP attributes it needs. In order to do that, along with the
"Eth QP blueflame" bit, we reserve a new "A0 steerable QP". According
to the combination of these bits, the PF tries to allocate a suitable QP.

In order to maintain backward compatibility (with older PFs), the PF
notifies which QP attributes it supports via QUERY_FUNC_CAP command.

Signed-off-by: Matan Barak <matanb@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

8 files changed, 300 insertions, 25 deletions

diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 506d1bdad227..cf000b7ad64f 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -807,8 +807,10 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
                  * VLAN insertion. */
                 if (init_attr->qp_type == IB_QPT_RAW_PACKET)
                         err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn,
-                                                    init_attr->cap.max_send_wr ?
-                                                    MLX4_RESERVE_ETH_BF_QP : 0);
+                                                    (init_attr->cap.max_send_wr ?
+                                                     MLX4_RESERVE_ETH_BF_QP : 0) |
+                                                    (init_attr->cap.max_recv_wr ?
+                                                     MLX4_RESERVE_A0_QP : 0));
                 else
                         if (qp->flags & MLX4_IB_QP_NETIF)
                                 err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index c67effb05b2f..568e1f41fdd4 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -595,7 +595,7 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
                 return 0;
         }
 
-        err = mlx4_qp_reserve_range(dev, 1, 1, qpn, 0);
+        err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);
         en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
         if (err) {
                 en_err(priv, "Failed to reserve qp for mac registration\n");
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index a850f24fabdf..a0474eb94aa3 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -1131,7 +1131,8 @@ int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
         int err;
         u32 qpn;
 
-        err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn, 0);
+        err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
+                                    MLX4_RESERVE_A0_QP);
         if (err) {
                 en_err(priv, "Failed reserving drop qpn\n");
                 return err;
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index 1469b5b5be64..622bffaa9d78 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -275,6 +275,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
 #define QUERY_FUNC_CAP_FLAG_VALID_MAILBOX       0x04
 
 #define QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG     (1UL << 31)
+#define QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG     (1UL << 30)
 
 /* when opcode modifier = 1 */
 #define QUERY_FUNC_CAP_PHYS_PORT_OFFSET         0x3
@@ -406,7 +407,8 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
                 MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
                 MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP);
 
-                size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG;
+                size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG |
+                        QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG;
                 MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET);
         } else
                 err = -EINVAL;
@@ -509,6 +511,8 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u8 gen_or_port,
                         MLX4_GET(size, outbox, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET);
                         if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG)
                                 func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_BF_RES_QP;
+                        if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_A0_QP_ALLOC_FLAG)
+                                func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_A0_RES_QP;
                 }
 
                 goto out;
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 6a9a941ddf58..3bfe90b95f96 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -436,6 +436,8 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                 (1 << dev->caps.log_num_vlans) *
                 dev->caps.num_ports;
         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
+        dev->caps.reserved_qps_cnt[MLX4_QP_REGION_RSS_RAW_ETH] =
+                MLX4_A0_STEERING_TABLE_SIZE;
 
         dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
@@ -469,7 +471,8 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         if (!mlx4_is_slave(dev)) {
                 mlx4_enable_cqe_eqe_stride(dev);
                 dev->caps.alloc_res_qp_mask =
-                        (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0);
+                        (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0) |
+                        MLX4_RESERVE_A0_QP;
         } else {
                 dev->caps.alloc_res_qp_mask = 0;
         }
@@ -826,6 +829,9 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
             dev->caps.bf_reg_size)
                 dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;
 
+        if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_A0_RES_QP)
+                dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_A0_QP;
+
         return 0;
 
 err_mem:
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index bc1505efa436..cebd1180702b 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -682,8 +682,19 @@ struct mlx4_srq_table {
         struct mlx4_icm_table   cmpt_table;
 };
 
+enum mlx4_qp_table_zones {
+        MLX4_QP_TABLE_ZONE_GENERAL,
+        MLX4_QP_TABLE_ZONE_RSS,
+        MLX4_QP_TABLE_ZONE_RAW_ETH,
+        MLX4_QP_TABLE_ZONE_NUM
+};
+
+#define MLX4_A0_STEERING_TABLE_SIZE    256
+
 struct mlx4_qp_table {
-        struct mlx4_bitmap      bitmap;
+        struct mlx4_bitmap      *bitmap_gen;
+        struct mlx4_zone_allocator *zones;
+        u32                     zones_uids[MLX4_QP_TABLE_ZONE_NUM];
         u32                     rdmarc_base;
         int                     rdmarc_shift;
         spinlock_t              lock;
diff --git a/drivers/net/ethernet/mellanox/mlx4/qp.c b/drivers/net/ethernet/mellanox/mlx4/qp.c
index 8720428c9807..d8d040c366f4 100644
--- a/drivers/net/ethernet/mellanox/mlx4/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx4/qp.c
@@ -213,6 +213,7 @@ EXPORT_SYMBOL_GPL(mlx4_qp_modify);
 int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
                             int *base, u8 flags)
 {
+        u32 uid;
         int bf_qp = !!(flags & (u8)MLX4_RESERVE_ETH_BF_QP);
 
         struct mlx4_priv *priv = mlx4_priv(dev);
@@ -221,8 +222,16 @@ int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
         if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp)
                 return -ENOMEM;
 
-        *base = mlx4_bitmap_alloc_range(&qp_table->bitmap, cnt, align,
-                                        bf_qp ? MLX4_BF_QP_SKIP_MASK : 0);
+        uid = MLX4_QP_TABLE_ZONE_GENERAL;
+        if (flags & (u8)MLX4_RESERVE_A0_QP) {
+                if (bf_qp)
+                        uid = MLX4_QP_TABLE_ZONE_RAW_ETH;
+                else
+                        uid = MLX4_QP_TABLE_ZONE_RSS;
+        }
+
+        *base = mlx4_zone_alloc_entries(qp_table->zones, uid, cnt, align,
+                                        bf_qp ? MLX4_BF_QP_SKIP_MASK