IB/mlx5: Create GSI transmission QPs when P_Key table is changed

Whenever the P_Key table is changed, we create the required GSI
transmission QPs on-demand.

Reviewed-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Haggai Eran <haggaie@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

3 files changed, 41 insertions, 0 deletions

diff --git a/drivers/infiniband/hw/mlx5/gsi.c b/drivers/infiniband/hw/mlx5/gsi.c
index 91bd20eb59b7..1648f539c836 100644
--- a/drivers/infiniband/hw/mlx5/gsi.c
+++ b/drivers/infiniband/hw/mlx5/gsi.c
@@ -341,3 +341,13 @@ int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
 
         return ib_post_recv(gsi->rx_qp, wr, bad_wr);
 }
+
+void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi)
+{
+        if (!gsi)
+                return;
+
+        mutex_lock(&gsi->mutex);
+        setup_qps(gsi);
+        mutex_unlock(&gsi->mutex);
+}
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 0b30dc53c925..d4224fab98f7 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -1721,6 +1721,17 @@ static struct device_attribute *mlx5_class_attributes[] = {
         &dev_attr_reg_pages,
 };
 
+static void pkey_change_handler(struct work_struct *work)
+{
+        struct mlx5_ib_port_resources *ports =
+                container_of(work, struct mlx5_ib_port_resources,
+                             pkey_change_work);
+
+        mutex_lock(&ports->devr->mutex);
+        mlx5_ib_gsi_pkey_change(ports->gsi);
+        mutex_unlock(&ports->devr->mutex);
+}
+
 static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
                           enum mlx5_dev_event event, unsigned long param)
 {
@@ -1757,6 +1768,8 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
         case MLX5_DEV_EVENT_PKEY_CHANGE:
                 ibev.event = IB_EVENT_PKEY_CHANGE;
                 port = (u8)param;
+
+                schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
                 break;
 
         case MLX5_DEV_EVENT_GUID_CHANGE:
@@ -1966,6 +1979,7 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
         struct ib_srq_init_attr attr;
         struct mlx5_ib_dev *dev;
         struct ib_cq_init_attr cq_attr = {.cqe = 1};
+        int port;
         int ret = 0;
 
         dev = container_of(devr, struct mlx5_ib_dev, devr);
@@ -2059,6 +2073,12 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
         atomic_inc(&devr->p0->usecnt);
         atomic_set(&devr->s0->usecnt, 0);
 
+        for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
+                INIT_WORK(&devr->ports[port].pkey_change_work,
+                          pkey_change_handler);
+                devr->ports[port].devr = devr;
+        }
+
         return 0;
 
 error5:
@@ -2077,12 +2097,20 @@ error0:
 
 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
 {
+        struct mlx5_ib_dev *dev =
+                container_of(devr, struct mlx5_ib_dev, devr);
+        int port;
+
         mlx5_ib_destroy_srq(devr->s1);
         mlx5_ib_destroy_srq(devr->s0);
         mlx5_ib_dealloc_xrcd(devr->x0);
         mlx5_ib_dealloc_xrcd(devr->x1);
         mlx5_ib_destroy_cq(devr->c0);
         mlx5_ib_dealloc_pd(devr->p0);
+
+        /* Make sure no change P_Key work items are still executing */
+        for (port = 0; port < dev->num_ports; ++port)
+                cancel_work_sync(&devr->ports[port].pkey_change_work);
 }
 
 static u32 get_core_cap_flags(struct ib_device *ibdev)
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index c68a9135831f..a8fc345c088a 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -510,7 +510,9 @@ struct mlx5_mr_cache {
 struct mlx5_ib_gsi_qp;
 
 struct mlx5_ib_port_resources {
+        struct mlx5_ib_resources *devr;
         struct mlx5_ib_gsi_qp *gsi;
+        struct work_struct pkey_change_work;
 };
 
 struct mlx5_ib_resources {
@@ -781,6 +783,7 @@ int mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr,
                           struct ib_send_wr **bad_wr);
 int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
                           struct ib_recv_wr **bad_wr);
+void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
 
 static inline void init_query_mad(struct ib_smp *mad)
 {