mlx4_core: Add IB port-state machine and port mgmt event propagation

For an IB port, a slave should not show port active until that slave
has a valid alias-guid (provided by the subnet manager).  Therefore
the port-up event should be passed to a slave only after both the port
is up, and the slave's alias-guid has been set.

Also, provide the infrastructure for propagating port-management
events (client-reregister, etc) to slaves.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Roland Dreier <roland@purestorage.com>

3 files changed, 250 insertions, 16 deletions

diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c
index 99a04648fab0..c425826b1fc0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/eq.c
@@ -200,6 +200,196 @@ static void mlx4_slave_event(struct mlx4_dev *dev, int slave,
         slave_event(dev, slave, eqe);
 }
 
+int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port)
+{
+        struct mlx4_eqe eqe;
+
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave];
+
+        if (!s_slave->active)
+                return 0;
+
+        memset(&eqe, 0, sizeof eqe);
+
+        eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
+        eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE;
+        eqe.event.port_mgmt_change.port = port;
+
+        return mlx4_GEN_EQE(dev, slave, &eqe);
+}
+EXPORT_SYMBOL(mlx4_gen_pkey_eqe);
+
+int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port)
+{
+        struct mlx4_eqe eqe;
+
+        /*don't send if we don't have the that slave */
+        if (dev->num_vfs < slave)
+                return 0;
+        memset(&eqe, 0, sizeof eqe);
+
+        eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
+        eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO;
+        eqe.event.port_mgmt_change.port = port;
+
+        return mlx4_GEN_EQE(dev, slave, &eqe);
+}
+EXPORT_SYMBOL(mlx4_gen_guid_change_eqe);
+
+int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port,
+                                   u8 port_subtype_change)
+{
+        struct mlx4_eqe eqe;
+
+        /*don't send if we don't have the that slave */
+        if (dev->num_vfs < slave)
+                return 0;
+        memset(&eqe, 0, sizeof eqe);
+
+        eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE;
+        eqe.subtype = port_subtype_change;
+        eqe.event.port_change.port = cpu_to_be32(port << 28);
+
+        mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__,
+                 port_subtype_change, slave, port);
+        return mlx4_GEN_EQE(dev, slave, &eqe);
+}
+EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe);
+
+enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
+        if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS) {
+                pr_err("%s: Error: asking for slave:%d, port:%d\n",
+                       __func__, slave, port);
+                return SLAVE_PORT_DOWN;
+        }
+        return s_state[slave].port_state[port];
+}
+EXPORT_SYMBOL(mlx4_get_slave_port_state);
+
+static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave, u8 port,
+                                     enum slave_port_state state)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
+
+        if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) {
+                pr_err("%s: Error: asking for slave:%d, port:%d\n",
+                       __func__, slave, port);
+                return -1;
+        }
+        s_state[slave].port_state[port] = state;
+
+        return 0;
+}
+
+static void set_all_slave_state(struct mlx4_dev *dev, u8 port, int event)
+{
+        int i;
+        enum slave_port_gen_event gen_event;
+
+        for (i = 0; i < dev->num_slaves; i++)
+                set_and_calc_slave_port_state(dev, i, port, event, &gen_event);
+}
+/**************************************************************************
+        The function get as input the new event to that port,
+        and according to the prev state change the slave's port state.
+        The events are:
+                MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
+                MLX4_PORT_STATE_DEV_EVENT_PORT_UP
+                MLX4_PORT_STATE_IB_EVENT_GID_VALID
+                MLX4_PORT_STATE_IB_EVENT_GID_INVALID
+***************************************************************************/
+int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave,
+                                  u8 port, int event,
+                                  enum slave_port_gen_event *gen_event)
+{
+        struct mlx4_priv *priv = mlx4_priv(dev);
+        struct mlx4_slave_state *ctx = NULL;
+        unsigned long flags;
+        int ret = -1;
+        enum slave_port_state cur_state =
+                mlx4_get_slave_port_state(dev, slave, port);
+
+        *gen_event = SLAVE_PORT_GEN_EVENT_NONE;
+
+        if (slave >= dev->num_slaves || port > MLX4_MAX_PORTS || port == 0) {
+                pr_err("%s: Error: asking for slave:%d, port:%d\n",
+                       __func__, slave, port);
+                return ret;
+        }
+
+        ctx = &priv->mfunc.master.slave_state[slave];
+        spin_lock_irqsave(&ctx->lock, flags);
+
+        mlx4_dbg(dev, "%s: slave: %d, current state: %d new event :%d\n",
+                 __func__, slave, cur_state, event);
+
+        switch (cur_state) {
+        case SLAVE_PORT_DOWN:
+                if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event)
+                        mlx4_set_slave_port_state(dev, slave, port,
+                                                  SLAVE_PENDING_UP);
+                break;
+        case SLAVE_PENDING_UP:
+                if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event)
+                        mlx4_set_slave_port_state(dev, slave, port,
+                                                  SLAVE_PORT_DOWN);
+                else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) {
+                        mlx4_set_slave_port_state(dev, slave, port,
+                                                  SLAVE_PORT_UP);
+                        *gen_event = SLAVE_PORT_GEN_EVENT_UP;
+                }
+                break;
+        case SLAVE_PORT_UP:
+                if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) {
+                        mlx4_set_slave_port_state(dev, slave, port,
+                                                  SLAVE_PORT_DOWN);
+                        *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
+                } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID ==
+                                event) {
+                        mlx4_set_slave_port_state(dev, slave, port,
+                                                  SLAVE_PENDING_UP);
+                        *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
+                }
+                break;
+        default:
+                pr_err("%s: BUG!!! UNKNOWN state: "
+                       "slave:%d, port:%d\n", __func__, slave, port);
+                        goto out;
+        }
+        ret = mlx4_get_slave_port_state(dev, slave, port);
+        mlx4_dbg(dev, "%s: slave: %d, current state: %d new event"
+                 " :%d gen_event: %d\n",
+                 __func__, slave, cur_state, event, *gen_event);
+
+out:
+        spin_unlock_irqrestore(&ctx->lock, flags);
+        return ret;
+}
+
+EXPORT_SYMBOL(set_and_calc_slave_port_state);
+
+int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr)
+{
+        struct mlx4_eqe eqe;
+
+        memset(&eqe, 0, sizeof eqe);
+
+        eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
+        eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO;
+        eqe.event.port_mgmt_change.port = port;
+        eqe.event.port_mgmt_change.params.port_info.changed_attr =
+                cpu_to_be32((u32) attr);
+
+        slave_event(dev, ALL_SLAVES, &eqe);
+        return 0;
+}
+EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev);
+
 void mlx4_master_handle_slave_flr(struct work_struct *work)
 {
         struct mlx4_mfunc_master_ctx *master =
@@ -251,6 +441,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
         u32 flr_slave;
         u8 update_slave_state;
         int i;
+        enum slave_port_gen_event gen_event;
 
         while ((eqe = next_eqe_sw(eq))) {
                 /*
@@ -347,35 +538,49 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
                 case MLX4_EVENT_TYPE_PORT_CHANGE:
                         port = be32_to_cpu(eqe->event.port_change.port) >> 28;
                         if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
-                                mlx4_dispatch_event(dev,
-                                                    MLX4_DEV_EVENT_PORT_DOWN,
+                                mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
                                                     port);
                                 mlx4_priv(dev)->sense.do_sense_port[port] = 1;
-                                if (mlx4_is_master(dev))
-                                        /*change the state of all slave's port
-                                        * to down:*/
-                                        for (i = 0; i < dev->num_slaves; i++) {
-                                                mlx4_dbg(dev, "%s: Sending "
-                                                         "MLX4_PORT_CHANGE_SUBTYPE_DOWN"
+                                if (!mlx4_is_master(dev))
+                                        break;
+                                for (i = 0; i < dev->num_slaves; i++) {
+                                        if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) {
+                                                if (i == mlx4_master_func_num(dev))
+                                                        continue;
+                                                mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN"
                                                          " to slave: %d, port:%d\n",
                                                          __func__, i, port);
-                                                if (i == dev->caps.function)
-                                                        continue;
                                                 mlx4_slave_event(dev, i, eqe);
+                                        } else {  /* IB port */
+                                                set_and_calc_slave_port_state(dev, i, port,
+                                                                              MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
+                                                                              &gen_event);
+                                                /*we can be in pending state, then do not send port_down event*/
+                                                if (SLAVE_PORT_GEN_EVENT_DOWN ==  gen_event) {
+                                                        if (i == mlx4_master_func_num(dev))
+                                                                continue;
+                                                        mlx4_slave_event(dev, i, eqe);
+                                                }
                                         }
+                                }
                         } else {
-                                mlx4_dispatch_event(dev,
-                                                    MLX4_DEV_EVENT_PORT_UP,
-                                                    port);
+                                mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port);
+
                                 mlx4_priv(dev)->sense.do_sense_port[port] = 0;
 
-                                if (mlx4_is_master(dev)) {
+                                if (!mlx4_is_master(dev))
+                                        break;
+                                if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
                                         for (i = 0; i < dev->num_slaves; i++) {
-                                                if (i == dev->caps.function)
+                                                if (i == mlx4_master_func_num(dev))
                                                         continue;
                                                 mlx4_slave_event(dev, i, eqe);
                                         }
-                                }
+                                else /* IB port */
+                                        /* port-up event will be sent to a slave when the
+                                         * slave's alias-guid is set. This is done in alias_GUID.c
+                                         */
+                                        set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP);
                         }
                         break;
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
index 7d27c3158d0c..23f74759f403 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h
@@ -452,6 +452,7 @@ struct mlx4_slave_state {
         /*initialized via the kzalloc*/
         u8 is_slave_going_down;
         u32 cookie;
+        enum slave_port_state port_state[MLX4_MAX_PORTS + 1];
 };
 
 struct slave_list {
diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h
index 9803fd5d3dba..380e01671d1e 100644
--- a/include/linux/mlx4/device.h
+++ b/include/linux/mlx4/device.h
@@ -192,6 +192,25 @@ enum {
         MLX4_FATAL_WARNING_SUBTYPE_WARMING = 0,
 };
 
+enum slave_port_state {
+        SLAVE_PORT_DOWN = 0,
+        SLAVE_PENDING_UP,
+        SLAVE_PORT_UP,
+};
+
+enum slave_port_gen_event {
+        SLAVE_PORT_GEN_EVENT_DOWN = 0,
+        SLAVE_PORT_GEN_EVENT_UP,
+        SLAVE_PORT_GEN_EVENT_NONE,
+};
+
+enum slave_port_state_event {
+        MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
+        MLX4_PORT_STATE_DEV_EVENT_PORT_UP,
+        MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID,
+        MLX4_PORT_STATE_IB_EVENT_GID_INVALID,
+};
+
 enum {
         MLX4_PERM_LOCAL_READ    = 1 << 10,
         MLX4_PERM_LOCAL_WRITE   = 1 << 11,
@@ -945,4 +964,13 @@ void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port,
 
 int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey);
 
+int mlx4_is_slave_active(struct mlx4_dev *dev, int slave);
+int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port);
+int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port);
+int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, u8 port, int attr);
+int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change);
+enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave, u8 port);
+int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave, u8 port, int event, enum slave_port_gen_event *gen_event);
+
+
 #endif /* MLX4_DEVICE_H */