mlx4_core: Multiple port type support

Multi-protocol adapters support different port types.  Each consumer
of mlx4_core queries for supported port types; in particular mlx4_ib
can no longer assume that all physical ports belong to it.  Port type
is configured through a sysfs interface.  When the type of a port is
changed, all mlx4 interfaces are unregistered, and then registered
again with the new port types.

Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

5 files changed, 239 insertions, 6 deletions

diff --git a/drivers/net/mlx4/fw.c b/drivers/net/mlx4/fw.c
index 8d402db9a03d..be09fdb79cb8 100644
--- a/drivers/net/mlx4/fw.c
+++ b/drivers/net/mlx4/fw.c
@@ -88,6 +88,7 @@ static void dump_dev_cap_flags(struct mlx4_dev *dev, u32 flags)
                 [ 8] = "P_Key violation counter",
                 [ 9] = "Q_Key violation counter",
                 [10] = "VMM",
+                [12] = "DPDP",
                 [16] = "MW support",
                 [17] = "APM support",
                 [18] = "Atomic ops support",
@@ -354,6 +355,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                         dev_cap->max_pkeys[i]      = 1 << (field & 0xf);
                 }
         } else {
+#define QUERY_PORT_SUPPORTED_TYPE_OFFSET        0x00
 #define QUERY_PORT_MTU_OFFSET                   0x01
 #define QUERY_PORT_ETH_MTU_OFFSET               0x02
 #define QUERY_PORT_WIDTH_OFFSET                 0x06
@@ -368,6 +370,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                         if (err)
                                 goto out;
 
+                        MLX4_GET(field, outbox, QUERY_PORT_SUPPORTED_TYPE_OFFSET);
+                        dev_cap->supported_port_types[i] = field & 3;
                         MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET);
                         dev_cap->ib_mtu[i]         = field & 0xf;
                         MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET);
diff --git a/drivers/net/mlx4/fw.h b/drivers/net/mlx4/fw.h
index d0913d4d262a..526d7f30c041 100644
--- a/drivers/net/mlx4/fw.h
+++ b/drivers/net/mlx4/fw.h
@@ -104,6 +104,7 @@ struct mlx4_dev_cap {
         u32 reserved_lkey;
         u64 max_icm_sz;
         int max_gso_sz;
+        u8  supported_port_types[MLX4_MAX_PORTS + 1];
         u8  log_max_macs[MLX4_MAX_PORTS + 1];
         u8  log_max_vlans[MLX4_MAX_PORTS + 1];
 };
diff --git a/drivers/net/mlx4/main.c b/drivers/net/mlx4/main.c
index 0a5c8bfb3f1f..c1d447873bf1 100644
--- a/drivers/net/mlx4/main.c
+++ b/drivers/net/mlx4/main.c
@@ -98,6 +98,44 @@ module_param_named(use_prio, use_prio, bool, 0444);
 MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
                   "(0/1, default 0)");
 
+static int mlx4_check_port_params(struct mlx4_dev *dev,
+                                  enum mlx4_port_type *port_type)
+{
+        int i;
+
+        for (i = 0; i < dev->caps.num_ports - 1; i++) {
+                if (port_type[i] != port_type[i+1] &&
+                    !(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
+                        mlx4_err(dev, "Only same port types supported "
+                                 "on this HCA, aborting.\n");
+                        return -EINVAL;
+                }
+        }
+        if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
+            (port_type[1] == MLX4_PORT_TYPE_IB)) {
+                mlx4_err(dev, "eth-ib configuration is not supported.\n");
+                return -EINVAL;
+        }
+
+        for (i = 0; i < dev->caps.num_ports; i++) {
+                if (!(port_type[i] & dev->caps.supported_type[i+1])) {
+                        mlx4_err(dev, "Requested port type for port %d is not "
+                                      "supported on this HCA\n", i + 1);
+                        return -EINVAL;
+                }
+        }
+        return 0;
+}
+
+static void mlx4_set_port_mask(struct mlx4_dev *dev)
+{
+        int i;
+
+        dev->caps.port_mask = 0;
+        for (i = 1; i <= dev->caps.num_ports; ++i)
+                if (dev->caps.port_type[i] == MLX4_PORT_TYPE_IB)
+                        dev->caps.port_mask |= 1 << (i - 1);
+}
 static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
 {
         int err;
@@ -139,6 +177,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                 dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
                 dev->caps.eth_mtu_cap[i]    = dev_cap->eth_mtu[i];
                 dev->caps.def_mac[i]        = dev_cap->def_mac[i];
+                dev->caps.supported_type[i] = dev_cap->supported_port_types[i];
         }
 
         dev->caps.num_uars           = dev_cap->uar_size / PAGE_SIZE;
@@ -182,6 +221,11 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         dev->caps.log_num_prios = use_prio ? 3 : 0;
 
         for (i = 1; i <= dev->caps.num_ports; ++i) {
+                if (dev->caps.supported_type[i] != MLX4_PORT_TYPE_ETH)
+                        dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
+                else
+                        dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
+
                 if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
                         dev->caps.log_num_macs = dev_cap->log_max_macs[i];
                         mlx4_warn(dev, "Requested number of MACs is too much "
@@ -196,6 +240,8 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
                 }
         }
 
+        mlx4_set_port_mask(dev);
+
         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
         dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
                 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
@@ -213,6 +259,95 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
         return 0;
 }
 
+/*
+ * Change the port configuration of the device.
+ * Every user of this function must hold the port mutex.
+ */
+static int mlx4_change_port_types(struct mlx4_dev *dev,
+                                  enum mlx4_port_type *port_types)
+{
+        int err = 0;
+        int change = 0;
+        int port;
+
+        for (port = 0; port <  dev->caps.num_ports; port++) {
+                if (port_types[port] != dev->caps.port_type[port + 1]) {
+                        change = 1;
+                        dev->caps.port_type[port + 1] = port_types[port];
+                }
+        }
+        if (change) {
+                mlx4_unregister_device(dev);
+                for (port = 1; port <= dev->caps.num_ports; port++) {
+                        mlx4_CLOSE_PORT(dev, port);
+                        err = mlx4_SET_PORT(dev, port);
+                        if (err) {
+                                mlx4_err(dev, "Failed to set port %d, "
+                                              "aborting\n", port);
+                                goto out;
+                        }
+                }
+                mlx4_set_port_mask(dev);
+                err = mlx4_register_device(dev);
+        }
+
+out:
+        return err;
+}
+
+static ssize_t show_port_type(struct device *dev,
+                              struct device_attribute *attr,
+                              char *buf)
+{
+        struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
+                                                   port_attr);
+        struct mlx4_dev *mdev = info->dev;
+
+        return sprintf(buf, "%s\n",
+                       mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB ?
+                       "ib" : "eth");
+}
+
+static ssize_t set_port_type(struct device *dev,
+                             struct device_attribute *attr,
+                             const char *buf, size_t count)
+{
+        struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
+                                                   port_attr);
+        struct mlx4_dev *mdev = info->dev;
+        struct mlx4_priv *priv = mlx4_priv(mdev);
+        enum mlx4_port_type types[MLX4_MAX_PORTS];
+        int i;
+        int err = 0;
+
+        if (!strcmp(buf, "ib\n"))
+                info->tmp_type = MLX4_PORT_TYPE_IB;
+        else if (!strcmp(buf, "eth\n"))
+                info->tmp_type = MLX4_PORT_TYPE_ETH;
+        else {
+                mlx4_err(mdev, "%s is not supported port type\n", buf);
+                return -EINVAL;
+        }
+
+        mutex_lock(&priv->port_mutex);
+        for (i = 0; i < mdev->caps.num_ports; i++)
+                types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
+                                        mdev->caps.port_type[i+1];
+
+        err = mlx4_check_port_params(mdev, types);
+        if (err)
+                goto out;
+
+        for (i = 1; i <= mdev->caps.num_ports; i++)
+                priv->port[i].tmp_type = 0;
+
+        err = mlx4_change_port_types(mdev, types);
+
+out:
+        mutex_unlock(&priv->port_mutex);
+        return err ? err : count;
+}
+
 static int mlx4_load_fw(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
@@ -617,6 +752,7 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
 {
         struct mlx4_priv *priv = mlx4_priv(dev);
         int err;
+        int port;
 
         err = mlx4_init_uar_table(dev);
         if (err) {
@@ -715,8 +851,20 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
                 goto err_qp_table_free;
         }
 
+        for (port = 1; port <= dev->caps.num_ports; port++) {
+                err = mlx4_SET_PORT(dev, port);
+                if (err) {
+                        mlx4_err(dev, "Failed to set port %d, aborting\n",
+                                port);
+                        goto err_mcg_table_free;
+                }
+        }
+
         return 0;
 
+err_mcg_table_free:
+        mlx4_cleanup_mcg_table(dev);
+
 err_qp_table_free:
         mlx4_cleanup_qp_table(dev);
 
@@ -780,14 +928,37 @@ no_msi:
                 priv->eq_table.eq[i].irq = dev->pdev->irq;
 }
 
-static void mlx4_init_port_info(struct mlx4_dev *dev, int port)
+static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
 {
         struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
+        int err = 0;
 
         info->dev = dev;
         info->port = port;
         mlx4_init_mac_table(dev, &info->mac_table);
         mlx4_init_vlan_table(dev, &info->vlan_table);
+
+        sprintf(info->dev_name, "mlx4_port%d", port);
+        info->port_attr.attr.name = info->dev_name;
+        info->port_attr.attr.mode = S_IRUGO | S_IWUSR;
+        info->port_attr.show      = show_port_type;
+        info->port_attr.store     = set_port_type;
+
+        err = device_create_file(&dev->pdev->dev, &info->port_attr);
+        if (err) {
+                mlx4_err(dev, "Failed to create file for port %d\n", port);
+                info->port = -1;
+        }
+
+        return err;
+}
+
+static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
+{
+        if (info->port < 0)
+                return;
+
+        device_remove_file(&info->dev->pdev->dev, &info->port_attr);
 }
 
 static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -870,6 +1041,8 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
         INIT_LIST_HEAD(&priv->ctx_list);
         spin_lock_init(&priv->ctx_lock);
 
+        mutex_init(&priv->port_mutex);
+
         INIT_LIST_HEAD(&priv->pgdir_list);
         mutex_init(&priv->pgdir_mutex);
 
@@ -905,18 +1078,24 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
         if (err)
                 goto err_close;
 
-        for (port = 1; port <= dev->caps.num_ports; port++)
-                mlx4_init_port_info(dev, port);
+        for (port = 1; port <= dev->caps.num_ports; port++) {
+                err = mlx4_init_port_info(dev, port);
+                if (err)
+                        goto err_port;
+        }
 
         err = mlx4_register_device(dev);
         if (err)
-                goto err_cleanup;
+                goto err_port;
 
         pci_set_drvdata(pdev, dev);
 
         return 0;
 
-err_cleanup:
+err_port:
+        for (port = 1; port <= dev->caps.num_ports; port++)
+                mlx4_cleanup_port_info(&priv->port[port]);
+
         mlx4_cleanup_mcg_table(dev);
         mlx4_cleanup_qp_table(dev);
         mlx4_cleanup_srq_table(dev);
@@ -973,8 +1152,10 @@ static void mlx4_remove_one(struct pci_dev *pdev)
         if (dev) {
                 mlx4_unregister_device(dev);
 
-                for (p = 1; p <= dev->caps.num_ports; ++p)
+                for (p = 1; p <= dev->caps.num_ports; p++) {
+                        mlx4_cleanup_port_info(&priv->port[p]);
                         mlx4_CLOSE_PORT(dev, p);
+                }
 
                 mlx4_cleanup_mcg_table(dev);
                 mlx4_cleanup_qp_table(dev);
@@ -1026,10 +1207,28 @@ static struct pci_driver mlx4_driver = {
         .remove         = __devexit_p(mlx4_remove_one)
 };
 
+static int __init mlx4_verify_params(void)
+{
+        if ((log_num_mac < 0) || (log_num_mac > 7)) {
+                printk(KERN_WARNING "mlx4_core: bad num_mac: %d\n", log_num_mac);
+                return -1;
+        }
+
+        if ((log_num_vlan < 0) || (log_num_vlan > 7)) {
+                printk(KERN_WARNING "mlx4_core: bad num_vlan: %d\n", log_num_vlan);
+                return -1;
+        }
+
+        return 0;
+}
+
 static int __init mlx4_init(void)
 {
         int ret;
 
+        if (mlx4_verify_params())
+                return -EINVAL;
+
         ret = mlx4_catas_init();
         if (ret)
                 return ret;
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 23309f381ee3..fa431fad0eec 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -277,6 +277,9 @@ struct mlx4_vlan_table {
 struct mlx4_port_info {
         struct mlx4_dev        *dev;
         int                     port;
+        char                    dev_name[16];
+        struct device_attribute port_attr;
+        enum mlx4_port_type     tmp_type;
         struct mlx4_mac_table   mac_table;
         struct mlx4_vlan_table  vlan_table;
 };
@@ -310,6 +313,7 @@ struct mlx4_priv {
         struct mlx4_uar         driver_uar;
         void __iomem           *kar;
         struct mlx4_port_info   port[MLX4_MAX_PORTS + 1];
+        struct mutex            port_mutex;
 };
 
 static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
@@ -383,4 +387,6 @@ void mlx4_handle_catas_err(struct mlx4_dev *dev);
 void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
 void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
 
+int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port);
+
 #endif /* MLX4_H */
diff --git a/drivers/net/mlx4/port.c b/drivers/net/mlx4/port.c
index 8644f3d978ee..e2fdab42c4ce 100644
--- a/drivers/net/mlx4/port.c
+++ b/drivers/net/mlx4/port.c
@@ -257,3 +257,26 @@ out:
         mutex_unlock(&table->mutex);
 }
 EXPORT_SYMBOL_GPL(mlx4_unregister_vlan);
+
+int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port)
+{
+        struct mlx4_cmd_mailbox *mailbox;
+        int err;
+        u8 is_eth = dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH;
+
+        mailbox = mlx4_alloc_cmd_mailbox(dev);
+        if (IS_ERR(mailbox))
+                return PTR_ERR(mailbox);
+
+        memset(mailbox->buf, 0, 256);
+        if (is_eth) {
+                ((u8 *) mailbox->buf)[3] = 6;
+                ((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15);
+                ((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15);
+        }
+        err = mlx4_cmd(dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT,
+                       MLX4_CMD_TIME_CLASS_B);
+
+        mlx4_free_cmd_mailbox(dev, mailbox);
+        return err;
+}