ixgbe: Implement PCI SR-IOV sysfs callback operation

Implement callbacks in the driver for the new PCI bus driver
interface that allows the user to enable/disable SR-IOV VFs
in a device via the sysfs interface.

Signed-off-by: Greg Rose <gregory.v.rose@intel.com>
CC: Don Dutile <ddutile@redhat.com>
Tested-by: Phil Schmitt <phillip.j.schmitt@intel.com>
Tested-by: Sibai Li <sibai.li@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>

4 files changed, 122 insertions, 6 deletions

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index cd4c83b1a977..8371ae4265fe 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -778,5 +778,8 @@ extern int ixgbe_ptp_hwtstamp_ioctl(struct ixgbe_adapter *adapter,
 extern void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_reset(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter, u32 eicr);
+#ifdef CONFIG_PCI_IOV
+void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter);
+#endif
 
 #endif /* _IXGBE_H_ */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 2472dc85db0d..5e3ed33215f7 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -6836,6 +6836,26 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 }
 
 #endif /* CONFIG_IXGBE_DCB */
+#ifdef CONFIG_PCI_IOV
+void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
+{
+        struct net_device *netdev = adapter->netdev;
+
+        rtnl_lock();
+#ifdef CONFIG_IXGBE_DCB
+        ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev));
+#else
+        if (netif_running(netdev))
+                ixgbe_close(netdev);
+        ixgbe_clear_interrupt_scheme(adapter);
+        ixgbe_init_interrupt_scheme(adapter);
+        if (netif_running(netdev))
+                ixgbe_open(netdev);
+#endif
+        rtnl_unlock();
+}
+
+#endif
 void ixgbe_do_reset(struct net_device *netdev)
 {
         struct ixgbe_adapter *adapter = netdev_priv(netdev);
@@ -7625,8 +7645,14 @@ static void ixgbe_remove(struct pci_dev *pdev)
         if (netdev->reg_state == NETREG_REGISTERED)
                 unregister_netdev(netdev);
 
-        ixgbe_disable_sriov(adapter);
-
+#ifdef CONFIG_PCI_IOV
+        /*
+         * Only disable SR-IOV on unload if the user specified the now
+         * deprecated max_vfs module parameter.
+         */
+        if (max_vfs)
+                ixgbe_disable_sriov(adapter);
+#endif
         ixgbe_clear_interrupt_scheme(adapter);
 
         ixgbe_release_hw_control(adapter);
@@ -7840,6 +7866,7 @@ static struct pci_driver ixgbe_driver = {
         .resume   = ixgbe_resume,
 #endif
         .shutdown = ixgbe_shutdown,
+        .sriov_configure = ixgbe_pci_sriov_configure,
         .err_handler = &ixgbe_err_handler
 };
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 8a284ac4b400..ee3507f0ea53 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -207,11 +207,12 @@ static bool ixgbe_vfs_are_assigned(struct ixgbe_adapter *adapter)
 }
 
 #endif /* #ifdef CONFIG_PCI_IOV */
-void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
+int ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 {
         struct ixgbe_hw *hw = &adapter->hw;
         u32 gpie;
         u32 vmdctl;
+        int rss;
 
         /* set num VFs to 0 to prevent access to vfinfo */
         adapter->num_vfs = 0;
@@ -226,7 +227,7 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 
         /* if SR-IOV is already disabled then there is nothing to do */
         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
-                return;
+                return 0;
 
 #ifdef CONFIG_PCI_IOV
         /*
@@ -236,7 +237,7 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
          */
         if (ixgbe_vfs_are_assigned(adapter)) {
                 e_dev_warn("Unloading driver while VFs are assigned - VFs will not be deallocated\n");
-                return;
+                return -EPERM;
         }
         /* disable iov and allow time for transactions to clear */
         pci_disable_sriov(adapter->pdev);
@@ -259,10 +260,94 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
                 adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
         adapter->ring_feature[RING_F_VMDQ].offset = 0;
 
+        rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus());
+        adapter->ring_feature[RING_F_RSS].limit = rss;
+
         /* take a breather then clean up driver data */
         msleep(100);
 
         adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
+        return 0;
+}
+
+static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
+{
+#ifdef CONFIG_PCI_IOV
+        struct ixgbe_adapter *adapter = pci_get_drvdata(dev);
+        int err = 0;
+        int i;
+        int pre_existing_vfs = pci_num_vf(dev);
+
+        if (pre_existing_vfs && pre_existing_vfs != num_vfs)
+                err = ixgbe_disable_sriov(adapter);
+        else if (pre_existing_vfs && pre_existing_vfs == num_vfs)
+                goto out;
+
+        if (err)
+                goto err_out;
+
+        /* While the SR-IOV capability structure reports total VFs to be
+         * 64 we limit the actual number that can be allocated to 63 so
+         * that some transmit/receive resources can be reserved to the
+         * PF.  The PCI bus driver already checks for other values out of
+         * range.
+         */
+        if (num_vfs > 63) {
+                err = -EPERM;
+                goto err_out;
+        }
+
+        adapter->num_vfs = num_vfs;
+
+        err = __ixgbe_enable_sriov(adapter);
+        if (err)
+                goto err_out;
+
+        for (i = 0; i < adapter->num_vfs; i++)
+                ixgbe_vf_configuration(dev, (i | 0x10000000));
+
+        err = pci_enable_sriov(dev, num_vfs);
+        if (err) {
+                e_dev_warn("Failed to enable PCI sriov: %d\n", err);
+                goto err_out;
+        }
+        ixgbe_sriov_reinit(adapter);
+
+out:
+        return num_vfs;
+
+err_out:
+        return err;
+#endif
+        return 0;
+}
+
+static int ixgbe_pci_sriov_disable(struct pci_dev *dev)
+{
+        struct ixgbe_adapter *adapter = pci_get_drvdata(dev);
+        int err;
+        u32 current_flags = adapter->flags;
+
+        err = ixgbe_disable_sriov(adapter);
+
+        /* Only reinit if no error and state changed */
+        if (!err && current_flags != adapter->flags) {
+                /* ixgbe_disable_sriov() doesn't clear VMDQ flag */
+                adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
+#ifdef CONFIG_PCI_IOV
+                ixgbe_sriov_reinit(adapter);
+#endif
+        }
+
+        return err;
+}
+
+int ixgbe_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
+{
+        if (num_vfs == 0)
+                return ixgbe_pci_sriov_disable(dev);
+        else
+                return ixgbe_pci_sriov_enable(dev, num_vfs);
 }
 
 static int ixgbe_set_vf_multicasts(struct ixgbe_adapter *adapter,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
index f693469b81ef..008f9cea68d1 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
@@ -41,10 +41,11 @@ int ixgbe_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting);
 int ixgbe_ndo_get_vf_config(struct net_device *netdev,
                             int vf, struct ifla_vf_info *ivi);
 void ixgbe_check_vf_rate_limit(struct ixgbe_adapter *adapter);
-void ixgbe_disable_sriov(struct ixgbe_adapter *adapter);
+int ixgbe_disable_sriov(struct ixgbe_adapter *adapter);
 #ifdef CONFIG_PCI_IOV
 void ixgbe_enable_sriov(struct ixgbe_adapter *adapter);
 #endif
+int ixgbe_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
 
 static inline void ixgbe_set_vmvir(struct ixgbe_adapter *adapter,
                                    u16 vid, u16 qos, u32 vf)