Merge branch 'pci/virtualization' into next

* pci/virtualization:
  PCI: Document reset method return values
  PCI: Detach driver before procfs & sysfs teardown on device remove
  PCI: Apply Cavium ThunderX ACS quirk to more Root Ports
  PCI: Set Cavium ACS capability quirk flags to assert RR/CR/SV/UF
  PCI: Restore ARI Capable Hierarchy before setting numVFs
  PCI: Create SR-IOV virtfn/physfn links before attaching driver
  PCI: Expose SR-IOV offset, stride, and VF device ID via sysfs
  PCI: Cache the VF device ID in the SR-IOV structure
  PCI: Add Kconfig PCI_IOV dependency for PCI_REALLOC_ENABLE_AUTO
  PCI: Remove unused function __pci_reset_function()
  PCI: Remove reset argument from pci_iov_{add,remove}_virtfn()

9 files changed, 95 insertions, 66 deletions

diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index 8b840191df59..4e1b433f6cb5 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -441,7 +441,7 @@ static void *eeh_add_virt_device(void *data, void *userdata)
         }
 
 #ifdef CONFIG_PPC_POWERNV
-        pci_iov_add_virtfn(edev->physfn, pdn->vf_index, 0);
+        pci_iov_add_virtfn(edev->physfn, pdn->vf_index);
 #endif
         return NULL;
 }
@@ -499,7 +499,7 @@ static void *eeh_rmv_device(void *data, void *userdata)
 #ifdef CONFIG_PPC_POWERNV
                 struct pci_dn *pdn = eeh_dev_to_pdn(edev);
 
-                pci_iov_remove_virtfn(edev->physfn, pdn->vf_index, 0);
+                pci_iov_remove_virtfn(edev->physfn, pdn->vf_index);
                 edev->pdev = NULL;
 
                 /*
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 658d40af887a..90944667ccea 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -51,13 +51,13 @@ config PCI_DEBUG
 config PCI_REALLOC_ENABLE_AUTO
         bool "Enable PCI resource re-allocation detection"
         depends on PCI
+        depends on PCI_IOV
         help
           Say Y here if you want the PCI core to detect if PCI resource
           re-allocation needs to be enabled. You can always use pci=realloc=on
-          or pci=realloc=off to override it.  Note this feature is a no-op
-          unless PCI_IOV support is also enabled; in that case it will
-          automatically re-allocate PCI resources if SR-IOV BARs have not
-          been allocated by the BIOS.
+          or pci=realloc=off to override it.  It will automatically
+          re-allocate PCI resources if SR-IOV BARs have not been allocated by
+          the BIOS.
 
           When in doubt, say N.
 
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index ac41c8be9200..6bacb8995e96 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -113,7 +113,7 @@ resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
         return dev->sriov->barsz[resno - PCI_IOV_RESOURCES];
 }
 
-int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
+int pci_iov_add_virtfn(struct pci_dev *dev, int id)
 {
         int i;
         int rc = -ENOMEM;
@@ -134,7 +134,7 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 
         virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
         virtfn->vendor = dev->vendor;
-        pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
+        virtfn->device = iov->vf_device;
         rc = pci_setup_device(virtfn);
         if (rc)
                 goto failed0;
@@ -157,12 +157,8 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
                 BUG_ON(rc);
         }
 
-        if (reset)
-                __pci_reset_function(virtfn);
-
         pci_device_add(virtfn, virtfn->bus);
 
-        pci_bus_add_device(virtfn);
         sprintf(buf, "virtfn%u", id);
         rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf);
         if (rc)
@@ -173,6 +169,8 @@ int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
 
         kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE);
 
+        pci_bus_add_device(virtfn);
+
         return 0;
 
 failed2:
@@ -187,7 +185,7 @@ failed:
         return rc;
 }
 
-void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
+void pci_iov_remove_virtfn(struct pci_dev *dev, int id)
 {
         char buf[VIRTFN_ID_LEN];
         struct pci_dev *virtfn;
@@ -198,11 +196,6 @@ void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset)
         if (!virtfn)
                 return;
 
-        if (reset) {
-                device_release_driver(&virtfn->dev);
-                __pci_reset_function(virtfn);
-        }
-
         sprintf(buf, "virtfn%u", id);
         sysfs_remove_link(&dev->dev.kobj, buf);
         /*
@@ -317,7 +310,7 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
         pci_cfg_access_unlock(dev);
 
         for (i = 0; i < initial; i++) {
-                rc = pci_iov_add_virtfn(dev, i, 0);
+                rc = pci_iov_add_virtfn(dev, i);
                 if (rc)
                         goto failed;
         }
@@ -329,7 +322,7 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 
 failed:
         while (i--)
-                pci_iov_remove_virtfn(dev, i, 0);
+                pci_iov_remove_virtfn(dev, i);
 
 err_pcibios:
         iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
@@ -356,7 +349,7 @@ static void sriov_disable(struct pci_dev *dev)
                 return;
 
         for (i = 0; i < iov->num_VFs; i++)
-                pci_iov_remove_virtfn(dev, i, 0);
+                pci_iov_remove_virtfn(dev, i);
 
         iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
         pci_cfg_access_lock(dev);
@@ -449,6 +442,7 @@ found:
         iov->nres = nres;
         iov->ctrl = ctrl;
         iov->total_VFs = total;
+        pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device);
         iov->pgsz = pgsz;
         iov->self = dev;
         iov->drivers_autoprobe = true;
@@ -504,6 +498,14 @@ static void sriov_restore_state(struct pci_dev *dev)
         if (ctrl & PCI_SRIOV_CTRL_VFE)
                 return;
 
+        /*
+         * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because
+         * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI.
+         */
+        ctrl &= ~PCI_SRIOV_CTRL_ARI;
+        ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI;
+        pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl);
+
         for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++)
                 pci_update_resource(dev, i);
 
@@ -724,7 +726,7 @@ int pci_vfs_assigned(struct pci_dev *dev)
          * determine the device ID for the VFs, the vendor ID will be the
          * same as the PF so there is no need to check for that one
          */
-        pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_VF_DID, &dev_id);
+        dev_id = dev->sriov->vf_device;
 
         /* loop through all the VFs to see if we own any that are assigned */
         vfdev = pci_get_device(dev->vendor, dev_id, NULL);
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 5aa81ad91a88..720090967444 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -648,6 +648,33 @@ exit:
         return count;
 }
 
+static ssize_t sriov_offset_show(struct device *dev,
+                                 struct device_attribute *attr,
+                                 char *buf)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+
+        return sprintf(buf, "%u\n", pdev->sriov->offset);
+}
+
+static ssize_t sriov_stride_show(struct device *dev,
+                                 struct device_attribute *attr,
+                                 char *buf)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+
+        return sprintf(buf, "%u\n", pdev->sriov->stride);
+}
+
+static ssize_t sriov_vf_device_show(struct device *dev,
+                                    struct device_attribute *attr,
+                                    char *buf)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+
+        return sprintf(buf, "%x\n", pdev->sriov->vf_device);
+}
+
 static ssize_t sriov_drivers_autoprobe_show(struct device *dev,
                                             struct device_attribute *attr,
                                             char *buf)
@@ -676,6 +703,9 @@ static struct device_attribute sriov_totalvfs_attr = __ATTR_RO(sriov_totalvfs);
 static struct device_attribute sriov_numvfs_attr =
                 __ATTR(sriov_numvfs, (S_IRUGO|S_IWUSR|S_IWGRP),
                        sriov_numvfs_show, sriov_numvfs_store);
+static struct device_attribute sriov_offset_attr = __ATTR_RO(sriov_offset);
+static struct device_attribute sriov_stride_attr = __ATTR_RO(sriov_stride);
+static struct device_attribute sriov_vf_device_attr = __ATTR_RO(sriov_vf_device);
 static struct device_attribute sriov_drivers_autoprobe_attr =
                 __ATTR(sriov_drivers_autoprobe, (S_IRUGO|S_IWUSR|S_IWGRP),
                        sriov_drivers_autoprobe_show, sriov_drivers_autoprobe_store);
@@ -1748,6 +1778,9 @@ static const struct attribute_group pci_dev_hp_attr_group = {
 static struct attribute *sriov_dev_attrs[] = {
         &sriov_totalvfs_attr.attr,
         &sriov_numvfs_attr.attr,
+        &sriov_offset_attr.attr,
+        &sriov_stride_attr.attr,
+        &sriov_vf_device_attr.attr,
         &sriov_drivers_autoprobe_attr.attr,
         NULL,
 };
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index c5172b01164d..50f148f154ab 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -4247,35 +4247,6 @@ static void pci_dev_restore(struct pci_dev *dev)
 }
 
 /**
- * __pci_reset_function - reset a PCI device function
- * @dev: PCI device to reset
- *
- * Some devices allow an individual function to be reset without affecting
- * other functions in the same device.  The PCI device must be responsive
- * to PCI config space in order to use this function.
- *
- * The device function is presumed to be unused when this function is called.
- * Resetting the device will make the contents of PCI configuration space
- * random, so any caller of this must be prepared to reinitialise the
- * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
- * etc.
- *
- * Returns 0 if the device function was successfully reset or negative if the
- * device doesn't support resetting a single function.
- */
-int __pci_reset_function(struct pci_dev *dev)
-{
-        int ret;
-
-        pci_dev_lock(dev);
-        ret = __pci_reset_function_locked(dev);
-        pci_dev_unlock(dev);
-
-        return ret;
-}
-EXPORT_SYMBOL_GPL(__pci_reset_function);
-
-/**
  * __pci_reset_function_locked - reset a PCI device function while holding
  * the @dev mutex lock.
  * @dev: PCI device to reset
@@ -4300,6 +4271,14 @@ int __pci_reset_function_locked(struct pci_dev *dev)
 
         might_sleep();
 
+        /*
+         * A reset method returns -ENOTTY if it doesn't support this device
+         * and we should try the next method.
+         *
+         * If it returns 0 (success), we're finished.  If it returns any
+         * other error, we're also finished: this indicates that further
+         * reset mechanisms might be broken on the device.
+         */
         rc = pci_dev_specific_reset(dev, 0);
         if (rc != -ENOTTY)
                 return rc;
@@ -4365,8 +4344,8 @@ int pci_probe_reset_function(struct pci_dev *dev)
  *
  * This function does not just reset the PCI portion of a device, but
  * clears all the state associated with the device.  This function differs
- * from __pci_reset_function in that it saves and restores device state
- * over the reset.
+ * from __pci_reset_function_locked() in that it saves and restores device state
+ * over the reset and takes the PCI device lock.
  *
  * Returns 0 if the device function was successfully reset or negative if the
  * device doesn't support resetting a single function.
@@ -4401,7 +4380,7 @@ EXPORT_SYMBOL_GPL(pci_reset_function);
  *
  * This function does not just reset the PCI portion of a device, but
  * clears all the state associated with the device.  This function differs
- * from __pci_reset_function() in that it saves and restores device state
+ * from __pci_reset_function_locked() in that it saves and restores device state
  * over the reset.  It also differs from pci_reset_function() in that it
  * requires the PCI device lock to be held.
  *
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 186278fa6917..39eaacd122d8 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -263,6 +263,7 @@ struct pci_sriov {
         u16 num_VFs;            /* number of VFs available */
         u16 offset;             /* first VF Routing ID offset */
         u16 stride;             /* following VF stride */
+        u16 vf_device;          /* VF device ID */
         u32 pgsz;               /* page size for BAR alignment */
         u8 link;                /* Function Dependency Link */
         u8 max_VF_buses;        /* max buses consumed by VFs */
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index a4d33619a7bb..0e22cce05742 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -4211,17 +4211,32 @@ static int pci_quirk_amd_sb_acs(struct pci_dev *dev, u16 acs_flags)
 #endif
 }
 
+static bool pci_quirk_cavium_acs_match(struct pci_dev *dev)
+{
+        /*
+         * Effectively selects all downstream ports for whole ThunderX 1
+         * family by 0xf800 mask (which represents 8 SoCs), while the lower
+         * bits of device ID are used to indicate which subdevice is used
+         * within the SoC.
+         */
+        return (pci_is_pcie(dev) &&
+                (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) &&
+                ((dev->device & 0xf800) == 0xa000));
+}
+
 static int pci_quirk_cavium_acs(struct pci_dev *dev, u16 acs_flags)
 {
         /*
-         * Cavium devices matching this quirk do not perform peer-to-peer
-         * with other functions, allowing masking out these bits as if they
-         * were unimplemented in the ACS capability.
+         * Cavium root ports don't advertise an ACS capability.  However,
+         * the RTL internally implements similar protection as if ACS had
+         * Request Redirection, Completion Redirection, Source Validation,
+         * and Upstream Forwarding features enabled.  Assert that the
+         * hardware implements and enables equivalent ACS functionality for
+         * these flags.
          */
-        acs_flags &= ~(PCI_ACS_SV | PCI_ACS_TB | PCI_ACS_RR |
-                       PCI_ACS_CR | PCI_ACS_UF | PCI_ACS_DT);
+        acs_flags &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_SV | PCI_ACS_UF);
 
-        if (!((dev->device >= 0xa000) && (dev->device <= 0xa0ff)))
+        if (!pci_quirk_cavium_acs_match(dev))
                 return -ENOTTY;
 
         return acs_flags ? 0 : 1;
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index 73a03d382590..2fa0dbde36b7 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -19,9 +19,9 @@ static void pci_stop_dev(struct pci_dev *dev)
         pci_pme_active(dev, false);
 
         if (dev->is_added) {
+                device_release_driver(&dev->dev);
                 pci_proc_detach_device(dev);
                 pci_remove_sysfs_dev_files(dev);
-                device_release_driver(&dev->dev);
                 dev->is_added = 0;
         }
 
diff --git a/include/linux/pci.h b/include/linux/pci.h
index f3b6909305ca..d85c626e1228 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1093,7 +1093,6 @@ int pcie_set_mps(struct pci_dev *dev, int mps);
 int pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed,
                           enum pcie_link_width *width);
 void pcie_flr(struct pci_dev *dev);
-int __pci_reset_function(struct pci_dev *dev);
 int __pci_reset_function_locked(struct pci_dev *dev);
 int pci_reset_function(struct pci_dev *dev);
 int pci_reset_function_locked(struct pci_dev *dev);
@@ -1965,8 +1964,8 @@ int pci_iov_virtfn_devfn(struct pci_dev *dev, int id);
 
 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
 void pci_disable_sriov(struct pci_dev *dev);
-int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset);
-void pci_iov_remove_virtfn(struct pci_dev *dev, int id, int reset);
+int pci_iov_add_virtfn(struct pci_dev *dev, int id);
+void pci_iov_remove_virtfn(struct pci_dev *dev, int id);
 int pci_num_vf(struct pci_dev *dev);
 int pci_vfs_assigned(struct pci_dev *dev);
 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
@@ -1983,12 +1982,12 @@ static inline int pci_iov_virtfn_devfn(struct pci_dev *dev, int id)
 }
 static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 { return -ENODEV; }
-static inline int pci_iov_add_virtfn(struct pci_dev *dev, int id, int reset)
+static inline int pci_iov_add_virtfn(struct pci_dev *dev, int id)
 {
         return -ENOSYS;
 }
 static inline void pci_iov_remove_virtfn(struct pci_dev *dev,
-                                         int id, int reset) { }
+                                         int id) { }
 static inline void pci_disable_sriov(struct pci_dev *dev) { }
 static inline int pci_num_vf(struct pci_dev *dev) { return 0; }
 static inline int pci_vfs_assigned(struct pci_dev *dev)