Merge branch 'dmaengine' into async-tx-next

Conflicts:
	crypto/async_tx/async_xor.c
	drivers/dma/ioat/dma_v2.h
	drivers/dma/ioat/pci.c
	drivers/md/raid5.c

10 files changed, 1211 insertions, 505 deletions

diff --git a/drivers/pci/pcie/aer/Kconfig b/drivers/pci/pcie/aer/Kconfig
index c3bde588aa13..50e94e02378a 100644
--- a/drivers/pci/pcie/aer/Kconfig
+++ b/drivers/pci/pcie/aer/Kconfig
@@ -10,3 +10,18 @@ config PCIEAER
           This enables PCI Express Root Port Advanced Error Reporting
           (AER) driver support. Error reporting messages sent to Root
           Port will be handled by PCI Express AER driver.
+
+
+#
+# PCI Express ECRC
+#
+config PCIE_ECRC
+        bool "PCI Express ECRC settings control"
+        depends on PCIEAER
+        help
+          Used to override firmware/bios settings for PCI Express ECRC
+          (transaction layer end-to-end CRC checking).
+
+          When in doubt, say N.
+
+source "drivers/pci/pcie/aer/Kconfig.debug"
diff --git a/drivers/pci/pcie/aer/Kconfig.debug b/drivers/pci/pcie/aer/Kconfig.debug
new file mode 100644
index 000000000000..b8c925c1f6aa
--- /dev/null
+++ b/drivers/pci/pcie/aer/Kconfig.debug
@@ -0,0 +1,18 @@
+#
+# PCI Express Root Port Device AER Debug Configuration
+#
+
+config PCIEAER_INJECT
+        tristate "PCIE AER error injector support"
+        depends on PCIEAER
+        default n
+        help
+          This enables PCI Express Root Port Advanced Error Reporting
+          (AER) software error injector.
+
+          Debuging PCIE AER code is quite difficult because it is hard
+          to trigger various real hardware errors. Software based
+          error injection can fake almost all kinds of errors with the
+          help of a user space helper tool aer-inject, which can be
+          gotten from:
+             http://www.kernel.org/pub/linux/utils/pci/aer-inject/
diff --git a/drivers/pci/pcie/aer/Makefile b/drivers/pci/pcie/aer/Makefile
index 8da3bd8455a8..2cba67510dc8 100644
--- a/drivers/pci/pcie/aer/Makefile
+++ b/drivers/pci/pcie/aer/Makefile
@@ -4,6 +4,9 @@
 
 obj-$(CONFIG_PCIEAER) += aerdriver.o
 
+obj-$(CONFIG_PCIE_ECRC) += ecrc.o
+
 aerdriver-objs := aerdrv_errprint.o aerdrv_core.o aerdrv.o
 aerdriver-$(CONFIG_ACPI) += aerdrv_acpi.o
 
+obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o
diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
new file mode 100644
index 000000000000..d92ae21a59d8
--- /dev/null
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -0,0 +1,473 @@
+/*
+ * PCIE AER software error injection support.
+ *
+ * Debuging PCIE AER code is quite difficult because it is hard to
+ * trigger various real hardware errors. Software based error
+ * injection can fake almost all kinds of errors with the help of a
+ * user space helper tool aer-inject, which can be gotten from:
+ *   http://www.kernel.org/pub/linux/utils/pci/aer-inject/
+ *
+ * Copyright 2009 Intel Corporation.
+ *     Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2
+ * of the License.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/miscdevice.h>
+#include <linux/pci.h>
+#include <linux/fs.h>
+#include <asm/uaccess.h>
+#include "aerdrv.h"
+
+struct aer_error_inj
+{
+        u8 bus;
+        u8 dev;
+        u8 fn;
+        u32 uncor_status;
+        u32 cor_status;
+        u32 header_log0;
+        u32 header_log1;
+        u32 header_log2;
+        u32 header_log3;
+};
+
+struct aer_error
+{
+        struct list_head list;
+        unsigned int bus;
+        unsigned int devfn;
+        int pos_cap_err;
+
+        u32 uncor_status;
+        u32 cor_status;
+        u32 header_log0;
+        u32 header_log1;
+        u32 header_log2;
+        u32 header_log3;
+        u32 root_status;
+        u32 source_id;
+};
+
+struct pci_bus_ops
+{
+        struct list_head list;
+        struct pci_bus *bus;
+        struct pci_ops *ops;
+};
+
+static LIST_HEAD(einjected);
+
+static LIST_HEAD(pci_bus_ops_list);
+
+/* Protect einjected and pci_bus_ops_list */
+static DEFINE_SPINLOCK(inject_lock);
+
+static void aer_error_init(struct aer_error *err, unsigned int bus,
+                           unsigned int devfn, int pos_cap_err)
+{
+        INIT_LIST_HEAD(&err->list);
+        err->bus = bus;
+        err->devfn = devfn;
+        err->pos_cap_err = pos_cap_err;
+}
+
+/* inject_lock must be held before calling */
+static struct aer_error *__find_aer_error(unsigned int bus, unsigned int devfn)
+{
+        struct aer_error *err;
+
+        list_for_each_entry(err, &einjected, list) {
+                if (bus == err->bus && devfn == err->devfn)
+                        return err;
+        }
+        return NULL;
+}
+
+/* inject_lock must be held before calling */
+static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
+{
+        return __find_aer_error(dev->bus->number, dev->devfn);
+}
+
+/* inject_lock must be held before calling */
+static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
+{
+        struct pci_bus_ops *bus_ops;
+
+        list_for_each_entry(bus_ops, &pci_bus_ops_list, list) {
+                if (bus_ops->bus == bus)
+                        return bus_ops->ops;
+        }
+        return NULL;
+}
+
+static struct pci_bus_ops *pci_bus_ops_pop(void)
+{
+        unsigned long flags;
+        struct pci_bus_ops *bus_ops = NULL;
+
+        spin_lock_irqsave(&inject_lock, flags);
+        if (list_empty(&pci_bus_ops_list))
+                bus_ops = NULL;
+        else {
+                struct list_head *lh = pci_bus_ops_list.next;
+                list_del(lh);
+                bus_ops = list_entry(lh, struct pci_bus_ops, list);
+        }
+        spin_unlock_irqrestore(&inject_lock, flags);
+        return bus_ops;
+}
+
+static u32 *find_pci_config_dword(struct aer_error *err, int where,
+                                  int *prw1cs)
+{
+        int rw1cs = 0;
+        u32 *target = NULL;
+
+        if (err->pos_cap_err == -1)
+                return NULL;
+
+        switch (where - err->pos_cap_err) {
+        case PCI_ERR_UNCOR_STATUS:
+                target = &err->uncor_status;
+                rw1cs = 1;
+                break;
+        case PCI_ERR_COR_STATUS:
+                target = &err->cor_status;
+                rw1cs = 1;
+                break;
+        case PCI_ERR_HEADER_LOG:
+                target = &err->header_log0;
+                break;
+        case PCI_ERR_HEADER_LOG+4:
+                target = &err->header_log1;
+                break;
+        case PCI_ERR_HEADER_LOG+8:
+                target = &err->header_log2;
+                break;
+        case PCI_ERR_HEADER_LOG+12:
+                target = &err->header_log3;
+                break;
+        case PCI_ERR_ROOT_STATUS:
+                target = &err->root_status;
+                rw1cs = 1;
+                break;
+        case PCI_ERR_ROOT_COR_SRC:
+                target = &err->source_id;
+                break;
+        }
+        if (prw1cs)
+                *prw1cs = rw1cs;
+        return target;
+}
+
+static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where,
+                        int size, u32 *val)
+{
+        u32 *sim;
+        struct aer_error *err;
+        unsigned long flags;
+        struct pci_ops *ops;
+
+        spin_lock_irqsave(&inject_lock, flags);
+        if (size != sizeof(u32))
+                goto out;
+        err = __find_aer_error(bus->number, devfn);
+        if (!err)
+                goto out;
+
+        sim = find_pci_config_dword(err, where, NULL);
+        if (sim) {
+                *val = *sim;
+                spin_unlock_irqrestore(&inject_lock, flags);
+                return 0;
+        }
+out:
+        ops = __find_pci_bus_ops(bus);
+        spin_unlock_irqrestore(&inject_lock, flags);
+        return ops->read(bus, devfn, where, size, val);
+}
+
+int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, int size,
+                  u32 val)
+{
+        u32 *sim;
+        struct aer_error *err;
+        unsigned long flags;
+        int rw1cs;
+        struct pci_ops *ops;
+
+        spin_lock_irqsave(&inject_lock, flags);
+        if (size != sizeof(u32))
+                goto out;
+        err = __find_aer_error(bus->number, devfn);
+        if (!err)
+                goto out;
+
+        sim = find_pci_config_dword(err, where, &rw1cs);
+        if (sim) {
+                if (rw1cs)
+                        *sim ^= val;
+                else
+                        *sim = val;
+                spin_unlock_irqrestore(&inject_lock, flags);
+                return 0;
+        }
+out:
+        ops = __find_pci_bus_ops(bus);
+        spin_unlock_irqrestore(&inject_lock, flags);
+        return ops->write(bus, devfn, where, size, val);
+}
+
+static struct pci_ops pci_ops_aer = {
+        .read = pci_read_aer,
+        .write = pci_write_aer,
+};
+
+static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
+                             struct pci_bus *bus,
+                             struct pci_ops *ops)
+{
+        INIT_LIST_HEAD(&bus_ops->list);
+        bus_ops->bus = bus;
+        bus_ops->ops = ops;
+}
+
+static int pci_bus_set_aer_ops(struct pci_bus *bus)
+{
+        struct pci_ops *ops;
+        struct pci_bus_ops *bus_ops;
+        unsigned long flags;
+
+        bus_ops = kmalloc(sizeof(*bus_ops), GFP_KERNEL);
+        if (!bus_ops)
+                return -ENOMEM;
+        ops = pci_bus_set_ops(bus, &pci_ops_aer);
+        spin_lock_irqsave(&inject_lock, flags);
+        if (ops == &pci_ops_aer)
+                goto out;
+        pci_bus_ops_init(bus_ops, bus, ops);
+        list_add(&bus_ops->list, &pci_bus_ops_list);
+        bus_ops = NULL;
+out:
+        spin_unlock_irqrestore(&inject_lock, flags);
+        if (bus_ops)
+                kfree(bus_ops);
+        return 0;
+}
+
+static struct pci_dev *pcie_find_root_port(struct pci_dev *dev)
+{
+        while (1) {
+                if (!dev->is_pcie)
+                        break;
+                if (dev->pcie_type == PCI_EXP_TYPE_ROOT_PORT)
+                        return dev;
+                if (!dev->bus->self)
+                        break;
+                dev = dev->bus->self;
+        }
+        return NULL;
+}
+
+static int find_aer_device_iter(struct device *device, void *data)
+{
+        struct pcie_device **result = data;
+        struct pcie_device *pcie_dev;
+
+        if (device->bus == &pcie_port_bus_type) {
+                pcie_dev = to_pcie_device(device);
+                if (pcie_dev->service & PCIE_PORT_SERVICE_AER) {
+                        *result = pcie_dev;
+                        return 1;
+                }
+        }
+        return 0;
+}
+
+static int find_aer_device(struct pci_dev *dev, struct pcie_device **result)
+{
+        return device_for_each_child(&dev->dev, result, find_aer_device_iter);
+}
+
+static int aer_inject(struct aer_error_inj *einj)
+{
+        struct aer_error *err, *rperr;
+        struct aer_error *err_alloc = NULL, *rperr_alloc = NULL;
+        struct pci_dev *dev, *rpdev;
+        struct pcie_device *edev;
+        unsigned long flags;
+        unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn);
+        int pos_cap_err, rp_pos_cap_err;
+        u32 sever;
+        int ret = 0;
+
+        dev = pci_get_bus_and_slot(einj->bus, devfn);
+        if (!dev)
+                return -EINVAL;
+        rpdev = pcie_find_root_port(dev);
+        if (!rpdev) {
+                ret = -EINVAL;
+                goto out_put;
+        }
+
+        pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+        if (!pos_cap_err) {
+                ret = -EIO;
+                goto out_put;
+        }
+        pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever);
+
+        rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR);
+        if (!rp_pos_cap_err) {
+                ret = -EIO;
+                goto out_put;
+        }
+
+        err_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
+        if (!err_alloc) {
+                ret = -ENOMEM;
+                goto out_put;
+        }
+        rperr_alloc =  kzalloc(sizeof(struct aer_error), GFP_KERNEL);
+        if (!rperr_alloc) {
+                ret = -ENOMEM;
+                goto out_put;
+        }
+
+        spin_lock_irqsave(&inject_lock, flags);
+
+        err = __find_aer_error_by_dev(dev);
+        if (!err) {
+                err = err_alloc;
+                err_alloc = NULL;
+                aer_error_init(err, einj->bus, devfn, pos_cap_err);
+                list_add(&err->list, &einjected);
+        }
+        err->uncor_status |= einj->uncor_status;
+        err->cor_status |= einj->cor_status;
+        err->header_log0 = einj->header_log0;
+        err->header_log1 = einj->header_log1;
+        err->header_log2 = einj->header_log2;
+        err->header_log3 = einj->header_log3;
+
+        rperr = __find_aer_error_by_dev(rpdev);
+        if (!rperr) {
+                rperr = rperr_alloc;
+                rperr_alloc = NULL;
+                aer_error_init(rperr, rpdev->bus->number, rpdev->devfn,
+                               rp_pos_cap_err);
+                list_add(&rperr->list, &einjected);
+        }
+        if (einj->cor_status) {
+                if (rperr->root_status & PCI_ERR_ROOT_COR_RCV)
+                        rperr->root_status |= PCI_ERR_ROOT_MULTI_COR_RCV;
+                else
+                        rperr->root_status |= PCI_ERR_ROOT_COR_RCV;
+                rperr->source_id &= 0xffff0000;
+                rperr->source_id |= (einj->bus << 8) | devfn;
+        }
+        if (einj->uncor_status) {
+                if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV)
+                        rperr->root_status |= PCI_ERR_ROOT_MULTI_UNCOR_RCV;
+                if (sever & einj->uncor_status) {
+                        rperr->root_status |= PCI_ERR_ROOT_FATAL_RCV;
+                        if (!(rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV))
+                                rperr->root_status |= PCI_ERR_ROOT_FIRST_FATAL;
+                } else
+                        rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV;
+                rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV;
+                rperr->source_id &= 0x0000ffff;
+                rperr->source_id |= ((einj->bus << 8) | devfn) << 16;
+        }
+        spin_unlock_irqrestore(&inject_lock, flags);
+
+        ret = pci_bus_set_aer_ops(dev->bus);
+        if (ret)
+                goto out_put;
+        ret = pci_bus_set_aer_ops(rpdev->bus);
+        if (ret)
+                goto out_put;
+
+        if (find_aer_device(rpdev, &edev))
+                aer_irq(-1, edev);
+        else
+                ret = -EINVAL;
+out_put:
+        if (err_alloc)
+                kfree(err_alloc);
+        if (rperr_alloc)
+                kfree(rperr_alloc);
+        pci_dev_put(dev);
+        return ret;
+}
+
+static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf,
+                                size_t usize, loff_t *off)
+{
+        struct aer_error_inj einj;
+        int ret;
+
+        if (!capable(CAP_SYS_ADMIN))
+                return -EPERM;
+
+        if (usize != sizeof(struct aer_error_inj))
+                return -EINVAL;
+
+        if (copy_from_user(&einj, ubuf, usize))
+                return -EFAULT;
+
+        ret = aer_inject(&einj);
+        return ret ? ret : usize;
+}
+
+static const struct file_operations aer_inject_fops = {
+        .write = aer_inject_write,
+        .owner = THIS_MODULE,
+};
+
+static struct miscdevice aer_inject_device = {
+        .minor = MISC_DYNAMIC_MINOR,
+        .name = "aer_inject",
+        .fops = &aer_inject_fops,
+};
+
+static int __init aer_inject_init(void)
+{
+        return misc_register(&aer_inject_device);
+}
+
+static void __exit aer_inject_exit(void)
+{
+        struct aer_error *err, *err_next;
+        unsigned long flags;
+        struct pci_bus_ops *bus_ops;
+
+        misc_deregister(&aer_inject_device);
+
+        while ((bus_ops = pci_bus_ops_pop())) {
+                pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
+                kfree(bus_ops);
+        }
+
+        spin_lock_irqsave(&inject_lock, flags);
+        list_for_each_entry_safe(err, err_next,
+                                 &pci_bus_ops_list, list) {
+                list_del(&err->list);
+                kfree(err);
+        }
+        spin_unlock_irqrestore(&inject_lock, flags);
+}
+
+module_init(aer_inject_init);
+module_exit(aer_inject_exit);
+
+MODULE_DESCRIPTION("PCIE AER software error injector");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c
index 32ade5af927e..4770f13b3ca1 100644
--- a/drivers/pci/pcie/aer/aerdrv.c
+++ b/drivers/pci/pcie/aer/aerdrv.c
@@ -77,7 +77,7 @@ void pci_no_aer(void)
  *
  * Invoked when Root Port detects AER messages.
  **/
-static irqreturn_t aer_irq(int irq, void *context)
+irqreturn_t aer_irq(int irq, void *context)
 {
         unsigned int status, id;
         struct pcie_device *pdev = (struct pcie_device *)context;
@@ -126,6 +126,7 @@ static irqreturn_t aer_irq(int irq, void *context)
 
         return IRQ_HANDLED;
 }
+EXPORT_SYMBOL_GPL(aer_irq);
 
 /**
  * aer_alloc_rpc - allocate Root Port data structure
diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h
index aa14482a4779..bbd7428ca2d0 100644
--- a/drivers/pci/pcie/aer/aerdrv.h
+++ b/drivers/pci/pcie/aer/aerdrv.h
@@ -11,6 +11,7 @@
 #include <linux/workqueue.h>
 #include <linux/pcieport_if.h>
 #include <linux/aer.h>
+#include <linux/interrupt.h>
 
 #define AER_NONFATAL                    0
 #define AER_FATAL                       1
@@ -56,7 +57,11 @@ struct header_log_regs {
         unsigned int dw3;
 };
 
+#define AER_MAX_MULTI_ERR_DEVICES       5       /* Not likely to have more */
 struct aer_err_info {
+        struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
+        int error_dev_num;
+        u16 id;
         int severity;                   /* 0:NONFATAL | 1:FATAL | 2:COR */
         int flags;
         unsigned int status;            /* COR/UNCOR Error Status */
@@ -120,6 +125,7 @@ extern void aer_delete_rootport(struct aer_rpc *rpc);
 extern int aer_init(struct pcie_device *dev);
 extern void aer_isr(struct work_struct *work);
 extern void aer_print_error(struct pci_dev *dev, struct aer_err_info *info);
+extern irqreturn_t aer_irq(int irq, void *context);
 
 #ifdef CONFIG_ACPI
 extern int aer_osc_setup(struct pcie_device *pciedev);
diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c
index 307452f30035..3d8872704a58 100644
--- a/drivers/pci/pcie/aer/aerdrv_core.c
+++ b/drivers/pci/pcie/aer/aerdrv_core.c
@@ -26,7 +26,9 @@
 #include "aerdrv.h"
 
 static int forceload;
+static int nosourceid;
 module_param(forceload, bool, 0);
+module_param(nosourceid, bool, 0);
 
 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
 {
@@ -109,19 +111,23 @@ int pci_cleanup_aer_correct_error_status(struct pci_dev *dev)
 #endif  /*  0  */
 
 
-static void set_device_error_reporting(struct pci_dev *dev, void *data)
+static int set_device_error_reporting(struct pci_dev *dev, void *data)
 {
         bool enable = *((bool *)data);
 
-        if (dev->pcie_type != PCIE_RC_PORT &&
-            dev->pcie_type != PCIE_SW_UPSTREAM_PORT &&
-            dev->pcie_type != PCIE_SW_DOWNSTREAM_PORT)
-                return;
+        if (dev->pcie_type == PCIE_RC_PORT ||
+            dev->pcie_type == PCIE_SW_UPSTREAM_PORT ||
+            dev->pcie_type == PCIE_SW_DOWNSTREAM_PORT) {
+                if (enable)
+                        pci_enable_pcie_error_reporting(dev);
+                else
+                        pci_disable_pcie_error_reporting(dev);
+        }
 
         if (enable)
-                pci_enable_pcie_error_reporting(dev);
-        else
-                pci_disable_pcie_error_reporting(dev);
+                pcie_set_ecrc_checking(dev);
+
+        return 0;
 }
 
 /**
@@ -139,73 +145,148 @@ static void set_downstream_devices_error_reporting(struct pci_dev *dev,
         pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable);
 }
 
-static int find_device_iter(struct device *device, void *data)
+static inline int compare_device_id(struct pci_dev *dev,
+                        struct aer_err_info *e_info)
 {
-        struct pci_dev *dev;
-        u16 id = *(unsigned long *)data;
-        u8 secondary, subordinate, d_bus = id >> 8;
+        if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) {
+                /*
+                 * Device ID match
+                 */
+                return 1;
+        }
 
-        if (device->bus == &pci_bus_type) {
-                dev = to_pci_dev(device);
-                if (id == ((dev->bus->number << 8) | dev->devfn)) {
-                        /*
-                         * Device ID match
-                         */
-                        *(unsigned long*)data = (unsigned long)device;
-                        return 1;
-                }
+        return 0;
+}
+
+static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
+{
+        if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
+                e_info->dev[e_info->error_dev_num] = dev;
+                e_info->error_dev_num++;
+                return 1;
+        } else
+                return 0;
+}
+
+
+#define PCI_BUS(x)      (((x) >> 8) & 0xff)
+
+static int find_device_iter(struct pci_dev *dev, void *data)
+{
+        int pos;
+        u32 status;
+        u32 mask;
+        u16 reg16;
+        int result;
+        struct aer_err_info *e_info = (struct aer_err_info *)data;
+
+        /*
+         * When bus id is equal to 0, it might be a bad id
+         * reported by root port.
+         */
+        if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
+                result = compare_device_id(dev, e_info);
+                if (result)
+                        add_error_device(e_info, dev);
 
                 /*
-                 * If device is P2P, check if it is an upstream?
+                 * If there is no multiple error, we stop
+                 * or continue based on the id comparing.
                  */
-                if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
-                        pci_read_config_byte(dev, PCI_SECONDARY_BUS,
-                                &secondary);
-                        pci_read_config_byte(dev, PCI_SUBORDINATE_BUS,
-                                &subordinate);
-                        if (d_bus >= secondary && d_bus <= subordinate) {
-                                *(unsigned long*)data = (unsigned long)device;
-                                return 1;
-                        }
+                if (!(e_info->flags & AER_MULTI_ERROR_VALID_FLAG))
+                        return result;
+
+                /*
+                 * If there are multiple errors and id does match,
+                 * We need continue to search other devices under
+                 * the root port. Return 0 means that.
+                 */
+                if (result)
+                        return 0;
+        }
+
+        /*
+         * When either
+         *      1) nosourceid==y;
+         *      2) bus id is equal to 0. Some ports might lose the bus
+         *              id of error source id;
+         *      3) There are multiple errors and prior id comparing fails;
+         * We check AER status registers to find the initial reporter.
+         */
+        if (atomic_read(&dev->enable_cnt) == 0)
+                return 0;
+        pos = pci_find_capability(dev, PCI_CAP_ID_EXP);
+        if (!pos)
+                return 0;
+        /* Check if AER is enabled */
+        pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, &reg16);
+        if (!(reg16 & (
+                PCI_EXP_DEVCTL_CERE |
+                PCI_EXP_DEVCTL_NFERE |
+                PCI_EXP_DEVCTL_FERE |
+                PCI_EXP_DEVCTL_URRE)))
+                return 0;
+        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+        if (!pos)
+                return 0;
+
+        status = 0;
+        mask = 0;
+        if (e_info->severity == AER_CORRECTABLE) {
+                pci_read_config_dword(dev,
+                                pos + PCI_ERR_COR_STATUS,
+                                &status);
+                pci_read_config_dword(dev,
+                                pos + PCI_ERR_COR_MASK,
+                                &mask);
+                if (status & ERR_CORRECTABLE_ERROR_MASK & ~mask) {
+                        add_error_device(e_info, dev);
+                        goto added;
+                }
+        } else {
+                pci_read_config_dword(dev,
+                                pos + PCI_ERR_UNCOR_STATUS,
+                                &status);
+                pci_read_config_dword(dev,
+                                pos + PCI_ERR_UNCOR_MASK,
+                                &mask);
+                if (status & ERR_UNCORRECTABLE_ERROR_MASK & ~mask) {
+                        add_error_device(e_info, dev);
+                        goto added;
                 }
         }
 
         return 0;
+
+added:
+        if (e_info->flags & AER_MULTI_ERROR_VALID_FLAG)
+                return 0;
+        else
+                return 1;
 }
 
 /**
  * find_source_device - search through device hierarchy for source device
  * @parent: pointer to Root Port pci_dev data structure
- * @id: device ID of agent who sends an error message to this Root Port
+ * @err_info: including detailed error information such like id
  *
  * Invoked when error is detected at the Root Port.
  */
-static struct device* find_source_device(struct pci_dev *parent, u16 id)
+static void find_source_device(struct pci_dev *parent,
+                struct aer_err_info *e_info)
 {
         struct pci_dev *dev = parent;
-        struct device *device;
-        unsigned long device_addr;
-        int status;
+        int result;
 
         /* Is Root Port an agent that sends error message? */
-        if (id == ((dev->bus->number << 8) | dev->devfn))
-                return &dev->dev;
-
-        do {
-                device_addr = id;
-                if ((status = device_for_each_child(&dev->dev,
-                        &device_addr, find_device_iter))) {
-                        device = (struct device*)device_addr;
-                        dev = to_pci_dev(device);
-                        if (id == ((dev->bus->number << 8) | dev->devfn))
-                                return device;
-                }
-        }while (status);
+        result = find_device_iter(dev, e_info);
+        if (result)
+                return;
 
-        return NULL;
+        pci_walk_bus(parent->subordinate, find_device_iter, e_info);
 }
 
-static void report_error_detected(struct pci_dev *dev, void *data)
+static int report_error_detected(struct pci_dev *dev, void *data)
 {
         pci_ers_result_t vote;
         struct pci_error_handlers *err_handler;
@@ -230,16 +311,16 @@ static void report_error_detected(struct pci_dev *dev, void *data)
                                    dev->driver ?
                                    "no AER-aware driver" : "no driver");
                 }
-                return;
+                return 0;
         }
 
         err_handler = dev->driver->err_handler;
         vote = err_handler->error_detected(dev, result_data->state);
         result_data->result = merge_result(result_data->result, vote);
-        return;
+        return 0;
 }
 
-static void report_mmio_enabled(struct pci_dev *dev, void *data)
+static int report_mmio_enabled(struct pci_dev *dev, void *data)
 {
         pci_ers_result_t vote;
         struct pci_error_handlers *err_handler;
@@ -249,15 +330,15 @@ static void report_mmio_enabled(struct pci_dev *dev, void *data)
         if (!dev->driver ||
                 !dev->driver->err_handler ||
                 !dev->driver->err_handler->mmio_enabled)
-                return;
+                return 0;
 
         err_handler = dev->driver->err_handler;
         vote = err_handler->mmio_enabled(dev);
         result_data->result = merge_result(result_data->result, vote);
-        return;
+        return 0;
 }
 
-static void report_slot_reset(struct pci_dev *dev, void *data)
+static int report_slot_reset(struct pci_dev *dev, void *data)
 {
         pci_ers_result_t vote;
         struct pci_error_handlers *err_handler;
@@ -267,15 +348,15 @@ static void report_slot_reset(struct pci_dev *dev, void *data)
         if (!dev->driver ||
                 !dev->driver->err_handler ||
                 !dev->driver->err_handler->slot_reset)
-                return;
+                return 0;
 
         err_handler = dev->driver->err_handler;
         vote = err_handler->slot_reset(dev);
         result_data->result = merge_result(result_data->result, vote);
-        return;
+        return 0;
 }
 
-static void report_resume(struct pci_dev *dev, void *data)
+static int report_resume(struct pci_dev *dev, void *data)
 {
         struct pci_error_handlers *err_handler;
 
@@ -284,11 +365,11 @@ static void report_resume(struct pci_dev *dev, void *data)
         if (!dev->driver ||
                 !dev->driver->err_handler ||
                 !dev->driver->err_handler->resume)
-                return;
+                return 0;
 
         err_handler = dev->driver->err_handler;
         err_handler->resume(dev);
-        return;
+        return 0;
 }
 
 /**
@@ -305,7 +386,7 @@ static void report_resume(struct pci_dev *dev, void *data)
 static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
         enum pci_channel_state state,
         char *error_mesg,
-        void (*cb)(struct pci_dev *, void *))
+        int (*cb)(struct pci_dev *, void *))
 {
         struct aer_broadcast_data result_data;
 
@@ -497,12 +578,12 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev,
  */
 static void handle_error_source(struct pcie_device * aerdev,
         struct pci_dev *dev,
-        struct aer_err_info info)
+        struct aer_err_info *info)
 {
         pci_ers_result_t status = 0;
         int pos;
 
-        if (info.severity == AER_CORRECTABLE) {
+        if (info->severity == AER_CORRECTABLE) {
                 /*
                  * Correctable error does not need software intevention.
                  * No need to go through error recovery process.
@@ -510,9 +591,9 @@ static void handle_error_source(struct pcie_device * aerdev,
                 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
                 if (pos)
                         pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
-                                        info.status);
+                                        info->status);
         } else {
-                status = do_recovery(aerdev, dev, info.severity);
+                status = do_recovery(aerdev, dev, info->severity);
                 if (status == PCI_ERS_RESULT_RECOVERED) {
                         dev_printk(KERN_DEBUG, &dev->dev, "AER driver "
                                    "successfully recovered\n");
@@ -661,6 +742,28 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
         return AER_SUCCESS;
 }
 
+static inline void aer_process_err_devices(struct pcie_device *p_device,
+                        struct aer_err_info *e_info)
+{
+        int i;
+
+        if (!e_info->dev[0]) {
+                dev_printk(KERN_DEBUG, &p_device->port->dev,
+                                "can't find device of ID%04x\n",
+                                e_info->id);
+        }
+
+        for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
+                if (get_device_error_info(e_info->dev[i], e_info) ==
+                                AER_SUCCESS) {
+                        aer_print_error(e_info->dev[i], e_info);
+                        handle_error_source(p_device,
+                                        e_info->dev[i],
+                                        e_info);
+                }
+        }
+}
+
 /**
  * aer_isr_one_error - consume an error detected by root port
  * @p_device: pointer to error root port service device
@@ -669,10 +772,16 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 static void aer_isr_one_error(struct pcie_device *p_device,
                 struct aer_err_source *e_src)
 {
-        struct device *s_device;
-        struct aer_err_info e_info = {0, 0, 0,};
+        struct aer_err_info *e_info;
         int i;
-        u16 id;
+
+        /* struct aer_err_info might be big, so we allocate it with slab */
+        e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL);
+        if (e_info == NULL) {
+                dev_printk(KERN_DEBUG, &p_device->port->dev,
+                        "Can't allocate mem when processing AER errors\n");
+                return;
+        }
 
         /*
          * There is a possibility that both correctable error and
@@ -684,31 +793,26 @@ static void aer_isr_one_error(struct pcie_device *p_device,
                 if (!(e_src->status & i))
                         continue;
 
+                memset(e_info, 0, sizeof(struct aer_err_info));
+
                 /* Init comprehensive error information */
                 if (i & PCI_ERR_ROOT_COR_RCV) {
-                        id = ERR_COR_ID(e_src->id);
-                        e_info.severity = AER_CORRECTABLE;
+                        e_info->id = ERR_COR_ID(e_src->id);
+                        e_info->severity = AER_CORRECTABLE;
                 } else {
-                        id = ERR_UNCOR_ID(e_src->id);
-                        e_info.severity = ((e_src->status >> 6) & 1);
+                        e_info->id = ERR_UNCOR_ID(e_src->id);
+                        e_info->severity = ((e_src->status >> 6) & 1);
                 }
                 if (e_src->status &
                         (PCI_ERR_ROOT_MULTI_COR_RCV |
                          PCI_ERR_ROOT_MULTI_UNCOR_RCV))
-                        e_info.flags |= AER_MULTI_ERROR_VALID_FLAG;
-                if (!(s_device = find_source_device(p_device->port, id))) {
-                        printk(KERN_DEBUG "%s->can't find device of ID%04x\n",
-                                __func__, id);
-                        continue;
-                }
-                if (get_device_error_info(to_pci_dev(s_device), &e_info) ==
-                                AER_SUCCESS) {
-                        aer_print_error(to_pci_dev(s_device), &e_info);
-                        handle_error_source(p_device,
-                                to_pci_dev(s_device),
-                                e_info);
-                }
+                        e_info->flags |= AER_MULTI_ERROR_VALID_FLAG;
+
+                find_source_device(p_device->port, e_info);
+                aer_process_err_devices(p_device, e_info);
         }
+
+        kfree(e_info);
 }
 
 /**
diff --git a/drivers/pci/pcie/aer/ecrc.c b/drivers/pci/pcie/aer/ecrc.c
new file mode 100644
index 000000000000..ece97df4df6d
--- /dev/null
+++ b/drivers/pci/pcie/aer/ecrc.c
@@ -0,0 +1,131 @@
+/*
+ *    Enables/disables PCIe ECRC checking.
+ *
+ *    (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
+ *    Andrew Patterson <andrew.patterson@hp.com>
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; version 2 of the License.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ *    General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with this program; if not, write to the Free Software
+ *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ *    02111-1307, USA.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include "../../pci.h"
+
+#define ECRC_POLICY_DEFAULT 0           /* ECRC set by BIOS */
+#define ECRC_POLICY_OFF     1           /* ECRC off for performance */
+#define ECRC_POLICY_ON      2           /* ECRC on for data integrity */
+
+static int ecrc_policy = ECRC_POLICY_DEFAULT;
+
+static const char *ecrc_policy_str[] = {
+        [ECRC_POLICY_DEFAULT] = "bios",
+        [ECRC_POLICY_OFF] = "off",
+        [ECRC_POLICY_ON] = "on"
+};
+
+/**
+ * enable_ercr_checking - enable PCIe ECRC checking for a device
+ * @dev: the PCI device
+ *
+ * Returns 0 on success, or negative on failure.
+ */
+static int enable_ecrc_checking(struct pci_dev *dev)
+{
+        int pos;
+        u32 reg32;
+
+        if (!dev->is_pcie)
+                return -ENODEV;
+
+        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+        if (!pos)
+                return -ENODEV;
+
+        pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+        if (reg32 & PCI_ERR_CAP_ECRC_GENC)
+                reg32 |= PCI_ERR_CAP_ECRC_GENE;
+        if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
+                reg32 |= PCI_ERR_CAP_ECRC_CHKE;
+        pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+
+        return 0;
+}
+
+/**
+ * disable_ercr_checking - disables PCIe ECRC checking for a device
+ * @dev: the PCI device
+ *
+ * Returns 0 on success, or negative on failure.
+ */
+static int disable_ecrc_checking(struct pci_dev *dev)
+{
+        int pos;
+        u32 reg32;
+
+        if (!dev->is_pcie)
+                return -ENODEV;
+
+        pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
+        if (!pos)
+                return -ENODEV;
+
+        pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
+        reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
+        pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
+
+        return 0;
+}
+
+/**
+ * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy
+ * @dev: the PCI device
+ */
+void pcie_set_ecrc_checking(struct pci_dev *dev)
+{
+        switch (ecrc_policy) {
+        case ECRC_POLICY_DEFAULT:
+                return;
+        case ECRC_POLICY_OFF:
+                disable_ecrc_checking(dev);
+                break;
+        case ECRC_POLICY_ON:
+                enable_ecrc_checking(dev);;
+                break;
+        default:
+                return;
+        }
+}
+
+/**
+ * pcie_ecrc_get_policy - parse kernel command-line ecrc option
+ */
+void pcie_ecrc_get_policy(char *str)
+{
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(ecrc_policy_str); i++)
+                if (!strncmp(str, ecrc_policy_str[i],
+                             strlen(ecrc_policy_str[i])))
+                        break;
+        if (i >= ARRAY_SIZE(ecrc_policy_str))
+                return;
+
+        ecrc_policy = i;
+}
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index b0367f168af4..3d27c97e0486 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -26,40 +26,36 @@
 #endif
 #define MODULE_PARAM_PREFIX "pcie_aspm."
 
-struct endpoint_state {
-        unsigned int l0s_acceptable_latency;
-        unsigned int l1_acceptable_latency;
+struct aspm_latency {
+        u32 l0s;                        /* L0s latency (nsec) */
+        u32 l1;                         /* L1 latency (nsec) */
 };
 
 struct pcie_link_state {
-        struct list_head sibiling;
-        struct pci_dev *pdev;
-        bool downstream_has_switch;
-
-        struct pcie_link_state *parent;
-        struct list_head children;
-        struct list_head link;
+        struct pci_dev *pdev;           /* Upstream component of the Link */
+        struct pcie_link_state *root;   /* pointer to the root port link */
+        struct pcie_link_state *parent; /* pointer to the parent Link state */
+        struct list_head sibling;       /* node in link_list */
+        struct list_head children;      /* list of child link states */
+        struct list_head link;          /* node in parent's children list */
 
         /* ASPM state */
-        unsigned int support_state;
-        unsigned int enabled_state;
-        unsigned int bios_aspm_state;
-        /* upstream component */
-        unsigned int l0s_upper_latency;
-        unsigned int l1_upper_latency;
-        /* downstream component */
-        unsigned int l0s_down_latency;
-        unsigned int l1_down_latency;
-        /* Clock PM state*/
-        unsigned int clk_pm_capable;
-        unsigned int clk_pm_enabled;
-        unsigned int bios_clk_state;
+        u32 aspm_support:2;             /* Supported ASPM state */
+        u32 aspm_enabled:2;             /* Enabled ASPM state */
+        u32 aspm_default:2;             /* Default ASPM state by BIOS */
+
+        /* Clock PM state */
+        u32 clkpm_capable:1;            /* Clock PM capable? */
+        u32 clkpm_enabled:1;            /* Current Clock PM state */
+        u32 clkpm_default:1;            /* Default Clock PM state by BIOS */
 
+        /* Latencies */
+        struct aspm_latency latency;    /* Exit latency */
         /*
-         * A pcie downstream port only has one slot under it, so at most there
-         * are 8 functions
+         * Endpoint acceptable latencies. A pcie downstream port only
+         * has one slot under it, so at most there are 8 functions.
          */
-        struct endpoint_state endpoints[8];
+        struct aspm_latency acceptable[8];
 };
 
 static int aspm_disabled, aspm_force;
@@ -78,27 +74,23 @@ static const char *policy_str[] = {
 
 #define LINK_RETRAIN_TIMEOUT HZ
 
-static int policy_to_aspm_state(struct pci_dev *pdev)
+static int policy_to_aspm_state(struct pcie_link_state *link)
 {
-        struct pcie_link_state *link_state = pdev->link_state;
-
         switch (aspm_policy) {
         case POLICY_PERFORMANCE:
                 /* Disable ASPM and Clock PM */
                 return 0;
         case POLICY_POWERSAVE:
                 /* Enable ASPM L0s/L1 */
-                return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+                return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
         case POLICY_DEFAULT:
-                return link_state->bios_aspm_state;
+                return link->aspm_default;
         }
         return 0;
 }
 
-static int policy_to_clkpm_state(struct pci_dev *pdev)
+static int policy_to_clkpm_state(struct pcie_link_state *link)
 {
-        struct pcie_link_state *link_state = pdev->link_state;
-
         switch (aspm_policy) {
         case POLICY_PERFORMANCE:
                 /* Disable ASPM and Clock PM */
@@ -107,73 +99,78 @@ static int policy_to_clkpm_state(struct pci_dev *pdev)
                 /* Disable Clock PM */
                 return 1;
         case POLICY_DEFAULT:
-                return link_state->bios_clk_state;
+                return link->clkpm_default;
         }
         return 0;
 }
 
-static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
 {
-        struct pci_dev *child_dev;
         int pos;
         u16 reg16;
-        struct pcie_link_state *link_state = pdev->link_state;
+        struct pci_dev *child;
+        struct pci_bus *linkbus = link->pdev->subordinate;
 
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                pos = pci_find_capability(child, PCI_CAP_ID_EXP);
                 if (!pos)
                         return;
-                pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, &reg16);
+                pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
                 if (enable)
                         reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
                 else
                         reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
-                pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+                pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
         }
-        link_state->clk_pm_enabled = !!enable;
+        link->clkpm_enabled = !!enable;
 }
 
-static void pcie_check_clock_pm(struct pci_dev *pdev, int blacklist)
+static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
 {
-        int pos;
+        /* Don't enable Clock PM if the link is not Clock PM capable */
+        if (!link->clkpm_capable && enable)
+                return;
+        /* Need nothing if the specified equals to current state */
+        if (link->clkpm_enabled == enable)
+                return;
+        pcie_set_clkpm_nocheck(link, enable);
+}
+
+static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
+{
+        int pos, capable = 1, enabled = 1;
         u32 reg32;
         u16 reg16;
-        int capable = 1, enabled = 1;
-        struct pci_dev *child_dev;
-        struct pcie_link_state *link_state = pdev->link_state;
+        struct pci_dev *child;
+        struct pci_bus *linkbus = link->pdev->subordinate;
 
         /* All functions should have the same cap and state, take the worst */
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                pos = pci_find_capability(child, PCI_CAP_ID_EXP);
                 if (!pos)
                         return;
-                pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, &reg32);
+                pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, &reg32);
                 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
                         capable = 0;
                         enabled = 0;
                         break;
                 }
-                pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, &reg16);
+                pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
                 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
                         enabled = 0;
         }
-        link_state->clk_pm_enabled = enabled;
-        link_state->bios_clk_state = enabled;
-        if (!blacklist) {
-                link_state->clk_pm_capable = capable;
-                pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
-        } else {
-                link_state->clk_pm_capable = 0;
-                pcie_set_clock_pm(pdev, 0);
-        }
+        link->clkpm_enabled = enabled;
+        link->clkpm_default = enabled;
+        link->clkpm_capable = (blacklist) ? 0 : capable;
 }
 
-static bool pcie_aspm_downstream_has_switch(struct pci_dev *pdev)
+static bool pcie_aspm_downstream_has_switch(struct pcie_link_state *link)
 {
-        struct pci_dev *child_dev;
+        struct pci_dev *child;
+        struct pci_bus *linkbus = link->pdev->subordinate;
 
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                if (child_dev->pcie_type == PCI_EXP_TYPE_UPSTREAM)
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                if (child->pcie_type == PCI_EXP_TYPE_UPSTREAM)
                         return true;
         }
         return false;
@@ -184,289 +181,263 @@ static bool pcie_aspm_downstream_has_switch(struct pci_dev *pdev)
  *   could use common clock. If they are, configure them to use the
  *   common clock. That will reduce the ASPM state exit latency.
  */
-static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
 {
-        int pos, child_pos, i = 0;
-        u16 reg16 = 0;
-        struct pci_dev *child_dev;
-        int same_clock = 1;
+        int ppos, cpos, same_clock = 1;
+        u16 reg16, parent_reg, child_reg[8];
         unsigned long start_jiffies;
-        u16 child_regs[8], parent_reg;
+        struct pci_dev *child, *parent = link->pdev;
+        struct pci_bus *linkbus = parent->subordinate;
         /*
-         * all functions of a slot should have the same Slot Clock
+         * All functions of a slot should have the same Slot Clock
          * Configuration, so just check one function
-         * */
-        child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
-                bus_list);
-        BUG_ON(!child_dev->is_pcie);
+         */
+        child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
+        BUG_ON(!child->is_pcie);
 
         /* Check downstream component if bit Slot Clock Configuration is 1 */
-        child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
-        pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, &reg16);
+        cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
+        pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, &reg16);
         if (!(reg16 & PCI_EXP_LNKSTA_SLC))
                 same_clock = 0;
 
         /* Check upstream component if bit Slot Clock Configuration is 1 */
-        pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
-        pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, &reg16);
+        ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
+        pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
         if (!(reg16 & PCI_EXP_LNKSTA_SLC))
                 same_clock = 0;
 
         /* Configure downstream component, all functions */
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
-                pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
-                        &reg16);
-                child_regs[i] = reg16;
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
+                pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, &reg16);
+                child_reg[PCI_FUNC(child->devfn)] = reg16;
                 if (same_clock)
                         reg16 |= PCI_EXP_LNKCTL_CCC;
                 else
                         reg16 &= ~PCI_EXP_LNKCTL_CCC;
-                pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
-                        reg16);
-                i++;
+                pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
         }
 
         /* Configure upstream component */
-        pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
+        pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, &reg16);
         parent_reg = reg16;
         if (same_clock)
                 reg16 |= PCI_EXP_LNKCTL_CCC;
         else
                 reg16 &= ~PCI_EXP_LNKCTL_CCC;
-        pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+        pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
 
-        /* retrain link */
+        /* Retrain link */
         reg16 |= PCI_EXP_LNKCTL_RL;
-        pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+        pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
 
-        /* Wait for link training end */
-        /* break out after waiting for timeout */
+        /* Wait for link training end. Break out after waiting for timeout */
         start_jiffies = jiffies;
         for (;;) {
-                pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, &reg16);
+                pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
                 if (!(reg16 & PCI_EXP_LNKSTA_LT))
                         break;
                 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
                         break;
                 msleep(1);
         }
-        /* training failed -> recover */
-        if (reg16 & PCI_EXP_LNKSTA_LT) {
-                dev_printk (KERN_ERR, &pdev->dev, "ASPM: Could not configure"
-                            " common clock\n");
-                i = 0;
-                list_for_each_entry(child_dev, &pdev->subordinate->devices,
-                                    bus_list) {
-                        child_pos = pci_find_capability(child_dev,
-                                                        PCI_CAP_ID_EXP);
-                        pci_write_config_word(child_dev,
-                                              child_pos + PCI_EXP_LNKCTL,
-                                              child_regs[i]);
-                        i++;
-                }
-                pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, parent_reg);
+        if (!(reg16 & PCI_EXP_LNKSTA_LT))
+                return;
+
+        /* Training failed. Restore common clock configurations */
+        dev_printk(KERN_ERR, &parent->dev,
+                   "ASPM: Could not configure common clock\n");
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
+                pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
+                                      child_reg[PCI_FUNC(child->devfn)]);
         }
+        pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
 }
 
-/*
- * calc_L0S_latency: Convert L0s latency encoding to ns
- */
-static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+/* Convert L0s latency encoding to ns */
+static u32 calc_l0s_latency(u32 encoding)
 {
-        unsigned int ns = 64;
+        if (encoding == 0x7)
+                return (5 * 1000);      /* > 4us */
+        return (64 << encoding);
+}
 
-        if (latency_encoding == 0x7) {
-                if (ac)
-                        ns = -1U;
-                else
-                        ns = 5*1000; /* > 4us */
-        } else
-                ns *= (1 << latency_encoding);
-        return ns;
+/* Convert L0s acceptable latency encoding to ns */
+static u32 calc_l0s_acceptable(u32 encoding)
+{
+        if (encoding == 0x7)
+                return -1U;
+        return (64 << encoding);
 }
 
-/*
- * calc_L1_latency: Convert L1 latency encoding to ns
- */
-static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+/* Convert L1 latency encoding to ns */
+static u32 calc_l1_latency(u32 encoding)
 {
-        unsigned int ns = 1000;
+        if (encoding == 0x7)
+                return (65 * 1000);     /* > 64us */
+        return (1000 << encoding);
+}
 
-        if (latency_encoding == 0x7) {
-                if (ac)
-                        ns = -1U;
-                else
-                        ns = 65*1000; /* > 64us */
-        } else
-                ns *= (1 << latency_encoding);
-        return ns;
+/* Convert L1 acceptable latency encoding to ns */
+static u32 calc_l1_acceptable(u32 encoding)
+{
+        if (encoding == 0x7)
+                return -1U;
+        return (1000 << encoding);
 }
 
 static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
-        unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+                                     u32 *l0s, u32 *l1, u32 *enabled)
 {
         int pos;
         u16 reg16;
-        u32 reg32;
-        unsigned int latency;
+        u32 reg32, encoding;
 
+        *l0s = *l1 = *enabled = 0;
         pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
         pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
         *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
         if (*state != PCIE_LINK_STATE_L0S &&
-                *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+            *state != (PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_L0S))
                 *state = 0;
         if (*state == 0)
                 return;
 
-        latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
-        *l0s = calc_L0S_latency(latency, 0);
+        encoding = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+        *l0s = calc_l0s_latency(encoding);
         if (*state & PCIE_LINK_STATE_L1) {
-                latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
-                *l1 = calc_L1_latency(latency, 0);
+                encoding = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+                *l1 = calc_l1_latency(encoding);
         }
         pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
-        *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+        *enabled = reg16 & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
 }
 
-static void pcie_aspm_cap_init(struct pci_dev *pdev)
+static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 {
-        struct pci_dev *child_dev;
-        u32 state, tmp;
-        struct pcie_link_state *link_state = pdev->link_state;
+        u32 support, l0s, l1, enabled;
+        struct pci_dev *child, *parent = link->pdev;
+        struct pci_bus *linkbus = parent->subordinate;
+
+        if (blacklist) {
+                /* Set support state to 0, so we will disable ASPM later */
+                link->aspm_support = 0;
+                link->aspm_default = 0;
+                link->aspm_enabled = PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
+                return;
+        }
+
+        /* Configure common clock before checking latencies */
+        pcie_aspm_configure_common_clock(link);
 
         /* upstream component states */
-        pcie_aspm_get_cap_device(pdev, &link_state->support_state,
-                &link_state->l0s_upper_latency,
-                &link_state->l1_upper_latency,
-                &link_state->enabled_state);
+        pcie_aspm_get_cap_device(parent, &support, &l0s, &l1, &enabled);
+        link->aspm_support = support;
+        link->latency.l0s = l0s;
+        link->latency.l1 = l1;
+        link->aspm_enabled = enabled;
+
         /* downstream component states, all functions have the same setting */
-        child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
-                bus_list);
-        pcie_aspm_get_cap_device(child_dev, &state,
-                &link_state->l0s_down_latency,
-                &link_state->l1_down_latency,
-                &tmp);
-        link_state->support_state &= state;
-        if (!link_state->support_state)
+        child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
+        pcie_aspm_get_cap_device(child, &support, &l0s, &l1, &enabled);
+        link->aspm_support &= support;
+        link->latency.l0s = max_t(u32, link->latency.l0s, l0s);
+        link->latency.l1 = max_t(u32, link->latency.l1, l1);
+
+        if (!link->aspm_support)
                 return;
-        link_state->enabled_state &= link_state->support_state;
-        link_state->bios_aspm_state = link_state->enabled_state;
+
+        link->aspm_enabled &= link->aspm_support;
+        link->aspm_default = link->aspm_enabled;
 
         /* ENDPOINT states*/
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
                 int pos;
-                u32 reg32;
-                unsigned int latency;
-                struct endpoint_state *ep_state =
-                        &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+                u32 reg32, encoding;
+                struct aspm_latency *acceptable =
+                        &link->acceptable[PCI_FUNC(child->devfn)];
 
-                if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
-                        child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+                if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+                    child->pcie_type != PCI_EXP_TYPE_LEG_END)
                         continue;
 
-                pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
-                pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, &reg32);
-                latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
-                latency = calc_L0S_latency(latency, 1);
-                ep_state->l0s_acceptable_latency = latency;
-                if (link_state->support_state & PCIE_LINK_STATE_L1) {
-                        latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
-                        latency = calc_L1_latency(latency, 1);
-                        ep_state->l1_acceptable_latency = latency;
+                pos = pci_find_capability(child, PCI_CAP_ID_EXP);
+                pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
+                encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+                acceptable->l0s = calc_l0s_acceptable(encoding);
+                if (link->aspm_support & PCIE_LINK_STATE_L1) {
+                        encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+                        acceptable->l1 = calc_l1_acceptable(encoding);
                 }
         }
 }
 
-static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
-        unsigned int state)
-{
-        struct pci_dev *parent_dev, *tmp_dev;
-        unsigned int latency, l1_latency = 0;
-        struct pcie_link_state *link_state;
-        struct endpoint_state *ep_state;
-
-        parent_dev = pdev->bus->self;
-        link_state = parent_dev->link_state;
-        state &= link_state->support_state;
-        if (state == 0)
-                return 0;
-        ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
-
-        /*
-         * Check latency for endpoint device.
-         * TBD: The latency from the endpoint to root complex vary per
-         * switch's upstream link state above the device. Here we just do a
-         * simple check which assumes all links above the device can be in L1
-         * state, that is we just consider the worst case. If switch's upstream
-         * link can't be put into L0S/L1, then our check is too strictly.
-         */
-        tmp_dev = pdev;
-        while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
-                parent_dev = tmp_dev->bus->self;
-                link_state = parent_dev->link_state;
-                if (state & PCIE_LINK_STATE_L0S) {
-                        latency = max_t(unsigned int,
-                                        link_state->l0s_upper_latency,
-                                        link_state->l0s_down_latency);
-                        if (latency > ep_state->l0s_acceptable_latency)
-                                state &= ~PCIE_LINK_STATE_L0S;
-                }
-                if (state & PCIE_LINK_STATE_L1) {
-                        latency = max_t(unsigned int,
-                                        link_state->l1_upper_latency,
-                                        link_state->l1_down_latency);
-                        if (latency + l1_latency >
-                                        ep_state->l1_acceptable_latency)
-                                state &= ~PCIE_LINK_STATE_L1;
-                }
-                if (!parent_dev->bus->self) /* parent_dev is a root port */
-                        break;
-                else {
-                        /*
-                         * parent_dev is the downstream port of a switch, make
-                         * tmp_dev the upstream port of the switch
-                         */
-                        tmp_dev = parent_dev->bus->self;
-                        /*
-                         * every switch on the path to root complex need 1 more
-                         * microsecond for L1. Spec doesn't mention L0S.
-                         */
-                        if (state & PCIE_LINK_STATE_L1)
-                                l1_latency += 1000;
-                }
+/**
+ * __pcie_aspm_check_state_one - check latency for endpoint device.
+ * @endpoint: pointer to the struct pci_dev of endpoint device
+ *
+ * TBD: The latency from the endpoint to root complex vary per switch's
+ * upstream link state above the device. Here we just do a simple check
+ * which assumes all links above the device can be in L1 state, that
+ * is we just consider the worst case. If switch's upstream link can't
+ * be put into L0S/L1, then our check is too strictly.
+ */
+static u32 __pcie_aspm_check_state_one(struct pci_dev *endpoint, u32 state)
+{
+        u32 l1_switch_latency = 0;
+        struct aspm_latency *acceptable;
+        struct pcie_link_state *link;
+
+        link = endpoint->bus->self->link_state;
+        state &= link->aspm_support;
+        acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
+
+        while (link && state) {
+                if ((state & PCIE_LINK_STATE_L0S) &&
+                    (link->latency.l0s > acceptable->l0s))
+                        state &= ~PCIE_LINK_STATE_L0S;
+                if ((state & PCIE_LINK_STATE_L1) &&
+                    (link->latency.l1 + l1_switch_latency > acceptable->l1))
+                        state &= ~PCIE_LINK_STATE_L1;
+                link = link->parent;
+                /*
+                 * Every switch on the path to root complex need 1
+                 * more microsecond for L1. Spec doesn't mention L0s.
+                 */
+                l1_switch_latency += 1000;
         }
         return state;
 }
 
-static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
-        unsigned int state)
+static u32 pcie_aspm_check_state(struct pcie_link_state *link, u32 state)
 {
-        struct pci_dev *child_dev;
+        pci_power_t power_state;
+        struct pci_dev *child;
+        struct pci_bus *linkbus = link->pdev->subordinate;
 
         /* If no child, ignore the link */
-        if (list_empty(&pdev->subordinate->devices))
+        if (list_empty(&linkbus->devices))
                 return state;
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
-                        /*
-                         * If downstream component of a link is pci bridge, we
-                         * disable ASPM for now for the link
-                         * */
-                        state = 0;
-                        break;
-                }
-                if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
-                        child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                /*
+                 * If downstream component of a link is pci bridge, we
+                 * disable ASPM for now for the link
+                 */
+                if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
+                        return 0;
+
+                if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+                     child->pcie_type != PCI_EXP_TYPE_LEG_END))
                         continue;
                 /* Device not in D0 doesn't need check latency */
-                if (child_dev->current_state == PCI_D1 ||
-                        child_dev->current_state == PCI_D2 ||
-                        child_dev->current_state == PCI_D3hot ||
-                        child_dev->current_state == PCI_D3cold)
+                power_state = child->current_state;
+                if (power_state == PCI_D1 || power_state == PCI_D2 ||
+                    power_state == PCI_D3hot || power_state == PCI_D3cold)
                         continue;
-                state = __pcie_aspm_check_state_one(child_dev, state);
+                state = __pcie_aspm_check_state_one(child, state);
         }
         return state;
 }
@@ -482,90 +453,71 @@ static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
         pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
 }
 
-static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+static void __pcie_aspm_config_link(struct pcie_link_state *link, u32 state)
 {
-        struct pci_dev *child_dev;
-        int valid = 1;
-        struct pcie_link_state *link_state = pdev->link_state;
+        struct pci_dev *child, *parent = link->pdev;
+        struct pci_bus *linkbus = parent->subordinate;
 
         /* If no child, disable the link */
-        if (list_empty(&pdev->subordinate->devices))
+        if (list_empty(&linkbus->devices))
                 state = 0;
         /*
-         * if the downstream component has pci bridge function, don't do ASPM
-         * now
+         * If the downstream component has pci bridge function, don't
+         * do ASPM now.
          */
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
-                        valid = 0;
-                        break;
-                }
+        list_for_each_entry(child, &linkbus->devices, bus_list) {
+                if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
+                        return;
         }
-        if (!valid)
-                return;
-
         /*
-         * spec 2.0 suggests all functions should be configured the same
-         * setting for ASPM. Enabling ASPM L1 should be done in upstream
-         * component first and then downstream, and vice versa for disabling
-         * ASPM L1. Spec doesn't mention L0S.
+         * Spec 2.0 suggests all functions should be configured the
+         * same setting for ASPM. Enabling ASPM L1 should be done in
+         * upstream component first and then downstream, and vice
+         * versa for disabling ASPM L1. Spec doesn't mention L0S.
          */
         if (state & PCIE_LINK_STATE_L1)
-                __pcie_aspm_config_one_dev(pdev, state);
+                __pcie_aspm_config_one_dev(parent, state);
 
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
-                __pcie_aspm_config_one_dev(child_dev, state);
+        list_for_each_entry(child, &linkbus->devices, bus_list)
+                __pcie_aspm_config_one_dev(child, state);
 
         if (!(state & PCIE_LINK_STATE_L1))
-                __pcie_aspm_config_one_dev(pdev, state);
+                __pcie_aspm_config_one_dev(parent, state);
 
-        link_state->enabled_state = state;
+        link->aspm_enabled = state;
 }
 
-static struct pcie_link_state *get_root_port_link(struct pcie_link_state *link)
+/* Check the whole hierarchy, and configure each link in the hierarchy */
+static void __pcie_aspm_configure_link_state(struct pcie_link_state *link,
+                                             u32 state)
 {
-        struct pcie_link_state *root_port_link = link;
-        while (root_port_link->parent)
-                root_port_link = root_port_link->parent;
-        return root_port_link;
-}
+        struct pcie_link_state *leaf, *root = link->root;
 
-/* check the whole hierarchy, and configure each link in the hierarchy */
-static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
-        unsigned int state)
-{
-        struct pcie_link_state *link_state = pdev->link_state;
-        struct pcie_link_state *root_port_link = get_root_port_link(link_state);
-        struct pcie_link_state *leaf;
+        state &= (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
 
-        state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
-
-        /* check all links who have specific root port link */
-        list_for_each_entry(leaf, &link_list, sibiling) {
-                if (!list_empty(&leaf->children) ||
-                        get_root_port_link(leaf) != root_port_link)
+        /* Check all links who have specific root port link */
+        list_for_each_entry(leaf, &link_list, sibling) {
+                if (!list_empty(&leaf->children) || (leaf->root != root))
                         continue;
-                state = pcie_aspm_check_state(leaf->pdev, state);
+                state = pcie_aspm_check_state(leaf, state);
         }
-        /* check root port link too in case it hasn't children */
-        state = pcie_aspm_check_state(root_port_link->pdev, state);
-
-        if (link_state->enabled_state == state)
+        /* Check root port link too in case it hasn't children */
+        state = pcie_aspm_check_state(root, state);
+        if (link->aspm_enabled == state)
                 return;
-
         /*
-         * we must change the hierarchy. See comments in
+         * We must change the hierarchy. See comments in
          * __pcie_aspm_config_link for the order
          **/
         if (state & PCIE_LINK_STATE_L1) {
-                list_for_each_entry(leaf, &link_list, sibiling) {
-                        if (get_root_port_link(leaf) == root_port_link)
-                                __pcie_aspm_config_link(leaf->pdev, state);
+                list_for_each_entry(leaf, &link_list, sibling) {
+                        if (leaf->root == root)
+                                __pcie_aspm_config_link(leaf, state);
                 }
         } else {
-                list_for_each_entry_reverse(leaf, &link_list, sibiling) {
-                        if (get_root_port_link(leaf) == root_port_link)
-                                __pcie_aspm_config_link(leaf->pdev, state);
+                list_for_each_entry_reverse(leaf, &link_list, sibling) {
+                        if (leaf->root == root)
+                                __pcie_aspm_config_link(leaf, state);
                 }
         }
 }
@@ -574,45 +526,42 @@ static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
  * pcie_aspm_configure_link_state: enable/disable PCI express link state
  * @pdev: the root port or switch downstream port
  */
-static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
-        unsigned int state)
+static void pcie_aspm_configure_link_state(struct pcie_link_state *link,
+                                           u32 state)
 {
         down_read(&pci_bus_sem);
         mutex_lock(&aspm_lock);
-        __pcie_aspm_configure_link_state(pdev, state);
+        __pcie_aspm_configure_link_state(link, state);
         mutex_unlock(&aspm_lock);
         up_read(&pci_bus_sem);
 }
 
-static void free_link_state(struct pci_dev *pdev)
+static void free_link_state(struct pcie_link_state *link)
 {
-        kfree(pdev->link_state);
-        pdev->link_state = NULL;
+        link->pdev->link_state = NULL;
+        kfree(link);
 }
 
 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
 {
-        struct pci_dev *child_dev;
-        int child_pos;
+        struct pci_dev *child;
+        int pos;
         u32 reg32;
-
         /*
-         * Some functions in a slot might not all be PCIE functions, very
-         * strange. Disable ASPM for the whole slot
+         * Some functions in a slot might not all be PCIE functions,
+         * very strange. Disable ASPM for the whole slot
          */
-        list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
-                child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
-                if (!child_pos)
+        list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
+                pos = pci_find_capability(child, PCI_CAP_ID_EXP);
+                if (!pos)
                         return -EINVAL;
-
                 /*
                  * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
                  * RBER bit to determine if a function is 1.1 version device
                  */
-                pci_read_config_dword(child_dev, child_pos + PCI_EXP_DEVCAP,
-                        &reg32);
+                pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
                 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
-                        dev_printk(KERN_INFO, &child_dev->dev, "disabling ASPM"
+                        dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
                                 " on pre-1.1 PCIe device.  You can enable it"
                                 " with 'pcie_aspm=force'\n");
                         return -EINVAL;
@@ -621,6 +570,47 @@ static int pcie_aspm_sanity_check(struct pci_dev *pdev)
         return 0;
 }
 
+static struct pcie_link_state *pcie_aspm_setup_link_state(struct pci_dev *pdev)
+{
+        struct pcie_link_state *link;
+        int blacklist = !!pcie_aspm_sanity_check(pdev);
+
+        link = kzalloc(sizeof(*link), GFP_KERNEL);
+        if (!link)
+                return NULL;
+        INIT_LIST_HEAD(&link->sibling);
+        INIT_LIST_HEAD(&link->children);
+        INIT_LIST_HEAD(&link->link);
+        link->pdev = pdev;
+        if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
+                struct pcie_link_state *parent;
+                parent = pdev->bus->parent->self->link_state;
+                if (!parent) {
+                        kfree(link);
+                        return NULL;
+                }
+                link->parent = parent;
+                list_add(&link->link, &parent->children);
+        }
+        /* Setup a pointer to the root port link */
+        if (!link->parent)
+                link->root = link;
+        else
+                link->root = link->parent->root;
+
+        list_add(&link->sibling, &link_list);
+
+        pdev->link_state = link;
+
+        /* Check ASPM capability */
+        pcie_aspm_cap_init(link, blacklist);
+
+        /* Check Clock PM capability */
+        pcie_clkpm_cap_init(link, blacklist);
+
+        return link;
+}
+
 /*
  * pcie_aspm_init_link_state: Initiate PCI express link state.
  * It is called after the pcie and its children devices are scaned.
@@ -628,75 +618,47 @@ static int pcie_aspm_sanity_check(struct pci_dev *pdev)
  */
 void pcie_aspm_init_link_state(struct pci_dev *pdev)
 {
-        unsigned int state;
-        struct pcie_link_state *link_state;
-        int error = 0;
-        int blacklist;
+        u32 state;
+        struct pcie_link_state *link;
 
         if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
                 return;
         if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
-                pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+            pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+                return;
+
+        /* VIA has a strange chipset, root port is under a bridge */
+        if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
+            pdev->bus->self)
                 return;
+
         down_read(&pci_bus_sem);
         if (list_empty(&pdev->subordinate->devices))
                 goto out;
 
-        blacklist = !!pcie_aspm_sanity_check(pdev);
-
         mutex_lock(&aspm_lock);
-
-        link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
-        if (!link_state)
-                goto unlock_out;
-
-        link_state->downstream_has_switch = pcie_aspm_downstream_has_switch(pdev);
-        INIT_LIST_HEAD(&link_state->children);
-        INIT_LIST_HEAD(&link_state->link);
-        if (pdev->bus->self) {/* this is a switch */
-                struct pcie_link_state *parent_link_state;
-
-                parent_link_state = pdev->bus->parent->self->link_state;
-                if (!parent_link_state) {
-                        kfree(link_state);
-                        goto unlock_out;
-                }
-                list_add(&link_state->link, &parent_link_state->children);
-                link_state->parent = parent_link_state;
-        }
-
-        pdev->link_state = link_state;
-
-        if (!blacklist) {
-                pcie_aspm_configure_common_clock(pdev);
-                pcie_aspm_cap_init(pdev);
+        link = pcie_aspm_setup_link_state(pdev);
+        if (!link)
+                goto unlock;
+        /*
+         * Setup initial ASPM state
+         *
+         * If link has switch, delay the link config. The leaf link
+         * initialization will config the whole hierarchy. But we must
+         * make sure BIOS doesn't set unsupported link state.
+         */
+        if (pcie_aspm_downstream_has_switch(link)) {
+                state = pcie_aspm_check_state(link, link->aspm_default);
+                __pcie_aspm_config_link(link, state);
         } else {
-                link_state->enabled_state = PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
-                link_state->bios_aspm_state = 0;
-                /* Set support state to 0, so we will disable ASPM later */
-                link_state->support_state = 0;
+                state = policy_to_aspm_state(link);
+                __pcie_aspm_configure_link_state(link, state);
         }
 
-        link_state->pdev = pdev;
-        list_add(&link_state->sibiling, &link_list);
-
-        if (link_state->downstream_has_switch) {
-                /*
-                 * If link has switch, delay the link config. The leaf link
-                 * initialization will config the whole hierarchy. but we must
-                 * make sure BIOS doesn't set unsupported link state
-                 **/
-                state = pcie_aspm_check_state(pdev, link_state->bios_aspm_state);
-                __pcie_aspm_config_link(pdev, state);
-        } else
-                __pcie_aspm_configure_link_state(pdev,
-                        policy_to_aspm_state(pdev));
-
-        pcie_check_clock_pm(pdev, blacklist);
-
-unlock_out:
-        if (error)
-                free_link_state(pdev);
+        /* Setup initial Clock PM state */
+        state = (link->clkpm_capable) ? policy_to_clkpm_state(link) : 0;
+        pcie_set_clkpm(link, state);
+unlock:
         mutex_unlock(&aspm_lock);
 out:
         up_read(&pci_bus_sem);
@@ -725,11 +687,11 @@ void pcie_aspm_exit_link_state(struct pci_dev *pdev)
 
         /* All functions are removed, so just disable ASPM for the link */
         __pcie_aspm_config_one_dev(parent, 0);
-        list_del(&link_state->sibiling);
+        list_del(&link_state->sibling);
         list_del(&link_state->link);
         /* Clock PM is for endpoint device */
 
-        free_link_state(parent);
+        free_link_state(link_state);
 out:
         mutex_unlock(&aspm_lock);
         up_read(&pci_bus_sem);
@@ -749,7 +711,7 @@ void pcie_aspm_pm_state_change(struct pci_dev *pdev)
          * devices changed PM state, we should recheck if latency meets all
          * functions' requirement
          */
-        pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+        pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
 }
 
 /*
@@ -772,14 +734,12 @@ void pci_disable_link_state(struct pci_dev *pdev, int state)
         down_read(&pci_bus_sem);
         mutex_lock(&aspm_lock);
         link_state = parent->link_state;
-        link_state->support_state &=
-                ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
-        if (state & PCIE_LINK_STATE_CLKPM)
-                link_state->clk_pm_capable = 0;
-
-        __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
-        if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
-                pcie_set_clock_pm(parent, 0);
+        link_state->aspm_support &= ~state;
+        __pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
+        if (state & PCIE_LINK_STATE_CLKPM) {
+                link_state->clkpm_capable = 0;
+                pcie_set_clkpm(link_state, 0);
+        }
         mutex_unlock(&aspm_lock);
         up_read(&pci_bus_sem);
 }
@@ -788,7 +748,6 @@ EXPORT_SYMBOL(pci_disable_link_state);
 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
 {
         int i;
-        struct pci_dev *pdev;
         struct pcie_link_state *link_state;
 
         for (i = 0; i < ARRAY_SIZE(policy_str); i++)
@@ -802,14 +761,10 @@ static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
         down_read(&pci_bus_sem);
         mutex_lock(&aspm_lock);
         aspm_policy = i;
-        list_for_each_entry(link_state, &link_list, sibiling) {
-                pdev = link_state->pdev;
-                __pcie_aspm_configure_link_state(pdev,
-                        policy_to_aspm_state(pdev));
-                if (link_state->clk_pm_capable &&
-                    link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
-                        pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
-
+        list_for_each_entry(link_state, &link_list, sibling) {
+                __pcie_aspm_configure_link_state(link_state,
+                        policy_to_aspm_state(link_state));
+                pcie_set_clkpm(link_state, policy_to_clkpm_state(link_state));
         }
         mutex_unlock(&aspm_lock);
         up_read(&pci_bus_sem);
@@ -838,7 +793,7 @@ static ssize_t link_state_show(struct device *dev,
         struct pci_dev *pci_device = to_pci_dev(dev);
         struct pcie_link_state *link_state = pci_device->link_state;
 
-        return sprintf(buf, "%d\n", link_state->enabled_state);
+        return sprintf(buf, "%d\n", link_state->aspm_enabled);
 }
 
 static ssize_t link_state_store(struct device *dev,
@@ -846,7 +801,7 @@ static ssize_t link_state_store(struct device *dev,
                 const char *buf,
                 size_t n)
 {
-        struct pci_dev *pci_device = to_pci_dev(dev);
+        struct pci_dev *pdev = to_pci_dev(dev);
         int state;
 
         if (n < 1)
@@ -854,7 +809,7 @@ static ssize_t link_state_store(struct device *dev,
         state = buf[0]-'0';
         if (state >= 0 && state <= 3) {
                 /* setup link aspm state */
-                pcie_aspm_configure_link_state(pci_device, state);
+                pcie_aspm_configure_link_state(pdev->link_state, state);
                 return n;
         }
 
@@ -868,7 +823,7 @@ static ssize_t clk_ctl_show(struct device *dev,
         struct pci_dev *pci_device = to_pci_dev(dev);
         struct pcie_link_state *link_state = pci_device->link_state;
 
-        return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+        return sprintf(buf, "%d\n", link_state->clkpm_enabled);
 }
 
 static ssize_t clk_ctl_store(struct device *dev,
@@ -876,7 +831,7 @@ static ssize_t clk_ctl_store(struct device *dev,
                 const char *buf,
                 size_t n)
 {
-        struct pci_dev *pci_device = to_pci_dev(dev);
+        struct pci_dev *pdev = to_pci_dev(dev);
         int state;
 
         if (n < 1)
@@ -885,7 +840,7 @@ static ssize_t clk_ctl_store(struct device *dev,
 
         down_read(&pci_bus_sem);
         mutex_lock(&aspm_lock);
-        pcie_set_clock_pm(pci_device, !!state);
+        pcie_set_clkpm_nocheck(pdev->link_state, !!state);
         mutex_unlock(&aspm_lock);
         up_read(&pci_bus_sem);
 
@@ -904,10 +859,10 @@ void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
                 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
                 return;
 
-        if (link_state->support_state)
+        if (link_state->aspm_support)
                 sysfs_add_file_to_group(&pdev->dev.kobj,
                         &dev_attr_link_state.attr, power_group);
-        if (link_state->clk_pm_capable)
+        if (link_state->clkpm_capable)
                 sysfs_add_file_to_group(&pdev->dev.kobj,
                         &dev_attr_clk_ctl.attr, power_group);
 }
@@ -920,10 +875,10 @@ void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
                 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
                 return;
 
-        if (link_state->support_state)
+        if (link_state->aspm_support)
                 sysfs_remove_file_from_group(&pdev->dev.kobj,
                         &dev_attr_link_state.attr, power_group);
-        if (link_state->clk_pm_capable)
+        if (link_state->clkpm_capable)
                 sysfs_remove_file_from_group(&pdev->dev.kobj,
                         &dev_attr_clk_ctl.attr, power_group);
 }
diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c
index e39982503863..13ffdc35ea0e 100644
--- a/drivers/pci/pcie/portdrv_core.c
+++ b/drivers/pci/pcie/portdrv_core.c
@@ -275,7 +275,7 @@ static void pcie_device_init(struct pci_dev *parent, struct pcie_device *dev,
         memset(device, 0, sizeof(struct device));
         device->bus = &pcie_port_bus_type;
         device->driver = NULL;
-        device->driver_data = NULL;
+        dev_set_drvdata(device, NULL);
         device->release = release_pcie_device;  /* callback to free pcie dev */
         dev_set_name(device, "%s:pcie%02x",
                  pci_name(parent), get_descriptor_id(port_type, service_type));