19 files changed, 952 insertions, 385 deletions

diff --git a/Documentation/devicetree/bindings/pci/designware-pcie.txt b/Documentation/devicetree/bindings/pci/designware-pcie.txt
index c124f9bc11f3..5561a1c060d0 100644
--- a/Documentation/devicetree/bindings/pci/designware-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/designware-pcie.txt
@@ -4,8 +4,11 @@ Required properties:
 - compatible:
         "snps,dw-pcie" for RC mode;
         "snps,dw-pcie-ep" for EP mode;
-- reg: Should contain the configuration address space.
-- reg-names: Must be "config" for the PCIe configuration space.
+- reg: For designware cores version < 4.80 contains the configuration
+       address space. For designware core version >= 4.80, contains
+       the configuration and ATU address space
+- reg-names: Must be "config" for the PCIe configuration space and "atu" for
+             the ATU address space.
     (The old way of getting the configuration address space from "ranges"
     is deprecated and should be avoided.)
 - num-lanes: number of lanes to use
diff --git a/Documentation/devicetree/bindings/pci/pci-keystone.txt b/Documentation/devicetree/bindings/pci/pci-keystone.txt
index 2030ee0dc4f9..47202a2938f2 100644
--- a/Documentation/devicetree/bindings/pci/pci-keystone.txt
+++ b/Documentation/devicetree/bindings/pci/pci-keystone.txt
@@ -11,16 +11,24 @@ described here as well as properties that are not applicable.
 
 Required Properties:-
 
-compatibility: "ti,keystone-pcie"
-reg:    index 1 is the base address and length of DW application registers.
-        index 2 is the base address and length of PCI device ID register.
+compatibility: Should be "ti,keystone-pcie" for RC on Keystone2 SoC
+               Should be "ti,am654-pcie-rc" for RC on AM654x SoC
+reg: Three register ranges as listed in the reg-names property
+reg-names: "dbics" for the DesignWare PCIe registers, "app" for the
+           TI specific application registers, "config" for the
+           configuration space address
 
 pcie_msi_intc : Interrupt controller device node for MSI IRQ chip
         interrupt-cells: should be set to 1
         interrupts: GIC interrupt lines connected to PCI MSI interrupt lines
+        (required if the compatible is "ti,keystone-pcie")
+msi-map: As specified in Documentation/devicetree/bindings/pci/pci-msi.txt
+         (required if the compatible is "ti,am654-pcie-rc".
 
 ti,syscon-pcie-id : phandle to the device control module required to set device
                     id and vendor id.
+ti,syscon-pcie-mode : phandle to the device control module required to configure
+                      PCI in either RC mode or EP mode.
 
  Example:
         pcie_msi_intc: msi-interrupt-controller {
@@ -61,3 +69,47 @@ Optional properties:-
 DesignWare DT Properties not applicable for Keystone PCI
 
 1. pcie_bus clock-names not used.  Instead, a phandle to phys is used.
+
+AM654 PCIe Endpoint
+===================
+
+Required Properties:-
+
+compatibility: Should be "ti,am654-pcie-ep" for EP on AM654x SoC
+reg: Four register ranges as listed in the reg-names property
+reg-names: "dbics" for the DesignWare PCIe registers, "app" for the
+           TI specific application registers, "atu" for the
+           Address Translation Unit configuration registers and
+           "addr_space" used to map remote RC address space
+num-ib-windows: As specified in
+                Documentation/devicetree/bindings/pci/designware-pcie.txt
+num-ob-windows: As specified in
+                Documentation/devicetree/bindings/pci/designware-pcie.txt
+num-lanes: As specified in
+           Documentation/devicetree/bindings/pci/designware-pcie.txt
+power-domains: As documented by the generic PM domain bindings in
+               Documentation/devicetree/bindings/power/power_domain.txt.
+ti,syscon-pcie-mode: phandle to the device control module required to configure
+                      PCI in either RC mode or EP mode.
+
+Optional properties:-
+
+phys: list of PHY specifiers (used by generic PHY framework)
+phy-names: must be "pcie-phy0", "pcie-phy1", "pcie-phyN".. based on the
+               number of lanes as specified in *num-lanes* property.
+("phys" and "phy-names" DT bindings are specified in
+Documentation/devicetree/bindings/phy/phy-bindings.txt)
+interrupts: platform interrupt for error interrupts.
+
+pcie-ep {
+        compatible = "ti,am654-pcie-ep";
+        reg =  <0x5500000 0x1000>, <0x5501000 0x1000>,
+               <0x10000000 0x8000000>, <0x5506000 0x1000>;
+        reg-names = "app", "dbics", "addr_space", "atu";
+        power-domains = <&k3_pds 120>;
+        ti,syscon-pcie-mode = <&pcie0_mode>;
+        num-lanes = <1>;
+        num-ib-windows = <16>;
+        num-ob-windows = <16>;
+        interrupts = <GIC_SPI 340 IRQ_TYPE_EDGE_RISING>;
+};
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index 29582fe57151..7b015f2a1c6f 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -75,6 +75,11 @@
 #define PCI_ENDPOINT_TEST_IRQ_TYPE              0x24
 #define PCI_ENDPOINT_TEST_IRQ_NUMBER            0x28
 
+#define PCI_DEVICE_ID_TI_AM654                  0xb00c
+
+#define is_am654_pci_dev(pdev)          \
+                ((pdev)->device == PCI_DEVICE_ID_TI_AM654)
+
 static DEFINE_IDA(pci_endpoint_test_ida);
 
 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
@@ -588,6 +593,7 @@ static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
         int ret = -EINVAL;
         enum pci_barno bar;
         struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
+        struct pci_dev *pdev = test->pdev;
 
         mutex_lock(&test->mutex);
         switch (cmd) {
@@ -595,6 +601,8 @@ static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
                 bar = arg;
                 if (bar < 0 || bar > 5)
                         goto ret;
+                if (is_am654_pci_dev(pdev) && bar == BAR_0)
+                        goto ret;
                 ret = pci_endpoint_test_bar(test, bar);
                 break;
         case PCITEST_LEGACY_IRQ:
@@ -662,6 +670,7 @@ static int pci_endpoint_test_probe(struct pci_dev *pdev,
         data = (struct pci_endpoint_test_data *)ent->driver_data;
         if (data) {
                 test_reg_bar = data->test_reg_bar;
+                test->test_reg_bar = test_reg_bar;
                 test->alignment = data->alignment;
                 irq_type = data->irq_type;
         }
@@ -785,11 +794,20 @@ static void pci_endpoint_test_remove(struct pci_dev *pdev)
         pci_disable_device(pdev);
 }
 
+static const struct pci_endpoint_test_data am654_data = {
+        .test_reg_bar = BAR_2,
+        .alignment = SZ_64K,
+        .irq_type = IRQ_TYPE_MSI,
+};
+
 static const struct pci_device_id pci_endpoint_test_tbl[] = {
         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) },
         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) },
         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0) },
         { PCI_DEVICE(PCI_VENDOR_ID_SYNOPSYS, 0xedda) },
+        { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
+          .driver_data = (kernel_ulong_t)&am654_data
+        },
         { }
 };
 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index 657d642fcc67..28cdd8c0213a 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -10,10 +10,10 @@ obj-$(CONFIG_PCI)		+= access.o bus.o probe.o host-bridge.o \
 ifdef CONFIG_PCI
 obj-$(CONFIG_PROC_FS)           += proc.o
 obj-$(CONFIG_SYSFS)             += slot.o
-obj-$(CONFIG_OF)                += of.o
 obj-$(CONFIG_ACPI)              += pci-acpi.o
 endif
 
+obj-$(CONFIG_OF)                += of.o
 obj-$(CONFIG_PCI_QUIRKS)        += quirks.o
 obj-$(CONFIG_PCIEPORTBUS)       += pcie/
 obj-$(CONFIG_HOTPLUG_PCI)       += hotplug/
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index 6ea74b1c0d94..a6ce1ee51b4c 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -103,15 +103,32 @@ config PCIE_SPEAR13XX
           Say Y here if you want PCIe support on SPEAr13XX SoCs.
 
 config PCI_KEYSTONE
-        bool "TI Keystone PCIe controller"
-        depends on ARCH_KEYSTONE || (ARM && COMPILE_TEST)
+        bool
+
+config PCI_KEYSTONE_HOST
+        bool "PCI Keystone Host Mode"
+        depends on ARCH_KEYSTONE || ARCH_K3 || ((ARM || ARM64) && COMPILE_TEST)
         depends on PCI_MSI_IRQ_DOMAIN
         select PCIE_DW_HOST
+        select PCI_KEYSTONE
+        default y
         help
-          Say Y here if you want to enable PCI controller support on Keystone
-          SoCs. The PCI controller on Keystone is based on DesignWare hardware
-          and therefore the driver re-uses the DesignWare core functions to
-          implement the driver.
+          Enables support for the PCIe controller in the Keystone SoC to
+          work in host mode. The PCI controller on Keystone is based on
+          DesignWare hardware and therefore the driver re-uses the
+          DesignWare core functions to implement the driver.
+
+config PCI_KEYSTONE_EP
+        bool "PCI Keystone Endpoint Mode"
+        depends on ARCH_KEYSTONE || ARCH_K3 || ((ARM || ARM64) && COMPILE_TEST)
+        depends on PCI_ENDPOINT
+        select PCIE_DW_EP
+        select PCI_KEYSTONE
+        help
+          Enables support for the PCIe controller in the Keystone SoC to
+          work in endpoint mode. The PCI controller on Keystone is based
+          on DesignWare hardware and therefore the driver re-uses the
+          DesignWare core functions to implement the driver.
 
 config PCI_LAYERSCAPE
         bool "Freescale Layerscape PCIe controller"
diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index ae84a69ae63a..b287dbf6914c 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -406,7 +406,7 @@ dra7xx_pcie_get_features(struct dw_pcie_ep *ep)
         return &dra7xx_pcie_epc_features;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
         .ep_init = dra7xx_pcie_ep_init,
         .raise_irq = dra7xx_pcie_raise_irq,
         .get_features = dra7xx_pcie_get_features,
diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c
index 14f2b0b4ed5e..af677254a072 100644
--- a/drivers/pci/controller/dwc/pci-keystone.c
+++ b/drivers/pci/controller/dwc/pci-keystone.c
@@ -11,6 +11,7 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/irqchip/chained_irq.h>
@@ -18,6 +19,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/msi.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/of_pci.h>
 #include <linux/phy/phy.h>
@@ -26,6 +28,7 @@
 #include <linux/resource.h>
 #include <linux/signal.h>
 
+#include "../../pci.h"
 #include "pcie-designware.h"
 
 #define PCIE_VENDORID_MASK      0xffff
@@ -44,28 +47,34 @@
 #define CFG_TYPE1                       BIT(24)
 
 #define OB_SIZE                         0x030
-#define SPACE0_REMOTE_CFG_OFFSET        0x1000
 #define OB_OFFSET_INDEX(n)              (0x200 + (8 * (n)))
 #define OB_OFFSET_HI(n)                 (0x204 + (8 * (n)))
 #define OB_ENABLEN                      BIT(0)
 #define OB_WIN_SIZE                     8       /* 8MB */
 
+#define PCIE_LEGACY_IRQ_ENABLE_SET(n)   (0x188 + (0x10 * ((n) - 1)))
+#define PCIE_LEGACY_IRQ_ENABLE_CLR(n)   (0x18c + (0x10 * ((n) - 1)))
+#define PCIE_EP_IRQ_SET                 0x64
+#define PCIE_EP_IRQ_CLR                 0x68
+#define INT_ENABLE                      BIT(0)
+
 /* IRQ register defines */
 #define IRQ_EOI                         0x050
-#define IRQ_STATUS                      0x184
-#define IRQ_ENABLE_SET                  0x188
-#define IRQ_ENABLE_CLR                  0x18c
 
 #define MSI_IRQ                         0x054
-#define MSI0_IRQ_STATUS                 0x104
-#define MSI0_IRQ_ENABLE_SET             0x108
-#define MSI0_IRQ_ENABLE_CLR             0x10c
-#define IRQ_STATUS                      0x184
+#define MSI_IRQ_STATUS(n)               (0x104 + ((n) << 4))
+#define MSI_IRQ_ENABLE_SET(n)           (0x108 + ((n) << 4))
+#define MSI_IRQ_ENABLE_CLR(n)           (0x10c + ((n) << 4))
 #define MSI_IRQ_OFFSET                  4
 
+#define IRQ_STATUS(n)                   (0x184 + ((n) << 4))
+#define IRQ_ENABLE_SET(n)               (0x188 + ((n) << 4))
+#define INTx_EN                         BIT(0)
+
 #define ERR_IRQ_STATUS                  0x1c4
 #define ERR_IRQ_ENABLE_SET              0x1c8
 #define ERR_AER                         BIT(5)  /* ECRC error */
+#define AM6_ERR_AER                     BIT(4)  /* AM6 ECRC error */
 #define ERR_AXI                         BIT(4)  /* AXI tag lookup fatal error */
 #define ERR_CORR                        BIT(3)  /* Correctable error */
 #define ERR_NONFATAL                    BIT(2)  /* Non-fatal error */
@@ -74,25 +83,45 @@
 #define ERR_IRQ_ALL                     (ERR_AER | ERR_AXI | ERR_CORR | \
                                          ERR_NONFATAL | ERR_FATAL | ERR_SYS)
 
-#define MAX_MSI_HOST_IRQS               8
 /* PCIE controller device IDs */
 #define PCIE_RC_K2HK                    0xb008
 #define PCIE_RC_K2E                     0xb009
 #define PCIE_RC_K2L                     0xb00a
 #define PCIE_RC_K2G                     0xb00b
 
+#define KS_PCIE_DEV_TYPE_MASK           (0x3 << 1)
+#define KS_PCIE_DEV_TYPE(mode)          ((mode) << 1)
+
+#define EP                              0x0
+#define LEG_EP                          0x1
+#define RC                              0x2
+
+#define EXP_CAP_ID_OFFSET               0x70
+
+#define KS_PCIE_SYSCLOCKOUTEN           BIT(0)
+
+#define AM654_PCIE_DEV_TYPE_MASK        0x3
+#define AM654_WIN_SIZE                  SZ_64K
+
+#define APP_ADDR_SPACE_0                (16 * SZ_1K)
+
 #define to_keystone_pcie(x)             dev_get_drvdata((x)->dev)
 
+struct ks_pcie_of_data {
+        enum dw_pcie_device_mode mode;
+        const struct dw_pcie_host_ops *host_ops;
+        const struct dw_pcie_ep_ops *ep_ops;
+        unsigned int version;
+};
+
 struct keystone_pcie {
         struct dw_pcie          *pci;
         /* PCI Device ID */
         u32                     device_id;
-        int                     num_legacy_host_irqs;
         int                     legacy_host_irqs[PCI_NUM_INTX];
         struct                  device_node *legacy_intc_np;
 
-        int                     num_msi_host_irqs;
-        int                     msi_host_irqs[MAX_MSI_HOST_IRQS];
+        int                     msi_host_irq;
         int                     num_lanes;
         u32                     num_viewport;
         struct phy              **phy;
@@ -101,28 +130,12 @@ struct keystone_pcie {
         struct irq_domain       *legacy_irq_domain;
         struct device_node      *np;
 
-        int error_irq;
-
         /* Application register space */
         void __iomem            *va_app_base;   /* DT 1st resource */
         struct resource         app;
+        bool                    is_am6;
 };
 
-static inline void update_reg_offset_bit_pos(u32 offset, u32 *reg_offset,
-                                             u32 *bit_pos)
-{
-        *reg_offset = offset % 8;
-        *bit_pos = offset >> 3;
-}
-
-static phys_addr_t ks_pcie_get_msi_addr(struct pcie_port *pp)
-{
-        struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
-
-        return ks_pcie->app.start + MSI_IRQ;
-}
-
 static u32 ks_pcie_app_readl(struct keystone_pcie *ks_pcie, u32 offset)
 {
         return readl(ks_pcie->va_app_base + offset);
@@ -134,81 +147,114 @@ static void ks_pcie_app_writel(struct keystone_pcie *ks_pcie, u32 offset,
         writel(val, ks_pcie->va_app_base + offset);
 }
 
-static void ks_pcie_handle_msi_irq(struct keystone_pcie *ks_pcie, int offset)
+static void ks_pcie_msi_irq_ack(struct irq_data *data)
 {
-        struct dw_pcie *pci = ks_pcie->pci;
-        struct pcie_port *pp = &pci->pp;
-        struct device *dev = pci->dev;
-        u32 pending, vector;
-        int src, virq;
+        struct pcie_port *pp  = irq_data_get_irq_chip_data(data);
+        struct keystone_pcie *ks_pcie;
+        u32 irq = data->hwirq;
+        struct dw_pcie *pci;
+        u32 reg_offset;
+        u32 bit_pos;
 
-        pending = ks_pcie_app_readl(ks_pcie, MSI0_IRQ_STATUS + (offset << 4));
+        pci = to_dw_pcie_from_pp(pp);
+        ks_pcie = to_keystone_pcie(pci);
 
-        /*
-         * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit
-         * shows 1, 9, 17, 25 and so forth
-         */
-        for (src = 0; src < 4; src++) {
-                if (BIT(src) & pending) {
-                        vector = offset + (src << 3);
-                        virq = irq_linear_revmap(pp->irq_domain, vector);
-                        dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n",
-                                src, vector, virq);
-                        generic_handle_irq(virq);
-                }
-        }
+        reg_offset = irq % 8;
+        bit_pos = irq >> 3;
+
+        ks_pcie_app_writel(ks_pcie, MSI_IRQ_STATUS(reg_offset),
+                           BIT(bit_pos));
+        ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET);
 }
 
-static void ks_pcie_msi_irq_ack(int irq, struct pcie_port *pp)
+static void ks_pcie_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
-        u32 reg_offset, bit_pos;
+        struct pcie_port *pp = irq_data_get_irq_chip_data(data);
         struct keystone_pcie *ks_pcie;
         struct dw_pcie *pci;
+        u64 msi_target;
 
         pci = to_dw_pcie_from_pp(pp);
         ks_pcie = to_keystone_pcie(pci);
-        update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
 
-        ks_pcie_app_writel(ks_pcie, MSI0_IRQ_STATUS + (reg_offset << 4),
-                           BIT(bit_pos));
-        ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET);
+        msi_target = ks_pcie->app.start + MSI_IRQ;
+        msg->address_lo = lower_32_bits(msi_target);
+        msg->address_hi = upper_32_bits(msi_target);
+        msg->data = data->hwirq;
+
+        dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
+                (int)data->hwirq, msg->address_hi, msg->address_lo);
 }
 
-static void ks_pcie_msi_set_irq(struct pcie_port *pp, int irq)
+static int ks_pcie_msi_set_affinity(struct irq_data *irq_data,
+                                    const struct cpumask *mask, bool force)
 {
-        u32 reg_offset, bit_pos;
-        struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+        return -EINVAL;
+}
 
-        update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
-        ks_pcie_app_writel(ks_pcie, MSI0_IRQ_ENABLE_SET + (reg_offset << 4),
+static void ks_pcie_msi_mask(struct irq_data *data)
+{
+        struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+        struct keystone_pcie *ks_pcie;
+        u32 irq = data->hwirq;
+        struct dw_pcie *pci;
+        unsigned long flags;
+        u32 reg_offset;
+        u32 bit_pos;
+
+        raw_spin_lock_irqsave(&pp->lock, flags);
+
+        pci = to_dw_pcie_from_pp(pp);
+        ks_pcie = to_keystone_pcie(pci);
+
+        reg_offset = irq % 8;
+        bit_pos = irq >> 3;
+
+        ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_CLR(reg_offset),
                            BIT(bit_pos));
+
+        raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
-static void ks_pcie_msi_clear_irq(struct pcie_port *pp, int irq)
+static void ks_pcie_msi_unmask(struct irq_data *data)
 {
-        u32 reg_offset, bit_pos;
-        struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
-        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+        struct pcie_port *pp = irq_data_get_irq_chip_data(data);
+        struct keystone_pcie *ks_pcie;
+        u32 irq = data->hwirq;
+        struct dw_pcie *pci;
+        unsigned long flags;
+        u32 reg_offset;
+        u32 bit_pos;
+
+        raw_spin_lock_irqsave(&pp->lock, flags);
 
-        update_reg_offset_bit_pos(irq, &reg_offset, &bit_pos);
-        ks_pcie_app_writel(ks_pcie, MSI0_IRQ_ENABLE_CLR + (reg_offset << 4),
+        pci = to_dw_pcie_from_pp(pp);
+        ks_pcie = to_keystone_pcie(pci);
+
+        reg_offset = irq % 8;
+        bit_pos = irq >> 3;
+
+        ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_SET(reg_offset),
                            BIT(bit_pos));
+
+        raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
+static struct irq_chip ks_pcie_msi_irq_chip = {
+        .name = "KEYSTONE-PCI-MSI",
+        .irq_ack = ks_pcie_msi_irq_ack,
+        .irq_compose_msi_msg = ks_pcie_compose_msi_msg,
+        .irq_set_affinity = ks_pcie_msi_set_affinity,
+        .irq_mask = ks_pcie_msi_mask,
+        .irq_unmask = ks_pcie_msi_unmask,
+};
+
 static int ks_pcie_msi_host_init(struct pcie_port *pp)
 {
+        pp->msi_irq_chip = &ks_pcie_msi_irq_chip;
         return dw_pcie_allocate_domains(pp);
 }
 
-static void ks_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie)
-{
-        int i;
-
-        for (i = 0; i < PCI_NUM_INTX; i++)
-                ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET + (i << 4), 0x1);
-}
-
 static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie,
                                       int offset)
 {
@@ -217,7 +263,7 @@ static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie,
         u32 pending;
         int virq;
 
-        pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS + (offset << 4));
+        pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS(offset));
 
         if (BIT(0) & pending) {
                 virq = irq_linear_revmap(ks_pcie->legacy_irq_domain, offset);
@@ -229,6 +275,14 @@ static void ks_pcie_handle_legacy_irq(struct keystone_pcie *ks_pcie,
         ks_pcie_app_writel(ks_pcie, IRQ_EOI, offset);
 }
 
+/*
+ * Dummy function so that DW core doesn't configure MSI
+ */
+static int ks_pcie_am654_msi_host_init(struct pcie_port *pp)
+{
+        return 0;
+}
+
 static void ks_pcie_enable_error_irq(struct keystone_pcie *ks_pcie)
 {
         ks_pcie_app_writel(ks_pcie, ERR_IRQ_ENABLE_SET, ERR_IRQ_ALL);
@@ -255,10 +309,10 @@ static irqreturn_t ks_pcie_handle_error_irq(struct keystone_pcie *ks_pcie)
         if (reg & ERR_CORR)
                 dev_dbg(dev, "Correctable Error\n");
 
-        if (reg & ERR_AXI)
+        if (!ks_pcie->is_am6 && (reg & ERR_AXI))
                 dev_err(dev, "AXI tag lookup fatal Error\n");
 
-        if (reg & ERR_AER)
+        if (reg & ERR_AER || (ks_pcie->is_am6 && (reg & AM6_ERR_AER)))
                 dev_err(dev, "ECRC Error\n");
 
         ks_pcie_app_writel(ks_pcie, ERR_IRQ_STATUS, reg);
@@ -356,6 +410,9 @@ static void ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie)
         dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0);
         ks_pcie_clear_dbi_mode(ks_pcie);
 
+        if (ks_pcie->is_am6)
+                return;
+
         val = ilog2(OB_WIN_SIZE);
         ks_pcie_app_writel(ks_pcie, OB_SIZE, val);
 
@@ -445,68 +502,33 @@ static int ks_pcie_link_up(struct dw_pcie *pci)
         return (val == PORT_LOGIC_LTSSM_STATE_L0);
 }
 
-static void ks_pcie_initiate_link_train(struct keystone_pcie *ks_pcie)
+static void ks_pcie_stop_link(struct dw_pcie *pci)
 {
+        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
         u32 val;
 
         /* Disable Link training */
         val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
         val &= ~LTSSM_EN_VAL;
         ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
-
-        /* Initiate Link Training */
-        val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
-        ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
 }
 
-/**
- * ks_pcie_dw_host_init() - initialize host for v3_65 dw hardware
- *
- * Ioremap the register resources, initialize legacy irq domain
- * and call dw_pcie_v3_65_host_init() API to initialize the Keystone
- * PCI host controller.
- */
-static int __init ks_pcie_dw_host_init(struct keystone_pcie *ks_pcie)
+static int ks_pcie_start_link(struct dw_pcie *pci)
 {
-        struct dw_pcie *pci = ks_pcie->pci;
-        struct pcie_port *pp = &pci->pp;
+        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
         struct device *dev = pci->dev;
-        struct platform_device *pdev = to_platform_device(dev);
-        struct resource *res;
-
-        /* Index 0 is the config reg. space address */
-        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-        pci->dbi_base = devm_pci_remap_cfg_resource(dev, res);
-        if (IS_ERR(pci->dbi_base))
-                return PTR_ERR(pci->dbi_base);
-
-        /*
-         * We set these same and is used in pcie rd/wr_other_conf
-         * functions
-         */
-        pp->va_cfg0_base = pci->dbi_base + SPACE0_REMOTE_CFG_OFFSET;
-        pp->va_cfg1_base = pp->va_cfg0_base;
-
-        /* Index 1 is the application reg. space address */
-        res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-        ks_pcie->va_app_base = devm_ioremap_resource(dev, res);
-        if (IS_ERR(ks_pcie->va_app_base))
-                return PTR_ERR(ks_pcie->va_app_base);
-
-        ks_pcie->app = *res;
+        u32 val;
 
-        /* Create legacy IRQ domain */
-        ks_pcie->legacy_irq_domain =
-                        irq_domain_add_linear(ks_pcie->legacy_intc_np,
-                                              PCI_NUM_INTX,
-                                              &ks_pcie_legacy_irq_domain_ops,
-                                              NULL);
-        if (!ks_pcie->legacy_irq_domain) {
-                dev_err(dev, "Failed to add irq domain for legacy irqs\n");
-                return -EINVAL;
+        if (dw_pcie_link_up(pci)) {
+                dev_dbg(dev, "link is already up\n");
+                return 0;
         }
 
-        return dw_pcie_host_init(pp);
+        /* Initiate Link Training */
+        val = ks_pcie_app_readl(ks_pcie, CMD_STATUS);
+        ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val);
+
+        return 0;
 }
 
 static void ks_pcie_quirk(struct pci_dev *dev)
@@ -552,34 +574,16 @@ static void ks_pcie_quirk(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, ks_pcie_quirk);
 
-static int ks_pcie_establish_link(struct keystone_pcie *ks_pcie)
-{
-        struct dw_pcie *pci = ks_pcie->pci;
-        struct device *dev = pci->dev;
-
-        if (dw_pcie_link_up(pci)) {
-                dev_info(dev, "Link already up\n");
-                return 0;
-        }
-
-        ks_pcie_initiate_link_train(ks_pcie);
-
-        /* check if the link is up or not */
-        if (!dw_pcie_wait_for_link(pci))
-                return 0;
-
-        dev_err(dev, "phy link never came up\n");
-        return -ETIMEDOUT;
-}
-
 static void ks_pcie_msi_irq_handler(struct irq_desc *desc)
 {
-        unsigned int irq = irq_desc_get_irq(desc);
+        unsigned int irq = desc->irq_data.hwirq;
         struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc);
-        u32 offset = irq - ks_pcie->msi_host_irqs[0];
+        u32 offset = irq - ks_pcie->msi_host_irq;
         struct dw_pcie *pci = ks_pcie->pci;
+        struct pcie_port *pp = &pci->pp;
         struct device *dev = pci->dev;
         struct irq_chip *chip = irq_desc_get_chip(desc);
+        u32 vector, virq, reg, pos;
 
         dev_dbg(dev, "%s, irq %d\n", __func__, irq);
 
@@ -589,7 +593,23 @@ static void ks_pcie_msi_irq_handler(struct irq_desc *desc)
          * ack operation.
          */
         chained_irq_enter(chip, desc);
-        ks_pcie_handle_msi_irq(ks_pcie, offset);
+
+        reg = ks_pcie_app_readl(ks_pcie, MSI_IRQ_STATUS(offset));
+        /*
+         * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit
+         * shows 1, 9, 17, 25 and so forth
+         */
+        for (pos = 0; pos < 4; pos++) {
+                if (!(reg & BIT(pos)))
+                        continue;
+
+                vector = offset + (pos << 3);
+                virq = irq_linear_revmap(pp->irq_domain, vector);
+                dev_dbg(dev, "irq: bit %d, vector %d, virq %d\n", pos, vector,
+                        virq);
+                generic_handle_irq(virq);
+        }
+
         chained_irq_exit(chip, desc);
 }
 
@@ -622,89 +642,119 @@ static void ks_pcie_legacy_irq_handler(struct irq_desc *desc)
         chained_irq_exit(chip, desc);
 }
 
-static int ks_pcie_get_irq_controller_info(struct keystone_pcie *ks_pcie,
-                                           char *controller, int *num_irqs)
+static int ks_pcie_config_msi_irq(struct keystone_pcie *ks_pcie)
 {
-        int temp, max_host_irqs, legacy = 1, *host_irqs;
         struct device *dev = ks_pcie->pci->dev;
-        struct device_node *np_pcie = dev->of_node, **np_temp;
-
-        if (!strcmp(controller, "msi-interrupt-controller"))
-                legacy = 0;
+        struct device_node *np = ks_pcie->np;
+        struct device_node *intc_np;
+        struct irq_data *irq_data;
+        int irq_count, irq, ret, i;
 
-        if (legacy) {
-                np_temp = &ks_pcie->legacy_intc_np;
-                max_host_irqs = PCI_NUM_INTX;
-                host_irqs = &ks_pcie->legacy_host_irqs[0];
-        } else {
-                np_temp = &ks_pcie->msi_intc_np;
-                max_host_irqs = MAX_MSI_HOST_IRQS;
-                host_irqs =  &ks_pcie->msi_host_irqs[0];
-        }
+        if (!IS_ENABLED(CONFIG_PCI_MSI))
+                return 0;
 
-        /* interrupt controller is in a child node */
-        *np_temp = of_get_child_by_name(np_pcie, controller);
-        if (!(*np_temp)) {
-                dev_err(dev, "Node for %s is absent\n", controller);
+        intc_np = of_get_child_by_name(np, "msi-interrupt-controller");
+        if (!intc_np) {
+                if (ks_pcie->is_am6)
+                        return 0;
+                dev_warn(dev, "msi-interrupt-controller node is absent\n");
                 return -EINVAL;
         }
 
-        temp = of_irq_count(*np_temp);
-        if (!temp) {
-                dev_err(dev, "No IRQ entries in %s\n", controller);
-                of_node_put(*np_temp);
-                return -EINVAL;
+        irq_count = of_irq_count(intc_np);
+        if (!irq_count) {
+                dev_err(dev, "No IRQ entries in msi-interrupt-controller\n");
+                ret = -EINVAL;
+                goto err;
         }
 
-        if (temp > max_host_irqs)
-                dev_warn(dev, "Too many %s interrupts defined %u\n",
-                        (legacy ? "legacy" : "MSI"), temp);
-
-        /*
-         * support upto max_host_irqs. In dt from index 0 to 3 (legacy) or 0 to
-         * 7 (MSI)
-         */
-        for (temp = 0; temp < max_host_irqs; temp++) {
-                host_irqs[temp] = irq_of_parse_and_map(*np_temp, temp);
-                if (!host_irqs[temp])
-                        break;
-        }
+        for (i = 0; i < irq_count; i++) {
+                irq = irq_of_parse_and_map(intc_np, i);
+                if (!irq) {
+                        ret = -EINVAL;
+                        goto err;
+                }
 
-        of_node_put(*np_temp);
+                if (!ks_pcie->msi_host_irq) {
+                        irq_data = irq_get_irq_data(irq);
+                        if (!irq_data) {
+                                ret = -EINVAL;
+                                goto err;
+                        }
+                        ks_pcie->msi_host_irq = irq_data->hwirq;
+                }
 
-        if (temp) {
-                *num_irqs = temp;
-                return 0;
+                irq_set_chained_handler_and_data(irq, ks_pcie_msi_irq_handler,
+                                                 ks_pcie);
         }
 
-        return -EINVAL;
+        of_node_put(intc_np);
+        return 0;
+
+err:
+        of_node_put(intc_np);
+        return ret;
 }
 
-static void ks_pcie_setup_interrupts(struct keystone_pcie *ks_pcie)
+static int ks_pcie_config_legacy_irq(struct keystone_pcie *ks_pcie)
 {
-        int i;
+        struct device *dev = ks_pcie->pci->dev;
+        struct irq_domain *legacy_irq_domain;
+        struct device_node *np = ks_pcie->np;
+        struct device_node *intc_np;
+        int irq_count, irq, ret = 0, i;
+
+        intc_np = of_get_child_by_name(np, "legacy-interrupt-controller");
+        if (!intc_np) {
+                /*
+                 * Since legacy interrupts are modeled as edge-interrupts in
+                 * AM6, keep it disabled for now.
+                 */
+                if (ks_pcie->is_am6)
+                        return 0;
+                dev_warn(dev, "legacy-interrupt-controller node is absent\n");
+                return -EINVAL;
+        }
 
-        /* Legacy IRQ */
-        for (i = 0; i < ks_pcie->num_legacy_host_irqs; i++) {
-                irq_set_chained_handler_and_data(ks_pcie->legacy_host_irqs[i],
+        irq_count = of_irq_count(intc_np);
+        if (!irq_count) {
+                dev_err(dev, "No IRQ entries in legacy-interrupt-controller\n");
+                ret = -EINVAL;
+                goto err;
+        }
+
+        for (i = 0; i < irq_count; i++) {
+                irq = irq_of_parse_and_map(intc_np, i);
+                if (!irq) {
+                        ret = -EINVAL;
+                        goto err;
+                }
+                ks_pcie->legacy_host_irqs[i] = irq;
+
+                irq_set_chained_handler_and_data(irq,
                                                  ks_pcie_legacy_irq_handler,
                                                  ks_pcie);
         }
-        ks_pcie_enable_legacy_irqs(ks_pcie);
 
-        /* MSI IRQ */
-        if (IS_ENABLED(CONFIG_PCI_MSI)) {
-                for (i = 0; i < ks_pcie->num_msi_host_irqs; i++) {
-                        irq_set_chained_handler_and_data(ks_pcie->msi_host_irqs[i],
-                                                         ks_pcie_msi_irq_handler,
-                                                         ks_pcie);
-                }
+        legacy_irq_domain =
+                irq_domain_add_linear(intc_np, PCI_NUM_INTX,
+                                      &ks_pcie_legacy_irq_domain_ops, NULL);
+        if (!legacy_irq_domain) {
+                dev_err(dev, "Failed to add irq domain for legacy irqs\n");
+                ret = -EINVAL;
+                goto err;
         }
+        ks_pcie->legacy_irq_domain = legacy_irq_domain;
+
+        for (i = 0; i < PCI_NUM_INTX; i++)
+                ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET(i), INTx_EN);
 
-        if (ks_pcie->error_irq > 0)
-                ks_pcie_enable_error_irq(ks_pcie);
+err:
+        of_node_put(intc_np);
+        return ret;
 }
 
+#ifdef CONFIG_ARM
 /*
  * When a PCI device does not exist during config cycles, keystone host gets a
  * bus error instead of returning 0xffffffff. This handler always returns 0
@@ -724,6 +774,7 @@ static int ks_pcie_fault(unsigned long addr, unsigned int fsr,
 
         return 0;
 }
+#endif
 
 static int __init ks_pcie_init_id(struct keystone_pcie *ks_pcie)
 {
@@ -742,8 +793,10 @@ static int __init ks_pcie_init_id(struct keystone_pcie *ks_pcie)
         if (ret)
                 return ret;
 
+        dw_pcie_dbi_ro_wr_en(pci);
         dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, id & PCIE_VENDORID_MASK);
         dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, id >> PCIE_DEVICEID_SHIFT);
+        dw_pcie_dbi_ro_wr_dis(pci);
 
         return 0;
 }
@@ -754,11 +807,18 @@ static int __init ks_pcie_host_init(struct pcie_port *pp)
         struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
         int ret;
 
+        ret = ks_pcie_config_legacy_irq(ks_pcie);
+        if (ret)
+                return ret;
+
+        ret = ks_pcie_config_msi_irq(ks_pcie);
+        if (ret)
+                return ret;
+
         dw_pcie_setup_rc(pp);
 
-        ks_pcie_establish_link(ks_pcie);
+        ks_pcie_stop_link(pci);
         ks_pcie_setup_rc_app_regs(ks_pcie);
-        ks_pcie_setup_interrupts(ks_pcie);
         writew(PCI_IO_RANGE_TYPE_32 | (PCI_IO_RANGE_TYPE_32 << 8),
                         pci->dbi_base + PCI_IO_BASE);
 
@@ -766,12 +826,17 @@ static int __init ks_pcie_host_init(struct pcie_port *pp)
         if (ret < 0)
                 return ret;
 
+#ifdef CONFIG_ARM
         /*
          * PCIe access errors that result into OCP errors are caught by ARM as
          * "External aborts"
          */
         hook_fault_code(17, ks_pcie_fault, SIGBUS, 0,
                         "Asynchronous external abort");
+#endif
+
+        ks_pcie_start_link(pci);
+        dw_pcie_wait_for_link(pci);
 
         return 0;
 }
@@ -780,14 +845,15 @@ static const struct dw_pcie_host_ops ks_pcie_host_ops = {
         .rd_other_conf = ks_pcie_rd_other_conf,
         .wr_other_conf = ks_pcie_wr_other_conf,
         .host_init = ks_pcie_host_init,
-        .msi_set_irq = ks_pcie_msi_set_irq,
-        .msi_clear_irq = ks_pcie_msi_clear_irq,
-        .get_msi_addr = ks_pcie_get_msi_addr,
         .msi_host_init = ks_pcie_msi_host_init,
-        .msi_irq_ack = ks_pcie_msi_irq_ack,
         .scan_bus = ks_pcie_v3_65_scan_bus,
 };
 
+static const struct dw_pcie_host_ops ks_pcie_am654_host_ops = {
+        .host_init = ks_pcie_host_init,
+        .msi_host_init = ks_pcie_am654_msi_host_init,
+};
+
 static irqreturn_t ks_pcie_err_irq_handler(int irq, void *priv)
 {
         struct keystone_pcie *ks_pcie = priv;
@@ -801,41 +867,17 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie,
         struct dw_pcie *pci = ks_pcie->pci;
         struct pcie_port *pp = &pci->pp;
         struct device *dev = &pdev->dev;
+        struct resource *res;
         int ret;
 
-        ret = ks_pcie_get_irq_controller_info(ks_pcie,
-                                        "legacy-interrupt-controller",
-                                        &ks_pcie->num_legacy_host_irqs);
-        if (ret)
-                return ret;
-
-        if (IS_ENABLED(CONFIG_PCI_MSI)) {
-                ret = ks_pcie_get_irq_controller_info(ks_pcie,
-                                                "msi-interrupt-controller",
-                                                &ks_pcie->num_msi_host_irqs);
-                if (ret)
-                        return ret;
-        }
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
+        pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
+        if (IS_ERR(pp->va_cfg0_base))
+                return PTR_ERR(pp->va_cfg0_base);
 
-        /*
-         * Index 0 is the platform interrupt for error interrupt
-         * from RC.  This is optional.
-         */
-        ks_pcie->error_irq = irq_of_parse_and_map(ks_pcie->np, 0);
-        if (ks_pcie->error_irq <= 0)
-                dev_info(dev, "no error IRQ defined\n");
-        else {
-                ret = request_irq(ks_pcie->error_irq, ks_pcie_err_irq_handler,
-                                  IRQF_SHARED, "pcie-error-irq", ks_pcie);
-                if (ret < 0) {
-                        dev_err(dev, "failed to request error IRQ %d\n",
-                                ks_pcie->error_irq);
-                        return ret;
-                }
-        }
+        pp->va_cfg1_base = pp->va_cfg0_base;
 
-        pp->ops = &ks_pcie_host_ops;
-        ret = ks_pcie_dw_host_init(ks_pcie);
+        ret = dw_pcie_host_init(pp);
         if (ret) {
                 dev_err(dev, "failed to initialize host\n");
                 return ret;
@@ -844,18 +886,139 @@ static int __init ks_pcie_add_pcie_port(struct keystone_pcie *ks_pcie,
         return 0;
 }
 
-static const struct of_device_id ks_pcie_of_match[] = {
-        {
-                .type = "pci",
-                .compatible = "ti,keystone-pcie",
-        },
-        { },
-};
+static u32 ks_pcie_am654_read_dbi2(struct dw_pcie *pci, void __iomem *base,
+                                   u32 reg, size_t size)
+{
+        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+        u32 val;
+
+        ks_pcie_set_dbi_mode(ks_pcie);
+        dw_pcie_read(base + reg, size, &val);
+        ks_pcie_clear_dbi_mode(ks_pcie);
+        return val;
+}
+
+static void ks_pcie_am654_write_dbi2(struct dw_pcie *pci, void __iomem *base,
+                                     u32 reg, size_t size, u32 val)
+{
+        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+        ks_pcie_set_dbi_mode(ks_pcie);
+        dw_pcie_write(base + reg, size, val);
+        ks_pcie_clear_dbi_mode(ks_pcie);
+}
 
 static const struct dw_pcie_ops ks_pcie_dw_pcie_ops = {
+        .start_link = ks_pcie_start_link,
+        .stop_link = ks_pcie_stop_link,
         .link_up = ks_pcie_link_up,
+        .read_dbi2 = ks_pcie_am654_read_dbi2,
+        .write_dbi2 = ks_pcie_am654_write_dbi2,
+};
+
+static void ks_pcie_am654_ep_init(struct dw_pcie_ep *ep)
+{
+        struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+        int flags;
+
+        ep->page_size = AM654_WIN_SIZE;
+        flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32;
+        dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, APP_ADDR_SPACE_0 - 1);
+        dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, flags);
+}
+
+static void ks_pcie_am654_raise_legacy_irq(struct keystone_pcie *ks_pcie)
+{
+        struct dw_pcie *pci = ks_pcie->pci;
+        u8 int_pin;
+
+        int_pin = dw_pcie_readb_dbi(pci, PCI_INTERRUPT_PIN);
+        if (int_pin == 0 || int_pin > 4)
+                return;
+
+        ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_SET(int_pin),
+                           INT_ENABLE);
+        ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_SET, INT_ENABLE);
+        mdelay(1);
+        ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_CLR, INT_ENABLE);
+        ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_CLR(int_pin),
+                           INT_ENABLE);
+}
+
+static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
+                                   enum pci_epc_irq_type type,
+                                   u16 interrupt_num)
+{
+        struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+        struct keystone_pcie *ks_pcie = to_keystone_pcie(pci);
+
+        switch (type) {
+        case PCI_EPC_IRQ_LEGACY:
+                ks_pcie_am654_raise_legacy_irq(ks_pcie);
+                break;
+        case PCI_EPC_IRQ_MSI:
+                dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
+                break;
+        default:
+                dev_err(pci->dev, "UNKNOWN IRQ type\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static const struct pci_epc_features ks_pcie_am654_epc_features = {
+        .linkup_notifier = false,
+        .msi_capable = true,
+        .msix_capable = false,
+        .reserved_bar = 1 << BAR_0 | 1 << BAR_1,
+        .bar_fixed_64bit = 1 << BAR_0,
+        .bar_fixed_size[2] = SZ_1M,
+        .bar_fixed_size[3] = SZ_64K,
+        .bar_fixed_size[4] = 256,
+        .bar_fixed_size[5] = SZ_1M,
+        .align = SZ_1M,
 };
 
+static const struct pci_epc_features*
+ks_pcie_am654_get_features(struct dw_pcie_ep *ep)
+{
+        return &ks_pcie_am654_epc_features;
+}
+
+static const struct dw_pcie_ep_ops ks_pcie_am654_ep_ops = {
+        .ep_init = ks_pcie_am654_ep_init,
+        .raise_irq = ks_pcie_am654_raise_irq,
+        .get_features = &ks_pcie_am654_get_features,
+};
+
+static int __init ks_pcie_add_pcie_ep(struct keystone_pcie *ks_pcie,
+                                      struct platform_device *pdev)
+{
+        int ret;
+        struct dw_pcie_ep *ep;
+        struct resource *res;
+        struct device *dev = &pdev->dev;
+        struct dw_pcie *pci = ks_pcie->pci;
+
+        ep = &pci->ep;
+
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
+        if (!res)
+                return -EINVAL;
+
+        ep->phys_base = res->start;
+        ep->addr_size = resource_size(res);
+
+        ret = dw_pcie_ep_init(ep);
+        if (ret) {
+                dev_err(dev, "failed to initialize endpoint\n");
+                return ret;
+        }
+
+        return 0;
+}
+
 static void ks_pcie_disable_phy(struct keystone_pcie *ks_pcie)
 {
         int num_lanes = ks_pcie->num_lanes;
@@ -873,6 +1036,10 @@ static int ks_pcie_enable_phy(struct keystone_pcie *ks_pcie)
         int num_lanes = ks_pcie->num_lanes;
 
         for (i = 0; i < num_lanes; i++) {
+                ret = phy_reset(ks_pcie->phy[i]);
+                if (ret < 0)
+                        goto err_phy;
+
                 ret = phy_init(ks_pcie->phy[i]);
                 if (ret < 0)
                         goto err_phy;
@@ -895,20 +1062,161 @@ err_phy:
         return ret;
 }
 
+static int ks_pcie_set_mode(struct device *dev)
+{
+        struct device_node *np = dev->of_node;
+        struct regmap *syscon;
+        u32 val;
+        u32 mask;
+        int ret = 0;
+
+        syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode");
+        if (IS_ERR(syscon))
+                return 0;
+
+        mask = KS_PCIE_DEV_TYPE_MASK | KS_PCIE_SYSCLOCKOUTEN;
+        val = KS_PCIE_DEV_TYPE(RC) | KS_PCIE_SYSCLOCKOUTEN;
+
+        ret = regmap_update_bits(syscon, 0, mask, val);
+        if (ret) {
+                dev_err(dev, "failed to set pcie mode\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+static int ks_pcie_am654_set_mode(struct device *dev,
+                                  enum dw_pcie_device_mode mode)
+{
+        struct device_node *np = dev->of_node;
+        struct regmap *syscon;
+        u32 val;
+        u32 mask;
+        int ret = 0;
+
+        syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode");
+        if (IS_ERR(syscon))
+                return 0;
+
+        mask = AM654_PCIE_DEV_TYPE_MASK;
+
+        switch (mode) {
+        case DW_PCIE_RC_TYPE:
+                val = RC;
+                break;
+        case DW_PCIE_EP_TYPE:
+                val = EP;
+                break;
+        default:
+                dev_err(dev, "INVALID device type %d\n", mode);
+                return -EINVAL;
+        }
+
+        ret = regmap_update_bits(syscon, 0, mask, val);
+        if (ret) {
+                dev_err(dev, "failed to set pcie mode\n");
+                return ret;
+        }
+
+        return 0;
+}
+
+static void ks_pcie_set_link_speed(struct dw_pcie *pci, int link_speed)
+{
+        u32 val;
+
+        dw_pcie_dbi_ro_wr_en(pci);
+
+        val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP);
+        if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) {
+                val &= ~((u32)PCI_EXP_LNKCAP_SLS);
+                val |= link_speed;
+                dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCAP,
+                                   val);
+        }
+
+        val = dw_pcie_readl_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2);
+        if ((val & PCI_EXP_LNKCAP_SLS) != link_speed) {
+                val &= ~((u32)PCI_EXP_LNKCAP_SLS);
+                val |= link_speed;
+                dw_pcie_writel_dbi(pci, EXP_CAP_ID_OFFSET + PCI_EXP_LNKCTL2,
+                                   val);
+        }
+
+        dw_pcie_dbi_ro_wr_dis(pci);
+}
+
+static const struct ks_pcie_of_data ks_pcie_rc_of_data = {
+        .host_ops = &ks_pcie_host_ops,
+        .version = 0x365A,
+};
+
+static const struct ks_pcie_of_data ks_pcie_am654_rc_of_data = {
+        .host_ops = &ks_pcie_am654_host_ops,
+        .mode = DW_PCIE_RC_TYPE,
+        .version = 0x490A,
+};
+
+static const struct ks_pcie_of_data ks_pcie_am654_ep_of_data = {
+        .ep_ops = &ks_pcie_am654_ep_ops,
+        .mode = DW_PCIE_EP_TYPE,
+        .version = 0x490A,
+};
+
+static const struct of_device_id ks_pcie_of_match[] = {
+        {
+                .type = "pci",
+                .data = &ks_pcie_rc_of_data,
+                .compatible = "ti,keystone-pcie",
+        },
+        {
+                .data = &ks_pcie_am654_rc_of_data,
+                .compatible = "ti,am654-pcie-rc",
+        },
+        {
+                .data = &ks_pcie_am654_ep_of_data,
+                .compatible = "ti,am654-pcie-ep",
+        },
+        { },
+};
+
 static int __init ks_pcie_probe(struct platform_device *pdev)
 {
+        const struct dw_pcie_host_ops *host_ops;
+        const struct dw_pcie_ep_ops *ep_ops;
         struct device *dev = &pdev->dev;
         struct device_node *np = dev->of_node;
+        const struct ks_pcie_of_data *data;
+        const struct of_device_id *match;
+        enum dw_pcie_device_mode mode;
         struct dw_pcie *pci;
         struct keystone_pcie *ks_pcie;
         struct device_link **link;
+        struct gpio_desc *gpiod;
+        void __iomem *atu_base;
+        struct resource *res;
+        unsigned int version;
+        void __iomem *base;
         u32 num_viewport;
         struct phy **phy;
+        int link_speed;
         u32 num_lanes;
         char name[10];
         int ret;
+        int irq;
         int i;
 
+        match = of_match_device(of_match_ptr(ks_pcie_of_match), dev);
+        data = (struct ks_pcie_of_data *)match->data;
+        if (!data)
+                return -EINVAL;
+
+        version = data->version;
+        host_ops = data->host_ops;
+        ep_ops = data->ep_ops;
+        mode = data->mode;
+
         ks_pcie = devm_kzalloc(dev, sizeof(*ks_pcie), GFP_KERNEL);
         if (!ks_pcie)
                 return -ENOMEM;
@@ -917,12 +1225,38 @@ static int __init ks_pcie_probe(struct platform_device *pdev)
         if (!pci)
                 return -ENOMEM;
 
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "app");
+        ks_pcie->va_app_base = devm_ioremap_resource(dev, res);
+        if (IS_ERR(ks_pcie->va_app_base))
+                return PTR_ERR(ks_pcie->va_app_base);
+
+        ks_pcie->app = *res;
+
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbics");
+        base = devm_pci_remap_cfg_resource(dev, res);
+        if (IS_ERR(base))
+                return PTR_ERR(base);
+
+        if (of_device_is_compatible(np, "ti,am654-pcie-rc"))
+                ks_pcie->is_am6 = true;
+
+        pci->dbi_base = base;
+        pci->dbi_base2 = base;
         pci->dev = dev;
         pci->ops = &ks_pcie_dw_pcie_ops;
+        pci->version = version;
+
+        irq = platform_get_irq(pdev, 0);
+        if (irq < 0) {
+                dev_err(dev, "missing IRQ resource: %d\n", irq);
+                return irq;
+        }
 
-        ret = of_property_read_u32(np, "num-viewport", &num_viewport);
+        ret = request_irq(irq, ks_pcie_err_irq_handler, IRQF_SHARED,
+                          "ks-pcie-error-irq", ks_pcie);
         if (ret < 0) {
-                dev_err(dev, "unable to read *num-viewport* property\n");
+                dev_err(dev, "failed to request error IRQ %d\n",
+                        irq);
                 return ret;
         }
 
@@ -960,9 +1294,17 @@ static int __init ks_pcie_probe(struct platform_device *pdev)
         ks_pcie->pci = pci;
         ks_pcie->link = link;
         ks_pcie->num_lanes = num_lanes;
-        ks_pcie->num_viewport = num_viewport;
         ks_pcie->phy = phy;
 
+        gpiod = devm_gpiod_get_optional(dev, "reset",
+                                        GPIOD_OUT_LOW);
+        if (IS_ERR(gpiod)) {
+                ret = PTR_ERR(gpiod);
+                if (ret != -EPROBE_DEFER)
+                        dev_err(dev, "Failed to get reset GPIO\n");
+                goto err_link;
+        }
+
         ret = ks_pcie_enable_phy(ks_pcie);
         if (ret) {
                 dev_err(dev, "failed to enable phy\n");
@@ -977,9 +1319,79 @@ static int __init ks_pcie_probe(struct platform_device *pdev)
                 goto err_get_sync;
         }
 
-        ret = ks_pcie_add_pcie_port(ks_pcie, pdev);
-        if (ret < 0)
-                goto err_get_sync;
+        if (pci->version >= 0x480A) {
+                res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu");
+                atu_base = devm_ioremap_resource(dev, res);
+                if (IS_ERR(atu_base)) {
+                        ret = PTR_ERR(atu_base);
+                        goto err_get_sync;
+                }
+
+                pci->atu_base = atu_base;
+
+                ret = ks_pcie_am654_set_mode(dev, mode);
+                if (ret < 0)
+                        goto err_get_sync;
+        } else {
+                ret = ks_pcie_set_mode(dev);
+                if (ret < 0)
+                        goto err_get_sync;
+        }
+
+        link_speed = of_pci_get_max_link_speed(np);
+        if (link_speed < 0)
+                link_speed = 2;
+
+        ks_pcie_set_link_speed(pci, link_speed);
+
+        switch (mode) {
+        case DW_PCIE_RC_TYPE:
+                if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_HOST)) {
+                        ret = -ENODEV;
+                        goto err_get_sync;
+                }
+
+                ret = of_property_read_u32(np, "num-viewport", &num_viewport);
+                if (ret < 0) {
+                        dev_err(dev, "unable to read *num-viewport* property\n");
+                        return ret;
+                }
+
+                /*
+                 * "Power Sequencing and Reset Signal Timings" table in
+                 * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 2.0
+                 * indicates PERST# should be deasserted after minimum of 100us
+                 * once REFCLK is stable. The REFCLK to the connector in RC
+                 * mode is selected while enabling the PHY. So deassert PERST#
+                 * after 100 us.
+                 */
+                if (gpiod) {
+                        usleep_range(100, 200);
+                        gpiod_set_value_cansleep(gpiod, 1);
+                }
+
+                ks_pcie->num_viewport = num_viewport;
+                pci->pp.ops = host_ops;
+                ret = ks_pcie_add_pcie_port(ks_pcie, pdev);
+                if (ret < 0)
+                        goto err_get_sync;
+                break;
+        case DW_PCIE_EP_TYPE:
+                if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_EP)) {
+                        ret = -ENODEV;
+                        goto err_get_sync;
+                }
+
+                pci->ep.ops = ep_ops;
+                ret = ks_pcie_add_pcie_ep(ks_pcie, pdev);
+                if (ret < 0)
+                        goto err_get_sync;
+                break;
+        default:
+                dev_err(dev, "INVALID device type %d\n", mode);
+        }
+
+        ks_pcie_enable_error_irq(ks_pcie);
 
         return 0;
 
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index a42c9c3ae1cc..be61d96cc95e 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -79,7 +79,7 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
         }
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
         .ep_init = ls_pcie_ep_init,
         .raise_irq = ls_pcie_ep_raise_irq,
         .get_features = ls_pcie_ep_get_features,
diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c
index dba83abfe764..d00252bd8fae 100644
--- a/drivers/pci/controller/dwc/pcie-artpec6.c
+++ b/drivers/pci/controller/dwc/pcie-artpec6.c
@@ -444,7 +444,7 @@ static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
         return 0;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
         .ep_init = artpec6_pcie_ep_init,
         .raise_irq = artpec6_pcie_raise_irq,
 };
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 24f5a775ad34..2bf5a35c0570 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -46,16 +46,19 @@ static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
         u8 cap_id, next_cap_ptr;
         u16 reg;
 
+        if (!cap_ptr)
+                return 0;
+
         reg = dw_pcie_readw_dbi(pci, cap_ptr);
-        next_cap_ptr = (reg & 0xff00) >> 8;
         cap_id = (reg & 0x00ff);
 
-        if (!next_cap_ptr || cap_id > PCI_CAP_ID_MAX)
+        if (cap_id > PCI_CAP_ID_MAX)
                 return 0;
 
         if (cap_id == cap)
                 return cap_ptr;
 
+        next_cap_ptr = (reg & 0xff00) >> 8;
         return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
 }
 
@@ -67,9 +70,6 @@ static u8 dw_pcie_ep_find_capability(struct dw_pcie *pci, u8 cap)
         reg = dw_pcie_readw_dbi(pci, PCI_CAPABILITY_LIST);
         next_cap_ptr = (reg & 0x00ff);
 
-        if (!next_cap_ptr)
-                return 0;
-
         return __dw_pcie_ep_find_next_cap(pci, next_cap_ptr, cap);
 }
 
@@ -397,6 +397,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 {
         struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
         struct pci_epc *epc = ep->epc;
+        unsigned int aligned_offset;
         u16 msg_ctrl, msg_data;
         u32 msg_addr_lower, msg_addr_upper, reg;
         u64 msg_addr;
@@ -422,13 +423,15 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
                 reg = ep->msi_cap + PCI_MSI_DATA_32;
                 msg_data = dw_pcie_readw_dbi(pci, reg);
         }
-        msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower;
+        aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
+        msg_addr = ((u64)msg_addr_upper) << 32 |
+                        (msg_addr_lower & ~aligned_offset);
         ret = dw_pcie_ep_map_addr(epc, func_no, ep->msi_mem_phys, msg_addr,
                                   epc->mem->page_size);
         if (ret)
                 return ret;
 
-        writel(msg_data | (interrupt_num - 1), ep->msi_mem);
+        writel(msg_data | (interrupt_num - 1), ep->msi_mem + aligned_offset);
 
         dw_pcie_ep_unmap_addr(epc, func_no, ep->msi_mem_phys);
 
@@ -504,10 +507,32 @@ void dw_pcie_ep_exit(struct dw_pcie_ep *ep)
         pci_epc_mem_exit(epc);
 }
 
+static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap)
+{
+        u32 header;
+        int pos = PCI_CFG_SPACE_SIZE;
+
+        while (pos) {
+                header = dw_pcie_readl_dbi(pci, pos);
+                if (PCI_EXT_CAP_ID(header) == cap)
+                        return pos;
+
+                pos = PCI_EXT_CAP_NEXT(header);
+                if (!pos)
+                        break;
+        }
+
+        return 0;
+}
+
 int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 {
+        int i;
         int ret;
+        u32 reg;
         void *addr;
+        unsigned int nbars;
+        unsigned int offset;
         struct pci_epc *epc;
         struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
         struct device *dev = pci->dev;
@@ -517,10 +542,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
                 dev_err(dev, "dbi_base/dbi_base2 is not populated\n");
                 return -EINVAL;
         }
-        if (pci->iatu_unroll_enabled && !pci->atu_base) {
-                dev_err(dev, "atu_base is not populated\n");
-                return -EINVAL;
-        }
 
         ret = of_property_read_u32(np, "num-ib-windows", &ep->num_ib_windows);
         if (ret < 0) {
@@ -595,6 +616,18 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 
         ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);
 
+        offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
+        if (offset) {
+                reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
+                nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >>
+                        PCI_REBAR_CTRL_NBAR_SHIFT;
+
+                dw_pcie_dbi_ro_wr_en(pci);
+                for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL)
+                        dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, 0x0);
+                dw_pcie_dbi_ro_wr_dis(pci);
+        }
+
         dw_pcie_setup(pci);
 
         return 0;
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 3e4169e738a5..77db32529319 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -126,18 +126,12 @@ static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
         struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
         u64 msi_target;
 
-        if (pp->ops->get_msi_addr)
-                msi_target = pp->ops->get_msi_addr(pp);
-        else
-                msi_target = (u64)pp->msi_data;
+        msi_target = (u64)pp->msi_data;
 
         msg->address_lo = lower_32_bits(msi_target);
         msg->address_hi = upper_32_bits(msi_target);
 
-        if (pp->ops->get_msi_data)
-                msg->data = pp->ops->get_msi_data(pp, d->hwirq);
-        else
-                msg->data = d->hwirq;
+        msg->data = d->hwirq;
 
         dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
                 (int)d->hwirq, msg->address_hi, msg->address_lo);
@@ -157,17 +151,13 @@ static void dw_pci_bottom_mask(struct irq_data *d)
 
         raw_spin_lock_irqsave(&pp->lock, flags);
 
-        if (pp->ops->msi_clear_irq) {
-                pp->ops->msi_clear_irq(pp, d->hwirq);
-        } else {
-                ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
-                res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
-                bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
+        ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
+        res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+        bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
-                pp->irq_mask[ctrl] |= BIT(bit);
-                dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
-                                    pp->irq_mask[ctrl]);
-        }
+        pp->irq_mask[ctrl] |= BIT(bit);
+        dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
+                            pp->irq_mask[ctrl]);
 
         raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
@@ -180,17 +170,13 @@ static void dw_pci_bottom_unmask(struct irq_data *d)
 
         raw_spin_lock_irqsave(&pp->lock, flags);
 
-        if (pp->ops->msi_set_irq) {
-                pp->ops->msi_set_irq(pp, d->hwirq);
-        } else {
-                ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
-                res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
-                bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
+        ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
+        res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
+        bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
-                pp->irq_mask[ctrl] &= ~BIT(bit);
-                dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
-                                    pp->irq_mask[ctrl]);
-        }
+        pp->irq_mask[ctrl] &= ~BIT(bit);
+        dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK + res, 4,
+                            pp->irq_mask[ctrl]);
 
         raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
@@ -199,20 +185,12 @@ static void dw_pci_bottom_ack(struct irq_data *d)
 {
         struct pcie_port *pp  = irq_data_get_irq_chip_data(d);
         unsigned int res, bit, ctrl;
-        unsigned long flags;
 
         ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
         res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
         bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
 
-        raw_spin_lock_irqsave(&pp->lock, flags);
-
         dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_STATUS + res, 4, BIT(bit));
-
-        if (pp->ops->msi_irq_ack)
-                pp->ops->msi_irq_ack(d->hwirq, pp);
-
-        raw_spin_unlock_irqrestore(&pp->lock, flags);
 }
 
 static struct irq_chip dw_pci_msi_bottom_irq_chip = {
@@ -245,7 +223,7 @@ static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
 
         for (i = 0; i < nr_irqs; i++)
                 irq_domain_set_info(domain, virq + i, bit + i,
-                                    &dw_pci_msi_bottom_irq_chip,
+                                    pp->msi_irq_chip,
                                     pp, handle_edge_irq,
                                     NULL, NULL);
 
@@ -462,6 +440,8 @@ int dw_pcie_host_init(struct pcie_port *pp)
                 }
 
                 if (!pp->ops->msi_host_init) {
+                        pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
+
                         ret = dw_pcie_allocate_domains(pp);
                         if (ret)
                                 return ret;
@@ -632,17 +612,6 @@ static struct pci_ops dw_pcie_ops = {
         .write = dw_pcie_wr_conf,
 };
 
-static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci)
-{
-        u32 val;
-
-        val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT);
-        if (val == 0xffffffff)
-                return 1;
-
-        return 0;
-}
-
 void dw_pcie_setup_rc(struct pcie_port *pp)
 {
         u32 val, ctrl, num_ctrls;
@@ -650,17 +619,19 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
 
         dw_pcie_setup(pci);
 
-        num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
-
-        /* Initialize IRQ Status array */
-        for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
-                pp->irq_mask[ctrl] = ~0;
-                dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK +
-                                        (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
-                                    4, pp->irq_mask[ctrl]);
-                dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE +
-                                        (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
-                                    4, ~0);
+        if (!pp->ops->msi_host_init) {
+                num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
+
+                /* Initialize IRQ Status array */
+                for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
+                        pp->irq_mask[ctrl] = ~0;
+                        dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_MASK +
+                                            (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
+                                            4, pp->irq_mask[ctrl]);
+                        dw_pcie_wr_own_conf(pp, PCIE_MSI_INTR0_ENABLE +
+                                            (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
+                                            4, ~0);
+                }
         }
 
         /* Setup RC BARs */
@@ -694,14 +665,6 @@ void dw_pcie_setup_rc(struct pcie_port *pp)
          * we should not program the ATU here.
          */
         if (!pp->ops->rd_other_conf) {
-                /* Get iATU unroll support */
-                pci->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pci);
-                dev_dbg(pci->dev, "iATU unroll: %s\n",
-                        pci->iatu_unroll_enabled ? "enabled" : "disabled");
-
-                if (pci->iatu_unroll_enabled && !pci->atu_base)
-                        pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET;
-
                 dw_pcie_prog_outbound_atu(pci, PCIE_ATU_REGION_INDEX0,
                                           PCIE_ATU_TYPE_MEM, pp->mem_base,
                                           pp->mem_bus_addr, pp->mem_size);
diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c
index 932dbd0b34b6..b58fdcbc664b 100644
--- a/drivers/pci/controller/dwc/pcie-designware-plat.c
+++ b/drivers/pci/controller/dwc/pcie-designware-plat.c
@@ -106,7 +106,7 @@ dw_plat_pcie_get_features(struct dw_pcie_ep *ep)
         return &dw_plat_pcie_epc_features;
 }
 
-static struct dw_pcie_ep_ops pcie_ep_ops = {
+static const struct dw_pcie_ep_ops pcie_ep_ops = {
         .ep_init = dw_plat_pcie_ep_init,
         .raise_irq = dw_plat_pcie_ep_raise_irq,
         .get_features = dw_plat_pcie_get_features,
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 086e87a40316..9d7c51c32b3b 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -83,6 +83,37 @@ void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
                 dev_err(pci->dev, "Write DBI address failed\n");
 }
 
+u32 __dw_pcie_read_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+                        size_t size)
+{
+        int ret;
+        u32 val;
+
+        if (pci->ops->read_dbi2)
+                return pci->ops->read_dbi2(pci, base, reg, size);
+
+        ret = dw_pcie_read(base + reg, size, &val);
+        if (ret)
+                dev_err(pci->dev, "read DBI address failed\n");
+
+        return val;
+}
+
+void __dw_pcie_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+                          size_t size, u32 val)
+{
+        int ret;
+
+        if (pci->ops->write_dbi2) {
+                pci->ops->write_dbi2(pci, base, reg, size, val);
+                return;
+        }
+
+        ret = dw_pcie_write(base + reg, size, val);
+        if (ret)
+                dev_err(pci->dev, "write DBI address failed\n");
+}
+
 static u32 dw_pcie_readl_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg)
 {
         u32 offset = PCIE_GET_ATU_OUTB_UNR_REG_OFFSET(index);
@@ -333,6 +364,17 @@ int dw_pcie_link_up(struct dw_pcie *pci)
                 (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING)));
 }
 
+static u8 dw_pcie_iatu_unroll_enabled(struct dw_pcie *pci)
+{
+        u32 val;
+
+        val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT);
+        if (val == 0xffffffff)
+                return 1;
+
+        return 0;
+}
+
 void dw_pcie_setup(struct dw_pcie *pci)
 {
         int ret;
@@ -341,6 +383,16 @@ void dw_pcie_setup(struct dw_pcie *pci)
         struct device *dev = pci->dev;
         struct device_node *np = dev->of_node;
 
+        if (pci->version >= 0x480A || (!pci->version &&
+                                       dw_pcie_iatu_unroll_enabled(pci))) {
+                pci->iatu_unroll_enabled = true;
+                if (!pci->atu_base)
+                        pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET;
+        }
+        dev_dbg(pci->dev, "iATU unroll: %s\n", pci->iatu_unroll_enabled ?
+                "enabled" : "disabled");
+
+
         ret = of_property_read_u32(np, "num-lanes", &lanes);
         if (ret)
                 lanes = 0;
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 351636d6d9b5..b8993f2b78df 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -148,14 +148,9 @@ struct dw_pcie_host_ops {
         int (*wr_other_conf)(struct pcie_port *pp, struct pci_bus *bus,
                              unsigned int devfn, int where, int size, u32 val);
         int (*host_init)(struct pcie_port *pp);
-        void (*msi_set_irq)(struct pcie_port *pp, int irq);
-        void (*msi_clear_irq)(struct pcie_port *pp, int irq);
-        phys_addr_t (*get_msi_addr)(struct pcie_port *pp);
-        u32 (*get_msi_data)(struct pcie_port *pp, int pos);
         void (*scan_bus)(struct pcie_port *pp);
         void (*set_num_vectors)(struct pcie_port *pp);
         int (*msi_host_init)(struct pcie_port *pp);
-        void (*msi_irq_ack)(int irq, struct pcie_port *pp);
 };
 
 struct pcie_port {
@@ -183,6 +178,7 @@ struct pcie_port {
         struct irq_domain       *msi_domain;
         dma_addr_t              msi_data;
         struct page             *msi_page;
+        struct irq_chip         *msi_irq_chip;
         u32                     num_vectors;
         u32                     irq_mask[MAX_MSI_CTRLS];
         struct pci_bus          *root_bus;
@@ -205,7 +201,7 @@ struct dw_pcie_ep_ops {
 
 struct dw_pcie_ep {
         struct pci_epc          *epc;
-        struct dw_pcie_ep_ops   *ops;
+        const struct dw_pcie_ep_ops *ops;
         phys_addr_t             phys_base;
         size_t                  addr_size;
         size_t                  page_size;
@@ -227,6 +223,10 @@ struct dw_pcie_ops {
                             size_t size);
         void    (*write_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
                              size_t size, u32 val);
+        u32     (*read_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
+                             size_t size);
+        void    (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg,
+                              size_t size, u32 val);
         int     (*link_up)(struct dw_pcie *pcie);
         int     (*start_link)(struct dw_pcie *pcie);
         void    (*stop_link)(struct dw_pcie *pcie);
@@ -243,6 +243,7 @@ struct dw_pcie {
         struct pcie_port        pp;
         struct dw_pcie_ep       ep;
         const struct dw_pcie_ops *ops;
+        unsigned int            version;
 };
 
 #define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp)
@@ -257,6 +258,10 @@ u32 __dw_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
                        size_t size);
 void __dw_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, u32 reg,
                          size_t size, u32 val);
+u32 __dw_pcie_read_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+                        size_t size);
+void __dw_pcie_write_dbi2(struct dw_pcie *pci, void __iomem *base, u32 reg,
+                          size_t size, u32 val);
 int dw_pcie_link_up(struct dw_pcie *pci);
 int dw_pcie_wait_for_link(struct dw_pcie *pci);
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
@@ -300,12 +305,12 @@ static inline u8 dw_pcie_readb_dbi(struct dw_pcie *pci, u32 reg)
 
 static inline void dw_pcie_writel_dbi2(struct dw_pcie *pci, u32 reg, u32 val)
 {
-        __dw_pcie_write_dbi(pci, pci->dbi_base2, reg, 0x4, val);
+        __dw_pcie_write_dbi2(pci, pci->dbi_base2, reg, 0x4, val);
 }
 
 static inline u32 dw_pcie_readl_dbi2(struct dw_pcie *pci, u32 reg)
 {
-        return __dw_pcie_read_dbi(pci, pci->dbi_base2, reg, 0x4);
+        return __dw_pcie_read_dbi2(pci, pci->dbi_base2, reg, 0x4);
 }
 
 static inline void dw_pcie_writel_atu(struct dw_pcie *pci, u32 reg, u32 val)
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index d0b91da49bf4..c0786ca74312 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -438,7 +438,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
         epc_features = epf_test->epc_features;
 
         base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg),
-                                   test_reg_bar);
+                                   test_reg_bar, epc_features->align);
         if (!base) {
                 dev_err(dev, "Failed to allocated register space\n");
                 return -ENOMEM;
@@ -453,7 +453,8 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
                 if (!!(epc_features->reserved_bar & (1 << bar)))
                         continue;
 
-                base = pci_epf_alloc_space(epf, bar_size[bar], bar);
+                base = pci_epf_alloc_space(epf, bar_size[bar], bar,
+                                           epc_features->align);
                 if (!base)
                         dev_err(dev, "Failed to allocate space for BAR%d\n",
                                 bar);
diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
index 8bfdcd291196..fb1306de8f40 100644
--- a/drivers/pci/endpoint/pci-epf-core.c
+++ b/drivers/pci/endpoint/pci-epf-core.c
@@ -109,10 +109,12 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space);
  * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
  * @size: the size of the memory that has to be allocated
  * @bar: the BAR number corresponding to the allocated register space
+ * @align: alignment size for the allocation region
  *
  * Invoke to allocate memory for the PCI EPF register space.
  */
-void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar)
+void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
+                          size_t align)
 {
         void *space;
         struct device *dev = epf->epc->dev.parent;
@@ -120,7 +122,11 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar)
 
         if (size < 128)
                 size = 128;
-        size = roundup_pow_of_two(size);
+
+        if (align)
+                size = ALIGN(size, align);
+        else
+                size = roundup_pow_of_two(size);
 
         space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
         if (!space) {
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 67376cf45880..73d5adec0a28 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -15,6 +15,7 @@
 #include <linux/of_pci.h>
 #include "pci.h"
 
+#ifdef CONFIG_PCI
 void pci_set_of_node(struct pci_dev *dev)
 {
         if (!dev->bus->dev.of_node)
@@ -203,27 +204,6 @@ int of_get_pci_domain_nr(struct device_node *node)
 EXPORT_SYMBOL_GPL(of_get_pci_domain_nr);
 
 /**
- * This function will try to find the limitation of link speed by finding
- * a property called "max-link-speed" of the given device node.
- *
- * @node: device tree node with the max link speed information
- *
- * Returns the associated max link speed from DT, or a negative value if the
- * required property is not found or is invalid.
- */
-int of_pci_get_max_link_speed(struct device_node *node)
-{
-        u32 max_link_speed;
-
-        if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
-            max_link_speed > 4)
-                return -EINVAL;
-
-        return max_link_speed;
-}
-EXPORT_SYMBOL_GPL(of_pci_get_max_link_speed);
-
-/**
  * of_pci_check_probe_only - Setup probe only mode if linux,pci-probe-only
  *                           is present and valid
  */
@@ -543,3 +523,25 @@ int pci_parse_request_of_pci_ranges(struct device *dev,
         return err;
 }
 
+#endif /* CONFIG_PCI */
+
+/**
+ * This function will try to find the limitation of link speed by finding
+ * a property called "max-link-speed" of the given device node.
+ *
+ * @node: device tree node with the max link speed information
+ *
+ * Returns the associated max link speed from DT, or a negative value if the
+ * required property is not found or is invalid.
+ */
+int of_pci_get_max_link_speed(struct device_node *node)
+{
+        u32 max_link_speed;
+
+        if (of_property_read_u32(node, "max-link-speed", &max_link_speed) ||
+            max_link_speed > 4)
+                return -EINVAL;
+
+        return max_link_speed;
+}
+EXPORT_SYMBOL_GPL(of_pci_get_max_link_speed);
diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
index c3ffa3917f88..f641badc2c61 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -109,6 +109,7 @@ struct pci_epc {
  * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver
  * @bar_fixed_64bit: bitmap to indicate fixed 64bit BARs
  * @bar_fixed_size: Array specifying the size supported by each BAR
+ * @align: alignment size required for BAR buffer allocation
  */
 struct pci_epc_features {
         unsigned int    linkup_notifier : 1;
@@ -117,6 +118,7 @@ struct pci_epc_features {
         u8      reserved_bar;
         u8      bar_fixed_64bit;
         u64     bar_fixed_size[BAR_5 + 1];
+        size_t  align;
 };
 
 #define to_pci_epc(device) container_of((device), struct pci_epc, dev)
diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h
index ec02f58758c8..2d6f07556682 100644
--- a/include/linux/pci-epf.h
+++ b/include/linux/pci-epf.h
@@ -149,7 +149,8 @@ void pci_epf_destroy(struct pci_epf *epf);
 int __pci_epf_register_driver(struct pci_epf_driver *driver,
                               struct module *owner);
 void pci_epf_unregister_driver(struct pci_epf_driver *driver);
-void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar);
+void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
+                          size_t align);
 void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar);
 int pci_epf_bind(struct pci_epf *epf);
 void pci_epf_unbind(struct pci_epf *epf);