1 files changed, 25 insertions, 1 deletions

diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index e36738291c32..572e9c9f1ea0 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1860,6 +1860,17 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
 
         return ret;
 }
+
+static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index,
+                unsigned long *hpa, enum dma_data_direction *direction)
+{
+        long ret = pnv_tce_xchg(tbl, index, hpa, direction);
+
+        if (!ret)
+                pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true);
+
+        return ret;
+}
 #endif
 
 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
@@ -1874,6 +1885,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = {
         .set = pnv_ioda1_tce_build,
 #ifdef CONFIG_IOMMU_API
         .exchange = pnv_ioda1_tce_xchg,
+        .exchange_rm = pnv_ioda1_tce_xchg_rm,
 #endif
         .clear = pnv_ioda1_tce_free,
         .get = pnv_tce_get,
@@ -1948,7 +1960,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
 {
         struct iommu_table_group_link *tgl;
 
-        list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
+        list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
                 struct pnv_ioda_pe *pe = container_of(tgl->table_group,
                                 struct pnv_ioda_pe, table_group);
                 struct pnv_phb *phb = pe->phb;
@@ -2004,6 +2016,17 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
 
         return ret;
 }
+
+static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index,
+                unsigned long *hpa, enum dma_data_direction *direction)
+{
+        long ret = pnv_tce_xchg(tbl, index, hpa, direction);
+
+        if (!ret)
+                pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true);
+
+        return ret;
+}
 #endif
 
 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
@@ -2024,6 +2047,7 @@ static struct iommu_table_ops pnv_ioda2_iommu_ops = {
         .set = pnv_ioda2_tce_build,
 #ifdef CONFIG_IOMMU_API
         .exchange = pnv_ioda2_tce_xchg,
+        .exchange_rm = pnv_ioda2_tce_xchg_rm,
 #endif
         .clear = pnv_ioda2_tce_free,
         .get = pnv_tce_get,