4 files changed, 36 insertions, 8 deletions

diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 62a400c5ba06..fb092f3f11cb 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1353,7 +1353,7 @@ void dmar_disable_qi(struct intel_iommu *iommu)
 
         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 
-        sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
+        sts =  readl(iommu->reg + DMAR_GSTS_REG);
         if (!(sts & DMA_GSTS_QIES))
                 goto end;
 
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index 50464833d0b8..d9939fa9b588 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -249,12 +249,30 @@ static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *s
 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 {
         struct intel_svm *svm = container_of(mn, struct intel_svm, notifier);
+        struct intel_svm_dev *sdev;
 
+        /* This might end up being called from exit_mmap(), *before* the page
+         * tables are cleared. And __mmu_notifier_release() will delete us from
+         * the list of notifiers so that our invalidate_range() callback doesn't
+         * get called when the page tables are cleared. So we need to protect
+         * against hardware accessing those page tables.
+         *
+         * We do it by clearing the entry in the PASID table and then flushing
+         * the IOTLB and the PASID table caches. This might upset hardware;
+         * perhaps we'll want to point the PASID to a dummy PGD (like the zero
+         * page) so that we end up taking a fault that the hardware really
+         * *has* to handle gracefully without affecting other processes.
+         */
         svm->iommu->pasid_table[svm->pasid].val = 0;
+        wmb();
+
+        rcu_read_lock();
+        list_for_each_entry_rcu(sdev, &svm->devs, list) {
+                intel_flush_pasid_dev(svm, sdev, svm->pasid);
+                intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);
+        }
+        rcu_read_unlock();
 
-        /* There's no need to do any flush because we can't get here if there
-         * are any devices left anyway. */
-        WARN_ON(!list_empty(&svm->devs));
 }
 
 static const struct mmu_notifier_ops intel_mmuops = {
@@ -379,7 +397,6 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
                                 goto out;
                         }
                         iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) | 1;
-                        mm = NULL;
                 } else
                         iommu->pasid_table[svm->pasid].val = (u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
                 wmb();
@@ -442,11 +459,11 @@ int intel_svm_unbind_mm(struct device *dev, int pasid)
                                 kfree_rcu(sdev, rcu);
 
                                 if (list_empty(&svm->devs)) {
-                                        mmu_notifier_unregister(&svm->notifier, svm->mm);
 
                                         idr_remove(&svm->iommu->pasid_idr, svm->pasid);
                                         if (svm->mm)
-                                                mmput(svm->mm);
+                                                mmu_notifier_unregister(&svm->notifier, svm->mm);
+
                                         /* We mandate that no page faults may be outstanding
                                          * for the PASID when intel_svm_unbind_mm() is called.
                                          * If that is not obeyed, subtle errors will happen.
@@ -507,6 +524,10 @@ static irqreturn_t prq_event_thread(int irq, void *d)
         struct intel_svm *svm = NULL;
         int head, tail, handled = 0;
 
+        /* Clear PPR bit before reading head/tail registers, to
+         * ensure that we get a new interrupt if needed. */
+        writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG);
+
         tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK;
         head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK;
         while (head != tail) {
@@ -551,6 +572,9 @@ static irqreturn_t prq_event_thread(int irq, void *d)
                  * any faults on kernel addresses. */
                 if (!svm->mm)
                         goto bad_req;
+                /* If the mm is already defunct, don't handle faults. */
+                if (!atomic_inc_not_zero(&svm->mm->mm_users))
+                        goto bad_req;
                 down_read(&svm->mm->mmap_sem);
                 vma = find_extend_vma(svm->mm, address);
                 if (!vma || address < vma->vm_start)
@@ -567,6 +591,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
                 result = QI_RESP_SUCCESS;
         invalid:
                 up_read(&svm->mm->mmap_sem);
+                mmput(svm->mm);
         bad_req:
                 /* Accounting for major/minor faults? */
                 rcu_read_lock();
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index c12ba4516df2..ac596928f6b4 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -629,7 +629,7 @@ static void iommu_disable_irq_remapping(struct intel_iommu *iommu)
 
         raw_spin_lock_irqsave(&iommu->register_lock, flags);
 
-        sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
+        sts = readl(iommu->reg + DMAR_GSTS_REG);
         if (!(sts & DMA_GSTS_IRES))
                 goto end;
 
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 821273ca4873..2d9b650047a5 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -235,6 +235,9 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
 /* low 64 bit */
 #define dma_frcd_page_addr(d) (d & (((u64)-1) << PAGE_SHIFT))
 
+/* PRS_REG */
+#define DMA_PRS_PPR     ((u32)1)
+
 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts)                     \
 do {                                                                    \
         cycles_t start_time = get_cycles();                             \