Merge branch 'linus' into cpumask-for-linus

Conflicts:
	arch/x86/kernel/cpu/common.c

4 files changed, 440 insertions, 241 deletions

diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index 5f333403c2ea..d313039e2fdf 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -31,6 +31,8 @@
 #include <linux/iova.h>
 #include <linux/intel-iommu.h>
 #include <linux/timer.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
 
 #undef PREFIX
 #define PREFIX "DMAR:"
@@ -509,6 +511,7 @@ int alloc_iommu(struct dmar_drhd_unit *drhd)
                 return -ENOMEM;
 
         iommu->seq_id = iommu_allocated++;
+        sprintf (iommu->name, "dmar%d", iommu->seq_id);
 
         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
         if (!iommu->reg) {
@@ -751,6 +754,42 @@ int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 }
 
 /*
+ * Disable Queued Invalidation interface.
+ */
+void dmar_disable_qi(struct intel_iommu *iommu)
+{
+        unsigned long flags;
+        u32 sts;
+        cycles_t start_time = get_cycles();
+
+        if (!ecap_qis(iommu->ecap))
+                return;
+
+        spin_lock_irqsave(&iommu->register_lock, flags);
+
+        sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
+        if (!(sts & DMA_GSTS_QIES))
+                goto end;
+
+        /*
+         * Give a chance to HW to complete the pending invalidation requests.
+         */
+        while ((readl(iommu->reg + DMAR_IQT_REG) !=
+                readl(iommu->reg + DMAR_IQH_REG)) &&
+                (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
+                cpu_relax();
+
+        iommu->gcmd &= ~DMA_GCMD_QIE;
+
+        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
+
+        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
+                      !(sts & DMA_GSTS_QIES), sts);
+end:
+        spin_unlock_irqrestore(&iommu->register_lock, flags);
+}
+
+/*
  * Enable Queued Invalidation interface. This is a must to support
  * interrupt-remapping. Also used by DMA-remapping, which replaces
  * register based IOTLB invalidation.
@@ -770,20 +809,20 @@ int dmar_enable_qi(struct intel_iommu *iommu)
         if (iommu->qi)
                 return 0;
 
-        iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
+        iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
         if (!iommu->qi)
                 return -ENOMEM;
 
         qi = iommu->qi;
 
-        qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
+        qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
         if (!qi->desc) {
                 kfree(qi);
                 iommu->qi = 0;
                 return -ENOMEM;
         }
 
-        qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
+        qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
         if (!qi->desc_status) {
                 free_page((unsigned long) qi->desc);
                 kfree(qi);
@@ -812,3 +851,254 @@ int dmar_enable_qi(struct intel_iommu *iommu)
 
         return 0;
 }
+
+/* iommu interrupt handling. Most stuff are MSI-like. */
+
+enum faulttype {
+        DMA_REMAP,
+        INTR_REMAP,
+        UNKNOWN,
+};
+
+static const char *dma_remap_fault_reasons[] =
+{
+        "Software",
+        "Present bit in root entry is clear",
+        "Present bit in context entry is clear",
+        "Invalid context entry",
+        "Access beyond MGAW",
+        "PTE Write access is not set",
+        "PTE Read access is not set",
+        "Next page table ptr is invalid",
+        "Root table address invalid",
+        "Context table ptr is invalid",
+        "non-zero reserved fields in RTP",
+        "non-zero reserved fields in CTP",
+        "non-zero reserved fields in PTE",
+};
+
+static const char *intr_remap_fault_reasons[] =
+{
+        "Detected reserved fields in the decoded interrupt-remapped request",
+        "Interrupt index exceeded the interrupt-remapping table size",
+        "Present field in the IRTE entry is clear",
+        "Error accessing interrupt-remapping table pointed by IRTA_REG",
+        "Detected reserved fields in the IRTE entry",
+        "Blocked a compatibility format interrupt request",
+        "Blocked an interrupt request due to source-id verification failure",
+};
+
+#define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
+
+const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
+{
+        if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
+                                     ARRAY_SIZE(intr_remap_fault_reasons))) {
+                *fault_type = INTR_REMAP;
+                return intr_remap_fault_reasons[fault_reason - 0x20];
+        } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
+                *fault_type = DMA_REMAP;
+                return dma_remap_fault_reasons[fault_reason];
+        } else {
+                *fault_type = UNKNOWN;
+                return "Unknown";
+        }
+}
+
+void dmar_msi_unmask(unsigned int irq)
+{
+        struct intel_iommu *iommu = get_irq_data(irq);
+        unsigned long flag;
+
+        /* unmask it */
+        spin_lock_irqsave(&iommu->register_lock, flag);
+        writel(0, iommu->reg + DMAR_FECTL_REG);
+        /* Read a reg to force flush the post write */
+        readl(iommu->reg + DMAR_FECTL_REG);
+        spin_unlock_irqrestore(&iommu->register_lock, flag);
+}
+
+void dmar_msi_mask(unsigned int irq)
+{
+        unsigned long flag;
+        struct intel_iommu *iommu = get_irq_data(irq);
+
+        /* mask it */
+        spin_lock_irqsave(&iommu->register_lock, flag);
+        writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
+        /* Read a reg to force flush the post write */
+        readl(iommu->reg + DMAR_FECTL_REG);
+        spin_unlock_irqrestore(&iommu->register_lock, flag);
+}
+
+void dmar_msi_write(int irq, struct msi_msg *msg)
+{
+        struct intel_iommu *iommu = get_irq_data(irq);
+        unsigned long flag;
+
+        spin_lock_irqsave(&iommu->register_lock, flag);
+        writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
+        writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
+        writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
+        spin_unlock_irqrestore(&iommu->register_lock, flag);
+}
+
+void dmar_msi_read(int irq, struct msi_msg *msg)
+{
+        struct intel_iommu *iommu = get_irq_data(irq);
+        unsigned long flag;
+
+        spin_lock_irqsave(&iommu->register_lock, flag);
+        msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
+        msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
+        msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
+        spin_unlock_irqrestore(&iommu->register_lock, flag);
+}
+
+static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
+                u8 fault_reason, u16 source_id, unsigned long long addr)
+{
+        const char *reason;
+        int fault_type;
+
+        reason = dmar_get_fault_reason(fault_reason, &fault_type);
+
+        if (fault_type == INTR_REMAP)
+                printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
+                       "fault index %llx\n"
+                        "INTR-REMAP:[fault reason %02d] %s\n",
+                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
+                        PCI_FUNC(source_id & 0xFF), addr >> 48,
+                        fault_reason, reason);
+        else
+                printk(KERN_ERR
+                       "DMAR:[%s] Request device [%02x:%02x.%d] "
+                       "fault addr %llx \n"
+                       "DMAR:[fault reason %02d] %s\n",
+                       (type ? "DMA Read" : "DMA Write"),
+                       (source_id >> 8), PCI_SLOT(source_id & 0xFF),
+                       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
+        return 0;
+}
+
+#define PRIMARY_FAULT_REG_LEN (16)
+irqreturn_t dmar_fault(int irq, void *dev_id)
+{
+        struct intel_iommu *iommu = dev_id;
+        int reg, fault_index;
+        u32 fault_status;
+        unsigned long flag;
+
+        spin_lock_irqsave(&iommu->register_lock, flag);
+        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
+        if (fault_status)
+                printk(KERN_ERR "DRHD: handling fault status reg %x\n",
+                       fault_status);
+
+        /* TBD: ignore advanced fault log currently */
+        if (!(fault_status & DMA_FSTS_PPF))
+                goto clear_rest;
+
+        fault_index = dma_fsts_fault_record_index(fault_status);
+        reg = cap_fault_reg_offset(iommu->cap);
+        while (1) {
+                u8 fault_reason;
+                u16 source_id;
+                u64 guest_addr;
+                int type;
+                u32 data;
+
+                /* highest 32 bits */
+                data = readl(iommu->reg + reg +
+                                fault_index * PRIMARY_FAULT_REG_LEN + 12);
+                if (!(data & DMA_FRCD_F))
+                        break;
+
+                fault_reason = dma_frcd_fault_reason(data);
+                type = dma_frcd_type(data);
+
+                data = readl(iommu->reg + reg +
+                                fault_index * PRIMARY_FAULT_REG_LEN + 8);
+                source_id = dma_frcd_source_id(data);
+
+                guest_addr = dmar_readq(iommu->reg + reg +
+                                fault_index * PRIMARY_FAULT_REG_LEN);
+                guest_addr = dma_frcd_page_addr(guest_addr);
+                /* clear the fault */
+                writel(DMA_FRCD_F, iommu->reg + reg +
+                        fault_index * PRIMARY_FAULT_REG_LEN + 12);
+
+                spin_unlock_irqrestore(&iommu->register_lock, flag);
+
+                dmar_fault_do_one(iommu, type, fault_reason,
+                                source_id, guest_addr);
+
+                fault_index++;
+                if (fault_index > cap_num_fault_regs(iommu->cap))
+                        fault_index = 0;
+                spin_lock_irqsave(&iommu->register_lock, flag);
+        }
+clear_rest:
+        /* clear all the other faults */
+        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
+        writel(fault_status, iommu->reg + DMAR_FSTS_REG);
+
+        spin_unlock_irqrestore(&iommu->register_lock, flag);
+        return IRQ_HANDLED;
+}
+
+int dmar_set_interrupt(struct intel_iommu *iommu)
+{
+        int irq, ret;
+
+        /*
+         * Check if the fault interrupt is already initialized.
+         */
+        if (iommu->irq)
+                return 0;
+
+        irq = create_irq();
+        if (!irq) {
+                printk(KERN_ERR "IOMMU: no free vectors\n");
+                return -EINVAL;
+        }
+
+        set_irq_data(irq, iommu);
+        iommu->irq = irq;
+
+        ret = arch_setup_dmar_msi(irq);
+        if (ret) {
+                set_irq_data(irq, NULL);
+                iommu->irq = 0;
+                destroy_irq(irq);
+                return 0;
+        }
+
+        ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
+        if (ret)
+                printk(KERN_ERR "IOMMU: can't request irq\n");
+        return ret;
+}
+
+int __init enable_drhd_fault_handling(void)
+{
+        struct dmar_drhd_unit *drhd;
+
+        /*
+         * Enable fault control interrupt.
+         */
+        for_each_drhd_unit(drhd) {
+                int ret;
+                struct intel_iommu *iommu = drhd->iommu;
+                ret = dmar_set_interrupt(iommu);
+
+                if (ret) {
+                        printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
+                               " interrupt, ret %d\n",
+                               (unsigned long long)drhd->reg_base_addr, ret);
+                        return -1;
+                }
+        }
+
+        return 0;
+}
diff --git a/drivers/pci/hotplug/cpqphp_sysfs.c b/drivers/pci/hotplug/cpqphp_sysfs.c
index a13abf55d784..8450f4a6568a 100644
--- a/drivers/pci/hotplug/cpqphp_sysfs.c
+++ b/drivers/pci/hotplug/cpqphp_sysfs.c
@@ -225,7 +225,8 @@ void cpqhp_shutdown_debugfs(void)
 
 void cpqhp_create_debugfs_files(struct controller *ctrl)
 {
-        ctrl->dentry = debugfs_create_file(ctrl->pci_dev->dev.bus_id, S_IRUGO, root, ctrl, &debug_ops);
+        ctrl->dentry = debugfs_create_file(dev_name(&ctrl->pci_dev->dev),
+                                           S_IRUGO, root, ctrl, &debug_ops);
 }
 
 void cpqhp_remove_debugfs_files(struct controller *ctrl)
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index f3f686581a90..49402c399232 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -1004,194 +1004,6 @@ static int iommu_disable_translation(struct intel_iommu *iommu)
         return 0;
 }
 
-/* iommu interrupt handling. Most stuff are MSI-like. */
-
-static const char *fault_reason_strings[] =
-{
-        "Software",
-        "Present bit in root entry is clear",
-        "Present bit in context entry is clear",
-        "Invalid context entry",
-        "Access beyond MGAW",
-        "PTE Write access is not set",
-        "PTE Read access is not set",
-        "Next page table ptr is invalid",
-        "Root table address invalid",
-        "Context table ptr is invalid",
-        "non-zero reserved fields in RTP",
-        "non-zero reserved fields in CTP",
-        "non-zero reserved fields in PTE",
-};
-#define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
-
-const char *dmar_get_fault_reason(u8 fault_reason)
-{
-        if (fault_reason > MAX_FAULT_REASON_IDX)
-                return "Unknown";
-        else
-                return fault_reason_strings[fault_reason];
-}
-
-void dmar_msi_unmask(unsigned int irq)
-{
-        struct intel_iommu *iommu = get_irq_data(irq);
-        unsigned long flag;
-
-        /* unmask it */
-        spin_lock_irqsave(&iommu->register_lock, flag);
-        writel(0, iommu->reg + DMAR_FECTL_REG);
-        /* Read a reg to force flush the post write */
-        readl(iommu->reg + DMAR_FECTL_REG);
-        spin_unlock_irqrestore(&iommu->register_lock, flag);
-}
-
-void dmar_msi_mask(unsigned int irq)
-{
-        unsigned long flag;
-        struct intel_iommu *iommu = get_irq_data(irq);
-
-        /* mask it */
-        spin_lock_irqsave(&iommu->register_lock, flag);
-        writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
-        /* Read a reg to force flush the post write */
-        readl(iommu->reg + DMAR_FECTL_REG);
-        spin_unlock_irqrestore(&iommu->register_lock, flag);
-}
-
-void dmar_msi_write(int irq, struct msi_msg *msg)
-{
-        struct intel_iommu *iommu = get_irq_data(irq);
-        unsigned long flag;
-
-        spin_lock_irqsave(&iommu->register_lock, flag);
-        writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
-        writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
-        writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
-        spin_unlock_irqrestore(&iommu->register_lock, flag);
-}
-
-void dmar_msi_read(int irq, struct msi_msg *msg)
-{
-        struct intel_iommu *iommu = get_irq_data(irq);
-        unsigned long flag;
-
-        spin_lock_irqsave(&iommu->register_lock, flag);
-        msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
-        msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
-        msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
-        spin_unlock_irqrestore(&iommu->register_lock, flag);
-}
-
-static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
-                u8 fault_reason, u16 source_id, unsigned long long addr)
-{
-        const char *reason;
-
-        reason = dmar_get_fault_reason(fault_reason);
-
-        printk(KERN_ERR
-                "DMAR:[%s] Request device [%02x:%02x.%d] "
-                "fault addr %llx \n"
-                "DMAR:[fault reason %02d] %s\n",
-                (type ? "DMA Read" : "DMA Write"),
-                (source_id >> 8), PCI_SLOT(source_id & 0xFF),
-                PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
-        return 0;
-}
-
-#define PRIMARY_FAULT_REG_LEN (16)
-static irqreturn_t iommu_page_fault(int irq, void *dev_id)
-{
-        struct intel_iommu *iommu = dev_id;
-        int reg, fault_index;
-        u32 fault_status;
-        unsigned long flag;
-
-        spin_lock_irqsave(&iommu->register_lock, flag);
-        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
-
-        /* TBD: ignore advanced fault log currently */
-        if (!(fault_status & DMA_FSTS_PPF))
-                goto clear_overflow;
-
-        fault_index = dma_fsts_fault_record_index(fault_status);
-        reg = cap_fault_reg_offset(iommu->cap);
-        while (1) {
-                u8 fault_reason;
-                u16 source_id;
-                u64 guest_addr;
-                int type;
-                u32 data;
-
-                /* highest 32 bits */
-                data = readl(iommu->reg + reg +
-                                fault_index * PRIMARY_FAULT_REG_LEN + 12);
-                if (!(data & DMA_FRCD_F))
-                        break;
-
-                fault_reason = dma_frcd_fault_reason(data);
-                type = dma_frcd_type(data);
-
-                data = readl(iommu->reg + reg +
-                                fault_index * PRIMARY_FAULT_REG_LEN + 8);
-                source_id = dma_frcd_source_id(data);
-
-                guest_addr = dmar_readq(iommu->reg + reg +
-                                fault_index * PRIMARY_FAULT_REG_LEN);
-                guest_addr = dma_frcd_page_addr(guest_addr);
-                /* clear the fault */
-                writel(DMA_FRCD_F, iommu->reg + reg +
-                        fault_index * PRIMARY_FAULT_REG_LEN + 12);
-
-                spin_unlock_irqrestore(&iommu->register_lock, flag);
-
-                iommu_page_fault_do_one(iommu, type, fault_reason,
-                                source_id, guest_addr);
-
-                fault_index++;
-                if (fault_index > cap_num_fault_regs(iommu->cap))
-                        fault_index = 0;
-                spin_lock_irqsave(&iommu->register_lock, flag);
-        }
-clear_overflow:
-        /* clear primary fault overflow */
-        fault_status = readl(iommu->reg + DMAR_FSTS_REG);
-        if (fault_status & DMA_FSTS_PFO)
-                writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
-
-        spin_unlock_irqrestore(&iommu->register_lock, flag);
-        return IRQ_HANDLED;
-}
-
-int dmar_set_interrupt(struct intel_iommu *iommu)
-{
-        int irq, ret;
-
-        irq = create_irq();
-        if (!irq) {
-                printk(KERN_ERR "IOMMU: no free vectors\n");
-                return -EINVAL;
-        }
-
-        set_irq_data(irq, iommu);
-        iommu->irq = irq;
-
-        ret = arch_setup_dmar_msi(irq);
-        if (ret) {
-                set_irq_data(irq, NULL);
-                iommu->irq = 0;
-                destroy_irq(irq);
-                return 0;
-        }
-
-        /* Force fault register is cleared */
-        iommu_page_fault(irq, iommu);
-
-        ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
-        if (ret)
-                printk(KERN_ERR "IOMMU: can't request irq\n");
-        return ret;
-}
 
 static int iommu_init_domains(struct intel_iommu *iommu)
 {
@@ -1987,7 +1799,7 @@ static int __init init_dmars(void)
         struct dmar_rmrr_unit *rmrr;
         struct pci_dev *pdev;
         struct intel_iommu *iommu;
-        int i, ret, unit = 0;
+        int i, ret;
 
         /*
          * for each drhd
@@ -2043,11 +1855,40 @@ static int __init init_dmars(void)
                 }
         }
 
+        /*
+         * Start from the sane iommu hardware state.
+         */
         for_each_drhd_unit(drhd) {
                 if (drhd->ignored)
                         continue;
 
                 iommu = drhd->iommu;
+
+                /*
+                 * If the queued invalidation is already initialized by us
+                 * (for example, while enabling interrupt-remapping) then
+                 * we got the things already rolling from a sane state.
+                 */
+                if (iommu->qi)
+                        continue;
+
+                /*
+                 * Clear any previous faults.
+                 */
+                dmar_fault(-1, iommu);
+                /*
+                 * Disable queued invalidation if supported and already enabled
+                 * before OS handover.
+                 */
+                dmar_disable_qi(iommu);
+        }
+
+        for_each_drhd_unit(drhd) {
+                if (drhd->ignored)
+                        continue;
+
+                iommu = drhd->iommu;
+
                 if (dmar_enable_qi(iommu)) {
                         /*
                          * Queued Invalidate not enabled, use Register Based
@@ -2109,7 +1950,6 @@ static int __init init_dmars(void)
                 if (drhd->ignored)
                         continue;
                 iommu = drhd->iommu;
-                sprintf (iommu->name, "dmar%d", unit++);
 
                 iommu_flush_write_buffer(iommu);
 
@@ -2284,11 +2124,13 @@ error:
         return 0;
 }
 
-dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
-                            size_t size, int dir)
+static dma_addr_t intel_map_page(struct device *dev, struct page *page,
+                                 unsigned long offset, size_t size,
+                                 enum dma_data_direction dir,
+                                 struct dma_attrs *attrs)
 {
-        return __intel_map_single(hwdev, paddr, size, dir,
-                                  to_pci_dev(hwdev)->dma_mask);
+        return __intel_map_single(dev, page_to_phys(page) + offset, size,
+                                  dir, to_pci_dev(dev)->dma_mask);
 }
 
 static void flush_unmaps(void)
@@ -2352,8 +2194,9 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova)
         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
 }
 
-void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
-                        int dir)
+static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
+                             size_t size, enum dma_data_direction dir,
+                             struct dma_attrs *attrs)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
         struct dmar_domain *domain;
@@ -2397,8 +2240,14 @@ void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
         }
 }
 
-void *intel_alloc_coherent(struct device *hwdev, size_t size,
-                           dma_addr_t *dma_handle, gfp_t flags)
+static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
+                               int dir)
+{
+        intel_unmap_page(dev, dev_addr, size, dir, NULL);
+}
+
+static void *intel_alloc_coherent(struct device *hwdev, size_t size,
+                                  dma_addr_t *dma_handle, gfp_t flags)
 {
         void *vaddr;
         int order;
@@ -2421,8 +2270,8 @@ void *intel_alloc_coherent(struct device *hwdev, size_t size,
         return NULL;
 }
 
-void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
-                         dma_addr_t dma_handle)
+static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
+                                dma_addr_t dma_handle)
 {
         int order;
 
@@ -2435,8 +2284,9 @@ void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
 
 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
 
-void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
-                    int nelems, int dir)
+static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
+                           int nelems, enum dma_data_direction dir,
+                           struct dma_attrs *attrs)
 {
         int i;
         struct pci_dev *pdev = to_pci_dev(hwdev);
@@ -2493,8 +2343,8 @@ static int intel_nontranslate_map_sg(struct device *hddev,
         return nelems;
 }
 
-int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
-                 int dir)
+static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
+                        enum dma_data_direction dir, struct dma_attrs *attrs)
 {
         void *addr;
         int i;
@@ -2574,13 +2424,19 @@ int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
         return nelems;
 }
 
-static struct dma_mapping_ops intel_dma_ops = {
+static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+        return !dma_addr;
+}
+
+struct dma_map_ops intel_dma_ops = {
         .alloc_coherent = intel_alloc_coherent,
         .free_coherent = intel_free_coherent,
-        .map_single = intel_map_single,
-        .unmap_single = intel_unmap_single,
         .map_sg = intel_map_sg,
         .unmap_sg = intel_unmap_sg,
+        .map_page = intel_map_page,
+        .unmap_page = intel_unmap_page,
+        .mapping_error = intel_mapping_error,
 };
 
 static inline int iommu_domain_cache_init(void)
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index 8e44db040db7..b041a409f4a7 100644
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -21,7 +21,7 @@ struct irq_2_iommu {
         u8  irte_mask;
 };
 
-#ifdef CONFIG_SPARSE_IRQ
+#ifdef CONFIG_GENERIC_HARDIRQS
 static struct irq_2_iommu *get_one_free_irq_2_iommu(int cpu)
 {
         struct irq_2_iommu *iommu;
@@ -117,21 +117,22 @@ int get_irte(int irq, struct irte *entry)
 {
         int index;
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
         if (!entry)
                 return -1;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
         index = irq_iommu->irte_index + irq_iommu->sub_handle;
         *entry = *(irq_iommu->iommu->ir_table->base + index);
 
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
         return 0;
 }
 
@@ -141,6 +142,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
         struct irq_2_iommu *irq_iommu;
         u16 index, start_index;
         unsigned int mask = 0;
+        unsigned long flags;
         int i;
 
         if (!count)
@@ -170,7 +172,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
                 return -1;
         }
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         do {
                 for (i = index; i < index + count; i++)
                         if  (table->base[i].present)
@@ -182,7 +184,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
                 index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
 
                 if (index == start_index) {
-                        spin_unlock(&irq_2_ir_lock);
+                        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                         printk(KERN_ERR "can't allocate an IRTE\n");
                         return -1;
                 }
@@ -193,7 +195,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
 
         irq_iommu = irq_2_iommu_alloc(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 printk(KERN_ERR "can't allocate irq_2_iommu\n");
                 return -1;
         }
@@ -203,7 +205,7 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
         irq_iommu->sub_handle = 0;
         irq_iommu->irte_mask = mask;
 
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return index;
 }
@@ -223,30 +225,32 @@ int map_irq_to_irte_handle(int irq, u16 *sub_handle)
 {
         int index;
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
         *sub_handle = irq_iommu->sub_handle;
         index = irq_iommu->irte_index;
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
         return index;
 }
 
 int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
 {
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
 
         irq_iommu = irq_2_iommu_alloc(irq);
 
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 printk(KERN_ERR "can't allocate irq_2_iommu\n");
                 return -1;
         }
@@ -256,7 +260,7 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
         irq_iommu->sub_handle = subhandle;
         irq_iommu->irte_mask = 0;
 
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return 0;
 }
@@ -264,11 +268,12 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
 int clear_irte_irq(int irq, struct intel_iommu *iommu, u16 index)
 {
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
@@ -277,7 +282,7 @@ int clear_irte_irq(int irq, struct intel_iommu *iommu, u16 index)
         irq_iommu->sub_handle = 0;
         irq_2_iommu(irq)->irte_mask = 0;
 
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return 0;
 }
@@ -289,11 +294,12 @@ int modify_irte(int irq, struct irte *irte_modified)
         struct irte *irte;
         struct intel_iommu *iommu;
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
@@ -302,11 +308,11 @@ int modify_irte(int irq, struct irte *irte_modified)
         index = irq_iommu->irte_index + irq_iommu->sub_handle;
         irte = &iommu->ir_table->base[index];
 
-        set_64bit((unsigned long *)irte, irte_modified->low | (1 << 1));
+        set_64bit((unsigned long *)irte, irte_modified->low);
         __iommu_flush_cache(iommu, irte, sizeof(*irte));
 
         rc = qi_flush_iec(iommu, index, 0);
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return rc;
 }
@@ -317,11 +323,12 @@ int flush_irte(int irq)
         int index;
         struct intel_iommu *iommu;
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
@@ -330,7 +337,7 @@ int flush_irte(int irq)
         index = irq_iommu->irte_index + irq_iommu->sub_handle;
 
         rc = qi_flush_iec(iommu, index, irq_iommu->irte_mask);
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return rc;
 }
@@ -363,11 +370,12 @@ int free_irte(int irq)
         struct irte *irte;
         struct intel_iommu *iommu;
         struct irq_2_iommu *irq_iommu;
+        unsigned long flags;
 
-        spin_lock(&irq_2_ir_lock);
+        spin_lock_irqsave(&irq_2_ir_lock, flags);
         irq_iommu = valid_irq_2_iommu(irq);
         if (!irq_iommu) {
-                spin_unlock(&irq_2_ir_lock);
+                spin_unlock_irqrestore(&irq_2_ir_lock, flags);
                 return -1;
         }
 
@@ -378,7 +386,7 @@ int free_irte(int irq)
 
         if (!irq_iommu->sub_handle) {
                 for (i = 0; i < (1 << irq_iommu->irte_mask); i++)
-                        set_64bit((unsigned long *)irte, 0);
+                        set_64bit((unsigned long *)(irte + i), 0);
                 rc = qi_flush_iec(iommu, index, irq_iommu->irte_mask);
         }
 
@@ -387,7 +395,7 @@ int free_irte(int irq)
         irq_iommu->sub_handle = 0;
         irq_iommu->irte_mask = 0;
 
-        spin_unlock(&irq_2_ir_lock);
+        spin_unlock_irqrestore(&irq_2_ir_lock, flags);
 
         return rc;
 }
@@ -439,12 +447,12 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
         struct page *pages;
 
         ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
-                                             GFP_KERNEL);
+                                             GFP_ATOMIC);
 
         if (!iommu->ir_table)
                 return -ENOMEM;
 
-        pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
+        pages = alloc_pages(GFP_ATOMIC | __GFP_ZERO, INTR_REMAP_PAGE_ORDER);
 
         if (!pages) {
                 printk(KERN_ERR "failed to allocate pages of order %d\n",
@@ -459,11 +467,55 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
         return 0;
 }
 
+/*
+ * Disable Interrupt Remapping.
+ */
+static void disable_intr_remapping(struct intel_iommu *iommu)
+{
+        unsigned long flags;
+        u32 sts;
+
+        if (!ecap_ir_support(iommu->ecap))
+                return;
+
+        spin_lock_irqsave(&iommu->register_lock, flags);
+
+        sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
+        if (!(sts & DMA_GSTS_IRES))
+                goto end;
+
+        iommu->gcmd &= ~DMA_GCMD_IRE;
+        writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
+
+        IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
+                      readl, !(sts & DMA_GSTS_IRES), sts);
+
+end:
+        spin_unlock_irqrestore(&iommu->register_lock, flags);
+}
+
 int __init enable_intr_remapping(int eim)
 {
         struct dmar_drhd_unit *drhd;
         int setup = 0;
 
+        for_each_drhd_unit(drhd) {
+                struct intel_iommu *iommu = drhd->iommu;
+
+                /*
+                 * Clear previous faults.
+                 */
+                dmar_fault(-1, iommu);
+
+                /*
+                 * Disable intr remapping and queued invalidation, if already
+                 * enabled prior to OS handover.
+                 */
+                disable_intr_remapping(iommu);
+
+                dmar_disable_qi(iommu);
+        }
+
         /*
          * check for the Interrupt-remapping support
          */