1 files changed, 217 insertions, 44 deletions

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 45be9499c973..98f230f6a28d 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -41,9 +41,13 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 static LIST_HEAD(iommu_pd_list);
 static DEFINE_SPINLOCK(iommu_pd_list_lock);
 
-#ifdef CONFIG_IOMMU_API
+/*
+ * Domain for untranslated devices - only allocated
+ * if iommu=pt passed on kernel cmd line.
+ */
+static struct protection_domain *pt_domain;
+
 static struct iommu_ops amd_iommu_ops;
-#endif
 
 /*
  * general struct to manage commands send to an IOMMU
@@ -61,14 +65,11 @@ static u64 *alloc_pte(struct protection_domain *domain,
 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
                                       unsigned long start_page,
                                       unsigned int pages);
+static void reset_iommu_command_buffer(struct amd_iommu *iommu);
 static u64 *fetch_pte(struct protection_domain *domain,
                       unsigned long address, int map_size);
 static void update_domain(struct protection_domain *domain);
 
-#ifndef BUS_NOTIFY_UNBOUND_DRIVER
-#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
-#endif
-
 #ifdef CONFIG_AMD_IOMMU_STATS
 
 /*
@@ -141,7 +142,25 @@ static int iommu_has_npcache(struct amd_iommu *iommu)
  *
  ****************************************************************************/
 
-static void iommu_print_event(void *__evt)
+static void dump_dte_entry(u16 devid)
+{
+        int i;
+
+        for (i = 0; i < 8; ++i)
+                pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
+                        amd_iommu_dev_table[devid].data[i]);
+}
+
+static void dump_command(unsigned long phys_addr)
+{
+        struct iommu_cmd *cmd = phys_to_virt(phys_addr);
+        int i;
+
+        for (i = 0; i < 4; ++i)
+                pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
+}
+
+static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 {
         u32 *event = __evt;
         int type  = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
@@ -150,7 +169,7 @@ static void iommu_print_event(void *__evt)
         int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
         u64 address = (u64)(((u64)event[3]) << 32) | event[2];
 
-        printk(KERN_ERR "AMD IOMMU: Event logged [");
+        printk(KERN_ERR "AMD-Vi: Event logged [");
 
         switch (type) {
         case EVENT_TYPE_ILL_DEV:
@@ -158,6 +177,7 @@ static void iommu_print_event(void *__evt)
                        "address=0x%016llx flags=0x%04x]\n",
                        PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
                        address, flags);
+                dump_dte_entry(devid);
                 break;
         case EVENT_TYPE_IO_FAULT:
                 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
@@ -179,6 +199,8 @@ static void iommu_print_event(void *__evt)
                 break;
         case EVENT_TYPE_ILL_CMD:
                 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
+                reset_iommu_command_buffer(iommu);
+                dump_command(address);
                 break;
         case EVENT_TYPE_CMD_HARD_ERR:
                 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
@@ -212,7 +234,7 @@ static void iommu_poll_events(struct amd_iommu *iommu)
         tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
 
         while (head != tail) {
-                iommu_print_event(iommu->evt_buf + head);
+                iommu_print_event(iommu, iommu->evt_buf + head);
                 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
         }
 
@@ -299,8 +321,11 @@ static void __iommu_wait_for_completion(struct amd_iommu *iommu)
         status &= ~MMIO_STATUS_COM_WAIT_INT_MASK;
         writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET);
 
-        if (unlikely(i == EXIT_LOOP_COUNT))
-                panic("AMD IOMMU: Completion wait loop failed\n");
+        if (unlikely(i == EXIT_LOOP_COUNT)) {
+                spin_unlock(&iommu->lock);
+                reset_iommu_command_buffer(iommu);
+                spin_lock(&iommu->lock);
+        }
 }
 
 /*
@@ -448,37 +473,67 @@ static void iommu_flush_tlb_pde(struct amd_iommu *iommu, u16 domid)
 }
 
 /*
+ * This function flushes one domain on one IOMMU
+ */
+static void flush_domain_on_iommu(struct amd_iommu *iommu, u16 domid)
+{
+        struct iommu_cmd cmd;
+        unsigned long flags;
+
+        __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+                                      domid, 1, 1);
+
+        spin_lock_irqsave(&iommu->lock, flags);
+        __iommu_queue_command(iommu, &cmd);
+        __iommu_completion_wait(iommu);
+        __iommu_wait_for_completion(iommu);
+        spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+static void flush_all_domains_on_iommu(struct amd_iommu *iommu)
+{
+        int i;
+
+        for (i = 1; i < MAX_DOMAIN_ID; ++i) {
+                if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
+                        continue;
+                flush_domain_on_iommu(iommu, i);
+        }
+
+}
+
+/*
  * This function is used to flush the IO/TLB for a given protection domain
  * on every IOMMU in the system
  */
 static void iommu_flush_domain(u16 domid)
 {
-        unsigned long flags;
         struct amd_iommu *iommu;
-        struct iommu_cmd cmd;
 
         INC_STATS_COUNTER(domain_flush_all);
 
-        __iommu_build_inv_iommu_pages(&cmd, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
-                                      domid, 1, 1);
-
-        for_each_iommu(iommu) {
-                spin_lock_irqsave(&iommu->lock, flags);
-                __iommu_queue_command(iommu, &cmd);
-                __iommu_completion_wait(iommu);
-                __iommu_wait_for_completion(iommu);
-                spin_unlock_irqrestore(&iommu->lock, flags);
-        }
+        for_each_iommu(iommu)
+                flush_domain_on_iommu(iommu, domid);
 }
 
 void amd_iommu_flush_all_domains(void)
 {
+        struct amd_iommu *iommu;
+
+        for_each_iommu(iommu)
+                flush_all_domains_on_iommu(iommu);
+}
+
+static void flush_all_devices_for_iommu(struct amd_iommu *iommu)
+{
         int i;
 
-        for (i = 1; i < MAX_DOMAIN_ID; ++i) {
-                if (!test_bit(i, amd_iommu_pd_alloc_bitmap))
+        for (i = 0; i <= amd_iommu_last_bdf; ++i) {
+                if (iommu != amd_iommu_rlookup_table[i])
                         continue;
-                iommu_flush_domain(i);
+
+                iommu_queue_inv_dev_entry(iommu, i);
+                iommu_completion_wait(iommu);
         }
 }
 
@@ -501,6 +556,22 @@ static void flush_devices_by_domain(struct protection_domain *domain)
         }
 }
 
+static void reset_iommu_command_buffer(struct amd_iommu *iommu)
+{
+        pr_err("AMD-Vi: Resetting IOMMU command buffer\n");
+
+        if (iommu->reset_in_progress)
+                panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n");
+
+        iommu->reset_in_progress = true;
+
+        amd_iommu_reset_cmd_buffer(iommu);
+        flush_all_devices_for_iommu(iommu);
+        flush_all_domains_on_iommu(iommu);
+
+        iommu->reset_in_progress = false;
+}
+
 void amd_iommu_flush_all_devices(void)
 {
         flush_devices_by_domain(NULL);
@@ -1077,19 +1148,36 @@ static struct protection_domain *domain_for_device(u16 devid)
 static void set_dte_entry(u16 devid, struct protection_domain *domain)
 {
         u64 pte_root = virt_to_phys(domain->pt_root);
-        unsigned long flags;
 
         pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
                     << DEV_ENTRY_MODE_SHIFT;
         pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
 
-        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
-        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
-        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
         amd_iommu_dev_table[devid].data[2] = domain->id;
+        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
+        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
 
         amd_iommu_pd_table[devid] = domain;
-        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
+}
+
+/*
+ * If a device is not yet associated with a domain, this function does
+ * assigns it visible for the hardware
+ */
+static void __attach_device(struct amd_iommu *iommu,
+                            struct protection_domain *domain,
+                            u16 devid)
+{
+        /* lock domain */
+        spin_lock(&domain->lock);
+
+        /* update DTE entry */
+        set_dte_entry(devid, domain);
+
+        domain->dev_cnt += 1;
+
+        /* ready */
+        spin_unlock(&domain->lock);
 }
 
 /*
@@ -1100,17 +1188,17 @@ static void attach_device(struct amd_iommu *iommu,
                           struct protection_domain *domain,
                           u16 devid)
 {
-        /* set the DTE entry */
-        set_dte_entry(devid, domain);
+        unsigned long flags;
 
-        /* increase reference counter */
-        domain->dev_cnt += 1;
+        write_lock_irqsave(&amd_iommu_devtable_lock, flags);
+        __attach_device(iommu, domain, devid);
+        write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
-       /*
-        * We might boot into a crash-kernel here. The crashed kernel
-        * left the caches in the IOMMU dirty. So we have to flush
-        * here to evict all dirty stuff.
-        */
+        /*
+         * We might boot into a crash-kernel here. The crashed kernel
+         * left the caches in the IOMMU dirty. So we have to flush
+         * here to evict all dirty stuff.
+         */
         iommu_queue_inv_dev_entry(iommu, devid);
         iommu_flush_tlb_pde(iommu, domain->id);
 }
@@ -1137,6 +1225,15 @@ static void __detach_device(struct protection_domain *domain, u16 devid)
 
         /* ready */
         spin_unlock(&domain->lock);
+
+        /*
+         * If we run in passthrough mode the device must be assigned to the
+         * passthrough domain if it is detached from any other domain
+         */
+        if (iommu_pass_through) {
+                struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+                __attach_device(iommu, pt_domain, devid);
+        }
 }
 
 /*
@@ -1182,6 +1279,8 @@ static int device_change_notifier(struct notifier_block *nb,
         case BUS_NOTIFY_UNBOUND_DRIVER:
                 if (!domain)
                         goto out;
+                if (iommu_pass_through)
+                        break;
                 detach_device(domain, devid);
                 break;
         case BUS_NOTIFY_ADD_DEVICE:
@@ -1312,12 +1411,15 @@ static int get_device_resources(struct device *dev,
 
 static void update_device_table(struct protection_domain *domain)
 {
+        unsigned long flags;
         int i;
 
         for (i = 0; i <= amd_iommu_last_bdf; ++i) {
                 if (amd_iommu_pd_table[i] != domain)
                         continue;
+                write_lock_irqsave(&amd_iommu_devtable_lock, flags);
                 set_dte_entry(i, domain);
+                write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
         }
 }
 
@@ -2058,19 +2160,47 @@ static void cleanup_domain(struct protection_domain *domain)
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 }
 
-static int amd_iommu_domain_init(struct iommu_domain *dom)
+static void protection_domain_free(struct protection_domain *domain)
+{
+        if (!domain)
+                return;
+
+        if (domain->id)
+                domain_id_free(domain->id);
+
+        kfree(domain);
+}
+
+static struct protection_domain *protection_domain_alloc(void)
 {
         struct protection_domain *domain;
 
         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
         if (!domain)
-                return -ENOMEM;
+                return NULL;
 
         spin_lock_init(&domain->lock);
-        domain->mode = PAGE_MODE_3_LEVEL;
         domain->id = domain_id_alloc();
         if (!domain->id)
+                goto out_err;
+
+        return domain;
+
+out_err:
+        kfree(domain);
+
+        return NULL;
+}
+
+static int amd_iommu_domain_init(struct iommu_domain *dom)
+{
+        struct protection_domain *domain;
+
+        domain = protection_domain_alloc();
+        if (!domain)
                 goto out_free;
+
+        domain->mode    = PAGE_MODE_3_LEVEL;
         domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
         if (!domain->pt_root)
                 goto out_free;
@@ -2080,7 +2210,7 @@ static int amd_iommu_domain_init(struct iommu_domain *dom)
         return 0;
 
 out_free:
-        kfree(domain);
+        protection_domain_free(domain);
 
         return -ENOMEM;
 }
@@ -2249,3 +2379,46 @@ static struct iommu_ops amd_iommu_ops = {
         .domain_has_cap = amd_iommu_domain_has_cap,
 };
 
+/*****************************************************************************
+ *
+ * The next functions do a basic initialization of IOMMU for pass through
+ * mode
+ *
+ * In passthrough mode the IOMMU is initialized and enabled but not used for
+ * DMA-API translation.
+ *
+ *****************************************************************************/
+
+int __init amd_iommu_init_passthrough(void)
+{
+        struct pci_dev *dev = NULL;
+        u16 devid, devid2;
+
+        /* allocate passthroug domain */
+        pt_domain = protection_domain_alloc();
+        if (!pt_domain)
+                return -ENOMEM;
+
+        pt_domain->mode |= PAGE_MODE_NONE;
+
+        while ((dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, dev)) != NULL) {
+                struct amd_iommu *iommu;
+
+                devid = calc_devid(dev->bus->number, dev->devfn);
+                if (devid > amd_iommu_last_bdf)
+                        continue;
+
+                devid2 = amd_iommu_alias_table[devid];
+
+                iommu = amd_iommu_rlookup_table[devid2];
+                if (!iommu)
+                        continue;
+
+                __attach_device(iommu, pt_domain, devid);
+                __attach_device(iommu, pt_domain, devid2);
+        }
+
+        pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
+
+        return 0;
+}