1 files changed, 272 insertions, 65 deletions

diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c
index 042fdc27bc92..a8fd9ebdc8e2 100644
--- a/arch/x86/kernel/amd_iommu.c
+++ b/arch/x86/kernel/amd_iommu.c
@@ -33,6 +33,10 @@
 
 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 
+/* A list of preallocated protection domains */
+static LIST_HEAD(iommu_pd_list);
+static DEFINE_SPINLOCK(iommu_pd_list_lock);
+
 /*
  * general struct to manage commands send to an IOMMU
  */
@@ -51,6 +55,102 @@ static int iommu_has_npcache(struct amd_iommu *iommu)
 
 /****************************************************************************
  *
+ * Interrupt handling functions
+ *
+ ****************************************************************************/
+
+static void iommu_print_event(void *__evt)
+{
+        u32 *event = __evt;
+        int type  = (event[1] >> EVENT_TYPE_SHIFT)  & EVENT_TYPE_MASK;
+        int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
+        int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
+        int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
+        u64 address = (u64)(((u64)event[3]) << 32) | event[2];
+
+        printk(KERN_ERR "AMD IOMMU: Event logged [");
+
+        switch (type) {
+        case EVENT_TYPE_ILL_DEV:
+                printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
+                       "address=0x%016llx flags=0x%04x]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       address, flags);
+                break;
+        case EVENT_TYPE_IO_FAULT:
+                printk("IO_PAGE_FAULT device=%02x:%02x.%x "
+                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       domid, address, flags);
+                break;
+        case EVENT_TYPE_DEV_TAB_ERR:
+                printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
+                       "address=0x%016llx flags=0x%04x]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       address, flags);
+                break;
+        case EVENT_TYPE_PAGE_TAB_ERR:
+                printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
+                       "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       domid, address, flags);
+                break;
+        case EVENT_TYPE_ILL_CMD:
+                printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
+                break;
+        case EVENT_TYPE_CMD_HARD_ERR:
+                printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
+                       "flags=0x%04x]\n", address, flags);
+                break;
+        case EVENT_TYPE_IOTLB_INV_TO:
+                printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
+                       "address=0x%016llx]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       address);
+                break;
+        case EVENT_TYPE_INV_DEV_REQ:
+                printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
+                       "address=0x%016llx flags=0x%04x]\n",
+                       PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
+                       address, flags);
+                break;
+        default:
+                printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
+        }
+}
+
+static void iommu_poll_events(struct amd_iommu *iommu)
+{
+        u32 head, tail;
+        unsigned long flags;
+
+        spin_lock_irqsave(&iommu->lock, flags);
+
+        head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
+        tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
+
+        while (head != tail) {
+                iommu_print_event(iommu->evt_buf + head);
+                head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
+        }
+
+        writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
+
+        spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
+irqreturn_t amd_iommu_int_handler(int irq, void *data)
+{
+        struct amd_iommu *iommu;
+
+        list_for_each_entry(iommu, &amd_iommu_list, list)
+                iommu_poll_events(iommu);
+
+        return IRQ_HANDLED;
+}
+
+/****************************************************************************
+ *
  * IOMMU command queuing functions
  *
  ****************************************************************************/
@@ -195,7 +295,7 @@ static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
                 u64 address, size_t size)
 {
         int s = 0;
-        unsigned pages = iommu_num_pages(address, size);
+        unsigned pages = iommu_num_pages(address, size, PAGE_SIZE);
 
         address &= PAGE_MASK;
 
@@ -213,6 +313,14 @@ static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
         return 0;
 }
 
+/* Flush the whole IO/TLB for a given protection domain */
+static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
+{
+        u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
+
+        iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
+}
+
 /****************************************************************************
  *
  * The functions below are used the create the page table mappings for
@@ -372,11 +480,6 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
  * efficient allocator.
  *
  ****************************************************************************/
-static unsigned long dma_mask_to_pages(unsigned long mask)
-{
-        return (mask >> PAGE_SHIFT) +
-                (PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
-}
 
 /*
  * The address allocator core function.
@@ -385,25 +488,31 @@ static unsigned long dma_mask_to_pages(unsigned long mask)
  */
 static unsigned long dma_ops_alloc_addresses(struct device *dev,
                                              struct dma_ops_domain *dom,
-                                             unsigned int pages)
+                                             unsigned int pages,
+                                             unsigned long align_mask,
+                                             u64 dma_mask)
 {
-        unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
+        unsigned long limit;
         unsigned long address;
-        unsigned long size = dom->aperture_size >> PAGE_SHIFT;
         unsigned long boundary_size;
 
         boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
                         PAGE_SIZE) >> PAGE_SHIFT;
-        limit = limit < size ? limit : size;
+        limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
+                                       dma_mask >> PAGE_SHIFT);
 
-        if (dom->next_bit >= limit)
+        if (dom->next_bit >= limit) {
                 dom->next_bit = 0;
+                dom->need_flush = true;
+        }
 
         address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
-                        0 , boundary_size, 0);
-        if (address == -1)
+                                   0 , boundary_size, align_mask);
+        if (address == -1) {
                 address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
-                                0, boundary_size, 0);
+                                0, boundary_size, align_mask);
+                dom->need_flush = true;
+        }
 
         if (likely(address != -1)) {
                 dom->next_bit = address + pages;
@@ -469,7 +578,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
         if (start_page + pages > last_page)
                 pages = last_page - start_page;
 
-        set_bit_string(dom->bitmap, start_page, pages);
+        iommu_area_reserve(dom->bitmap, start_page, pages);
 }
 
 static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
@@ -563,12 +672,16 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
         dma_dom->bitmap[0] = 1;
         dma_dom->next_bit = 0;
 
+        dma_dom->need_flush = false;
+        dma_dom->target_dev = 0xffff;
+
         /* Intialize the exclusion range if necessary */
         if (iommu->exclusion_start &&
             iommu->exclusion_start < dma_dom->aperture_size) {
                 unsigned long startpage = iommu->exclusion_start >> PAGE_SHIFT;
                 int pages = iommu_num_pages(iommu->exclusion_start,
-                                            iommu->exclusion_length);
+                                            iommu->exclusion_length,
+                                            PAGE_SIZE);
                 dma_ops_reserve_addresses(dma_dom, startpage, pages);
         }
 
@@ -633,12 +746,13 @@ static void set_device_domain(struct amd_iommu *iommu,
 
         u64 pte_root = virt_to_phys(domain->pt_root);
 
-        pte_root |= (domain->mode & 0x07) << 9;
-        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | 2;
+        pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
+                    << DEV_ENTRY_MODE_SHIFT;
+        pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
 
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
-        amd_iommu_dev_table[devid].data[0] = pte_root;
-        amd_iommu_dev_table[devid].data[1] = pte_root >> 32;
+        amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
+        amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
         amd_iommu_dev_table[devid].data[2] = domain->id;
 
         amd_iommu_pd_table[devid] = domain;
@@ -656,6 +770,45 @@ static void set_device_domain(struct amd_iommu *iommu,
  *****************************************************************************/
 
 /*
+ * This function checks if the driver got a valid device from the caller to
+ * avoid dereferencing invalid pointers.
+ */
+static bool check_device(struct device *dev)
+{
+        if (!dev || !dev->dma_mask)
+                return false;
+
+        return true;
+}
+
+/*
+ * In this function the list of preallocated protection domains is traversed to
+ * find the domain for a specific device
+ */
+static struct dma_ops_domain *find_protection_domain(u16 devid)
+{
+        struct dma_ops_domain *entry, *ret = NULL;
+        unsigned long flags;
+
+        if (list_empty(&iommu_pd_list))
+                return NULL;
+
+        spin_lock_irqsave(&iommu_pd_list_lock, flags);
+
+        list_for_each_entry(entry, &iommu_pd_list, list) {
+                if (entry->target_dev == devid) {
+                        ret = entry;
+                        list_del(&ret->list);
+                        break;
+                }
+        }
+
+        spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
+
+        return ret;
+}
+
+/*
  * In the dma_ops path we only have the struct device. This function
  * finds the corresponding IOMMU, the protection domain and the
  * requestor id for a given device.
@@ -671,27 +824,30 @@ static int get_device_resources(struct device *dev,
         struct pci_dev *pcidev;
         u16 _bdf;
 
-        BUG_ON(!dev || dev->bus != &pci_bus_type || !dev->dma_mask);
+        *iommu = NULL;
+        *domain = NULL;
+        *bdf = 0xffff;
+
+        if (dev->bus != &pci_bus_type)
+                return 0;
 
         pcidev = to_pci_dev(dev);
         _bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
 
         /* device not translated by any IOMMU in the system? */
-        if (_bdf > amd_iommu_last_bdf) {
-                *iommu = NULL;
-                *domain = NULL;
-                *bdf = 0xffff;
+        if (_bdf > amd_iommu_last_bdf)
                 return 0;
-        }
 
         *bdf = amd_iommu_alias_table[_bdf];
 
         *iommu = amd_iommu_rlookup_table[*bdf];
         if (*iommu == NULL)
                 return 0;
-        dma_dom = (*iommu)->default_dom;
         *domain = domain_for_device(*bdf);
         if (*domain == NULL) {
+                dma_dom = find_protection_domain(*bdf);
+                if (!dma_dom)
+                        dma_dom = (*iommu)->default_dom;
                 *domain = &dma_dom->domain;
                 set_device_domain(*iommu, *domain, *bdf);
                 printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
@@ -770,17 +926,24 @@ static dma_addr_t __map_single(struct device *dev,
                                struct dma_ops_domain *dma_dom,
                                phys_addr_t paddr,
                                size_t size,
-                               int dir)
+                               int dir,
+                               bool align,
+                               u64 dma_mask)
 {
         dma_addr_t offset = paddr & ~PAGE_MASK;
         dma_addr_t address, start;
         unsigned int pages;
+        unsigned long align_mask = 0;
         int i;
 
-        pages = iommu_num_pages(paddr, size);
+        pages = iommu_num_pages(paddr, size, PAGE_SIZE);
         paddr &= PAGE_MASK;
 
-        address = dma_ops_alloc_addresses(dev, dma_dom, pages);
+        if (align)
+                align_mask = (1UL << get_order(size)) - 1;
+
+        address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
+                                          dma_mask);
         if (unlikely(address == bad_dma_address))
                 goto out;
 
@@ -792,6 +955,12 @@ static dma_addr_t __map_single(struct device *dev,
         }
         address += offset;
 
+        if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
+                iommu_flush_tlb(iommu, dma_dom->domain.id);
+                dma_dom->need_flush = false;
+        } else if (unlikely(iommu_has_npcache(iommu)))
+                iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
+
 out:
         return address;
 }
@@ -812,7 +981,7 @@ static void __unmap_single(struct amd_iommu *iommu,
         if ((dma_addr == 0) || (dma_addr + size > dma_dom->aperture_size))
                 return;
 
-        pages = iommu_num_pages(dma_addr, size);
+        pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
         dma_addr &= PAGE_MASK;
         start = dma_addr;
 
@@ -822,6 +991,9 @@ static void __unmap_single(struct amd_iommu *iommu,
         }
 
         dma_ops_free_addresses(dma_dom, dma_addr, pages);
+
+        if (amd_iommu_unmap_flush)
+                iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
 }
 
 /*
@@ -835,6 +1007,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
         struct protection_domain *domain;
         u16 devid;
         dma_addr_t addr;
+        u64 dma_mask;
+
+        if (!check_device(dev))
+                return bad_dma_address;
+
+        dma_mask = *dev->dma_mask;
 
         get_device_resources(dev, &iommu, &domain, &devid);
 
@@ -843,14 +1021,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
                 return (dma_addr_t)paddr;
 
         spin_lock_irqsave(&domain->lock, flags);
-        addr = __map_single(dev, iommu, domain->priv, paddr, size, dir);
+        addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
+                            dma_mask);
         if (addr == bad_dma_address)
                 goto out;
 
-        if (iommu_has_npcache(iommu))
-                iommu_flush_pages(iommu, domain->id, addr, size);
-
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
 out:
@@ -870,7 +1046,8 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
         struct protection_domain *domain;
         u16 devid;
 
-        if (!get_device_resources(dev, &iommu, &domain, &devid))
+        if (!check_device(dev) ||
+            !get_device_resources(dev, &iommu, &domain, &devid))
                 /* device not handled by any AMD IOMMU */
                 return;
 
@@ -878,9 +1055,7 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
 
         __unmap_single(iommu, domain->priv, dma_addr, size, dir);
 
-        iommu_flush_pages(iommu, domain->id, dma_addr, size);
-
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
         spin_unlock_irqrestore(&domain->lock, flags);
@@ -919,6 +1094,12 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
         struct scatterlist *s;
         phys_addr_t paddr;
         int mapped_elems = 0;
+        u64 dma_mask;
+
+        if (!check_device(dev))
+                return 0;
+
+        dma_mask = *dev->dma_mask;
 
         get_device_resources(dev, &iommu, &domain, &devid);
 
@@ -931,19 +1112,17 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
                 paddr = sg_phys(s);
 
                 s->dma_address = __map_single(dev, iommu, domain->priv,
-                                              paddr, s->length, dir);
+                                              paddr, s->length, dir, false,
+                                              dma_mask);
 
                 if (s->dma_address) {
                         s->dma_length = s->length;
                         mapped_elems++;
                 } else
                         goto unmap;
-                if (iommu_has_npcache(iommu))
-                        iommu_flush_pages(iommu, domain->id, s->dma_address,
-                                          s->dma_length);
         }
 
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
 out:
@@ -977,7 +1156,8 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
         u16 devid;
         int i;
 
-        if (!get_device_resources(dev, &iommu, &domain, &devid))
+        if (!check_device(dev) ||
+            !get_device_resources(dev, &iommu, &domain, &devid))
                 return;
 
         spin_lock_irqsave(&domain->lock, flags);
@@ -985,12 +1165,10 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
         for_each_sg(sglist, s, nelems, i) {
                 __unmap_single(iommu, domain->priv, s->dma_address,
                                s->dma_length, dir);
-                iommu_flush_pages(iommu, domain->id, s->dma_address,
-                                  s->dma_length);
                 s->dma_address = s->dma_length = 0;
         }
 
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
         spin_unlock_irqrestore(&domain->lock, flags);
@@ -1008,25 +1186,33 @@ static void *alloc_coherent(struct device *dev, size_t size,
         struct protection_domain *domain;
         u16 devid;
         phys_addr_t paddr;
+        u64 dma_mask = dev->coherent_dma_mask;
+
+        if (!check_device(dev))
+                return NULL;
+
+        if (!get_device_resources(dev, &iommu, &domain, &devid))
+                flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
 
+        flag |= __GFP_ZERO;
         virt_addr = (void *)__get_free_pages(flag, get_order(size));
         if (!virt_addr)
                 return 0;
 
-        memset(virt_addr, 0, size);
         paddr = virt_to_phys(virt_addr);
 
-        get_device_resources(dev, &iommu, &domain, &devid);
-
         if (!iommu || !domain) {
                 *dma_addr = (dma_addr_t)paddr;
                 return virt_addr;
         }
 
+        if (!dma_mask)
+                dma_mask = *dev->dma_mask;
+
         spin_lock_irqsave(&domain->lock, flags);
 
         *dma_addr = __map_single(dev, iommu, domain->priv, paddr,
-                                 size, DMA_BIDIRECTIONAL);
+                                 size, DMA_BIDIRECTIONAL, true, dma_mask);
 
         if (*dma_addr == bad_dma_address) {
                 free_pages((unsigned long)virt_addr, get_order(size));
@@ -1034,10 +1220,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
                 goto out;
         }
 
-        if (iommu_has_npcache(iommu))
-                iommu_flush_pages(iommu, domain->id, *dma_addr, size);
-
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
 out:
@@ -1048,8 +1231,6 @@ out:
 
 /*
  * The exported free_coherent function for dma_ops.
- * FIXME: fix the generic x86 DMA layer so that it actually calls that
- *        function.
  */
 static void free_coherent(struct device *dev, size_t size,
                           void *virt_addr, dma_addr_t dma_addr)
@@ -1059,6 +1240,9 @@ static void free_coherent(struct device *dev, size_t size,
         struct protection_domain *domain;
         u16 devid;
 
+        if (!check_device(dev))
+                return;
+
         get_device_resources(dev, &iommu, &domain, &devid);
 
         if (!iommu || !domain)
@@ -1067,9 +1251,8 @@ static void free_coherent(struct device *dev, size_t size,
         spin_lock_irqsave(&domain->lock, flags);
 
         __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
-        iommu_flush_pages(iommu, domain->id, dma_addr, size);
 
-        if (iommu->need_sync)
+        if (unlikely(iommu->need_sync))
                 iommu_completion_wait(iommu);
 
         spin_unlock_irqrestore(&domain->lock, flags);
@@ -1079,6 +1262,30 @@ free_mem:
 }
 
 /*
+ * This function is called by the DMA layer to find out if we can handle a
+ * particular device. It is part of the dma_ops.
+ */
+static int amd_iommu_dma_supported(struct device *dev, u64 mask)
+{
+        u16 bdf;
+        struct pci_dev *pcidev;
+
+        /* No device or no PCI device */
+        if (!dev || dev->bus != &pci_bus_type)
+                return 0;
+
+        pcidev = to_pci_dev(dev);
+
+        bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
+
+        /* Out of our scope? */
+        if (bdf > amd_iommu_last_bdf)
+                return 0;
+
+        return 1;
+}
+
+/*
  * The function for pre-allocating protection domains.
  *
  * If the driver core informs the DMA layer if a driver grabs a device
@@ -1107,10 +1314,9 @@ void prealloc_protection_domains(void)
                 if (!dma_dom)
                         continue;
                 init_unity_mappings_for_device(dma_dom, devid);
-                set_device_domain(iommu, &dma_dom->domain, devid);
-                printk(KERN_INFO "AMD IOMMU: Allocated domain %d for device ",
-                       dma_dom->domain.id);
-                print_devid(devid, 1);
+                dma_dom->target_dev = devid;
+
+                list_add_tail(&dma_dom->list, &iommu_pd_list);
         }
 }
 
@@ -1121,6 +1327,7 @@ static struct dma_mapping_ops amd_iommu_dma_ops = {
         .unmap_single = unmap_single,
         .map_sg = map_sg,
         .unmap_sg = unmap_sg,
+        .dma_supported = amd_iommu_dma_supported,
 };
 
 /*