9 files changed, 177 insertions, 28 deletions

diff --git a/include/asm-x86/amd_iommu.h b/include/asm-x86/amd_iommu.h
index 783f43e58052..041d0db7da27 100644
--- a/include/asm-x86/amd_iommu.h
+++ b/include/asm-x86/amd_iommu.h
@@ -20,10 +20,13 @@
 #ifndef ASM_X86__AMD_IOMMU_H
 #define ASM_X86__AMD_IOMMU_H
 
+#include <linux/irqreturn.h>
+
 #ifdef CONFIG_AMD_IOMMU
 extern int amd_iommu_init(void);
 extern int amd_iommu_init_dma_ops(void);
 extern void amd_iommu_detect(void);
+extern irqreturn_t amd_iommu_int_handler(int irq, void *data);
 #else
 static inline int amd_iommu_init(void) { return -ENODEV; }
 static inline void amd_iommu_detect(void) { }
diff --git a/include/asm-x86/amd_iommu_types.h b/include/asm-x86/amd_iommu_types.h
index 1ffa4e53c989..b3085869a17b 100644
--- a/include/asm-x86/amd_iommu_types.h
+++ b/include/asm-x86/amd_iommu_types.h
@@ -37,6 +37,7 @@
 /* Capability offsets used by the driver */
 #define MMIO_CAP_HDR_OFFSET     0x00
 #define MMIO_RANGE_OFFSET       0x0c
+#define MMIO_MISC_OFFSET        0x10
 
 /* Masks, shifts and macros to parse the device range capability */
 #define MMIO_RANGE_LD_MASK      0xff000000
@@ -48,6 +49,7 @@
 #define MMIO_GET_LD(x)  (((x) & MMIO_RANGE_LD_MASK) >> MMIO_RANGE_LD_SHIFT)
 #define MMIO_GET_FD(x)  (((x) & MMIO_RANGE_FD_MASK) >> MMIO_RANGE_FD_SHIFT)
 #define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
+#define MMIO_MSI_NUM(x) ((x) & 0x1f)
 
 /* Flag masks for the AMD IOMMU exclusion range */
 #define MMIO_EXCL_ENABLE_MASK 0x01ULL
@@ -69,6 +71,25 @@
 /* MMIO status bits */
 #define MMIO_STATUS_COM_WAIT_INT_MASK   0x04
 
+/* event logging constants */
+#define EVENT_ENTRY_SIZE        0x10
+#define EVENT_TYPE_SHIFT        28
+#define EVENT_TYPE_MASK         0xf
+#define EVENT_TYPE_ILL_DEV      0x1
+#define EVENT_TYPE_IO_FAULT     0x2
+#define EVENT_TYPE_DEV_TAB_ERR  0x3
+#define EVENT_TYPE_PAGE_TAB_ERR 0x4
+#define EVENT_TYPE_ILL_CMD      0x5
+#define EVENT_TYPE_CMD_HARD_ERR 0x6
+#define EVENT_TYPE_IOTLB_INV_TO 0x7
+#define EVENT_TYPE_INV_DEV_REQ  0x8
+#define EVENT_DEVID_MASK        0xffff
+#define EVENT_DEVID_SHIFT       0
+#define EVENT_DOMID_MASK        0xffff
+#define EVENT_DOMID_SHIFT       0
+#define EVENT_FLAGS_MASK        0xfff
+#define EVENT_FLAGS_SHIFT       0x10
+
 /* feature control bits */
 #define CONTROL_IOMMU_EN        0x00ULL
 #define CONTROL_HT_TUN_EN       0x01ULL
@@ -109,6 +130,8 @@
 #define DEV_ENTRY_NMI_PASS      0xba
 #define DEV_ENTRY_LINT0_PASS    0xbe
 #define DEV_ENTRY_LINT1_PASS    0xbf
+#define DEV_ENTRY_MODE_MASK     0x07
+#define DEV_ENTRY_MODE_SHIFT    0x09
 
 /* constants to configure the command buffer */
 #define CMD_BUFFER_SIZE    8192
@@ -116,6 +139,10 @@
 #define MMIO_CMD_SIZE_SHIFT 56
 #define MMIO_CMD_SIZE_512 (0x9ULL << MMIO_CMD_SIZE_SHIFT)
 
+/* constants for event buffer handling */
+#define EVT_BUFFER_SIZE         8192 /* 512 entries */
+#define EVT_LEN_MASK            (0x9ULL << 56)
+
 #define PAGE_MODE_1_LEVEL 0x01
 #define PAGE_MODE_2_LEVEL 0x02
 #define PAGE_MODE_3_LEVEL 0x03
@@ -134,6 +161,7 @@
 #define IOMMU_MAP_SIZE_L3 (1ULL << 39)
 
 #define IOMMU_PTE_P  (1ULL << 0)
+#define IOMMU_PTE_TV (1ULL << 1)
 #define IOMMU_PTE_U  (1ULL << 59)
 #define IOMMU_PTE_FC (1ULL << 60)
 #define IOMMU_PTE_IR (1ULL << 61)
@@ -159,6 +187,9 @@
 
 #define MAX_DOMAIN_ID 65536
 
+/* FIXME: move this macro to <linux/pci.h> */
+#define PCI_BUS(x) (((x) >> 8) & 0xff)
+
 /*
  * This structure contains generic data for  IOMMU protection domains
  * independent of their use.
@@ -196,6 +227,15 @@ struct dma_ops_domain {
          * just calculate its address in constant time.
          */
         u64 **pte_pages;
+
+        /* This will be set to true when TLB needs to be flushed */
+        bool need_flush;
+
+        /*
+         * if this is a preallocated domain, keep the device for which it was
+         * preallocated in this variable
+         */
+        u16 target_dev;
 };
 
 /*
@@ -208,8 +248,9 @@ struct amd_iommu {
         /* locks the accesses to the hardware */
         spinlock_t lock;
 
-        /* device id of this IOMMU */
-        u16 devid;
+        /* Pointer to PCI device of this IOMMU */
+        struct pci_dev *dev;
+
         /*
          * Capability pointer. There could be more than one IOMMU per PCI
          * device function if there are more than one AMD IOMMU capability
@@ -225,6 +266,9 @@ struct amd_iommu {
         /* capabilities of that IOMMU read from ACPI */
         u32 cap;
 
+        /* pci domain of this IOMMU */
+        u16 pci_seg;
+
         /* first device this IOMMU handles. read from PCI */
         u16 first_device;
         /* last device this IOMMU handles. read from PCI */
@@ -240,9 +284,19 @@ struct amd_iommu {
         /* size of command buffer */
         u32 cmd_buf_size;
 
+        /* event buffer virtual address */
+        u8 *evt_buf;
+        /* size of event buffer */
+        u32 evt_buf_size;
+        /* MSI number for event interrupt */
+        u16 evt_msi_num;
+
         /* if one, we need to send a completion wait command */
         int need_sync;
 
+        /* true if interrupts for this IOMMU are already enabled */
+        bool int_enabled;
+
         /* default dma_ops domain for that IOMMU */
         struct dma_ops_domain *default_dom;
 };
@@ -322,6 +376,12 @@ extern unsigned long *amd_iommu_pd_alloc_bitmap;
 /* will be 1 if device isolation is enabled */
 extern int amd_iommu_isolate;
 
+/*
+ * If true, the addresses will be flushed on unmap time, not when
+ * they are reused
+ */
+extern bool amd_iommu_unmap_flush;
+
 /* takes a PCI device id and prints it out in a readable form */
 static inline void print_devid(u16 devid, int nl)
 {
diff --git a/include/asm-x86/bitops.h b/include/asm-x86/bitops.h
index 61989b93b475..451a74762bd4 100644
--- a/include/asm-x86/bitops.h
+++ b/include/asm-x86/bitops.h
@@ -424,16 +424,6 @@ static inline int fls(int x)
 
 #undef ADDR
 
-static inline void set_bit_string(unsigned long *bitmap,
-                unsigned long i, int len)
-{
-        unsigned long end = i + len;
-        while (i < end) {
-                __set_bit(i, bitmap);
-                i++;
-        }
-}
-
 #ifdef __KERNEL__
 
 #include <asm-generic/bitops/sched.h>
diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h
index 5d200e78bd81..219c33d6361c 100644
--- a/include/asm-x86/dma-mapping.h
+++ b/include/asm-x86/dma-mapping.h
@@ -9,12 +9,12 @@
 #include <linux/scatterlist.h>
 #include <asm/io.h>
 #include <asm/swiotlb.h>
+#include <asm-generic/dma-coherent.h>
 
 extern dma_addr_t bad_dma_address;
 extern int iommu_merge;
-extern struct device fallback_dev;
+extern struct device x86_dma_fallback_dev;
 extern int panic_on_overflow;
-extern int force_iommu;
 
 struct dma_mapping_ops {
         int             (*mapping_error)(struct device *dev,
@@ -25,9 +25,6 @@ struct dma_mapping_ops {
                                 void *vaddr, dma_addr_t dma_handle);
         dma_addr_t      (*map_single)(struct device *hwdev, phys_addr_t ptr,
                                 size_t size, int direction);
-        /* like map_single, but doesn't check the device mask */
-        dma_addr_t      (*map_simple)(struct device *hwdev, phys_addr_t ptr,
-                                size_t size, int direction);
         void            (*unmap_single)(struct device *dev, dma_addr_t addr,
                                 size_t size, int direction);
         void            (*sync_single_for_cpu)(struct device *hwdev,
@@ -68,7 +65,7 @@ static inline struct dma_mapping_ops *get_dma_ops(struct device *dev)
                 return dma_ops;
         else
                 return dev->archdata.dma_ops;
-#endif
+#endif /* ASM_X86__DMA_MAPPING_H */
 }
 
 /* Make sure we keep the same behaviour */
@@ -87,17 +84,14 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 
 #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
 #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
-
-void *dma_alloc_coherent(struct device *dev, size_t size,
-                           dma_addr_t *dma_handle, gfp_t flag);
-
-void dma_free_coherent(struct device *dev, size_t size,
-                         void *vaddr, dma_addr_t dma_handle);
-
+#define dma_is_consistent(d, h) (1)
 
 extern int dma_supported(struct device *hwdev, u64 mask);
 extern int dma_set_mask(struct device *dev, u64 mask);
 
+extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
+                                        dma_addr_t *dma_addr, gfp_t flag);
+
 static inline dma_addr_t
 dma_map_single(struct device *hwdev, void *ptr, size_t size,
                int direction)
@@ -247,7 +241,68 @@ static inline int dma_get_cache_alignment(void)
         return boot_cpu_data.x86_clflush_size;
 }
 
-#define dma_is_consistent(d, h) (1)
+static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
+                                                    gfp_t gfp)
+{
+        unsigned long dma_mask = 0;
 
-#include <asm-generic/dma-coherent.h>
-#endif /* ASM_X86__DMA_MAPPING_H */
+        dma_mask = dev->coherent_dma_mask;
+        if (!dma_mask)
+                dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
+
+        return dma_mask;
+}
+
+static inline gfp_t dma_alloc_coherent_gfp_flags(struct device *dev, gfp_t gfp)
+{
+#ifdef CONFIG_X86_64
+        unsigned long dma_mask = dma_alloc_coherent_mask(dev, gfp);
+
+        if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA))
+                gfp |= GFP_DMA32;
+#endif
+       return gfp;
+}
+
+static inline void *
+dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
+                gfp_t gfp)
+{
+        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        void *memory;
+
+        gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
+
+        if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
+                return memory;
+
+        if (!dev) {
+                dev = &x86_dma_fallback_dev;
+                gfp |= GFP_DMA;
+        }
+
+        if (!is_device_dma_capable(dev))
+                return NULL;
+
+        if (!ops->alloc_coherent)
+                return NULL;
+
+        return ops->alloc_coherent(dev, size, dma_handle,
+                                   dma_alloc_coherent_gfp_flags(dev, gfp));
+}
+
+static inline void dma_free_coherent(struct device *dev, size_t size,
+                                     void *vaddr, dma_addr_t bus)
+{
+        struct dma_mapping_ops *ops = get_dma_ops(dev);
+
+        WARN_ON(irqs_disabled());       /* for portability */
+
+        if (dma_release_from_coherent(dev, get_order(size), vaddr))
+                return;
+
+        if (ops->free_coherent)
+                ops->free_coherent(dev, size, vaddr, bus);
+}
+
+#endif
diff --git a/include/asm-x86/gart.h b/include/asm-x86/gart.h
index baa54faba892..605edb39ef9e 100644
--- a/include/asm-x86/gart.h
+++ b/include/asm-x86/gart.h
@@ -29,6 +29,8 @@ extern int fix_aperture;
 #define AMD64_GARTCACHECTL      0x9c
 #define AMD64_GARTEN            (1<<0)
 
+extern int agp_amd64_init(void);
+
 static inline void enable_gart_translation(struct pci_dev *dev, u64 addr)
 {
         u32 tmp, ctl;
diff --git a/include/asm-x86/iommu.h b/include/asm-x86/iommu.h
index e86f44148c66..546ad3110fea 100644
--- a/include/asm-x86/iommu.h
+++ b/include/asm-x86/iommu.h
@@ -6,6 +6,7 @@ extern void no_iommu_init(void);
 extern struct dma_mapping_ops nommu_dma_ops;
 extern int force_iommu, no_iommu;
 extern int iommu_detected;
+extern int dmar_disabled;
 
 extern unsigned long iommu_num_pages(unsigned long addr, unsigned long len);
 
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 952e0f857ac9..ba9114ec5d3a 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -48,6 +48,11 @@ static inline int is_device_dma_capable(struct device *dev)
         return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE;
 }
 
+static inline int is_buffer_dma_capable(u64 mask, dma_addr_t addr, size_t size)
+{
+        return addr + size <= mask;
+}
+
 #ifdef CONFIG_HAS_DMA
 #include <asm/dma-mapping.h>
 #else
@@ -58,6 +63,13 @@ static inline int is_device_dma_capable(struct device *dev)
 #define dma_sync_single         dma_sync_single_for_cpu
 #define dma_sync_sg             dma_sync_sg_for_cpu
 
+static inline u64 dma_get_mask(struct device *dev)
+{
+        if (dev && dev->dma_mask && *dev->dma_mask)
+                return *dev->dma_mask;
+        return DMA_32BIT_MASK;
+}
+
 extern u64 dma_get_required_mask(struct device *dev);
 
 static inline unsigned int dma_get_max_seg_size(struct device *dev)
diff --git a/include/linux/iommu-helper.h b/include/linux/iommu-helper.h
index c975caf75385..a6d0586e2bf7 100644
--- a/include/linux/iommu-helper.h
+++ b/include/linux/iommu-helper.h
@@ -1,6 +1,20 @@
+#ifndef _LINUX_IOMMU_HELPER_H
+#define _LINUX_IOMMU_HELPER_H
+
+static inline unsigned long iommu_device_max_index(unsigned long size,
+                                                   unsigned long offset,
+                                                   u64 dma_mask)
+{
+        if (size + offset > dma_mask)
+                return dma_mask - offset + 1;
+        else
+                return size;
+}
+
 extern int iommu_is_span_boundary(unsigned int index, unsigned int nr,
                                   unsigned long shift,
                                   unsigned long boundary_size);
+extern void iommu_area_reserve(unsigned long *map, unsigned long i, int len);
 extern unsigned long iommu_area_alloc(unsigned long *map, unsigned long size,
                                       unsigned long start, unsigned int nr,
                                       unsigned long shift,
@@ -8,3 +22,5 @@ extern unsigned long iommu_area_alloc(unsigned long *map, unsigned long size,
                                       unsigned long align_mask);
 extern void iommu_area_free(unsigned long *map, unsigned long start,
                             unsigned int nr);
+
+#endif
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index f1624b396754..c114103af987 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -497,6 +497,16 @@
 #define PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP 0x1101
 #define PCI_DEVICE_ID_AMD_K8_NB_MEMCTL  0x1102
 #define PCI_DEVICE_ID_AMD_K8_NB_MISC    0x1103
+#define PCI_DEVICE_ID_AMD_10H_NB_HT     0x1200
+#define PCI_DEVICE_ID_AMD_10H_NB_MAP    0x1201
+#define PCI_DEVICE_ID_AMD_10H_NB_DRAM   0x1202
+#define PCI_DEVICE_ID_AMD_10H_NB_MISC   0x1203
+#define PCI_DEVICE_ID_AMD_10H_NB_LINK   0x1204
+#define PCI_DEVICE_ID_AMD_11H_NB_HT     0x1300
+#define PCI_DEVICE_ID_AMD_11H_NB_MAP    0x1301
+#define PCI_DEVICE_ID_AMD_11H_NB_DRAM   0x1302
+#define PCI_DEVICE_ID_AMD_11H_NB_MISC   0x1303
+#define PCI_DEVICE_ID_AMD_11H_NB_LINK   0x1304
 #define PCI_DEVICE_ID_AMD_LANCE         0x2000
 #define PCI_DEVICE_ID_AMD_LANCE_HOME    0x2001
 #define PCI_DEVICE_ID_AMD_SCSI          0x2020