1 files changed, 170 insertions, 51 deletions

diff --git a/arch/x86/kernel/amd_iommu_init.c b/arch/x86/kernel/amd_iommu_init.c
index 5a170cbbbed8..bfc8453bd98d 100644
--- a/arch/x86/kernel/amd_iommu_init.c
+++ b/arch/x86/kernel/amd_iommu_init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
+ * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
  * Author: Joerg Roedel <joerg.roedel@amd.com>
  *         Leo Duran <leo.duran@amd.com>
  *
@@ -21,7 +21,7 @@
 #include <linux/acpi.h>
 #include <linux/list.h>
 #include <linux/slab.h>
-#include <linux/sysdev.h>
+#include <linux/syscore_ops.h>
 #include <linux/interrupt.h>
 #include <linux/msi.h>
 #include <asm/pci-direct.h>
@@ -31,7 +31,7 @@
 #include <asm/iommu.h>
 #include <asm/gart.h>
 #include <asm/x86_init.h>
-
+#include <asm/iommu_table.h>
 /*
  * definitions for the ACPI scanning code
  */
@@ -137,6 +137,7 @@ int amd_iommus_present;
 
 /* IOMMUs have a non-present cache? */
 bool amd_iommu_np_cache __read_mostly;
+bool amd_iommu_iotlb_sup __read_mostly = true;
 
 /*
  * The ACPI table parsing functions set this variable on an error
@@ -180,6 +181,12 @@ static u32 dev_table_size;	/* size of the device table */
 static u32 alias_table_size;    /* size of the alias table */
 static u32 rlookup_table_size;  /* size if the rlookup table */
 
+/*
+ * This function flushes all internal caches of
+ * the IOMMU used by this driver.
+ */
+extern void iommu_flush_all_caches(struct amd_iommu *iommu);
+
 static inline void update_last_devid(u16 devid)
 {
         if (devid > amd_iommu_last_bdf)
@@ -194,6 +201,39 @@ static inline unsigned long tbl_size(int entry_size)
         return 1UL << shift;
 }
 
+/* Access to l1 and l2 indexed register spaces */
+
+static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
+{
+        u32 val;
+
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
+        pci_read_config_dword(iommu->dev, 0xfc, &val);
+        return val;
+}
+
+static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
+{
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
+        pci_write_config_dword(iommu->dev, 0xfc, val);
+        pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
+}
+
+static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
+{
+        u32 val;
+
+        pci_write_config_dword(iommu->dev, 0xf0, address);
+        pci_read_config_dword(iommu->dev, 0xf4, &val);
+        return val;
+}
+
+static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
+{
+        pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
+        pci_write_config_dword(iommu->dev, 0xf4, val);
+}
+
 /****************************************************************************
  *
  * AMD IOMMU MMIO register space handling functions
@@ -260,9 +300,23 @@ static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
 /* Function to enable the hardware */
 static void iommu_enable(struct amd_iommu *iommu)
 {
-        printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx\n",
+        static const char * const feat_str[] = {
+                "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
+                "IA", "GA", "HE", "PC", NULL
+        };
+        int i;
+
+        printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx",
                dev_name(&iommu->dev->dev), iommu->cap_ptr);
 
+        if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
+                printk(KERN_CONT " extended features: ");
+                for (i = 0; feat_str[i]; ++i)
+                        if (iommu_feature(iommu, (1ULL << i)))
+                                printk(KERN_CONT " %s", feat_str[i]);
+        }
+        printk(KERN_CONT "\n");
+
         iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
 }
 
@@ -618,7 +672,8 @@ static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
 static void __init init_iommu_from_pci(struct amd_iommu *iommu)
 {
         int cap_ptr = iommu->cap_ptr;
-        u32 range, misc;
+        u32 range, misc, low, high;
+        int i, j;
 
         pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
                               &iommu->cap);
@@ -633,12 +688,38 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
                                         MMIO_GET_LD(range));
         iommu->evt_msi_num = MMIO_MSI_NUM(misc);
 
-        if (is_rd890_iommu(iommu->dev)) {
-                pci_read_config_dword(iommu->dev, 0xf0, &iommu->cache_cfg[0]);
-                pci_read_config_dword(iommu->dev, 0xf4, &iommu->cache_cfg[1]);
-                pci_read_config_dword(iommu->dev, 0xf8, &iommu->cache_cfg[2]);
-                pci_read_config_dword(iommu->dev, 0xfc, &iommu->cache_cfg[3]);
-        }
+        if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
+                amd_iommu_iotlb_sup = false;
+
+        /* read extended feature bits */
+        low  = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
+        high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
+
+        iommu->features = ((u64)high << 32) | low;
+
+        if (!is_rd890_iommu(iommu->dev))
+                return;
+
+        /*
+         * Some rd890 systems may not be fully reconfigured by the BIOS, so
+         * it's necessary for us to store this information so it can be
+         * reprogrammed on resume
+         */
+
+        pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                              &iommu->stored_addr_lo);
+        pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
+                              &iommu->stored_addr_hi);
+
+        /* Low bit locks writes to configuration space */
+        iommu->stored_addr_lo &= ~1;
+
+        for (i = 0; i < 6; i++)
+                for (j = 0; j < 0x12; j++)
+                        iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
+
+        for (i = 0; i < 0x83; i++)
+                iommu->stored_l2[i] = iommu_read_l2(iommu, i);
 }
 
 /*
@@ -650,8 +731,8 @@ static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
 {
         u8 *p = (u8 *)h;
         u8 *end = p, flags = 0;
-        u16 dev_i, devid = 0, devid_start = 0, devid_to = 0;
-        u32 ext_flags = 0;
+        u16 devid = 0, devid_start = 0, devid_to = 0;
+        u32 dev_i, ext_flags = 0;
         bool alias = false;
         struct ivhd_entry *e;
 
@@ -806,7 +887,7 @@ static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
 /* Initializes the device->iommu mapping for the driver */
 static int __init init_iommu_devices(struct amd_iommu *iommu)
 {
-        u16 i;
+        u32 i;
 
         for (i = iommu->first_device; i <= iommu->last_device; ++i)
                 set_iommu_for_device(iommu, i);
@@ -953,10 +1034,11 @@ static int iommu_setup_msi(struct amd_iommu *iommu)
         if (pci_enable_msi(iommu->dev))
                 return 1;
 
-        r = request_irq(iommu->dev->irq, amd_iommu_int_handler,
-                        IRQF_SAMPLE_RANDOM,
-                        "AMD-Vi",
-                        NULL);
+        r = request_threaded_irq(iommu->dev->irq,
+                                 amd_iommu_int_handler,
+                                 amd_iommu_int_thread,
+                                 0, "AMD-Vi",
+                                 iommu->dev);
 
         if (r) {
                 pci_disable_msi(iommu->dev);
@@ -1095,7 +1177,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table)
  */
 static void init_device_table(void)
 {
-        u16 devid;
+        u32 devid;
 
         for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
                 set_dev_entry_bit(devid, DEV_ENTRY_VALID);
@@ -1127,14 +1209,53 @@ static void iommu_init_flags(struct amd_iommu *iommu)
         iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
 }
 
-static void iommu_apply_quirks(struct amd_iommu *iommu)
+static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
 {
-        if (is_rd890_iommu(iommu->dev)) {
-                pci_write_config_dword(iommu->dev, 0xf0, iommu->cache_cfg[0]);
-                pci_write_config_dword(iommu->dev, 0xf4, iommu->cache_cfg[1]);
-                pci_write_config_dword(iommu->dev, 0xf8, iommu->cache_cfg[2]);
-                pci_write_config_dword(iommu->dev, 0xfc, iommu->cache_cfg[3]);
-        }
+        int i, j;
+        u32 ioc_feature_control;
+        struct pci_dev *pdev = NULL;
+
+        /* RD890 BIOSes may not have completely reconfigured the iommu */
+        if (!is_rd890_iommu(iommu->dev))
+                return;
+
+        /*
+         * First, we need to ensure that the iommu is enabled. This is
+         * controlled by a register in the northbridge
+         */
+        pdev = pci_get_bus_and_slot(iommu->dev->bus->number, PCI_DEVFN(0, 0));
+
+        if (!pdev)
+                return;
+
+        /* Select Northbridge indirect register 0x75 and enable writing */
+        pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
+        pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
+
+        /* Enable the iommu */
+        if (!(ioc_feature_control & 0x1))
+                pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
+
+        pci_dev_put(pdev);
+
+        /* Restore the iommu BAR */
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                               iommu->stored_addr_lo);
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
+                               iommu->stored_addr_hi);
+
+        /* Restore the l1 indirect regs for each of the 6 l1s */
+        for (i = 0; i < 6; i++)
+                for (j = 0; j < 0x12; j++)
+                        iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
+
+        /* Restore the l2 indirect regs */
+        for (i = 0; i < 0x83; i++)
+                iommu_write_l2(iommu, i, iommu->stored_l2[i]);
+
+        /* Lock PCI setup registers */
+        pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
+                               iommu->stored_addr_lo | 1);
 }
 
 /*
@@ -1147,7 +1268,6 @@ static void enable_iommus(void)
 
         for_each_iommu(iommu) {
                 iommu_disable(iommu);
-                iommu_apply_quirks(iommu);
                 iommu_init_flags(iommu);
                 iommu_set_device_table(iommu);
                 iommu_enable_command_buffer(iommu);
@@ -1155,6 +1275,7 @@ static void enable_iommus(void)
                 iommu_set_exclusion_range(iommu);
                 iommu_init_msi(iommu);
                 iommu_enable(iommu);
+                iommu_flush_all_caches(iommu);
         }
 }
 
@@ -1171,8 +1292,13 @@ static void disable_iommus(void)
  * disable suspend until real resume implemented
  */
 
-static int amd_iommu_resume(struct sys_device *dev)
+static void amd_iommu_resume(void)
 {
+        struct amd_iommu *iommu;
+
+        for_each_iommu(iommu)
+                iommu_apply_resume_quirks(iommu);
+
         /* re-load the hardware */
         enable_iommus();
 
@@ -1180,13 +1306,11 @@ static int amd_iommu_resume(struct sys_device *dev)
          * we have to flush after the IOMMUs are enabled because a
          * disabled IOMMU will never execute the commands we send
          */
-        amd_iommu_flush_all_devices();
-        amd_iommu_flush_all_domains();
-
-        return 0;
+        for_each_iommu(iommu)
+                iommu_flush_all_caches(iommu);
 }
 
-static int amd_iommu_suspend(struct sys_device *dev, pm_message_t state)
+static int amd_iommu_suspend(void)
 {
         /* disable IOMMUs to go out of the way for BIOS */
         disable_iommus();
@@ -1194,17 +1318,11 @@ static int amd_iommu_suspend(struct sys_device *dev, pm_message_t state)
         return 0;
 }
 
-static struct sysdev_class amd_iommu_sysdev_class = {
-        .name = "amd_iommu",
+static struct syscore_ops amd_iommu_syscore_ops = {
         .suspend = amd_iommu_suspend,
         .resume = amd_iommu_resume,
 };
 
-static struct sys_device device_amd_iommu = {
-        .id = 0,
-        .cls = &amd_iommu_sysdev_class,
-};
-
 /*
  * This is the core init function for AMD IOMMU hardware in the system.
  * This function is called from the generic x86 DMA layer initialization
@@ -1321,14 +1439,6 @@ static int __init amd_iommu_init(void)
                 goto free;
         }
 
-        ret = sysdev_class_register(&amd_iommu_sysdev_class);
-        if (ret)
-                goto free;
-
-        ret = sysdev_register(&device_amd_iommu);
-        if (ret)
-                goto free;
-
         ret = amd_iommu_init_devices();
         if (ret)
                 goto free;
@@ -1347,6 +1457,8 @@ static int __init amd_iommu_init(void)
 
         amd_iommu_init_notifier();
 
+        register_syscore_ops(&amd_iommu_syscore_ops);
+
         if (iommu_pass_through)
                 goto out;
 
@@ -1405,13 +1517,13 @@ static int __init early_amd_iommu_detect(struct acpi_table_header *table)
         return 0;
 }
 
-void __init amd_iommu_detect(void)
+int __init amd_iommu_detect(void)
 {
         if (no_iommu || (iommu_detected && !gart_iommu_aperture))
-                return;
+                return -ENODEV;
 
         if (amd_iommu_disabled)
-                return;
+                return -ENODEV;
 
         if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {
                 iommu_detected = 1;
@@ -1420,7 +1532,9 @@ void __init amd_iommu_detect(void)
 
                 /* Make sure ACS will be enabled */
                 pci_request_acs();
+                return 1;
         }
+        return -ENODEV;
 }
 
 /****************************************************************************
@@ -1451,3 +1565,8 @@ static int __init parse_amd_iommu_options(char *str)
 
 __setup("amd_iommu_dump", parse_amd_iommu_dump);
 __setup("amd_iommu=", parse_amd_iommu_options);
+
+IOMMU_INIT_FINISH(amd_iommu_detect,
+                  gart_iommu_hole_init,
+                  0,
+                  0);