1 files changed, 845 insertions, 38 deletions

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 4ee277a8521a..cce1f03b8895 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -17,6 +17,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  */
 
+#include <linux/ratelimit.h>
 #include <linux/pci.h>
 #include <linux/pci-ats.h>
 #include <linux/bitmap.h>
@@ -28,6 +29,8 @@
 #include <linux/iommu.h>
 #include <linux/delay.h>
 #include <linux/amd-iommu.h>
+#include <linux/notifier.h>
+#include <linux/export.h>
 #include <asm/msidef.h>
 #include <asm/proto.h>
 #include <asm/iommu.h>
@@ -41,6 +44,24 @@
 
 #define LOOP_TIMEOUT    100000
 
+/*
+ * This bitmap is used to advertise the page sizes our hardware support
+ * to the IOMMU core, which will then use this information to split
+ * physically contiguous memory regions it is mapping into page sizes
+ * that we support.
+ *
+ * Traditionally the IOMMU core just handed us the mappings directly,
+ * after making sure the size is an order of a 4KiB page and that the
+ * mapping has natural alignment.
+ *
+ * To retain this behavior, we currently advertise that we support
+ * all page sizes that are an order of 4KiB.
+ *
+ * If at some point we'd like to utilize the IOMMU core's new behavior,
+ * we could change this to advertise the real page sizes we support.
+ */
+#define AMD_IOMMU_PGSIZES       (~0xFFFUL)
+
 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
 
 /* A list of preallocated protection domains */
@@ -59,6 +80,9 @@ static struct protection_domain *pt_domain;
 
 static struct iommu_ops amd_iommu_ops;
 
+static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
+int amd_iommu_max_glx_val = -1;
+
 /*
  * general struct to manage commands send to an IOMMU
  */
@@ -67,6 +91,7 @@ struct iommu_cmd {
 };
 
 static void update_domain(struct protection_domain *domain);
+static int __init alloc_passthrough_domain(void);
 
 /****************************************************************************
  *
@@ -147,6 +172,33 @@ static struct iommu_dev_data *get_dev_data(struct device *dev)
         return dev->archdata.iommu;
 }
 
+static bool pci_iommuv2_capable(struct pci_dev *pdev)
+{
+        static const int caps[] = {
+                PCI_EXT_CAP_ID_ATS,
+                PCI_EXT_CAP_ID_PRI,
+                PCI_EXT_CAP_ID_PASID,
+        };
+        int i, pos;
+
+        for (i = 0; i < 3; ++i) {
+                pos = pci_find_ext_capability(pdev, caps[i]);
+                if (pos == 0)
+                        return false;
+        }
+
+        return true;
+}
+
+static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
+{
+        struct iommu_dev_data *dev_data;
+
+        dev_data = get_dev_data(&pdev->dev);
+
+        return dev_data->errata & (1 << erratum) ? true : false;
+}
+
 /*
  * In this function the list of preallocated protection domains is traversed to
  * find the domain for a specific device
@@ -204,6 +256,7 @@ static bool check_device(struct device *dev)
 
 static int iommu_init_device(struct device *dev)
 {
+        struct pci_dev *pdev = to_pci_dev(dev);
         struct iommu_dev_data *dev_data;
         u16 alias;
 
@@ -228,6 +281,13 @@ static int iommu_init_device(struct device *dev)
                 dev_data->alias_data = alias_data;
         }
 
+        if (pci_iommuv2_capable(pdev)) {
+                struct amd_iommu *iommu;
+
+                iommu              = amd_iommu_rlookup_table[dev_data->devid];
+                dev_data->iommu_v2 = iommu->is_iommu_v2;
+        }
+
         dev->archdata.iommu = dev_data;
 
         return 0;
@@ -317,6 +377,11 @@ DECLARE_STATS_COUNTER(domain_flush_single);
 DECLARE_STATS_COUNTER(domain_flush_all);
 DECLARE_STATS_COUNTER(alloced_io_mem);
 DECLARE_STATS_COUNTER(total_map_requests);
+DECLARE_STATS_COUNTER(complete_ppr);
+DECLARE_STATS_COUNTER(invalidate_iotlb);
+DECLARE_STATS_COUNTER(invalidate_iotlb_all);
+DECLARE_STATS_COUNTER(pri_requests);
+
 
 static struct dentry *stats_dir;
 static struct dentry *de_fflush;
@@ -351,6 +416,10 @@ static void amd_iommu_stats_init(void)
         amd_iommu_stats_add(&domain_flush_all);
         amd_iommu_stats_add(&alloced_io_mem);
         amd_iommu_stats_add(&total_map_requests);
+        amd_iommu_stats_add(&complete_ppr);
+        amd_iommu_stats_add(&invalidate_iotlb);
+        amd_iommu_stats_add(&invalidate_iotlb_all);
+        amd_iommu_stats_add(&pri_requests);
 }
 
 #endif
@@ -365,8 +434,8 @@ static void dump_dte_entry(u16 devid)
 {
         int i;
 
-        for (i = 0; i < 8; ++i)
-                pr_err("AMD-Vi: DTE[%d]: %08x\n", i,
+        for (i = 0; i < 4; ++i)
+                pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
                         amd_iommu_dev_table[devid].data[i]);
 }
 
@@ -461,12 +530,84 @@ static void iommu_poll_events(struct amd_iommu *iommu)
         spin_unlock_irqrestore(&iommu->lock, flags);
 }
 
+static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u32 head)
+{
+        struct amd_iommu_fault fault;
+        volatile u64 *raw;
+        int i;
+
+        INC_STATS_COUNTER(pri_requests);
+
+        raw = (u64 *)(iommu->ppr_log + head);
+
+        /*
+         * Hardware bug: Interrupt may arrive before the entry is written to
+         * memory. If this happens we need to wait for the entry to arrive.
+         */
+        for (i = 0; i < LOOP_TIMEOUT; ++i) {
+                if (PPR_REQ_TYPE(raw[0]) != 0)
+                        break;
+                udelay(1);
+        }
+
+        if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
+                pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
+                return;
+        }
+
+        fault.address   = raw[1];
+        fault.pasid     = PPR_PASID(raw[0]);
+        fault.device_id = PPR_DEVID(raw[0]);
+        fault.tag       = PPR_TAG(raw[0]);
+        fault.flags     = PPR_FLAGS(raw[0]);
+
+        /*
+         * To detect the hardware bug we need to clear the entry
+         * to back to zero.
+         */
+        raw[0] = raw[1] = 0;
+
+        atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
+}
+
+static void iommu_poll_ppr_log(struct amd_iommu *iommu)
+{
+        unsigned long flags;
+        u32 head, tail;
+
+        if (iommu->ppr_log == NULL)
+                return;
+
+        spin_lock_irqsave(&iommu->lock, flags);
+
+        head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
+        tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
+
+        while (head != tail) {
+
+                /* Handle PPR entry */
+                iommu_handle_ppr_entry(iommu, head);
+
+                /* Update and refresh ring-buffer state*/
+                head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
+                writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
+                tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
+        }
+
+        /* enable ppr interrupts again */
+        writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
+
+        spin_unlock_irqrestore(&iommu->lock, flags);
+}
+
 irqreturn_t amd_iommu_int_thread(int irq, void *data)
 {
         struct amd_iommu *iommu;
 
-        for_each_iommu(iommu)
+        for_each_iommu(iommu) {
                 iommu_poll_events(iommu);
+                iommu_poll_ppr_log(iommu);
+        }
 
         return IRQ_HANDLED;
 }
@@ -595,6 +736,60 @@ static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
                 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
 }
 
+static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
+                                  u64 address, bool size)
+{
+        memset(cmd, 0, sizeof(*cmd));
+
+        address &= ~(0xfffULL);
+
+        cmd->data[0]  = pasid & PASID_MASK;
+        cmd->data[1]  = domid;
+        cmd->data[2]  = lower_32_bits(address);
+        cmd->data[3]  = upper_32_bits(address);
+        cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
+        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
+        if (size)
+                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+        CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
+}
+
+static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
+                                  int qdep, u64 address, bool size)
+{
+        memset(cmd, 0, sizeof(*cmd));
+
+        address &= ~(0xfffULL);
+
+        cmd->data[0]  = devid;
+        cmd->data[0] |= (pasid & 0xff) << 16;
+        cmd->data[0] |= (qdep  & 0xff) << 24;
+        cmd->data[1]  = devid;
+        cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
+        cmd->data[2]  = lower_32_bits(address);
+        cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
+        cmd->data[3]  = upper_32_bits(address);
+        if (size)
+                cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
+        CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
+}
+
+static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
+                               int status, int tag, bool gn)
+{
+        memset(cmd, 0, sizeof(*cmd));
+
+        cmd->data[0]  = devid;
+        if (gn) {
+                cmd->data[1]  = pasid & PASID_MASK;
+                cmd->data[2]  = CMD_INV_IOMMU_PAGES_GN_MASK;
+        }
+        cmd->data[3]  = tag & 0x1ff;
+        cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
+
+        CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
+}
+
 static void build_inv_all(struct iommu_cmd *cmd)
 {
         memset(cmd, 0, sizeof(*cmd));
@@ -1496,6 +1691,48 @@ static void free_pagetable(struct protection_domain *domain)
         domain->pt_root = NULL;
 }
 
+static void free_gcr3_tbl_level1(u64 *tbl)
+{
+        u64 *ptr;
+        int i;
+
+        for (i = 0; i < 512; ++i) {
+                if (!(tbl[i] & GCR3_VALID))
+                        continue;
+
+                ptr = __va(tbl[i] & PAGE_MASK);
+
+                free_page((unsigned long)ptr);
+        }
+}
+
+static void free_gcr3_tbl_level2(u64 *tbl)
+{
+        u64 *ptr;
+        int i;
+
+        for (i = 0; i < 512; ++i) {
+                if (!(tbl[i] & GCR3_VALID))
+                        continue;
+
+                ptr = __va(tbl[i] & PAGE_MASK);
+
+                free_gcr3_tbl_level1(ptr);
+        }
+}
+
+static void free_gcr3_table(struct protection_domain *domain)
+{
+        if (domain->glx == 2)
+                free_gcr3_tbl_level2(domain->gcr3_tbl);
+        else if (domain->glx == 1)
+                free_gcr3_tbl_level1(domain->gcr3_tbl);
+        else if (domain->glx != 0)
+                BUG();
+
+        free_page((unsigned long)domain->gcr3_tbl);
+}
+
 /*
  * Free a domain, only used if something went wrong in the
  * allocation path and we need to free an already allocated page table
@@ -1582,20 +1819,52 @@ static bool dma_ops_domain(struct protection_domain *domain)
 
 static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 {
-        u64 pte_root = virt_to_phys(domain->pt_root);
-        u32 flags = 0;
+        u64 pte_root = 0;
+        u64 flags = 0;
+
+        if (domain->mode != PAGE_MODE_NONE)
+                pte_root = virt_to_phys(domain->pt_root);
 
         pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
                     << DEV_ENTRY_MODE_SHIFT;
         pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
 
+        flags = amd_iommu_dev_table[devid].data[1];
+
         if (ats)
                 flags |= DTE_FLAG_IOTLB;
 
-        amd_iommu_dev_table[devid].data[3] |= flags;
-        amd_iommu_dev_table[devid].data[2]  = domain->id;
-        amd_iommu_dev_table[devid].data[1]  = upper_32_bits(pte_root);
-        amd_iommu_dev_table[devid].data[0]  = lower_32_bits(pte_root);
+        if (domain->flags & PD_IOMMUV2_MASK) {
+                u64 gcr3 = __pa(domain->gcr3_tbl);
+                u64 glx  = domain->glx;
+                u64 tmp;
+
+                pte_root |= DTE_FLAG_GV;
+                pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
+
+                /* First mask out possible old values for GCR3 table */
+                tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
+                flags    &= ~tmp;
+
+                tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
+                flags    &= ~tmp;
+
+                /* Encode GCR3 table into DTE */
+                tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
+                pte_root |= tmp;
+
+                tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
+                flags    |= tmp;
+
+                tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
+                flags    |= tmp;
+        }
+
+        flags &= ~(0xffffUL);
+        flags |= domain->id;
+
+        amd_iommu_dev_table[devid].data[1]  = flags;
+        amd_iommu_dev_table[devid].data[0]  = pte_root;
 }
 
 static void clear_dte_entry(u16 devid)
@@ -1603,7 +1872,6 @@ static void clear_dte_entry(u16 devid)
         /* remove entry from the device table seen by the hardware */
         amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
         amd_iommu_dev_table[devid].data[1] = 0;
-        amd_iommu_dev_table[devid].data[2] = 0;
 
         amd_iommu_apply_erratum_63(devid);
 }
@@ -1696,6 +1964,93 @@ out_unlock:
         return ret;
 }
 
+
+static void pdev_iommuv2_disable(struct pci_dev *pdev)
+{
+        pci_disable_ats(pdev);
+        pci_disable_pri(pdev);
+        pci_disable_pasid(pdev);
+}
+
+/* FIXME: Change generic reset-function to do the same */
+static int pri_reset_while_enabled(struct pci_dev *pdev)
+{
+        u16 control;
+        int pos;
+
+        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
+        if (!pos)
+                return -EINVAL;
+
+        pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
+        control |= PCI_PRI_CTRL_RESET;
+        pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
+
+        return 0;
+}
+
+static int pdev_iommuv2_enable(struct pci_dev *pdev)
+{
+        bool reset_enable;
+        int reqs, ret;
+
+        /* FIXME: Hardcode number of outstanding requests for now */
+        reqs = 32;
+        if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
+                reqs = 1;
+        reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
+
+        /* Only allow access to user-accessible pages */
+        ret = pci_enable_pasid(pdev, 0);
+        if (ret)
+                goto out_err;
+
+        /* First reset the PRI state of the device */
+        ret = pci_reset_pri(pdev);
+        if (ret)
+                goto out_err;
+
+        /* Enable PRI */
+        ret = pci_enable_pri(pdev, reqs);
+        if (ret)
+                goto out_err;
+
+        if (reset_enable) {
+                ret = pri_reset_while_enabled(pdev);
+                if (ret)
+                        goto out_err;
+        }
+
+        ret = pci_enable_ats(pdev, PAGE_SHIFT);
+        if (ret)
+                goto out_err;
+
+        return 0;
+
+out_err:
+        pci_disable_pri(pdev);
+        pci_disable_pasid(pdev);
+
+        return ret;
+}
+
+/* FIXME: Move this to PCI code */
+#define PCI_PRI_TLP_OFF         (1 << 2)
+
+bool pci_pri_tlp_required(struct pci_dev *pdev)
+{
+        u16 control;
+        int pos;
+
+        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
+        if (!pos)
+                return false;
+
+        pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
+
+        return (control & PCI_PRI_TLP_OFF) ? true : false;
+}
+
 /*
  * If a device is not yet associated with a domain, this function does
  * assigns it visible for the hardware
@@ -1710,7 +2065,18 @@ static int attach_device(struct device *dev,
 
         dev_data = get_dev_data(dev);
 
-        if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
+        if (domain->flags & PD_IOMMUV2_MASK) {
+                if (!dev_data->iommu_v2 || !dev_data->passthrough)
+                        return -EINVAL;
+
+                if (pdev_iommuv2_enable(pdev) != 0)
+                        return -EINVAL;
+
+                dev_data->ats.enabled = true;
+                dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
+                dev_data->pri_tlp     = pci_pri_tlp_required(pdev);
+        } else if (amd_iommu_iotlb_sup &&
+                   pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
                 dev_data->ats.enabled = true;
                 dev_data->ats.qdep    = pci_ats_queue_depth(pdev);
         }
@@ -1760,7 +2126,7 @@ static void __detach_device(struct iommu_dev_data *dev_data)
          * passthrough domain if it is detached from any other domain.
          * Make sure we can deassign from the pt_domain itself.
          */
-        if (iommu_pass_through &&
+        if (dev_data->passthrough &&
             (dev_data->domain == NULL && domain != pt_domain))
                 __attach_device(dev_data, pt_domain);
 }
@@ -1770,20 +2136,24 @@ static void __detach_device(struct iommu_dev_data *dev_data)
  */
 static void detach_device(struct device *dev)
 {
+        struct protection_domain *domain;
         struct iommu_dev_data *dev_data;
         unsigned long flags;
 
         dev_data = get_dev_data(dev);
+        domain   = dev_data->domain;
 
         /* lock device table */
         write_lock_irqsave(&amd_iommu_devtable_lock, flags);
         __detach_device(dev_data);
         write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
 
-        if (dev_data->ats.enabled) {
+        if (domain->flags & PD_IOMMUV2_MASK)
+                pdev_iommuv2_disable(to_pci_dev(dev));
+        else if (dev_data->ats.enabled)
                 pci_disable_ats(to_pci_dev(dev));
-                dev_data->ats.enabled = false;
-        }
+
+        dev_data->ats.enabled = false;
 }
 
 /*
@@ -1818,18 +2188,20 @@ static struct protection_domain *domain_for_device(struct device *dev)
 static int device_change_notifier(struct notifier_block *nb,
                                   unsigned long action, void *data)
 {
-        struct device *dev = data;
-        u16 devid;
-        struct protection_domain *domain;
         struct dma_ops_domain *dma_domain;
+        struct protection_domain *domain;
+        struct iommu_dev_data *dev_data;
+        struct device *dev = data;
         struct amd_iommu *iommu;
         unsigned long flags;
+        u16 devid;
 
         if (!check_device(dev))
                 return 0;
 
-        devid  = get_device_id(dev);
-        iommu  = amd_iommu_rlookup_table[devid];
+        devid    = get_device_id(dev);
+        iommu    = amd_iommu_rlookup_table[devid];
+        dev_data = get_dev_data(dev);
 
         switch (action) {
         case BUS_NOTIFY_UNBOUND_DRIVER:
@@ -1838,7 +2210,7 @@ static int device_change_notifier(struct notifier_block *nb,
 
                 if (!domain)
                         goto out;
-                if (iommu_pass_through)
+                if (dev_data->passthrough)
                         break;
                 detach_device(dev);
                 break;
@@ -2434,8 +2806,9 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
  */
 static void prealloc_protection_domains(void)
 {
-        struct pci_dev *dev = NULL;
+        struct iommu_dev_data *dev_data;
         struct dma_ops_domain *dma_dom;
+        struct pci_dev *dev = NULL;
         u16 devid;
 
         for_each_pci_dev(dev) {
@@ -2444,6 +2817,16 @@ static void prealloc_protection_domains(void)
                 if (!check_device(&dev->dev))
                         continue;
 
+                dev_data = get_dev_data(&dev->dev);
+                if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
+                        /* Make sure passthrough domain is allocated */
+                        alloc_passthrough_domain();
+                        dev_data->passthrough = true;
+                        attach_device(&dev->dev, pt_domain);
+                        pr_info("AMD-Vi: Using passthough domain for device %s\n",
+                                dev_name(&dev->dev));
+                }
+
                 /* Is there already any domain for it? */
                 if (domain_for_device(&dev->dev))
                         continue;
@@ -2474,6 +2857,7 @@ static struct dma_map_ops amd_iommu_dma_ops = {
 
 static unsigned device_dma_ops_init(void)
 {
+        struct iommu_dev_data *dev_data;
         struct pci_dev *pdev = NULL;
         unsigned unhandled = 0;
 
@@ -2483,7 +2867,12 @@ static unsigned device_dma_ops_init(void)
                         continue;
                 }
 
-                pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
+                dev_data = get_dev_data(&pdev->dev);
+
+                if (!dev_data->passthrough)
+                        pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
+                else
+                        pdev->dev.archdata.dma_ops = &nommu_dma_ops;
         }
 
         return unhandled;
@@ -2610,6 +2999,20 @@ out_err:
         return NULL;
 }
 
+static int __init alloc_passthrough_domain(void)
+{
+        if (pt_domain != NULL)
+                return 0;
+
+        /* allocate passthrough domain */
+        pt_domain = protection_domain_alloc();
+        if (!pt_domain)
+                return -ENOMEM;
+
+        pt_domain->mode = PAGE_MODE_NONE;
+
+        return 0;
+}
 static int amd_iommu_domain_init(struct iommu_domain *dom)
 {
         struct protection_domain *domain;
@@ -2623,6 +3026,8 @@ static int amd_iommu_domain_init(struct iommu_domain *dom)
         if (!domain->pt_root)
                 goto out_free;
 
+        domain->iommu_domain = dom;
+
         dom->priv = domain;
 
         return 0;
@@ -2645,7 +3050,11 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
 
         BUG_ON(domain->dev_cnt != 0);
 
-        free_pagetable(domain);
+        if (domain->mode != PAGE_MODE_NONE)
+                free_pagetable(domain);
+
+        if (domain->flags & PD_IOMMUV2_MASK)
+                free_gcr3_table(domain);
 
         protection_domain_free(domain);
 
@@ -2702,13 +3111,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 }
 
 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
-                         phys_addr_t paddr, int gfp_order, int iommu_prot)
+                         phys_addr_t paddr, size_t page_size, int iommu_prot)
 {
-        unsigned long page_size = 0x1000UL << gfp_order;
         struct protection_domain *domain = dom->priv;
         int prot = 0;
         int ret;
 
+        if (domain->mode == PAGE_MODE_NONE)
+                return -EINVAL;
+
         if (iommu_prot & IOMMU_READ)
                 prot |= IOMMU_PROT_IR;
         if (iommu_prot & IOMMU_WRITE)
@@ -2721,13 +3132,14 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
         return ret;
 }
 
-static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
-                           int gfp_order)
+static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
+                           size_t page_size)
 {
         struct protection_domain *domain = dom->priv;
-        unsigned long page_size, unmap_size;
+        size_t unmap_size;
 
-        page_size  = 0x1000UL << gfp_order;
+        if (domain->mode == PAGE_MODE_NONE)
+                return -EINVAL;
 
         mutex_lock(&domain->api_lock);
         unmap_size = iommu_unmap_page(domain, iova, page_size);
@@ -2735,7 +3147,7 @@ static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
 
         domain_flush_tlb_pde(domain);
 
-        return get_order(unmap_size);
+        return unmap_size;
 }
 
 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
@@ -2746,6 +3158,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
         phys_addr_t paddr;
         u64 *pte, __pte;
 
+        if (domain->mode == PAGE_MODE_NONE)
+                return iova;
+
         pte = fetch_pte(domain, iova);
 
         if (!pte || !IOMMU_PTE_PRESENT(*pte))
@@ -2773,6 +3188,26 @@ static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
         return 0;
 }
 
+static int amd_iommu_device_group(struct device *dev, unsigned int *groupid)
+{
+        struct iommu_dev_data *dev_data = dev->archdata.iommu;
+        struct pci_dev *pdev = to_pci_dev(dev);
+        u16 devid;
+
+        if (!dev_data)
+                return -ENODEV;
+
+        if (pdev->is_virtfn || !iommu_group_mf)
+                devid = dev_data->devid;
+        else
+                devid = calc_devid(pdev->bus->number,
+                                   PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
+
+        *groupid = amd_iommu_alias_table[devid];
+
+        return 0;
+}
+
 static struct iommu_ops amd_iommu_ops = {
         .domain_init = amd_iommu_domain_init,
         .domain_destroy = amd_iommu_domain_destroy,
@@ -2782,6 +3217,8 @@ static struct iommu_ops amd_iommu_ops = {
         .unmap = amd_iommu_unmap,
         .iova_to_phys = amd_iommu_iova_to_phys,
         .domain_has_cap = amd_iommu_domain_has_cap,
+        .device_group = amd_iommu_device_group,
+        .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
 };
 
 /*****************************************************************************
@@ -2796,21 +3233,23 @@ static struct iommu_ops amd_iommu_ops = {
 
 int __init amd_iommu_init_passthrough(void)
 {
-        struct amd_iommu *iommu;
+        struct iommu_dev_data *dev_data;
         struct pci_dev *dev = NULL;
+        struct amd_iommu *iommu;
         u16 devid;
+        int ret;
 
-        /* allocate passthrough domain */
-        pt_domain = protection_domain_alloc();
-        if (!pt_domain)
-                return -ENOMEM;
-
-        pt_domain->mode |= PAGE_MODE_NONE;
+        ret = alloc_passthrough_domain();
+        if (ret)
+                return ret;
 
         for_each_pci_dev(dev) {
                 if (!check_device(&dev->dev))
                         continue;
 
+                dev_data = get_dev_data(&dev->dev);
+                dev_data->passthrough = true;
+
                 devid = get_device_id(&dev->dev);
 
                 iommu = amd_iommu_rlookup_table[devid];
@@ -2820,7 +3259,375 @@ int __init amd_iommu_init_passthrough(void)
                 attach_device(&dev->dev, pt_domain);
         }
 
+        amd_iommu_stats_init();
+
         pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
 
         return 0;
 }
+
+/* IOMMUv2 specific functions */
+int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
+{
+        return atomic_notifier_chain_register(&ppr_notifier, nb);
+}
+EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
+
+int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
+{
+        return atomic_notifier_chain_unregister(&ppr_notifier, nb);
+}
+EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
+
+void amd_iommu_domain_direct_map(struct iommu_domain *dom)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+
+        spin_lock_irqsave(&domain->lock, flags);
+
+        /* Update data structure */
+        domain->mode    = PAGE_MODE_NONE;
+        domain->updated = true;
+
+        /* Make changes visible to IOMMUs */
+        update_domain(domain);
+
+        /* Page-table is not visible to IOMMU anymore, so free it */
+        free_pagetable(domain);
+
+        spin_unlock_irqrestore(&domain->lock, flags);
+}
+EXPORT_SYMBOL(amd_iommu_domain_direct_map);
+
+int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+        int levels, ret;
+
+        if (pasids <= 0 || pasids > (PASID_MASK + 1))
+                return -EINVAL;
+
+        /* Number of GCR3 table levels required */
+        for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
+                levels += 1;
+
+        if (levels > amd_iommu_max_glx_val)
+                return -EINVAL;
+
+        spin_lock_irqsave(&domain->lock, flags);
+
+        /*
+         * Save us all sanity checks whether devices already in the
+         * domain support IOMMUv2. Just force that the domain has no
+         * devices attached when it is switched into IOMMUv2 mode.
+         */
+        ret = -EBUSY;
+        if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
+                goto out;
+
+        ret = -ENOMEM;
+        domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
+        if (domain->gcr3_tbl == NULL)
+                goto out;
+
+        domain->glx      = levels;
+        domain->flags   |= PD_IOMMUV2_MASK;
+        domain->updated  = true;
+
+        update_domain(domain);
+
+        ret = 0;
+
+out:
+        spin_unlock_irqrestore(&domain->lock, flags);
+
+        return ret;
+}
+EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
+
+static int __flush_pasid(struct protection_domain *domain, int pasid,
+                         u64 address, bool size)
+{
+        struct iommu_dev_data *dev_data;
+        struct iommu_cmd cmd;
+        int i, ret;
+
+        if (!(domain->flags & PD_IOMMUV2_MASK))
+                return -EINVAL;
+
+        build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
+
+        /*
+         * IOMMU TLB needs to be flushed before Device TLB to
+         * prevent device TLB refill from IOMMU TLB
+         */
+        for (i = 0; i < amd_iommus_present; ++i) {
+                if (domain->dev_iommu[i] == 0)
+                        continue;
+
+                ret = iommu_queue_command(amd_iommus[i], &cmd);
+                if (ret != 0)
+                        goto out;
+        }
+
+        /* Wait until IOMMU TLB flushes are complete */
+        domain_flush_complete(domain);
+
+        /* Now flush device TLBs */
+        list_for_each_entry(dev_data, &domain->dev_list, list) {
+                struct amd_iommu *iommu;
+                int qdep;
+
+                BUG_ON(!dev_data->ats.enabled);
+
+                qdep  = dev_data->ats.qdep;
+                iommu = amd_iommu_rlookup_table[dev_data->devid];
+
+                build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
+                                      qdep, address, size);
+
+                ret = iommu_queue_command(iommu, &cmd);
+                if (ret != 0)
+                        goto out;
+        }
+
+        /* Wait until all device TLBs are flushed */
+        domain_flush_complete(domain);
+
+        ret = 0;
+
+out:
+
+        return ret;
+}
+
+static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
+                                  u64 address)
+{
+        INC_STATS_COUNTER(invalidate_iotlb);
+
+        return __flush_pasid(domain, pasid, address, false);
+}
+
+int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
+                         u64 address)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+        int ret;
+
+        spin_lock_irqsave(&domain->lock, flags);
+        ret = __amd_iommu_flush_page(domain, pasid, address);
+        spin_unlock_irqrestore(&domain->lock, flags);
+
+        return ret;
+}
+EXPORT_SYMBOL(amd_iommu_flush_page);
+
+static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
+{
+        INC_STATS_COUNTER(invalidate_iotlb_all);
+
+        return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
+                             true);
+}
+
+int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+        int ret;
+
+        spin_lock_irqsave(&domain->lock, flags);
+        ret = __amd_iommu_flush_tlb(domain, pasid);
+        spin_unlock_irqrestore(&domain->lock, flags);
+
+        return ret;
+}
+EXPORT_SYMBOL(amd_iommu_flush_tlb);
+
+static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
+{
+        int index;
+        u64 *pte;
+
+        while (true) {
+
+                index = (pasid >> (9 * level)) & 0x1ff;
+                pte   = &root[index];
+
+                if (level == 0)
+                        break;
+
+                if (!(*pte & GCR3_VALID)) {
+                        if (!alloc)
+                                return NULL;
+
+                        root = (void *)get_zeroed_page(GFP_ATOMIC);
+                        if (root == NULL)
+                                return NULL;
+
+                        *pte = __pa(root) | GCR3_VALID;
+                }
+
+                root = __va(*pte & PAGE_MASK);
+
+                level -= 1;
+        }
+
+        return pte;
+}
+
+static int __set_gcr3(struct protection_domain *domain, int pasid,
+                      unsigned long cr3)
+{
+        u64 *pte;
+
+        if (domain->mode != PAGE_MODE_NONE)
+                return -EINVAL;
+
+        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
+        if (pte == NULL)
+                return -ENOMEM;
+
+        *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
+
+        return __amd_iommu_flush_tlb(domain, pasid);
+}
+
+static int __clear_gcr3(struct protection_domain *domain, int pasid)
+{
+        u64 *pte;
+
+        if (domain->mode != PAGE_MODE_NONE)
+                return -EINVAL;
+
+        pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
+        if (pte == NULL)
+                return 0;
+
+        *pte = 0;
+
+        return __amd_iommu_flush_tlb(domain, pasid);
+}
+
+int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
+                              unsigned long cr3)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+        int ret;
+
+        spin_lock_irqsave(&domain->lock, flags);
+        ret = __set_gcr3(domain, pasid, cr3);
+        spin_unlock_irqrestore(&domain->lock, flags);
+
+        return ret;
+}
+EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
+
+int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
+{
+        struct protection_domain *domain = dom->priv;
+        unsigned long flags;
+        int ret;
+
+        spin_lock_irqsave(&domain->lock, flags);
+        ret = __clear_gcr3(domain, pasid);
+        spin_unlock_irqrestore(&domain->lock, flags);
+
+        return ret;
+}
+EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
+
+int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
+                           int status, int tag)
+{
+        struct iommu_dev_data *dev_data;
+        struct amd_iommu *iommu;
+        struct iommu_cmd cmd;
+
+        INC_STATS_COUNTER(complete_ppr);
+
+        dev_data = get_dev_data(&pdev->dev);
+        iommu    = amd_iommu_rlookup_table[dev_data->devid];
+
+        build_complete_ppr(&cmd, dev_data->devid, pasid, status,
+                           tag, dev_data->pri_tlp);
+
+        return iommu_queue_command(iommu, &cmd);
+}
+EXPORT_SYMBOL(amd_iommu_complete_ppr);
+
+struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
+{
+        struct protection_domain *domain;
+
+        domain = get_domain(&pdev->dev);
+        if (IS_ERR(domain))
+                return NULL;
+
+        /* Only return IOMMUv2 domains */
+        if (!(domain->flags & PD_IOMMUV2_MASK))
+                return NULL;
+
+        return domain->iommu_domain;
+}
+EXPORT_SYMBOL(amd_iommu_get_v2_domain);
+
+void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
+{
+        struct iommu_dev_data *dev_data;
+
+        if (!amd_iommu_v2_supported())
+                return;
+
+        dev_data = get_dev_data(&pdev->dev);
+        dev_data->errata |= (1 << erratum);
+}
+EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
+
+int amd_iommu_device_info(struct pci_dev *pdev,
+                          struct amd_iommu_device_info *info)
+{
+        int max_pasids;
+        int pos;
+
+        if (pdev == NULL || info == NULL)
+                return -EINVAL;
+
+        if (!amd_iommu_v2_supported())
+                return -EINVAL;
+
+        memset(info, 0, sizeof(*info));
+
+        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
+        if (pos)
+                info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
+
+        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
+        if (pos)
+                info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
+
+        pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
+        if (pos) {
+                int features;
+
+                max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
+                max_pasids = min(max_pasids, (1 << 20));
+
+                info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
+                info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
+
+                features = pci_pasid_features(pdev);
+                if (features & PCI_PASID_CAP_EXEC)
+                        info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
+                if (features & PCI_PASID_CAP_PRIV)
+                        info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL(amd_iommu_device_info);