aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig13
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c862
-rw-r--r--drivers/iommu/amd_iommu_init.c133
-rw-r--r--drivers/iommu/amd_iommu_proto.h24
-rw-r--r--drivers/iommu/amd_iommu_types.h118
-rw-r--r--drivers/iommu/amd_iommu_v2.c994
-rw-r--r--drivers/iommu/intel-iommu.c30
-rw-r--r--drivers/iommu/iommu.c119
-rw-r--r--drivers/iommu/msm_iommu.c25
-rw-r--r--drivers/iommu/omap-iommu.c80
-rw-r--r--drivers/iommu/omap-iovmm.c48
12 files changed, 2295 insertions, 152 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 5414253b185a..6bea6962f8ee 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -34,7 +34,9 @@ config AMD_IOMMU
34 bool "AMD IOMMU support" 34 bool "AMD IOMMU support"
35 select SWIOTLB 35 select SWIOTLB
36 select PCI_MSI 36 select PCI_MSI
37 select PCI_IOV 37 select PCI_ATS
38 select PCI_PRI
39 select PCI_PASID
38 select IOMMU_API 40 select IOMMU_API
39 depends on X86_64 && PCI && ACPI 41 depends on X86_64 && PCI && ACPI
40 ---help--- 42 ---help---
@@ -58,6 +60,15 @@ config AMD_IOMMU_STATS
58 information to userspace via debugfs. 60 information to userspace via debugfs.
59 If unsure, say N. 61 If unsure, say N.
60 62
63config AMD_IOMMU_V2
64 tristate "AMD IOMMU Version 2 driver (EXPERIMENTAL)"
65 depends on AMD_IOMMU && PROFILING && EXPERIMENTAL
66 select MMU_NOTIFIER
67 ---help---
68 This option enables support for the AMD IOMMUv2 features of the IOMMU
69 hardware. Select this option if you want to use devices that support
70 the the PCI PRI and PASID interface.
71
61# Intel IOMMU support 72# Intel IOMMU support
62config DMAR_TABLE 73config DMAR_TABLE
63 bool 74 bool
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 2f4448794bc7..0e36b4934aff 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,6 +1,7 @@
1obj-$(CONFIG_IOMMU_API) += iommu.o 1obj-$(CONFIG_IOMMU_API) += iommu.o
2obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o 2obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
3obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o 3obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
4obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
4obj-$(CONFIG_DMAR_TABLE) += dmar.o 5obj-$(CONFIG_DMAR_TABLE) += dmar.o
5obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o 6obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o
6obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o 7obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 4ee277a8521a..edd291070b0b 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -17,6 +17,7 @@
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */ 18 */
19 19
20#include <linux/ratelimit.h>
20#include <linux/pci.h> 21#include <linux/pci.h>
21#include <linux/pci-ats.h> 22#include <linux/pci-ats.h>
22#include <linux/bitmap.h> 23#include <linux/bitmap.h>
@@ -28,6 +29,8 @@
28#include <linux/iommu.h> 29#include <linux/iommu.h>
29#include <linux/delay.h> 30#include <linux/delay.h>
30#include <linux/amd-iommu.h> 31#include <linux/amd-iommu.h>
32#include <linux/notifier.h>
33#include <linux/export.h>
31#include <asm/msidef.h> 34#include <asm/msidef.h>
32#include <asm/proto.h> 35#include <asm/proto.h>
33#include <asm/iommu.h> 36#include <asm/iommu.h>
@@ -41,6 +44,24 @@
41 44
42#define LOOP_TIMEOUT 100000 45#define LOOP_TIMEOUT 100000
43 46
47/*
48 * This bitmap is used to advertise the page sizes our hardware support
49 * to the IOMMU core, which will then use this information to split
50 * physically contiguous memory regions it is mapping into page sizes
51 * that we support.
52 *
53 * Traditionally the IOMMU core just handed us the mappings directly,
54 * after making sure the size is an order of a 4KiB page and that the
55 * mapping has natural alignment.
56 *
57 * To retain this behavior, we currently advertise that we support
58 * all page sizes that are an order of 4KiB.
59 *
60 * If at some point we'd like to utilize the IOMMU core's new behavior,
61 * we could change this to advertise the real page sizes we support.
62 */
63#define AMD_IOMMU_PGSIZES (~0xFFFUL)
64
44static DEFINE_RWLOCK(amd_iommu_devtable_lock); 65static DEFINE_RWLOCK(amd_iommu_devtable_lock);
45 66
46/* A list of preallocated protection domains */ 67/* A list of preallocated protection domains */
@@ -59,6 +80,9 @@ static struct protection_domain *pt_domain;
59 80
60static struct iommu_ops amd_iommu_ops; 81static struct iommu_ops amd_iommu_ops;
61 82
83static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
84int amd_iommu_max_glx_val = -1;
85
62/* 86/*
63 * general struct to manage commands send to an IOMMU 87 * general struct to manage commands send to an IOMMU
64 */ 88 */
@@ -67,6 +91,7 @@ struct iommu_cmd {
67}; 91};
68 92
69static void update_domain(struct protection_domain *domain); 93static void update_domain(struct protection_domain *domain);
94static int __init alloc_passthrough_domain(void);
70 95
71/**************************************************************************** 96/****************************************************************************
72 * 97 *
@@ -147,6 +172,33 @@ static struct iommu_dev_data *get_dev_data(struct device *dev)
147 return dev->archdata.iommu; 172 return dev->archdata.iommu;
148} 173}
149 174
175static bool pci_iommuv2_capable(struct pci_dev *pdev)
176{
177 static const int caps[] = {
178 PCI_EXT_CAP_ID_ATS,
179 PCI_EXT_CAP_ID_PRI,
180 PCI_EXT_CAP_ID_PASID,
181 };
182 int i, pos;
183
184 for (i = 0; i < 3; ++i) {
185 pos = pci_find_ext_capability(pdev, caps[i]);
186 if (pos == 0)
187 return false;
188 }
189
190 return true;
191}
192
193static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
194{
195 struct iommu_dev_data *dev_data;
196
197 dev_data = get_dev_data(&pdev->dev);
198
199 return dev_data->errata & (1 << erratum) ? true : false;
200}
201
150/* 202/*
151 * In this function the list of preallocated protection domains is traversed to 203 * In this function the list of preallocated protection domains is traversed to
152 * find the domain for a specific device 204 * find the domain for a specific device
@@ -204,6 +256,7 @@ static bool check_device(struct device *dev)
204 256
205static int iommu_init_device(struct device *dev) 257static int iommu_init_device(struct device *dev)
206{ 258{
259 struct pci_dev *pdev = to_pci_dev(dev);
207 struct iommu_dev_data *dev_data; 260 struct iommu_dev_data *dev_data;
208 u16 alias; 261 u16 alias;
209 262
@@ -228,6 +281,13 @@ static int iommu_init_device(struct device *dev)
228 dev_data->alias_data = alias_data; 281 dev_data->alias_data = alias_data;
229 } 282 }
230 283
284 if (pci_iommuv2_capable(pdev)) {
285 struct amd_iommu *iommu;
286
287 iommu = amd_iommu_rlookup_table[dev_data->devid];
288 dev_data->iommu_v2 = iommu->is_iommu_v2;
289 }
290
231 dev->archdata.iommu = dev_data; 291 dev->archdata.iommu = dev_data;
232 292
233 return 0; 293 return 0;
@@ -317,6 +377,11 @@ DECLARE_STATS_COUNTER(domain_flush_single);
317DECLARE_STATS_COUNTER(domain_flush_all); 377DECLARE_STATS_COUNTER(domain_flush_all);
318DECLARE_STATS_COUNTER(alloced_io_mem); 378DECLARE_STATS_COUNTER(alloced_io_mem);
319DECLARE_STATS_COUNTER(total_map_requests); 379DECLARE_STATS_COUNTER(total_map_requests);
380DECLARE_STATS_COUNTER(complete_ppr);
381DECLARE_STATS_COUNTER(invalidate_iotlb);
382DECLARE_STATS_COUNTER(invalidate_iotlb_all);
383DECLARE_STATS_COUNTER(pri_requests);
384
320 385
321static struct dentry *stats_dir; 386static struct dentry *stats_dir;
322static struct dentry *de_fflush; 387static struct dentry *de_fflush;
@@ -351,6 +416,10 @@ static void amd_iommu_stats_init(void)
351 amd_iommu_stats_add(&domain_flush_all); 416 amd_iommu_stats_add(&domain_flush_all);
352 amd_iommu_stats_add(&alloced_io_mem); 417 amd_iommu_stats_add(&alloced_io_mem);
353 amd_iommu_stats_add(&total_map_requests); 418 amd_iommu_stats_add(&total_map_requests);
419 amd_iommu_stats_add(&complete_ppr);
420 amd_iommu_stats_add(&invalidate_iotlb);
421 amd_iommu_stats_add(&invalidate_iotlb_all);
422 amd_iommu_stats_add(&pri_requests);
354} 423}
355 424
356#endif 425#endif
@@ -365,8 +434,8 @@ static void dump_dte_entry(u16 devid)
365{ 434{
366 int i; 435 int i;
367 436
368 for (i = 0; i < 8; ++i) 437 for (i = 0; i < 4; ++i)
369 pr_err("AMD-Vi: DTE[%d]: %08x\n", i, 438 pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
370 amd_iommu_dev_table[devid].data[i]); 439 amd_iommu_dev_table[devid].data[i]);
371} 440}
372 441
@@ -461,12 +530,84 @@ static void iommu_poll_events(struct amd_iommu *iommu)
461 spin_unlock_irqrestore(&iommu->lock, flags); 530 spin_unlock_irqrestore(&iommu->lock, flags);
462} 531}
463 532
533static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u32 head)
534{
535 struct amd_iommu_fault fault;
536 volatile u64 *raw;
537 int i;
538
539 INC_STATS_COUNTER(pri_requests);
540
541 raw = (u64 *)(iommu->ppr_log + head);
542
543 /*
544 * Hardware bug: Interrupt may arrive before the entry is written to
545 * memory. If this happens we need to wait for the entry to arrive.
546 */
547 for (i = 0; i < LOOP_TIMEOUT; ++i) {
548 if (PPR_REQ_TYPE(raw[0]) != 0)
549 break;
550 udelay(1);
551 }
552
553 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
554 pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
555 return;
556 }
557
558 fault.address = raw[1];
559 fault.pasid = PPR_PASID(raw[0]);
560 fault.device_id = PPR_DEVID(raw[0]);
561 fault.tag = PPR_TAG(raw[0]);
562 fault.flags = PPR_FLAGS(raw[0]);
563
564 /*
565 * To detect the hardware bug we need to clear the entry
566 * to back to zero.
567 */
568 raw[0] = raw[1] = 0;
569
570 atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
571}
572
573static void iommu_poll_ppr_log(struct amd_iommu *iommu)
574{
575 unsigned long flags;
576 u32 head, tail;
577
578 if (iommu->ppr_log == NULL)
579 return;
580
581 spin_lock_irqsave(&iommu->lock, flags);
582
583 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
584 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
585
586 while (head != tail) {
587
588 /* Handle PPR entry */
589 iommu_handle_ppr_entry(iommu, head);
590
591 /* Update and refresh ring-buffer state*/
592 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
593 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
594 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
595 }
596
597 /* enable ppr interrupts again */
598 writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
599
600 spin_unlock_irqrestore(&iommu->lock, flags);
601}
602
464irqreturn_t amd_iommu_int_thread(int irq, void *data) 603irqreturn_t amd_iommu_int_thread(int irq, void *data)
465{ 604{
466 struct amd_iommu *iommu; 605 struct amd_iommu *iommu;
467 606
468 for_each_iommu(iommu) 607 for_each_iommu(iommu) {
469 iommu_poll_events(iommu); 608 iommu_poll_events(iommu);
609 iommu_poll_ppr_log(iommu);
610 }
470 611
471 return IRQ_HANDLED; 612 return IRQ_HANDLED;
472} 613}
@@ -595,6 +736,60 @@ static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
595 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; 736 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
596} 737}
597 738
739static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
740 u64 address, bool size)
741{
742 memset(cmd, 0, sizeof(*cmd));
743
744 address &= ~(0xfffULL);
745
746 cmd->data[0] = pasid & PASID_MASK;
747 cmd->data[1] = domid;
748 cmd->data[2] = lower_32_bits(address);
749 cmd->data[3] = upper_32_bits(address);
750 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
751 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
752 if (size)
753 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
754 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
755}
756
757static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
758 int qdep, u64 address, bool size)
759{
760 memset(cmd, 0, sizeof(*cmd));
761
762 address &= ~(0xfffULL);
763
764 cmd->data[0] = devid;
765 cmd->data[0] |= (pasid & 0xff) << 16;
766 cmd->data[0] |= (qdep & 0xff) << 24;
767 cmd->data[1] = devid;
768 cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
769 cmd->data[2] = lower_32_bits(address);
770 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
771 cmd->data[3] = upper_32_bits(address);
772 if (size)
773 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
774 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
775}
776
777static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
778 int status, int tag, bool gn)
779{
780 memset(cmd, 0, sizeof(*cmd));
781
782 cmd->data[0] = devid;
783 if (gn) {
784 cmd->data[1] = pasid & PASID_MASK;
785 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
786 }
787 cmd->data[3] = tag & 0x1ff;
788 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT;
789
790 CMD_SET_TYPE(cmd, CMD_COMPLETE_PPR);
791}
792
598static void build_inv_all(struct iommu_cmd *cmd) 793static void build_inv_all(struct iommu_cmd *cmd)
599{ 794{
600 memset(cmd, 0, sizeof(*cmd)); 795 memset(cmd, 0, sizeof(*cmd));
@@ -1496,6 +1691,48 @@ static void free_pagetable(struct protection_domain *domain)
1496 domain->pt_root = NULL; 1691 domain->pt_root = NULL;
1497} 1692}
1498 1693
1694static void free_gcr3_tbl_level1(u64 *tbl)
1695{
1696 u64 *ptr;
1697 int i;
1698
1699 for (i = 0; i < 512; ++i) {
1700 if (!(tbl[i] & GCR3_VALID))
1701 continue;
1702
1703 ptr = __va(tbl[i] & PAGE_MASK);
1704
1705 free_page((unsigned long)ptr);
1706 }
1707}
1708
1709static void free_gcr3_tbl_level2(u64 *tbl)
1710{
1711 u64 *ptr;
1712 int i;
1713
1714 for (i = 0; i < 512; ++i) {
1715 if (!(tbl[i] & GCR3_VALID))
1716 continue;
1717
1718 ptr = __va(tbl[i] & PAGE_MASK);
1719
1720 free_gcr3_tbl_level1(ptr);
1721 }
1722}
1723
1724static void free_gcr3_table(struct protection_domain *domain)
1725{
1726 if (domain->glx == 2)
1727 free_gcr3_tbl_level2(domain->gcr3_tbl);
1728 else if (domain->glx == 1)
1729 free_gcr3_tbl_level1(domain->gcr3_tbl);
1730 else if (domain->glx != 0)
1731 BUG();
1732
1733 free_page((unsigned long)domain->gcr3_tbl);
1734}
1735
1499/* 1736/*
1500 * Free a domain, only used if something went wrong in the 1737 * Free a domain, only used if something went wrong in the
1501 * allocation path and we need to free an already allocated page table 1738 * allocation path and we need to free an already allocated page table
@@ -1582,20 +1819,52 @@ static bool dma_ops_domain(struct protection_domain *domain)
1582 1819
1583static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats) 1820static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
1584{ 1821{
1585 u64 pte_root = virt_to_phys(domain->pt_root); 1822 u64 pte_root = 0;
1586 u32 flags = 0; 1823 u64 flags = 0;
1824
1825 if (domain->mode != PAGE_MODE_NONE)
1826 pte_root = virt_to_phys(domain->pt_root);
1587 1827
1588 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) 1828 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1589 << DEV_ENTRY_MODE_SHIFT; 1829 << DEV_ENTRY_MODE_SHIFT;
1590 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; 1830 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1591 1831
1832 flags = amd_iommu_dev_table[devid].data[1];
1833
1592 if (ats) 1834 if (ats)
1593 flags |= DTE_FLAG_IOTLB; 1835 flags |= DTE_FLAG_IOTLB;
1594 1836
1595 amd_iommu_dev_table[devid].data[3] |= flags; 1837 if (domain->flags & PD_IOMMUV2_MASK) {
1596 amd_iommu_dev_table[devid].data[2] = domain->id; 1838 u64 gcr3 = __pa(domain->gcr3_tbl);
1597 amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); 1839 u64 glx = domain->glx;
1598 amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); 1840 u64 tmp;
1841
1842 pte_root |= DTE_FLAG_GV;
1843 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1844
1845 /* First mask out possible old values for GCR3 table */
1846 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1847 flags &= ~tmp;
1848
1849 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1850 flags &= ~tmp;
1851
1852 /* Encode GCR3 table into DTE */
1853 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1854 pte_root |= tmp;
1855
1856 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1857 flags |= tmp;
1858
1859 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1860 flags |= tmp;
1861 }
1862
1863 flags &= ~(0xffffUL);
1864 flags |= domain->id;
1865
1866 amd_iommu_dev_table[devid].data[1] = flags;
1867 amd_iommu_dev_table[devid].data[0] = pte_root;
1599} 1868}
1600 1869
1601static void clear_dte_entry(u16 devid) 1870static void clear_dte_entry(u16 devid)
@@ -1603,7 +1872,6 @@ static void clear_dte_entry(u16 devid)
1603 /* remove entry from the device table seen by the hardware */ 1872 /* remove entry from the device table seen by the hardware */
1604 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV; 1873 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1605 amd_iommu_dev_table[devid].data[1] = 0; 1874 amd_iommu_dev_table[devid].data[1] = 0;
1606 amd_iommu_dev_table[devid].data[2] = 0;
1607 1875
1608 amd_iommu_apply_erratum_63(devid); 1876 amd_iommu_apply_erratum_63(devid);
1609} 1877}
@@ -1696,6 +1964,93 @@ out_unlock:
1696 return ret; 1964 return ret;
1697} 1965}
1698 1966
1967
1968static void pdev_iommuv2_disable(struct pci_dev *pdev)
1969{
1970 pci_disable_ats(pdev);
1971 pci_disable_pri(pdev);
1972 pci_disable_pasid(pdev);
1973}
1974
1975/* FIXME: Change generic reset-function to do the same */
1976static int pri_reset_while_enabled(struct pci_dev *pdev)
1977{
1978 u16 control;
1979 int pos;
1980
1981 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
1982 if (!pos)
1983 return -EINVAL;
1984
1985 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
1986 control |= PCI_PRI_CTRL_RESET;
1987 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
1988
1989 return 0;
1990}
1991
1992static int pdev_iommuv2_enable(struct pci_dev *pdev)
1993{
1994 bool reset_enable;
1995 int reqs, ret;
1996
1997 /* FIXME: Hardcode number of outstanding requests for now */
1998 reqs = 32;
1999 if (pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_LIMIT_REQ_ONE))
2000 reqs = 1;
2001 reset_enable = pdev_pri_erratum(pdev, AMD_PRI_DEV_ERRATUM_ENABLE_RESET);
2002
2003 /* Only allow access to user-accessible pages */
2004 ret = pci_enable_pasid(pdev, 0);
2005 if (ret)
2006 goto out_err;
2007
2008 /* First reset the PRI state of the device */
2009 ret = pci_reset_pri(pdev);
2010 if (ret)
2011 goto out_err;
2012
2013 /* Enable PRI */
2014 ret = pci_enable_pri(pdev, reqs);
2015 if (ret)
2016 goto out_err;
2017
2018 if (reset_enable) {
2019 ret = pri_reset_while_enabled(pdev);
2020 if (ret)
2021 goto out_err;
2022 }
2023
2024 ret = pci_enable_ats(pdev, PAGE_SHIFT);
2025 if (ret)
2026 goto out_err;
2027
2028 return 0;
2029
2030out_err:
2031 pci_disable_pri(pdev);
2032 pci_disable_pasid(pdev);
2033
2034 return ret;
2035}
2036
2037/* FIXME: Move this to PCI code */
2038#define PCI_PRI_TLP_OFF (1 << 2)
2039
2040bool pci_pri_tlp_required(struct pci_dev *pdev)
2041{
2042 u16 control;
2043 int pos;
2044
2045 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
2046 if (!pos)
2047 return false;
2048
2049 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
2050
2051 return (control & PCI_PRI_TLP_OFF) ? true : false;
2052}
2053
1699/* 2054/*
1700 * If a device is not yet associated with a domain, this function does 2055 * If a device is not yet associated with a domain, this function does
1701 * assigns it visible for the hardware 2056 * assigns it visible for the hardware
@@ -1710,7 +2065,18 @@ static int attach_device(struct device *dev,
1710 2065
1711 dev_data = get_dev_data(dev); 2066 dev_data = get_dev_data(dev);
1712 2067
1713 if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) { 2068 if (domain->flags & PD_IOMMUV2_MASK) {
2069 if (!dev_data->iommu_v2 || !dev_data->passthrough)
2070 return -EINVAL;
2071
2072 if (pdev_iommuv2_enable(pdev) != 0)
2073 return -EINVAL;
2074
2075 dev_data->ats.enabled = true;
2076 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
2077 dev_data->pri_tlp = pci_pri_tlp_required(pdev);
2078 } else if (amd_iommu_iotlb_sup &&
2079 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
1714 dev_data->ats.enabled = true; 2080 dev_data->ats.enabled = true;
1715 dev_data->ats.qdep = pci_ats_queue_depth(pdev); 2081 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
1716 } 2082 }
@@ -1760,7 +2126,7 @@ static void __detach_device(struct iommu_dev_data *dev_data)
1760 * passthrough domain if it is detached from any other domain. 2126 * passthrough domain if it is detached from any other domain.
1761 * Make sure we can deassign from the pt_domain itself. 2127 * Make sure we can deassign from the pt_domain itself.
1762 */ 2128 */
1763 if (iommu_pass_through && 2129 if (dev_data->passthrough &&
1764 (dev_data->domain == NULL && domain != pt_domain)) 2130 (dev_data->domain == NULL && domain != pt_domain))
1765 __attach_device(dev_data, pt_domain); 2131 __attach_device(dev_data, pt_domain);
1766} 2132}
@@ -1770,20 +2136,24 @@ static void __detach_device(struct iommu_dev_data *dev_data)
1770 */ 2136 */
1771static void detach_device(struct device *dev) 2137static void detach_device(struct device *dev)
1772{ 2138{
2139 struct protection_domain *domain;
1773 struct iommu_dev_data *dev_data; 2140 struct iommu_dev_data *dev_data;
1774 unsigned long flags; 2141 unsigned long flags;
1775 2142
1776 dev_data = get_dev_data(dev); 2143 dev_data = get_dev_data(dev);
2144 domain = dev_data->domain;
1777 2145
1778 /* lock device table */ 2146 /* lock device table */
1779 write_lock_irqsave(&amd_iommu_devtable_lock, flags); 2147 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1780 __detach_device(dev_data); 2148 __detach_device(dev_data);
1781 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); 2149 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1782 2150
1783 if (dev_data->ats.enabled) { 2151 if (domain->flags & PD_IOMMUV2_MASK)
2152 pdev_iommuv2_disable(to_pci_dev(dev));
2153 else if (dev_data->ats.enabled)
1784 pci_disable_ats(to_pci_dev(dev)); 2154 pci_disable_ats(to_pci_dev(dev));
1785 dev_data->ats.enabled = false; 2155
1786 } 2156 dev_data->ats.enabled = false;
1787} 2157}
1788 2158
1789/* 2159/*
@@ -1818,18 +2188,20 @@ static struct protection_domain *domain_for_device(struct device *dev)
1818static int device_change_notifier(struct notifier_block *nb, 2188static int device_change_notifier(struct notifier_block *nb,
1819 unsigned long action, void *data) 2189 unsigned long action, void *data)
1820{ 2190{
1821 struct device *dev = data;
1822 u16 devid;
1823 struct protection_domain *domain;
1824 struct dma_ops_domain *dma_domain; 2191 struct dma_ops_domain *dma_domain;
2192 struct protection_domain *domain;
2193 struct iommu_dev_data *dev_data;
2194 struct device *dev = data;
1825 struct amd_iommu *iommu; 2195 struct amd_iommu *iommu;
1826 unsigned long flags; 2196 unsigned long flags;
2197 u16 devid;
1827 2198
1828 if (!check_device(dev)) 2199 if (!check_device(dev))
1829 return 0; 2200 return 0;
1830 2201
1831 devid = get_device_id(dev); 2202 devid = get_device_id(dev);
1832 iommu = amd_iommu_rlookup_table[devid]; 2203 iommu = amd_iommu_rlookup_table[devid];
2204 dev_data = get_dev_data(dev);
1833 2205
1834 switch (action) { 2206 switch (action) {
1835 case BUS_NOTIFY_UNBOUND_DRIVER: 2207 case BUS_NOTIFY_UNBOUND_DRIVER:
@@ -1838,7 +2210,7 @@ static int device_change_notifier(struct notifier_block *nb,
1838 2210
1839 if (!domain) 2211 if (!domain)
1840 goto out; 2212 goto out;
1841 if (iommu_pass_through) 2213 if (dev_data->passthrough)
1842 break; 2214 break;
1843 detach_device(dev); 2215 detach_device(dev);
1844 break; 2216 break;
@@ -2434,8 +2806,9 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2434 */ 2806 */
2435static void prealloc_protection_domains(void) 2807static void prealloc_protection_domains(void)
2436{ 2808{
2437 struct pci_dev *dev = NULL; 2809 struct iommu_dev_data *dev_data;
2438 struct dma_ops_domain *dma_dom; 2810 struct dma_ops_domain *dma_dom;
2811 struct pci_dev *dev = NULL;
2439 u16 devid; 2812 u16 devid;
2440 2813
2441 for_each_pci_dev(dev) { 2814 for_each_pci_dev(dev) {
@@ -2444,6 +2817,16 @@ static void prealloc_protection_domains(void)
2444 if (!check_device(&dev->dev)) 2817 if (!check_device(&dev->dev))
2445 continue; 2818 continue;
2446 2819
2820 dev_data = get_dev_data(&dev->dev);
2821 if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
2822 /* Make sure passthrough domain is allocated */
2823 alloc_passthrough_domain();
2824 dev_data->passthrough = true;
2825 attach_device(&dev->dev, pt_domain);
2826 pr_info("AMD-Vi: Using passthough domain for device %s\n",
2827 dev_name(&dev->dev));
2828 }
2829
2447 /* Is there already any domain for it? */ 2830 /* Is there already any domain for it? */
2448 if (domain_for_device(&dev->dev)) 2831 if (domain_for_device(&dev->dev))
2449 continue; 2832 continue;
@@ -2474,6 +2857,7 @@ static struct dma_map_ops amd_iommu_dma_ops = {
2474 2857
2475static unsigned device_dma_ops_init(void) 2858static unsigned device_dma_ops_init(void)
2476{ 2859{
2860 struct iommu_dev_data *dev_data;
2477 struct pci_dev *pdev = NULL; 2861 struct pci_dev *pdev = NULL;
2478 unsigned unhandled = 0; 2862 unsigned unhandled = 0;
2479 2863
@@ -2483,7 +2867,12 @@ static unsigned device_dma_ops_init(void)
2483 continue; 2867 continue;
2484 } 2868 }
2485 2869
2486 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; 2870 dev_data = get_dev_data(&pdev->dev);
2871
2872 if (!dev_data->passthrough)
2873 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
2874 else
2875 pdev->dev.archdata.dma_ops = &nommu_dma_ops;
2487 } 2876 }
2488 2877
2489 return unhandled; 2878 return unhandled;
@@ -2610,6 +2999,20 @@ out_err:
2610 return NULL; 2999 return NULL;
2611} 3000}
2612 3001
3002static int __init alloc_passthrough_domain(void)
3003{
3004 if (pt_domain != NULL)
3005 return 0;
3006
3007 /* allocate passthrough domain */
3008 pt_domain = protection_domain_alloc();
3009 if (!pt_domain)
3010 return -ENOMEM;
3011
3012 pt_domain->mode = PAGE_MODE_NONE;
3013
3014 return 0;
3015}
2613static int amd_iommu_domain_init(struct iommu_domain *dom) 3016static int amd_iommu_domain_init(struct iommu_domain *dom)
2614{ 3017{
2615 struct protection_domain *domain; 3018 struct protection_domain *domain;
@@ -2623,6 +3026,8 @@ static int amd_iommu_domain_init(struct iommu_domain *dom)
2623 if (!domain->pt_root) 3026 if (!domain->pt_root)
2624 goto out_free; 3027 goto out_free;
2625 3028
3029 domain->iommu_domain = dom;
3030
2626 dom->priv = domain; 3031 dom->priv = domain;
2627 3032
2628 return 0; 3033 return 0;
@@ -2645,7 +3050,11 @@ static void amd_iommu_domain_destroy(struct iommu_domain *dom)
2645 3050
2646 BUG_ON(domain->dev_cnt != 0); 3051 BUG_ON(domain->dev_cnt != 0);
2647 3052
2648 free_pagetable(domain); 3053 if (domain->mode != PAGE_MODE_NONE)
3054 free_pagetable(domain);
3055
3056 if (domain->flags & PD_IOMMUV2_MASK)
3057 free_gcr3_table(domain);
2649 3058
2650 protection_domain_free(domain); 3059 protection_domain_free(domain);
2651 3060
@@ -2702,13 +3111,15 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
2702} 3111}
2703 3112
2704static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, 3113static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
2705 phys_addr_t paddr, int gfp_order, int iommu_prot) 3114 phys_addr_t paddr, size_t page_size, int iommu_prot)
2706{ 3115{
2707 unsigned long page_size = 0x1000UL << gfp_order;
2708 struct protection_domain *domain = dom->priv; 3116 struct protection_domain *domain = dom->priv;
2709 int prot = 0; 3117 int prot = 0;
2710 int ret; 3118 int ret;
2711 3119
3120 if (domain->mode == PAGE_MODE_NONE)
3121 return -EINVAL;
3122
2712 if (iommu_prot & IOMMU_READ) 3123 if (iommu_prot & IOMMU_READ)
2713 prot |= IOMMU_PROT_IR; 3124 prot |= IOMMU_PROT_IR;
2714 if (iommu_prot & IOMMU_WRITE) 3125 if (iommu_prot & IOMMU_WRITE)
@@ -2721,13 +3132,14 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
2721 return ret; 3132 return ret;
2722} 3133}
2723 3134
2724static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, 3135static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
2725 int gfp_order) 3136 size_t page_size)
2726{ 3137{
2727 struct protection_domain *domain = dom->priv; 3138 struct protection_domain *domain = dom->priv;
2728 unsigned long page_size, unmap_size; 3139 size_t unmap_size;
2729 3140
2730 page_size = 0x1000UL << gfp_order; 3141 if (domain->mode == PAGE_MODE_NONE)
3142 return -EINVAL;
2731 3143
2732 mutex_lock(&domain->api_lock); 3144 mutex_lock(&domain->api_lock);
2733 unmap_size = iommu_unmap_page(domain, iova, page_size); 3145 unmap_size = iommu_unmap_page(domain, iova, page_size);
@@ -2735,7 +3147,7 @@ static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
2735 3147
2736 domain_flush_tlb_pde(domain); 3148 domain_flush_tlb_pde(domain);
2737 3149
2738 return get_order(unmap_size); 3150 return unmap_size;
2739} 3151}
2740 3152
2741static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, 3153static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
@@ -2746,6 +3158,9 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2746 phys_addr_t paddr; 3158 phys_addr_t paddr;
2747 u64 *pte, __pte; 3159 u64 *pte, __pte;
2748 3160
3161 if (domain->mode == PAGE_MODE_NONE)
3162 return iova;
3163
2749 pte = fetch_pte(domain, iova); 3164 pte = fetch_pte(domain, iova);
2750 3165
2751 if (!pte || !IOMMU_PTE_PRESENT(*pte)) 3166 if (!pte || !IOMMU_PTE_PRESENT(*pte))
@@ -2782,6 +3197,7 @@ static struct iommu_ops amd_iommu_ops = {
2782 .unmap = amd_iommu_unmap, 3197 .unmap = amd_iommu_unmap,
2783 .iova_to_phys = amd_iommu_iova_to_phys, 3198 .iova_to_phys = amd_iommu_iova_to_phys,
2784 .domain_has_cap = amd_iommu_domain_has_cap, 3199 .domain_has_cap = amd_iommu_domain_has_cap,
3200 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
2785}; 3201};
2786 3202
2787/***************************************************************************** 3203/*****************************************************************************
@@ -2796,21 +3212,23 @@ static struct iommu_ops amd_iommu_ops = {
2796 3212
2797int __init amd_iommu_init_passthrough(void) 3213int __init amd_iommu_init_passthrough(void)
2798{ 3214{
2799 struct amd_iommu *iommu; 3215 struct iommu_dev_data *dev_data;
2800 struct pci_dev *dev = NULL; 3216 struct pci_dev *dev = NULL;
3217 struct amd_iommu *iommu;
2801 u16 devid; 3218 u16 devid;
3219 int ret;
2802 3220
2803 /* allocate passthrough domain */ 3221 ret = alloc_passthrough_domain();
2804 pt_domain = protection_domain_alloc(); 3222 if (ret)
2805 if (!pt_domain) 3223 return ret;
2806 return -ENOMEM;
2807
2808 pt_domain->mode |= PAGE_MODE_NONE;
2809 3224
2810 for_each_pci_dev(dev) { 3225 for_each_pci_dev(dev) {
2811 if (!check_device(&dev->dev)) 3226 if (!check_device(&dev->dev))
2812 continue; 3227 continue;
2813 3228
3229 dev_data = get_dev_data(&dev->dev);
3230 dev_data->passthrough = true;
3231
2814 devid = get_device_id(&dev->dev); 3232 devid = get_device_id(&dev->dev);
2815 3233
2816 iommu = amd_iommu_rlookup_table[devid]; 3234 iommu = amd_iommu_rlookup_table[devid];
@@ -2820,7 +3238,375 @@ int __init amd_iommu_init_passthrough(void)
2820 attach_device(&dev->dev, pt_domain); 3238 attach_device(&dev->dev, pt_domain);
2821 } 3239 }
2822 3240
3241 amd_iommu_stats_init();
3242
2823 pr_info("AMD-Vi: Initialized for Passthrough Mode\n"); 3243 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
2824 3244
2825 return 0; 3245 return 0;
2826} 3246}
3247
3248/* IOMMUv2 specific functions */
3249int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3250{
3251 return atomic_notifier_chain_register(&ppr_notifier, nb);
3252}
3253EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3254
3255int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3256{
3257 return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3258}
3259EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3260
3261void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3262{
3263 struct protection_domain *domain = dom->priv;
3264 unsigned long flags;
3265
3266 spin_lock_irqsave(&domain->lock, flags);
3267
3268 /* Update data structure */
3269 domain->mode = PAGE_MODE_NONE;
3270 domain->updated = true;
3271
3272 /* Make changes visible to IOMMUs */
3273 update_domain(domain);
3274
3275 /* Page-table is not visible to IOMMU anymore, so free it */
3276 free_pagetable(domain);
3277
3278 spin_unlock_irqrestore(&domain->lock, flags);
3279}
3280EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3281
3282int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3283{
3284 struct protection_domain *domain = dom->priv;
3285 unsigned long flags;
3286 int levels, ret;
3287
3288 if (pasids <= 0 || pasids > (PASID_MASK + 1))
3289 return -EINVAL;
3290
3291 /* Number of GCR3 table levels required */
3292 for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3293 levels += 1;
3294
3295 if (levels > amd_iommu_max_glx_val)
3296 return -EINVAL;
3297
3298 spin_lock_irqsave(&domain->lock, flags);
3299
3300 /*
3301 * Save us all sanity checks whether devices already in the
3302 * domain support IOMMUv2. Just force that the domain has no
3303 * devices attached when it is switched into IOMMUv2 mode.
3304 */
3305 ret = -EBUSY;
3306 if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
3307 goto out;
3308
3309 ret = -ENOMEM;
3310 domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3311 if (domain->gcr3_tbl == NULL)
3312 goto out;
3313
3314 domain->glx = levels;
3315 domain->flags |= PD_IOMMUV2_MASK;
3316 domain->updated = true;
3317
3318 update_domain(domain);
3319
3320 ret = 0;
3321
3322out:
3323 spin_unlock_irqrestore(&domain->lock, flags);
3324
3325 return ret;
3326}
3327EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
3328
3329static int __flush_pasid(struct protection_domain *domain, int pasid,
3330 u64 address, bool size)
3331{
3332 struct iommu_dev_data *dev_data;
3333 struct iommu_cmd cmd;
3334 int i, ret;
3335
3336 if (!(domain->flags & PD_IOMMUV2_MASK))
3337 return -EINVAL;
3338
3339 build_inv_iommu_pasid(&cmd, domain->id, pasid, address, size);
3340
3341 /*
3342 * IOMMU TLB needs to be flushed before Device TLB to
3343 * prevent device TLB refill from IOMMU TLB
3344 */
3345 for (i = 0; i < amd_iommus_present; ++i) {
3346 if (domain->dev_iommu[i] == 0)
3347 continue;
3348
3349 ret = iommu_queue_command(amd_iommus[i], &cmd);
3350 if (ret != 0)
3351 goto out;
3352 }
3353
3354 /* Wait until IOMMU TLB flushes are complete */
3355 domain_flush_complete(domain);
3356
3357 /* Now flush device TLBs */
3358 list_for_each_entry(dev_data, &domain->dev_list, list) {
3359 struct amd_iommu *iommu;
3360 int qdep;
3361
3362 BUG_ON(!dev_data->ats.enabled);
3363
3364 qdep = dev_data->ats.qdep;
3365 iommu = amd_iommu_rlookup_table[dev_data->devid];
3366
3367 build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,
3368 qdep, address, size);
3369
3370 ret = iommu_queue_command(iommu, &cmd);
3371 if (ret != 0)
3372 goto out;
3373 }
3374
3375 /* Wait until all device TLBs are flushed */
3376 domain_flush_complete(domain);
3377
3378 ret = 0;
3379
3380out:
3381
3382 return ret;
3383}
3384
3385static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid,
3386 u64 address)
3387{
3388 INC_STATS_COUNTER(invalidate_iotlb);
3389
3390 return __flush_pasid(domain, pasid, address, false);
3391}
3392
3393int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
3394 u64 address)
3395{
3396 struct protection_domain *domain = dom->priv;
3397 unsigned long flags;
3398 int ret;
3399
3400 spin_lock_irqsave(&domain->lock, flags);
3401 ret = __amd_iommu_flush_page(domain, pasid, address);
3402 spin_unlock_irqrestore(&domain->lock, flags);
3403
3404 return ret;
3405}
3406EXPORT_SYMBOL(amd_iommu_flush_page);
3407
3408static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid)
3409{
3410 INC_STATS_COUNTER(invalidate_iotlb_all);
3411
3412 return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
3413 true);
3414}
3415
3416int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid)
3417{
3418 struct protection_domain *domain = dom->priv;
3419 unsigned long flags;
3420 int ret;
3421
3422 spin_lock_irqsave(&domain->lock, flags);
3423 ret = __amd_iommu_flush_tlb(domain, pasid);
3424 spin_unlock_irqrestore(&domain->lock, flags);
3425
3426 return ret;
3427}
3428EXPORT_SYMBOL(amd_iommu_flush_tlb);
3429
3430static u64 *__get_gcr3_pte(u64 *root, int level, int pasid, bool alloc)
3431{
3432 int index;
3433 u64 *pte;
3434
3435 while (true) {
3436
3437 index = (pasid >> (9 * level)) & 0x1ff;
3438 pte = &root[index];
3439
3440 if (level == 0)
3441 break;
3442
3443 if (!(*pte & GCR3_VALID)) {
3444 if (!alloc)
3445 return NULL;
3446
3447 root = (void *)get_zeroed_page(GFP_ATOMIC);
3448 if (root == NULL)
3449 return NULL;
3450
3451 *pte = __pa(root) | GCR3_VALID;
3452 }
3453
3454 root = __va(*pte & PAGE_MASK);
3455
3456 level -= 1;
3457 }
3458
3459 return pte;
3460}
3461
3462static int __set_gcr3(struct protection_domain *domain, int pasid,
3463 unsigned long cr3)
3464{
3465 u64 *pte;
3466
3467 if (domain->mode != PAGE_MODE_NONE)
3468 return -EINVAL;
3469
3470 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, true);
3471 if (pte == NULL)
3472 return -ENOMEM;
3473
3474 *pte = (cr3 & PAGE_MASK) | GCR3_VALID;
3475
3476 return __amd_iommu_flush_tlb(domain, pasid);
3477}
3478
3479static int __clear_gcr3(struct protection_domain *domain, int pasid)
3480{
3481 u64 *pte;
3482
3483 if (domain->mode != PAGE_MODE_NONE)
3484 return -EINVAL;
3485
3486 pte = __get_gcr3_pte(domain->gcr3_tbl, domain->glx, pasid, false);
3487 if (pte == NULL)
3488 return 0;
3489
3490 *pte = 0;
3491
3492 return __amd_iommu_flush_tlb(domain, pasid);
3493}
3494
3495int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
3496 unsigned long cr3)
3497{
3498 struct protection_domain *domain = dom->priv;
3499 unsigned long flags;
3500 int ret;
3501
3502 spin_lock_irqsave(&domain->lock, flags);
3503 ret = __set_gcr3(domain, pasid, cr3);
3504 spin_unlock_irqrestore(&domain->lock, flags);
3505
3506 return ret;
3507}
3508EXPORT_SYMBOL(amd_iommu_domain_set_gcr3);
3509
3510int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid)
3511{
3512 struct protection_domain *domain = dom->priv;
3513 unsigned long flags;
3514 int ret;
3515
3516 spin_lock_irqsave(&domain->lock, flags);
3517 ret = __clear_gcr3(domain, pasid);
3518 spin_unlock_irqrestore(&domain->lock, flags);
3519
3520 return ret;
3521}
3522EXPORT_SYMBOL(amd_iommu_domain_clear_gcr3);
3523
3524int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
3525 int status, int tag)
3526{
3527 struct iommu_dev_data *dev_data;
3528 struct amd_iommu *iommu;
3529 struct iommu_cmd cmd;
3530
3531 INC_STATS_COUNTER(complete_ppr);
3532
3533 dev_data = get_dev_data(&pdev->dev);
3534 iommu = amd_iommu_rlookup_table[dev_data->devid];
3535
3536 build_complete_ppr(&cmd, dev_data->devid, pasid, status,
3537 tag, dev_data->pri_tlp);
3538
3539 return iommu_queue_command(iommu, &cmd);
3540}
3541EXPORT_SYMBOL(amd_iommu_complete_ppr);
3542
3543struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev)
3544{
3545 struct protection_domain *domain;
3546
3547 domain = get_domain(&pdev->dev);
3548 if (IS_ERR(domain))
3549 return NULL;
3550
3551 /* Only return IOMMUv2 domains */
3552 if (!(domain->flags & PD_IOMMUV2_MASK))
3553 return NULL;
3554
3555 return domain->iommu_domain;
3556}
3557EXPORT_SYMBOL(amd_iommu_get_v2_domain);
3558
3559void amd_iommu_enable_device_erratum(struct pci_dev *pdev, u32 erratum)
3560{
3561 struct iommu_dev_data *dev_data;
3562
3563 if (!amd_iommu_v2_supported())
3564 return;
3565
3566 dev_data = get_dev_data(&pdev->dev);
3567 dev_data->errata |= (1 << erratum);
3568}
3569EXPORT_SYMBOL(amd_iommu_enable_device_erratum);
3570
3571int amd_iommu_device_info(struct pci_dev *pdev,
3572 struct amd_iommu_device_info *info)
3573{
3574 int max_pasids;
3575 int pos;
3576
3577 if (pdev == NULL || info == NULL)
3578 return -EINVAL;
3579
3580 if (!amd_iommu_v2_supported())
3581 return -EINVAL;
3582
3583 memset(info, 0, sizeof(*info));
3584
3585 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS);
3586 if (pos)
3587 info->flags |= AMD_IOMMU_DEVICE_FLAG_ATS_SUP;
3588
3589 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
3590 if (pos)
3591 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRI_SUP;
3592
3593 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
3594 if (pos) {
3595 int features;
3596
3597 max_pasids = 1 << (9 * (amd_iommu_max_glx_val + 1));
3598 max_pasids = min(max_pasids, (1 << 20));
3599
3600 info->flags |= AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
3601 info->max_pasids = min(pci_max_pasids(pdev), max_pasids);
3602
3603 features = pci_pasid_features(pdev);
3604 if (features & PCI_PASID_CAP_EXEC)
3605 info->flags |= AMD_IOMMU_DEVICE_FLAG_EXEC_SUP;
3606 if (features & PCI_PASID_CAP_PRIV)
3607 info->flags |= AMD_IOMMU_DEVICE_FLAG_PRIV_SUP;
3608 }
3609
3610 return 0;
3611}
3612EXPORT_SYMBOL(amd_iommu_device_info);
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 82d2410f4205..bdea288dc185 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -25,6 +25,7 @@
25#include <linux/interrupt.h> 25#include <linux/interrupt.h>
26#include <linux/msi.h> 26#include <linux/msi.h>
27#include <linux/amd-iommu.h> 27#include <linux/amd-iommu.h>
28#include <linux/export.h>
28#include <asm/pci-direct.h> 29#include <asm/pci-direct.h>
29#include <asm/iommu.h> 30#include <asm/iommu.h>
30#include <asm/gart.h> 31#include <asm/gart.h>
@@ -141,6 +142,12 @@ int amd_iommus_present;
141bool amd_iommu_np_cache __read_mostly; 142bool amd_iommu_np_cache __read_mostly;
142bool amd_iommu_iotlb_sup __read_mostly = true; 143bool amd_iommu_iotlb_sup __read_mostly = true;
143 144
145u32 amd_iommu_max_pasids __read_mostly = ~0;
146
147bool amd_iommu_v2_present __read_mostly;
148
149bool amd_iommu_force_isolation __read_mostly;
150
144/* 151/*
145 * The ACPI table parsing functions set this variable on an error 152 * The ACPI table parsing functions set this variable on an error
146 */ 153 */
@@ -299,6 +306,16 @@ static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
299 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); 306 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
300} 307}
301 308
309static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
310{
311 u32 ctrl;
312
313 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
314 ctrl &= ~CTRL_INV_TO_MASK;
315 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
316 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
317}
318
302/* Function to enable the hardware */ 319/* Function to enable the hardware */
303static void iommu_enable(struct amd_iommu *iommu) 320static void iommu_enable(struct amd_iommu *iommu)
304{ 321{
@@ -581,21 +598,69 @@ static void __init free_event_buffer(struct amd_iommu *iommu)
581 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); 598 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
582} 599}
583 600
601/* allocates the memory where the IOMMU will log its events to */
602static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
603{
604 iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
605 get_order(PPR_LOG_SIZE));
606
607 if (iommu->ppr_log == NULL)
608 return NULL;
609
610 return iommu->ppr_log;
611}
612
613static void iommu_enable_ppr_log(struct amd_iommu *iommu)
614{
615 u64 entry;
616
617 if (iommu->ppr_log == NULL)
618 return;
619
620 entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
621
622 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
623 &entry, sizeof(entry));
624
625 /* set head and tail to zero manually */
626 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
627 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
628
629 iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
630 iommu_feature_enable(iommu, CONTROL_PPR_EN);
631}
632
633static void __init free_ppr_log(struct amd_iommu *iommu)
634{
635 if (iommu->ppr_log == NULL)
636 return;
637
638 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
639}
640
641static void iommu_enable_gt(struct amd_iommu *iommu)
642{
643 if (!iommu_feature(iommu, FEATURE_GT))
644 return;
645
646 iommu_feature_enable(iommu, CONTROL_GT_EN);
647}
648
584/* sets a specific bit in the device table entry. */ 649/* sets a specific bit in the device table entry. */
585static void set_dev_entry_bit(u16 devid, u8 bit) 650static void set_dev_entry_bit(u16 devid, u8 bit)
586{ 651{
587 int i = (bit >> 5) & 0x07; 652 int i = (bit >> 6) & 0x03;
588 int _bit = bit & 0x1f; 653 int _bit = bit & 0x3f;
589 654
590 amd_iommu_dev_table[devid].data[i] |= (1 << _bit); 655 amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);
591} 656}
592 657
593static int get_dev_entry_bit(u16 devid, u8 bit) 658static int get_dev_entry_bit(u16 devid, u8 bit)
594{ 659{
595 int i = (bit >> 5) & 0x07; 660 int i = (bit >> 6) & 0x03;
596 int _bit = bit & 0x1f; 661 int _bit = bit & 0x3f;
597 662
598 return (amd_iommu_dev_table[devid].data[i] & (1 << _bit)) >> _bit; 663 return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
599} 664}
600 665
601 666
@@ -699,6 +764,32 @@ static void __init init_iommu_from_pci(struct amd_iommu *iommu)
699 764
700 iommu->features = ((u64)high << 32) | low; 765 iommu->features = ((u64)high << 32) | low;
701 766
767 if (iommu_feature(iommu, FEATURE_GT)) {
768 int glxval;
769 u32 pasids;
770 u64 shift;
771
772 shift = iommu->features & FEATURE_PASID_MASK;
773 shift >>= FEATURE_PASID_SHIFT;
774 pasids = (1 << shift);
775
776 amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
777
778 glxval = iommu->features & FEATURE_GLXVAL_MASK;
779 glxval >>= FEATURE_GLXVAL_SHIFT;
780
781 if (amd_iommu_max_glx_val == -1)
782 amd_iommu_max_glx_val = glxval;
783 else
784 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
785 }
786
787 if (iommu_feature(iommu, FEATURE_GT) &&
788 iommu_feature(iommu, FEATURE_PPR)) {
789 iommu->is_iommu_v2 = true;
790 amd_iommu_v2_present = true;
791 }
792
702 if (!is_rd890_iommu(iommu->dev)) 793 if (!is_rd890_iommu(iommu->dev))
703 return; 794 return;
704 795
@@ -901,6 +992,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu)
901{ 992{
902 free_command_buffer(iommu); 993 free_command_buffer(iommu);
903 free_event_buffer(iommu); 994 free_event_buffer(iommu);
995 free_ppr_log(iommu);
904 iommu_unmap_mmio_space(iommu); 996 iommu_unmap_mmio_space(iommu);
905} 997}
906 998
@@ -964,6 +1056,12 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
964 init_iommu_from_acpi(iommu, h); 1056 init_iommu_from_acpi(iommu, h);
965 init_iommu_devices(iommu); 1057 init_iommu_devices(iommu);
966 1058
1059 if (iommu_feature(iommu, FEATURE_PPR)) {
1060 iommu->ppr_log = alloc_ppr_log(iommu);
1061 if (!iommu->ppr_log)
1062 return -ENOMEM;
1063 }
1064
967 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) 1065 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
968 amd_iommu_np_cache = true; 1066 amd_iommu_np_cache = true;
969 1067
@@ -1050,6 +1148,9 @@ static int iommu_setup_msi(struct amd_iommu *iommu)
1050 iommu->int_enabled = true; 1148 iommu->int_enabled = true;
1051 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); 1149 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
1052 1150
1151 if (iommu->ppr_log != NULL)
1152 iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
1153
1053 return 0; 1154 return 0;
1054} 1155}
1055 1156
@@ -1209,6 +1310,9 @@ static void iommu_init_flags(struct amd_iommu *iommu)
1209 * make IOMMU memory accesses cache coherent 1310 * make IOMMU memory accesses cache coherent
1210 */ 1311 */
1211 iommu_feature_enable(iommu, CONTROL_COHERENT_EN); 1312 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
1313
1314 /* Set IOTLB invalidation timeout to 1s */
1315 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
1212} 1316}
1213 1317
1214static void iommu_apply_resume_quirks(struct amd_iommu *iommu) 1318static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
@@ -1274,6 +1378,8 @@ static void enable_iommus(void)
1274 iommu_set_device_table(iommu); 1378 iommu_set_device_table(iommu);
1275 iommu_enable_command_buffer(iommu); 1379 iommu_enable_command_buffer(iommu);
1276 iommu_enable_event_buffer(iommu); 1380 iommu_enable_event_buffer(iommu);
1381 iommu_enable_ppr_log(iommu);
1382 iommu_enable_gt(iommu);
1277 iommu_set_exclusion_range(iommu); 1383 iommu_set_exclusion_range(iommu);
1278 iommu_init_msi(iommu); 1384 iommu_init_msi(iommu);
1279 iommu_enable(iommu); 1385 iommu_enable(iommu);
@@ -1303,13 +1409,6 @@ static void amd_iommu_resume(void)
1303 1409
1304 /* re-load the hardware */ 1410 /* re-load the hardware */
1305 enable_iommus(); 1411 enable_iommus();
1306
1307 /*
1308 * we have to flush after the IOMMUs are enabled because a
1309 * disabled IOMMU will never execute the commands we send
1310 */
1311 for_each_iommu(iommu)
1312 iommu_flush_all_caches(iommu);
1313} 1412}
1314 1413
1315static int amd_iommu_suspend(void) 1414static int amd_iommu_suspend(void)
@@ -1560,6 +1659,8 @@ static int __init parse_amd_iommu_options(char *str)
1560 amd_iommu_unmap_flush = true; 1659 amd_iommu_unmap_flush = true;
1561 if (strncmp(str, "off", 3) == 0) 1660 if (strncmp(str, "off", 3) == 0)
1562 amd_iommu_disabled = true; 1661 amd_iommu_disabled = true;
1662 if (strncmp(str, "force_isolation", 15) == 0)
1663 amd_iommu_force_isolation = true;
1563 } 1664 }
1564 1665
1565 return 1; 1666 return 1;
@@ -1572,3 +1673,9 @@ IOMMU_INIT_FINISH(amd_iommu_detect,
1572 gart_iommu_hole_init, 1673 gart_iommu_hole_init,
1573 0, 1674 0,
1574 0); 1675 0);
1676
1677bool amd_iommu_v2_supported(void)
1678{
1679 return amd_iommu_v2_present;
1680}
1681EXPORT_SYMBOL(amd_iommu_v2_supported);
diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h
index 7ffaa64410b0..1a7f41c6cc66 100644
--- a/drivers/iommu/amd_iommu_proto.h
+++ b/drivers/iommu/amd_iommu_proto.h
@@ -31,6 +31,30 @@ extern int amd_iommu_init_devices(void);
31extern void amd_iommu_uninit_devices(void); 31extern void amd_iommu_uninit_devices(void);
32extern void amd_iommu_init_notifier(void); 32extern void amd_iommu_init_notifier(void);
33extern void amd_iommu_init_api(void); 33extern void amd_iommu_init_api(void);
34
35/* IOMMUv2 specific functions */
36struct iommu_domain;
37
38extern bool amd_iommu_v2_supported(void);
39extern int amd_iommu_register_ppr_notifier(struct notifier_block *nb);
40extern int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb);
41extern void amd_iommu_domain_direct_map(struct iommu_domain *dom);
42extern int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids);
43extern int amd_iommu_flush_page(struct iommu_domain *dom, int pasid,
44 u64 address);
45extern int amd_iommu_flush_tlb(struct iommu_domain *dom, int pasid);
46extern int amd_iommu_domain_set_gcr3(struct iommu_domain *dom, int pasid,
47 unsigned long cr3);
48extern int amd_iommu_domain_clear_gcr3(struct iommu_domain *dom, int pasid);
49extern struct iommu_domain *amd_iommu_get_v2_domain(struct pci_dev *pdev);
50
51#define PPR_SUCCESS 0x0
52#define PPR_INVALID 0x1
53#define PPR_FAILURE 0xf
54
55extern int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid,
56 int status, int tag);
57
34#ifndef CONFIG_AMD_IOMMU_STATS 58#ifndef CONFIG_AMD_IOMMU_STATS
35 59
36static inline void amd_iommu_stats_init(void) { } 60static inline void amd_iommu_stats_init(void) { }
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 5b9c5075e81a..2452f3b71736 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -69,11 +69,14 @@
69#define MMIO_EXCL_BASE_OFFSET 0x0020 69#define MMIO_EXCL_BASE_OFFSET 0x0020
70#define MMIO_EXCL_LIMIT_OFFSET 0x0028 70#define MMIO_EXCL_LIMIT_OFFSET 0x0028
71#define MMIO_EXT_FEATURES 0x0030 71#define MMIO_EXT_FEATURES 0x0030
72#define MMIO_PPR_LOG_OFFSET 0x0038
72#define MMIO_CMD_HEAD_OFFSET 0x2000 73#define MMIO_CMD_HEAD_OFFSET 0x2000
73#define MMIO_CMD_TAIL_OFFSET 0x2008 74#define MMIO_CMD_TAIL_OFFSET 0x2008
74#define MMIO_EVT_HEAD_OFFSET 0x2010 75#define MMIO_EVT_HEAD_OFFSET 0x2010
75#define MMIO_EVT_TAIL_OFFSET 0x2018 76#define MMIO_EVT_TAIL_OFFSET 0x2018
76#define MMIO_STATUS_OFFSET 0x2020 77#define MMIO_STATUS_OFFSET 0x2020
78#define MMIO_PPR_HEAD_OFFSET 0x2030
79#define MMIO_PPR_TAIL_OFFSET 0x2038
77 80
78 81
79/* Extended Feature Bits */ 82/* Extended Feature Bits */
@@ -87,8 +90,17 @@
87#define FEATURE_HE (1ULL<<8) 90#define FEATURE_HE (1ULL<<8)
88#define FEATURE_PC (1ULL<<9) 91#define FEATURE_PC (1ULL<<9)
89 92
93#define FEATURE_PASID_SHIFT 32
94#define FEATURE_PASID_MASK (0x1fULL << FEATURE_PASID_SHIFT)
95
96#define FEATURE_GLXVAL_SHIFT 14
97#define FEATURE_GLXVAL_MASK (0x03ULL << FEATURE_GLXVAL_SHIFT)
98
99#define PASID_MASK 0x000fffff
100
90/* MMIO status bits */ 101/* MMIO status bits */
91#define MMIO_STATUS_COM_WAIT_INT_MASK 0x04 102#define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2)
103#define MMIO_STATUS_PPR_INT_MASK (1 << 6)
92 104
93/* event logging constants */ 105/* event logging constants */
94#define EVENT_ENTRY_SIZE 0x10 106#define EVENT_ENTRY_SIZE 0x10
@@ -115,6 +127,7 @@
115#define CONTROL_EVT_LOG_EN 0x02ULL 127#define CONTROL_EVT_LOG_EN 0x02ULL
116#define CONTROL_EVT_INT_EN 0x03ULL 128#define CONTROL_EVT_INT_EN 0x03ULL
117#define CONTROL_COMWAIT_EN 0x04ULL 129#define CONTROL_COMWAIT_EN 0x04ULL
130#define CONTROL_INV_TIMEOUT 0x05ULL
118#define CONTROL_PASSPW_EN 0x08ULL 131#define CONTROL_PASSPW_EN 0x08ULL
119#define CONTROL_RESPASSPW_EN 0x09ULL 132#define CONTROL_RESPASSPW_EN 0x09ULL
120#define CONTROL_COHERENT_EN 0x0aULL 133#define CONTROL_COHERENT_EN 0x0aULL
@@ -122,18 +135,34 @@
122#define CONTROL_CMDBUF_EN 0x0cULL 135#define CONTROL_CMDBUF_EN 0x0cULL
123#define CONTROL_PPFLOG_EN 0x0dULL 136#define CONTROL_PPFLOG_EN 0x0dULL
124#define CONTROL_PPFINT_EN 0x0eULL 137#define CONTROL_PPFINT_EN 0x0eULL
138#define CONTROL_PPR_EN 0x0fULL
139#define CONTROL_GT_EN 0x10ULL
140
141#define CTRL_INV_TO_MASK (7 << CONTROL_INV_TIMEOUT)
142#define CTRL_INV_TO_NONE 0
143#define CTRL_INV_TO_1MS 1
144#define CTRL_INV_TO_10MS 2
145#define CTRL_INV_TO_100MS 3
146#define CTRL_INV_TO_1S 4
147#define CTRL_INV_TO_10S 5
148#define CTRL_INV_TO_100S 6
125 149
126/* command specific defines */ 150/* command specific defines */
127#define CMD_COMPL_WAIT 0x01 151#define CMD_COMPL_WAIT 0x01
128#define CMD_INV_DEV_ENTRY 0x02 152#define CMD_INV_DEV_ENTRY 0x02
129#define CMD_INV_IOMMU_PAGES 0x03 153#define CMD_INV_IOMMU_PAGES 0x03
130#define CMD_INV_IOTLB_PAGES 0x04 154#define CMD_INV_IOTLB_PAGES 0x04
155#define CMD_COMPLETE_PPR 0x07
131#define CMD_INV_ALL 0x08 156#define CMD_INV_ALL 0x08
132 157
133#define CMD_COMPL_WAIT_STORE_MASK 0x01 158#define CMD_COMPL_WAIT_STORE_MASK 0x01
134#define CMD_COMPL_WAIT_INT_MASK 0x02 159#define CMD_COMPL_WAIT_INT_MASK 0x02
135#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01 160#define CMD_INV_IOMMU_PAGES_SIZE_MASK 0x01
136#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02 161#define CMD_INV_IOMMU_PAGES_PDE_MASK 0x02
162#define CMD_INV_IOMMU_PAGES_GN_MASK 0x04
163
164#define PPR_STATUS_MASK 0xf
165#define PPR_STATUS_SHIFT 12
137 166
138#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL 167#define CMD_INV_IOMMU_ALL_PAGES_ADDRESS 0x7fffffffffffffffULL
139 168
@@ -165,6 +194,23 @@
165#define EVT_BUFFER_SIZE 8192 /* 512 entries */ 194#define EVT_BUFFER_SIZE 8192 /* 512 entries */
166#define EVT_LEN_MASK (0x9ULL << 56) 195#define EVT_LEN_MASK (0x9ULL << 56)
167 196
197/* Constants for PPR Log handling */
198#define PPR_LOG_ENTRIES 512
199#define PPR_LOG_SIZE_SHIFT 56
200#define PPR_LOG_SIZE_512 (0x9ULL << PPR_LOG_SIZE_SHIFT)
201#define PPR_ENTRY_SIZE 16
202#define PPR_LOG_SIZE (PPR_ENTRY_SIZE * PPR_LOG_ENTRIES)
203
204#define PPR_REQ_TYPE(x) (((x) >> 60) & 0xfULL)
205#define PPR_FLAGS(x) (((x) >> 48) & 0xfffULL)
206#define PPR_DEVID(x) ((x) & 0xffffULL)
207#define PPR_TAG(x) (((x) >> 32) & 0x3ffULL)
208#define PPR_PASID1(x) (((x) >> 16) & 0xffffULL)
209#define PPR_PASID2(x) (((x) >> 42) & 0xfULL)
210#define PPR_PASID(x) ((PPR_PASID2(x) << 16) | PPR_PASID1(x))
211
212#define PPR_REQ_FAULT 0x01
213
168#define PAGE_MODE_NONE 0x00 214#define PAGE_MODE_NONE 0x00
169#define PAGE_MODE_1_LEVEL 0x01 215#define PAGE_MODE_1_LEVEL 0x01
170#define PAGE_MODE_2_LEVEL 0x02 216#define PAGE_MODE_2_LEVEL 0x02
@@ -230,7 +276,24 @@
230#define IOMMU_PTE_IR (1ULL << 61) 276#define IOMMU_PTE_IR (1ULL << 61)
231#define IOMMU_PTE_IW (1ULL << 62) 277#define IOMMU_PTE_IW (1ULL << 62)
232 278
233#define DTE_FLAG_IOTLB 0x01 279#define DTE_FLAG_IOTLB (0x01UL << 32)
280#define DTE_FLAG_GV (0x01ULL << 55)
281#define DTE_GLX_SHIFT (56)
282#define DTE_GLX_MASK (3)
283
284#define DTE_GCR3_VAL_A(x) (((x) >> 12) & 0x00007ULL)
285#define DTE_GCR3_VAL_B(x) (((x) >> 15) & 0x0ffffULL)
286#define DTE_GCR3_VAL_C(x) (((x) >> 31) & 0xfffffULL)
287
288#define DTE_GCR3_INDEX_A 0
289#define DTE_GCR3_INDEX_B 1
290#define DTE_GCR3_INDEX_C 1
291
292#define DTE_GCR3_SHIFT_A 58
293#define DTE_GCR3_SHIFT_B 16
294#define DTE_GCR3_SHIFT_C 43
295
296#define GCR3_VALID 0x01ULL
234 297
235#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) 298#define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)
236#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P) 299#define IOMMU_PTE_PRESENT(pte) ((pte) & IOMMU_PTE_P)
@@ -257,6 +320,7 @@
257 domain for an IOMMU */ 320 domain for an IOMMU */
258#define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page 321#define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page
259 translation */ 322 translation */
323#define PD_IOMMUV2_MASK (1UL << 3) /* domain has gcr3 table */
260 324
261extern bool amd_iommu_dump; 325extern bool amd_iommu_dump;
262#define DUMP_printk(format, arg...) \ 326#define DUMP_printk(format, arg...) \
@@ -285,6 +349,29 @@ extern bool amd_iommu_iotlb_sup;
285#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT) 349#define APERTURE_RANGE_INDEX(a) ((a) >> APERTURE_RANGE_SHIFT)
286#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL) 350#define APERTURE_PAGE_INDEX(a) (((a) >> 21) & 0x3fULL)
287 351
352
353/*
354 * This struct is used to pass information about
355 * incoming PPR faults around.
356 */
357struct amd_iommu_fault {
358 u64 address; /* IO virtual address of the fault*/
359 u32 pasid; /* Address space identifier */
360 u16 device_id; /* Originating PCI device id */
361 u16 tag; /* PPR tag */
362 u16 flags; /* Fault flags */
363
364};
365
366#define PPR_FAULT_EXEC (1 << 1)
367#define PPR_FAULT_READ (1 << 2)
368#define PPR_FAULT_WRITE (1 << 5)
369#define PPR_FAULT_USER (1 << 6)
370#define PPR_FAULT_RSVD (1 << 7)
371#define PPR_FAULT_GN (1 << 8)
372
373struct iommu_domain;
374
288/* 375/*
289 * This structure contains generic data for IOMMU protection domains 376 * This structure contains generic data for IOMMU protection domains
290 * independent of their use. 377 * independent of their use.
@@ -297,11 +384,15 @@ struct protection_domain {
297 u16 id; /* the domain id written to the device table */ 384 u16 id; /* the domain id written to the device table */
298 int mode; /* paging mode (0-6 levels) */ 385 int mode; /* paging mode (0-6 levels) */
299 u64 *pt_root; /* page table root pointer */ 386 u64 *pt_root; /* page table root pointer */
387 int glx; /* Number of levels for GCR3 table */
388 u64 *gcr3_tbl; /* Guest CR3 table */
300 unsigned long flags; /* flags to find out type of domain */ 389 unsigned long flags; /* flags to find out type of domain */
301 bool updated; /* complete domain flush required */ 390 bool updated; /* complete domain flush required */
302 unsigned dev_cnt; /* devices assigned to this domain */ 391 unsigned dev_cnt; /* devices assigned to this domain */
303 unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ 392 unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
304 void *priv; /* private data */ 393 void *priv; /* private data */
394 struct iommu_domain *iommu_domain; /* Pointer to generic
395 domain structure */
305 396
306}; 397};
307 398
@@ -315,10 +406,15 @@ struct iommu_dev_data {
315 struct protection_domain *domain; /* Domain the device is bound to */ 406 struct protection_domain *domain; /* Domain the device is bound to */
316 atomic_t bind; /* Domain attach reverent count */ 407 atomic_t bind; /* Domain attach reverent count */
317 u16 devid; /* PCI Device ID */ 408 u16 devid; /* PCI Device ID */
409 bool iommu_v2; /* Device can make use of IOMMUv2 */
410 bool passthrough; /* Default for device is pt_domain */
318 struct { 411 struct {
319 bool enabled; 412 bool enabled;
320 int qdep; 413 int qdep;
321 } ats; /* ATS state */ 414 } ats; /* ATS state */
415 bool pri_tlp; /* PASID TLB required for
416 PPR completions */
417 u32 errata; /* Bitmap for errata to apply */
322}; 418};
323 419
324/* 420/*
@@ -399,6 +495,9 @@ struct amd_iommu {
399 /* Extended features */ 495 /* Extended features */
400 u64 features; 496 u64 features;
401 497
498 /* IOMMUv2 */
499 bool is_iommu_v2;
500
402 /* 501 /*
403 * Capability pointer. There could be more than one IOMMU per PCI 502 * Capability pointer. There could be more than one IOMMU per PCI
404 * device function if there are more than one AMD IOMMU capability 503 * device function if there are more than one AMD IOMMU capability
@@ -431,6 +530,9 @@ struct amd_iommu {
431 /* MSI number for event interrupt */ 530 /* MSI number for event interrupt */
432 u16 evt_msi_num; 531 u16 evt_msi_num;
433 532
533 /* Base of the PPR log, if present */
534 u8 *ppr_log;
535
434 /* true if interrupts for this IOMMU are already enabled */ 536 /* true if interrupts for this IOMMU are already enabled */
435 bool int_enabled; 537 bool int_enabled;
436 538
@@ -484,7 +586,7 @@ extern struct list_head amd_iommu_pd_list;
484 * Structure defining one entry in the device table 586 * Structure defining one entry in the device table
485 */ 587 */
486struct dev_table_entry { 588struct dev_table_entry {
487 u32 data[8]; 589 u64 data[4];
488}; 590};
489 591
490/* 592/*
@@ -549,6 +651,16 @@ extern unsigned long *amd_iommu_pd_alloc_bitmap;
549 */ 651 */
550extern bool amd_iommu_unmap_flush; 652extern bool amd_iommu_unmap_flush;
551 653
654/* Smallest number of PASIDs supported by any IOMMU in the system */
655extern u32 amd_iommu_max_pasids;
656
657extern bool amd_iommu_v2_present;
658
659extern bool amd_iommu_force_isolation;
660
661/* Max levels of glxval supported */
662extern int amd_iommu_max_glx_val;
663
552/* takes bus and device/function and returns the device id 664/* takes bus and device/function and returns the device id
553 * FIXME: should that be in generic PCI code? */ 665 * FIXME: should that be in generic PCI code? */
554static inline u16 calc_devid(u8 bus, u8 devfn) 666static inline u16 calc_devid(u8 bus, u8 devfn)
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
new file mode 100644
index 000000000000..8add9f125d3e
--- /dev/null
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -0,0 +1,994 @@
1/*
2 * Copyright (C) 2010-2012 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published
7 * by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */
18
19#include <linux/mmu_notifier.h>
20#include <linux/amd-iommu.h>
21#include <linux/mm_types.h>
22#include <linux/profile.h>
23#include <linux/module.h>
24#include <linux/sched.h>
25#include <linux/iommu.h>
26#include <linux/wait.h>
27#include <linux/pci.h>
28#include <linux/gfp.h>
29
30#include "amd_iommu_types.h"
31#include "amd_iommu_proto.h"
32
33MODULE_LICENSE("GPL v2");
34MODULE_AUTHOR("Joerg Roedel <joerg.roedel@amd.com>");
35
36#define MAX_DEVICES 0x10000
37#define PRI_QUEUE_SIZE 512
38
39struct pri_queue {
40 atomic_t inflight;
41 bool finish;
42 int status;
43};
44
45struct pasid_state {
46 struct list_head list; /* For global state-list */
47 atomic_t count; /* Reference count */
48 struct task_struct *task; /* Task bound to this PASID */
49 struct mm_struct *mm; /* mm_struct for the faults */
50 struct mmu_notifier mn; /* mmu_otifier handle */
51 struct pri_queue pri[PRI_QUEUE_SIZE]; /* PRI tag states */
52 struct device_state *device_state; /* Link to our device_state */
53 int pasid; /* PASID index */
54 spinlock_t lock; /* Protect pri_queues */
55 wait_queue_head_t wq; /* To wait for count == 0 */
56};
57
58struct device_state {
59 atomic_t count;
60 struct pci_dev *pdev;
61 struct pasid_state **states;
62 struct iommu_domain *domain;
63 int pasid_levels;
64 int max_pasids;
65 amd_iommu_invalid_ppr_cb inv_ppr_cb;
66 amd_iommu_invalidate_ctx inv_ctx_cb;
67 spinlock_t lock;
68 wait_queue_head_t wq;
69};
70
71struct fault {
72 struct work_struct work;
73 struct device_state *dev_state;
74 struct pasid_state *state;
75 struct mm_struct *mm;
76 u64 address;
77 u16 devid;
78 u16 pasid;
79 u16 tag;
80 u16 finish;
81 u16 flags;
82};
83
84struct device_state **state_table;
85static spinlock_t state_lock;
86
87/* List and lock for all pasid_states */
88static LIST_HEAD(pasid_state_list);
89static DEFINE_SPINLOCK(ps_lock);
90
91static struct workqueue_struct *iommu_wq;
92
93/*
94 * Empty page table - Used between
95 * mmu_notifier_invalidate_range_start and
96 * mmu_notifier_invalidate_range_end
97 */
98static u64 *empty_page_table;
99
100static void free_pasid_states(struct device_state *dev_state);
101static void unbind_pasid(struct device_state *dev_state, int pasid);
102static int task_exit(struct notifier_block *nb, unsigned long e, void *data);
103
104static u16 device_id(struct pci_dev *pdev)
105{
106 u16 devid;
107
108 devid = pdev->bus->number;
109 devid = (devid << 8) | pdev->devfn;
110
111 return devid;
112}
113
114static struct device_state *get_device_state(u16 devid)
115{
116 struct device_state *dev_state;
117 unsigned long flags;
118
119 spin_lock_irqsave(&state_lock, flags);
120 dev_state = state_table[devid];
121 if (dev_state != NULL)
122 atomic_inc(&dev_state->count);
123 spin_unlock_irqrestore(&state_lock, flags);
124
125 return dev_state;
126}
127
128static void free_device_state(struct device_state *dev_state)
129{
130 /*
131 * First detach device from domain - No more PRI requests will arrive
132 * from that device after it is unbound from the IOMMUv2 domain.
133 */
134 iommu_detach_device(dev_state->domain, &dev_state->pdev->dev);
135
136 /* Everything is down now, free the IOMMUv2 domain */
137 iommu_domain_free(dev_state->domain);
138
139 /* Finally get rid of the device-state */
140 kfree(dev_state);
141}
142
143static void put_device_state(struct device_state *dev_state)
144{
145 if (atomic_dec_and_test(&dev_state->count))
146 wake_up(&dev_state->wq);
147}
148
149static void put_device_state_wait(struct device_state *dev_state)
150{
151 DEFINE_WAIT(wait);
152
153 prepare_to_wait(&dev_state->wq, &wait, TASK_UNINTERRUPTIBLE);
154 if (!atomic_dec_and_test(&dev_state->count))
155 schedule();
156 finish_wait(&dev_state->wq, &wait);
157
158 free_device_state(dev_state);
159}
160
161static struct notifier_block profile_nb = {
162 .notifier_call = task_exit,
163};
164
165static void link_pasid_state(struct pasid_state *pasid_state)
166{
167 spin_lock(&ps_lock);
168 list_add_tail(&pasid_state->list, &pasid_state_list);
169 spin_unlock(&ps_lock);
170}
171
172static void __unlink_pasid_state(struct pasid_state *pasid_state)
173{
174 list_del(&pasid_state->list);
175}
176
177static void unlink_pasid_state(struct pasid_state *pasid_state)
178{
179 spin_lock(&ps_lock);
180 __unlink_pasid_state(pasid_state);
181 spin_unlock(&ps_lock);
182}
183
184/* Must be called under dev_state->lock */
185static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
186 int pasid, bool alloc)
187{
188 struct pasid_state **root, **ptr;
189 int level, index;
190
191 level = dev_state->pasid_levels;
192 root = dev_state->states;
193
194 while (true) {
195
196 index = (pasid >> (9 * level)) & 0x1ff;
197 ptr = &root[index];
198
199 if (level == 0)
200 break;
201
202 if (*ptr == NULL) {
203 if (!alloc)
204 return NULL;
205
206 *ptr = (void *)get_zeroed_page(GFP_ATOMIC);
207 if (*ptr == NULL)
208 return NULL;
209 }
210
211 root = (struct pasid_state **)*ptr;
212 level -= 1;
213 }
214
215 return ptr;
216}
217
218static int set_pasid_state(struct device_state *dev_state,
219 struct pasid_state *pasid_state,
220 int pasid)
221{
222 struct pasid_state **ptr;
223 unsigned long flags;
224 int ret;
225
226 spin_lock_irqsave(&dev_state->lock, flags);
227 ptr = __get_pasid_state_ptr(dev_state, pasid, true);
228
229 ret = -ENOMEM;
230 if (ptr == NULL)
231 goto out_unlock;
232
233 ret = -ENOMEM;
234 if (*ptr != NULL)
235 goto out_unlock;
236
237 *ptr = pasid_state;
238
239 ret = 0;
240
241out_unlock:
242 spin_unlock_irqrestore(&dev_state->lock, flags);
243
244 return ret;
245}
246
247static void clear_pasid_state(struct device_state *dev_state, int pasid)
248{
249 struct pasid_state **ptr;
250 unsigned long flags;
251
252 spin_lock_irqsave(&dev_state->lock, flags);
253 ptr = __get_pasid_state_ptr(dev_state, pasid, true);
254
255 if (ptr == NULL)
256 goto out_unlock;
257
258 *ptr = NULL;
259
260out_unlock:
261 spin_unlock_irqrestore(&dev_state->lock, flags);
262}
263
264static struct pasid_state *get_pasid_state(struct device_state *dev_state,
265 int pasid)
266{
267 struct pasid_state **ptr, *ret = NULL;
268 unsigned long flags;
269
270 spin_lock_irqsave(&dev_state->lock, flags);
271 ptr = __get_pasid_state_ptr(dev_state, pasid, false);
272
273 if (ptr == NULL)
274 goto out_unlock;
275
276 ret = *ptr;
277 if (ret)
278 atomic_inc(&ret->count);
279
280out_unlock:
281 spin_unlock_irqrestore(&dev_state->lock, flags);
282
283 return ret;
284}
285
286static void free_pasid_state(struct pasid_state *pasid_state)
287{
288 kfree(pasid_state);
289}
290
291static void put_pasid_state(struct pasid_state *pasid_state)
292{
293 if (atomic_dec_and_test(&pasid_state->count)) {
294 put_device_state(pasid_state->device_state);
295 wake_up(&pasid_state->wq);
296 }
297}
298
299static void put_pasid_state_wait(struct pasid_state *pasid_state)
300{
301 DEFINE_WAIT(wait);
302
303 prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE);
304
305 if (atomic_dec_and_test(&pasid_state->count))
306 put_device_state(pasid_state->device_state);
307 else
308 schedule();
309
310 finish_wait(&pasid_state->wq, &wait);
311 mmput(pasid_state->mm);
312 free_pasid_state(pasid_state);
313}
314
315static void __unbind_pasid(struct pasid_state *pasid_state)
316{
317 struct iommu_domain *domain;
318
319 domain = pasid_state->device_state->domain;
320
321 amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);
322 clear_pasid_state(pasid_state->device_state, pasid_state->pasid);
323
324 /* Make sure no more pending faults are in the queue */
325 flush_workqueue(iommu_wq);
326
327 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
328
329 put_pasid_state(pasid_state); /* Reference taken in bind() function */
330}
331
332static void unbind_pasid(struct device_state *dev_state, int pasid)
333{
334 struct pasid_state *pasid_state;
335
336 pasid_state = get_pasid_state(dev_state, pasid);
337 if (pasid_state == NULL)
338 return;
339
340 unlink_pasid_state(pasid_state);
341 __unbind_pasid(pasid_state);
342 put_pasid_state_wait(pasid_state); /* Reference taken in this function */
343}
344
345static void free_pasid_states_level1(struct pasid_state **tbl)
346{
347 int i;
348
349 for (i = 0; i < 512; ++i) {
350 if (tbl[i] == NULL)
351 continue;
352
353 free_page((unsigned long)tbl[i]);
354 }
355}
356
357static void free_pasid_states_level2(struct pasid_state **tbl)
358{
359 struct pasid_state **ptr;
360 int i;
361
362 for (i = 0; i < 512; ++i) {
363 if (tbl[i] == NULL)
364 continue;
365
366 ptr = (struct pasid_state **)tbl[i];
367 free_pasid_states_level1(ptr);
368 }
369}
370
371static void free_pasid_states(struct device_state *dev_state)
372{
373 struct pasid_state *pasid_state;
374 int i;
375
376 for (i = 0; i < dev_state->max_pasids; ++i) {
377 pasid_state = get_pasid_state(dev_state, i);
378 if (pasid_state == NULL)
379 continue;
380
381 put_pasid_state(pasid_state);
382 unbind_pasid(dev_state, i);
383 }
384
385 if (dev_state->pasid_levels == 2)
386 free_pasid_states_level2(dev_state->states);
387 else if (dev_state->pasid_levels == 1)
388 free_pasid_states_level1(dev_state->states);
389 else if (dev_state->pasid_levels != 0)
390 BUG();
391
392 free_page((unsigned long)dev_state->states);
393}
394
395static struct pasid_state *mn_to_state(struct mmu_notifier *mn)
396{
397 return container_of(mn, struct pasid_state, mn);
398}
399
400static void __mn_flush_page(struct mmu_notifier *mn,
401 unsigned long address)
402{
403 struct pasid_state *pasid_state;
404 struct device_state *dev_state;
405
406 pasid_state = mn_to_state(mn);
407 dev_state = pasid_state->device_state;
408
409 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);
410}
411
412static int mn_clear_flush_young(struct mmu_notifier *mn,
413 struct mm_struct *mm,
414 unsigned long address)
415{
416 __mn_flush_page(mn, address);
417
418 return 0;
419}
420
421static void mn_change_pte(struct mmu_notifier *mn,
422 struct mm_struct *mm,
423 unsigned long address,
424 pte_t pte)
425{
426 __mn_flush_page(mn, address);
427}
428
429static void mn_invalidate_page(struct mmu_notifier *mn,
430 struct mm_struct *mm,
431 unsigned long address)
432{
433 __mn_flush_page(mn, address);
434}
435
436static void mn_invalidate_range_start(struct mmu_notifier *mn,
437 struct mm_struct *mm,
438 unsigned long start, unsigned long end)
439{
440 struct pasid_state *pasid_state;
441 struct device_state *dev_state;
442
443 pasid_state = mn_to_state(mn);
444 dev_state = pasid_state->device_state;
445
446 amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
447 __pa(empty_page_table));
448}
449
450static void mn_invalidate_range_end(struct mmu_notifier *mn,
451 struct mm_struct *mm,
452 unsigned long start, unsigned long end)
453{
454 struct pasid_state *pasid_state;
455 struct device_state *dev_state;
456
457 pasid_state = mn_to_state(mn);
458 dev_state = pasid_state->device_state;
459
460 amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid,
461 __pa(pasid_state->mm->pgd));
462}
463
464static struct mmu_notifier_ops iommu_mn = {
465 .clear_flush_young = mn_clear_flush_young,
466 .change_pte = mn_change_pte,
467 .invalidate_page = mn_invalidate_page,
468 .invalidate_range_start = mn_invalidate_range_start,
469 .invalidate_range_end = mn_invalidate_range_end,
470};
471
472static void set_pri_tag_status(struct pasid_state *pasid_state,
473 u16 tag, int status)
474{
475 unsigned long flags;
476
477 spin_lock_irqsave(&pasid_state->lock, flags);
478 pasid_state->pri[tag].status = status;
479 spin_unlock_irqrestore(&pasid_state->lock, flags);
480}
481
482static void finish_pri_tag(struct device_state *dev_state,
483 struct pasid_state *pasid_state,
484 u16 tag)
485{
486 unsigned long flags;
487
488 spin_lock_irqsave(&pasid_state->lock, flags);
489 if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&
490 pasid_state->pri[tag].finish) {
491 amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,
492 pasid_state->pri[tag].status, tag);
493 pasid_state->pri[tag].finish = false;
494 pasid_state->pri[tag].status = PPR_SUCCESS;
495 }
496 spin_unlock_irqrestore(&pasid_state->lock, flags);
497}
498
499static void do_fault(struct work_struct *work)
500{
501 struct fault *fault = container_of(work, struct fault, work);
502 int npages, write;
503 struct page *page;
504
505 write = !!(fault->flags & PPR_FAULT_WRITE);
506
507 npages = get_user_pages(fault->state->task, fault->state->mm,
508 fault->address, 1, write, 0, &page, NULL);
509
510 if (npages == 1) {
511 put_page(page);
512 } else if (fault->dev_state->inv_ppr_cb) {
513 int status;
514
515 status = fault->dev_state->inv_ppr_cb(fault->dev_state->pdev,
516 fault->pasid,
517 fault->address,
518 fault->flags);
519 switch (status) {
520 case AMD_IOMMU_INV_PRI_RSP_SUCCESS:
521 set_pri_tag_status(fault->state, fault->tag, PPR_SUCCESS);
522 break;
523 case AMD_IOMMU_INV_PRI_RSP_INVALID:
524 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
525 break;
526 case AMD_IOMMU_INV_PRI_RSP_FAIL:
527 set_pri_tag_status(fault->state, fault->tag, PPR_FAILURE);
528 break;
529 default:
530 BUG();
531 }
532 } else {
533 set_pri_tag_status(fault->state, fault->tag, PPR_INVALID);
534 }
535
536 finish_pri_tag(fault->dev_state, fault->state, fault->tag);
537
538 put_pasid_state(fault->state);
539
540 kfree(fault);
541}
542
543static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
544{
545 struct amd_iommu_fault *iommu_fault;
546 struct pasid_state *pasid_state;
547 struct device_state *dev_state;
548 unsigned long flags;
549 struct fault *fault;
550 bool finish;
551 u16 tag;
552 int ret;
553
554 iommu_fault = data;
555 tag = iommu_fault->tag & 0x1ff;
556 finish = (iommu_fault->tag >> 9) & 1;
557
558 ret = NOTIFY_DONE;
559 dev_state = get_device_state(iommu_fault->device_id);
560 if (dev_state == NULL)
561 goto out;
562
563 pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);
564 if (pasid_state == NULL) {
565 /* We know the device but not the PASID -> send INVALID */
566 amd_iommu_complete_ppr(dev_state->pdev, iommu_fault->pasid,
567 PPR_INVALID, tag);
568 goto out_drop_state;
569 }
570
571 spin_lock_irqsave(&pasid_state->lock, flags);
572 atomic_inc(&pasid_state->pri[tag].inflight);
573 if (finish)
574 pasid_state->pri[tag].finish = true;
575 spin_unlock_irqrestore(&pasid_state->lock, flags);
576
577 fault = kzalloc(sizeof(*fault), GFP_ATOMIC);
578 if (fault == NULL) {
579 /* We are OOM - send success and let the device re-fault */
580 finish_pri_tag(dev_state, pasid_state, tag);
581 goto out_drop_state;
582 }
583
584 fault->dev_state = dev_state;
585 fault->address = iommu_fault->address;
586 fault->state = pasid_state;
587 fault->tag = tag;
588 fault->finish = finish;
589 fault->flags = iommu_fault->flags;
590 INIT_WORK(&fault->work, do_fault);
591
592 queue_work(iommu_wq, &fault->work);
593
594 ret = NOTIFY_OK;
595
596out_drop_state:
597 put_device_state(dev_state);
598
599out:
600 return ret;
601}
602
603static struct notifier_block ppr_nb = {
604 .notifier_call = ppr_notifier,
605};
606
607static int task_exit(struct notifier_block *nb, unsigned long e, void *data)
608{
609 struct pasid_state *pasid_state;
610 struct task_struct *task;
611
612 task = data;
613
614 /*
615 * Using this notifier is a hack - but there is no other choice
616 * at the moment. What I really want is a sleeping notifier that
617 * is called when an MM goes down. But such a notifier doesn't
618 * exist yet. The notifier needs to sleep because it has to make
619 * sure that the device does not use the PASID and the address
620 * space anymore before it is destroyed. This includes waiting
621 * for pending PRI requests to pass the workqueue. The
622 * MMU-Notifiers would be a good fit, but they use RCU and so
623 * they are not allowed to sleep. Lets see how we can solve this
624 * in a more intelligent way in the future.
625 */
626again:
627 spin_lock(&ps_lock);
628 list_for_each_entry(pasid_state, &pasid_state_list, list) {
629 struct device_state *dev_state;
630 int pasid;
631
632 if (pasid_state->task != task)
633 continue;
634
635 /* Drop Lock and unbind */
636 spin_unlock(&ps_lock);
637
638 dev_state = pasid_state->device_state;
639 pasid = pasid_state->pasid;
640
641 if (pasid_state->device_state->inv_ctx_cb)
642 dev_state->inv_ctx_cb(dev_state->pdev, pasid);
643
644 unbind_pasid(dev_state, pasid);
645
646 /* Task may be in the list multiple times */
647 goto again;
648 }
649 spin_unlock(&ps_lock);
650
651 return NOTIFY_OK;
652}
653
654int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
655 struct task_struct *task)
656{
657 struct pasid_state *pasid_state;
658 struct device_state *dev_state;
659 u16 devid;
660 int ret;
661
662 might_sleep();
663
664 if (!amd_iommu_v2_supported())
665 return -ENODEV;
666
667 devid = device_id(pdev);
668 dev_state = get_device_state(devid);
669
670 if (dev_state == NULL)
671 return -EINVAL;
672
673 ret = -EINVAL;
674 if (pasid < 0 || pasid >= dev_state->max_pasids)
675 goto out;
676
677 ret = -ENOMEM;
678 pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);
679 if (pasid_state == NULL)
680 goto out;
681
682 atomic_set(&pasid_state->count, 1);
683 init_waitqueue_head(&pasid_state->wq);
684 pasid_state->task = task;
685 pasid_state->mm = get_task_mm(task);
686 pasid_state->device_state = dev_state;
687 pasid_state->pasid = pasid;
688 pasid_state->mn.ops = &iommu_mn;
689
690 if (pasid_state->mm == NULL)
691 goto out_free;
692
693 mmu_notifier_register(&pasid_state->mn, pasid_state->mm);
694
695 ret = set_pasid_state(dev_state, pasid_state, pasid);
696 if (ret)
697 goto out_unregister;
698
699 ret = amd_iommu_domain_set_gcr3(dev_state->domain, pasid,
700 __pa(pasid_state->mm->pgd));
701 if (ret)
702 goto out_clear_state;
703
704 link_pasid_state(pasid_state);
705
706 return 0;
707
708out_clear_state:
709 clear_pasid_state(dev_state, pasid);
710
711out_unregister:
712 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
713
714out_free:
715 free_pasid_state(pasid_state);
716
717out:
718 put_device_state(dev_state);
719
720 return ret;
721}
722EXPORT_SYMBOL(amd_iommu_bind_pasid);
723
724void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
725{
726 struct device_state *dev_state;
727 u16 devid;
728
729 might_sleep();
730
731 if (!amd_iommu_v2_supported())
732 return;
733
734 devid = device_id(pdev);
735 dev_state = get_device_state(devid);
736 if (dev_state == NULL)
737 return;
738
739 if (pasid < 0 || pasid >= dev_state->max_pasids)
740 goto out;
741
742 unbind_pasid(dev_state, pasid);
743
744out:
745 put_device_state(dev_state);
746}
747EXPORT_SYMBOL(amd_iommu_unbind_pasid);
748
749int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
750{
751 struct device_state *dev_state;
752 unsigned long flags;
753 int ret, tmp;
754 u16 devid;
755
756 might_sleep();
757
758 if (!amd_iommu_v2_supported())
759 return -ENODEV;
760
761 if (pasids <= 0 || pasids > (PASID_MASK + 1))
762 return -EINVAL;
763
764 devid = device_id(pdev);
765
766 dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL);
767 if (dev_state == NULL)
768 return -ENOMEM;
769
770 spin_lock_init(&dev_state->lock);
771 init_waitqueue_head(&dev_state->wq);
772 dev_state->pdev = pdev;
773
774 tmp = pasids;
775 for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
776 dev_state->pasid_levels += 1;
777
778 atomic_set(&dev_state->count, 1);
779 dev_state->max_pasids = pasids;
780
781 ret = -ENOMEM;
782 dev_state->states = (void *)get_zeroed_page(GFP_KERNEL);
783 if (dev_state->states == NULL)
784 goto out_free_dev_state;
785
786 dev_state->domain = iommu_domain_alloc(&pci_bus_type);
787 if (dev_state->domain == NULL)
788 goto out_free_states;
789
790 amd_iommu_domain_direct_map(dev_state->domain);
791
792 ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids);
793 if (ret)
794 goto out_free_domain;
795
796 ret = iommu_attach_device(dev_state->domain, &pdev->dev);
797 if (ret != 0)
798 goto out_free_domain;
799
800 spin_lock_irqsave(&state_lock, flags);
801
802 if (state_table[devid] != NULL) {
803 spin_unlock_irqrestore(&state_lock, flags);
804 ret = -EBUSY;
805 goto out_free_domain;
806 }
807
808 state_table[devid] = dev_state;
809
810 spin_unlock_irqrestore(&state_lock, flags);
811
812 return 0;
813
814out_free_domain:
815 iommu_domain_free(dev_state->domain);
816
817out_free_states:
818 free_page((unsigned long)dev_state->states);
819
820out_free_dev_state:
821 kfree(dev_state);
822
823 return ret;
824}
825EXPORT_SYMBOL(amd_iommu_init_device);
826
827void amd_iommu_free_device(struct pci_dev *pdev)
828{
829 struct device_state *dev_state;
830 unsigned long flags;
831 u16 devid;
832
833 if (!amd_iommu_v2_supported())
834 return;
835
836 devid = device_id(pdev);
837
838 spin_lock_irqsave(&state_lock, flags);
839
840 dev_state = state_table[devid];
841 if (dev_state == NULL) {
842 spin_unlock_irqrestore(&state_lock, flags);
843 return;
844 }
845
846 state_table[devid] = NULL;
847
848 spin_unlock_irqrestore(&state_lock, flags);
849
850 /* Get rid of any remaining pasid states */
851 free_pasid_states(dev_state);
852
853 put_device_state_wait(dev_state);
854}
855EXPORT_SYMBOL(amd_iommu_free_device);
856
857int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
858 amd_iommu_invalid_ppr_cb cb)
859{
860 struct device_state *dev_state;
861 unsigned long flags;
862 u16 devid;
863 int ret;
864
865 if (!amd_iommu_v2_supported())
866 return -ENODEV;
867
868 devid = device_id(pdev);
869
870 spin_lock_irqsave(&state_lock, flags);
871
872 ret = -EINVAL;
873 dev_state = state_table[devid];
874 if (dev_state == NULL)
875 goto out_unlock;
876
877 dev_state->inv_ppr_cb = cb;
878
879 ret = 0;
880
881out_unlock:
882 spin_unlock_irqrestore(&state_lock, flags);
883
884 return ret;
885}
886EXPORT_SYMBOL(amd_iommu_set_invalid_ppr_cb);
887
888int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
889 amd_iommu_invalidate_ctx cb)
890{
891 struct device_state *dev_state;
892 unsigned long flags;
893 u16 devid;
894 int ret;
895
896 if (!amd_iommu_v2_supported())
897 return -ENODEV;
898
899 devid = device_id(pdev);
900
901 spin_lock_irqsave(&state_lock, flags);
902
903 ret = -EINVAL;
904 dev_state = state_table[devid];
905 if (dev_state == NULL)
906 goto out_unlock;
907
908 dev_state->inv_ctx_cb = cb;
909
910 ret = 0;
911
912out_unlock:
913 spin_unlock_irqrestore(&state_lock, flags);
914
915 return ret;
916}
917EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
918
919static int __init amd_iommu_v2_init(void)
920{
921 size_t state_table_size;
922 int ret;
923
924 pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>");
925
926 spin_lock_init(&state_lock);
927
928 state_table_size = MAX_DEVICES * sizeof(struct device_state *);
929 state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
930 get_order(state_table_size));
931 if (state_table == NULL)
932 return -ENOMEM;
933
934 ret = -ENOMEM;
935 iommu_wq = create_workqueue("amd_iommu_v2");
936 if (iommu_wq == NULL)
937 goto out_free;
938
939 ret = -ENOMEM;
940 empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
941 if (empty_page_table == NULL)
942 goto out_destroy_wq;
943
944 amd_iommu_register_ppr_notifier(&ppr_nb);
945 profile_event_register(PROFILE_TASK_EXIT, &profile_nb);
946
947 return 0;
948
949out_destroy_wq:
950 destroy_workqueue(iommu_wq);
951
952out_free:
953 free_pages((unsigned long)state_table, get_order(state_table_size));
954
955 return ret;
956}
957
958static void __exit amd_iommu_v2_exit(void)
959{
960 struct device_state *dev_state;
961 size_t state_table_size;
962 int i;
963
964 profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb);
965 amd_iommu_unregister_ppr_notifier(&ppr_nb);
966
967 flush_workqueue(iommu_wq);
968
969 /*
970 * The loop below might call flush_workqueue(), so call
971 * destroy_workqueue() after it
972 */
973 for (i = 0; i < MAX_DEVICES; ++i) {
974 dev_state = get_device_state(i);
975
976 if (dev_state == NULL)
977 continue;
978
979 WARN_ON_ONCE(1);
980
981 put_device_state(dev_state);
982 amd_iommu_free_device(dev_state->pdev);
983 }
984
985 destroy_workqueue(iommu_wq);
986
987 state_table_size = MAX_DEVICES * sizeof(struct device_state *);
988 free_pages((unsigned long)state_table, get_order(state_table_size));
989
990 free_page((unsigned long)empty_page_table);
991}
992
993module_init(amd_iommu_v2_init);
994module_exit(amd_iommu_v2_exit);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index bdc447fd4766..aea72e11e166 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -78,6 +78,24 @@
78#define LEVEL_STRIDE (9) 78#define LEVEL_STRIDE (9)
79#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1) 79#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
80 80
81/*
82 * This bitmap is used to advertise the page sizes our hardware support
83 * to the IOMMU core, which will then use this information to split
84 * physically contiguous memory regions it is mapping into page sizes
85 * that we support.
86 *
87 * Traditionally the IOMMU core just handed us the mappings directly,
88 * after making sure the size is an order of a 4KiB page and that the
89 * mapping has natural alignment.
90 *
91 * To retain this behavior, we currently advertise that we support
92 * all page sizes that are an order of 4KiB.
93 *
94 * If at some point we'd like to utilize the IOMMU core's new behavior,
95 * we could change this to advertise the real page sizes we support.
96 */
97#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
98
81static inline int agaw_to_level(int agaw) 99static inline int agaw_to_level(int agaw)
82{ 100{
83 return agaw + 2; 101 return agaw + 2;
@@ -3984,12 +4002,11 @@ static void intel_iommu_detach_device(struct iommu_domain *domain,
3984 4002
3985static int intel_iommu_map(struct iommu_domain *domain, 4003static int intel_iommu_map(struct iommu_domain *domain,
3986 unsigned long iova, phys_addr_t hpa, 4004 unsigned long iova, phys_addr_t hpa,
3987 int gfp_order, int iommu_prot) 4005 size_t size, int iommu_prot)
3988{ 4006{
3989 struct dmar_domain *dmar_domain = domain->priv; 4007 struct dmar_domain *dmar_domain = domain->priv;
3990 u64 max_addr; 4008 u64 max_addr;
3991 int prot = 0; 4009 int prot = 0;
3992 size_t size;
3993 int ret; 4010 int ret;
3994 4011
3995 if (iommu_prot & IOMMU_READ) 4012 if (iommu_prot & IOMMU_READ)
@@ -3999,7 +4016,6 @@ static int intel_iommu_map(struct iommu_domain *domain,
3999 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping) 4016 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4000 prot |= DMA_PTE_SNP; 4017 prot |= DMA_PTE_SNP;
4001 4018
4002 size = PAGE_SIZE << gfp_order;
4003 max_addr = iova + size; 4019 max_addr = iova + size;
4004 if (dmar_domain->max_addr < max_addr) { 4020 if (dmar_domain->max_addr < max_addr) {
4005 u64 end; 4021 u64 end;
@@ -4022,11 +4038,10 @@ static int intel_iommu_map(struct iommu_domain *domain,
4022 return ret; 4038 return ret;
4023} 4039}
4024 4040
4025static int intel_iommu_unmap(struct iommu_domain *domain, 4041static size_t intel_iommu_unmap(struct iommu_domain *domain,
4026 unsigned long iova, int gfp_order) 4042 unsigned long iova, size_t size)
4027{ 4043{
4028 struct dmar_domain *dmar_domain = domain->priv; 4044 struct dmar_domain *dmar_domain = domain->priv;
4029 size_t size = PAGE_SIZE << gfp_order;
4030 int order; 4045 int order;
4031 4046
4032 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT, 4047 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
@@ -4035,7 +4050,7 @@ static int intel_iommu_unmap(struct iommu_domain *domain,
4035 if (dmar_domain->max_addr == iova + size) 4050 if (dmar_domain->max_addr == iova + size)
4036 dmar_domain->max_addr = iova; 4051 dmar_domain->max_addr = iova;
4037 4052
4038 return order; 4053 return PAGE_SIZE << order;
4039} 4054}
4040 4055
4041static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, 4056static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
@@ -4074,6 +4089,7 @@ static struct iommu_ops intel_iommu_ops = {
4074 .unmap = intel_iommu_unmap, 4089 .unmap = intel_iommu_unmap,
4075 .iova_to_phys = intel_iommu_iova_to_phys, 4090 .iova_to_phys = intel_iommu_iova_to_phys,
4076 .domain_has_cap = intel_iommu_domain_has_cap, 4091 .domain_has_cap = intel_iommu_domain_has_cap,
4092 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4077}; 4093};
4078 4094
4079static void __devinit quirk_iommu_rwbf(struct pci_dev *dev) 4095static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 5b5fa5cdaa31..14d6f908196b 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -16,6 +16,8 @@
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 */ 17 */
18 18
19#define pr_fmt(fmt) "%s: " fmt, __func__
20
19#include <linux/device.h> 21#include <linux/device.h>
20#include <linux/kernel.h> 22#include <linux/kernel.h>
21#include <linux/bug.h> 23#include <linux/bug.h>
@@ -157,32 +159,125 @@ int iommu_domain_has_cap(struct iommu_domain *domain,
157EXPORT_SYMBOL_GPL(iommu_domain_has_cap); 159EXPORT_SYMBOL_GPL(iommu_domain_has_cap);
158 160
159int iommu_map(struct iommu_domain *domain, unsigned long iova, 161int iommu_map(struct iommu_domain *domain, unsigned long iova,
160 phys_addr_t paddr, int gfp_order, int prot) 162 phys_addr_t paddr, size_t size, int prot)
161{ 163{
162 size_t size; 164 unsigned long orig_iova = iova;
165 unsigned int min_pagesz;
166 size_t orig_size = size;
167 int ret = 0;
163 168
164 if (unlikely(domain->ops->map == NULL)) 169 if (unlikely(domain->ops->map == NULL))
165 return -ENODEV; 170 return -ENODEV;
166 171
167 size = PAGE_SIZE << gfp_order; 172 /* find out the minimum page size supported */
173 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
174
175 /*
176 * both the virtual address and the physical one, as well as
177 * the size of the mapping, must be aligned (at least) to the
178 * size of the smallest page supported by the hardware
179 */
180 if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
181 pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz "
182 "0x%x\n", iova, (unsigned long)paddr,
183 (unsigned long)size, min_pagesz);
184 return -EINVAL;
185 }
186
187 pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova,
188 (unsigned long)paddr, (unsigned long)size);
189
190 while (size) {
191 unsigned long pgsize, addr_merge = iova | paddr;
192 unsigned int pgsize_idx;
193
194 /* Max page size that still fits into 'size' */
195 pgsize_idx = __fls(size);
196
197 /* need to consider alignment requirements ? */
198 if (likely(addr_merge)) {
199 /* Max page size allowed by both iova and paddr */
200 unsigned int align_pgsize_idx = __ffs(addr_merge);
201
202 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
203 }
204
205 /* build a mask of acceptable page sizes */
206 pgsize = (1UL << (pgsize_idx + 1)) - 1;
207
208 /* throw away page sizes not supported by the hardware */
209 pgsize &= domain->ops->pgsize_bitmap;
168 210
169 BUG_ON(!IS_ALIGNED(iova | paddr, size)); 211 /* make sure we're still sane */
212 BUG_ON(!pgsize);
170 213
171 return domain->ops->map(domain, iova, paddr, gfp_order, prot); 214 /* pick the biggest page */
215 pgsize_idx = __fls(pgsize);
216 pgsize = 1UL << pgsize_idx;
217
218 pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova,
219 (unsigned long)paddr, pgsize);
220
221 ret = domain->ops->map(domain, iova, paddr, pgsize, prot);
222 if (ret)
223 break;
224
225 iova += pgsize;
226 paddr += pgsize;
227 size -= pgsize;
228 }
229
230 /* unroll mapping in case something went wrong */
231 if (ret)
232 iommu_unmap(domain, orig_iova, orig_size - size);
233
234 return ret;
172} 235}
173EXPORT_SYMBOL_GPL(iommu_map); 236EXPORT_SYMBOL_GPL(iommu_map);
174 237
175int iommu_unmap(struct iommu_domain *domain, unsigned long iova, int gfp_order) 238size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
176{ 239{
177 size_t size; 240 size_t unmapped_page, unmapped = 0;
241 unsigned int min_pagesz;
178 242
179 if (unlikely(domain->ops->unmap == NULL)) 243 if (unlikely(domain->ops->unmap == NULL))
180 return -ENODEV; 244 return -ENODEV;
181 245
182 size = PAGE_SIZE << gfp_order; 246 /* find out the minimum page size supported */
183 247 min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap);
184 BUG_ON(!IS_ALIGNED(iova, size)); 248
185 249 /*
186 return domain->ops->unmap(domain, iova, gfp_order); 250 * The virtual address, as well as the size of the mapping, must be
251 * aligned (at least) to the size of the smallest page supported
252 * by the hardware
253 */
254 if (!IS_ALIGNED(iova | size, min_pagesz)) {
255 pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n",
256 iova, (unsigned long)size, min_pagesz);
257 return -EINVAL;
258 }
259
260 pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova,
261 (unsigned long)size);
262
263 /*
264 * Keep iterating until we either unmap 'size' bytes (or more)
265 * or we hit an area that isn't mapped.
266 */
267 while (unmapped < size) {
268 size_t left = size - unmapped;
269
270 unmapped_page = domain->ops->unmap(domain, iova, left);
271 if (!unmapped_page)
272 break;
273
274 pr_debug("unmapped: iova 0x%lx size %lx\n", iova,
275 (unsigned long)unmapped_page);
276
277 iova += unmapped_page;
278 unmapped += unmapped_page;
279 }
280
281 return unmapped;
187} 282}
188EXPORT_SYMBOL_GPL(iommu_unmap); 283EXPORT_SYMBOL_GPL(iommu_unmap);
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 5865dd2e28f9..08a90b88e40d 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -42,6 +42,9 @@ __asm__ __volatile__ ( \
42#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) 42#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
43#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) 43#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
44 44
45/* bitmap of the page sizes currently supported */
46#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
47
45static int msm_iommu_tex_class[4]; 48static int msm_iommu_tex_class[4];
46 49
47DEFINE_SPINLOCK(msm_iommu_lock); 50DEFINE_SPINLOCK(msm_iommu_lock);
@@ -352,7 +355,7 @@ fail:
352} 355}
353 356
354static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, 357static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
355 phys_addr_t pa, int order, int prot) 358 phys_addr_t pa, size_t len, int prot)
356{ 359{
357 struct msm_priv *priv; 360 struct msm_priv *priv;
358 unsigned long flags; 361 unsigned long flags;
@@ -363,7 +366,6 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long va,
363 unsigned long *sl_pte; 366 unsigned long *sl_pte;
364 unsigned long sl_offset; 367 unsigned long sl_offset;
365 unsigned int pgprot; 368 unsigned int pgprot;
366 size_t len = 0x1000UL << order;
367 int ret = 0, tex, sh; 369 int ret = 0, tex, sh;
368 370
369 spin_lock_irqsave(&msm_iommu_lock, flags); 371 spin_lock_irqsave(&msm_iommu_lock, flags);
@@ -463,8 +465,8 @@ fail:
463 return ret; 465 return ret;
464} 466}
465 467
466static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, 468static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
467 int order) 469 size_t len)
468{ 470{
469 struct msm_priv *priv; 471 struct msm_priv *priv;
470 unsigned long flags; 472 unsigned long flags;
@@ -474,7 +476,6 @@ static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
474 unsigned long *sl_table; 476 unsigned long *sl_table;
475 unsigned long *sl_pte; 477 unsigned long *sl_pte;
476 unsigned long sl_offset; 478 unsigned long sl_offset;
477 size_t len = 0x1000UL << order;
478 int i, ret = 0; 479 int i, ret = 0;
479 480
480 spin_lock_irqsave(&msm_iommu_lock, flags); 481 spin_lock_irqsave(&msm_iommu_lock, flags);
@@ -544,15 +545,12 @@ static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va,
544 545
545 ret = __flush_iotlb(domain); 546 ret = __flush_iotlb(domain);
546 547
547 /*
548 * the IOMMU API requires us to return the order of the unmapped
549 * page (on success).
550 */
551 if (!ret)
552 ret = order;
553fail: 548fail:
554 spin_unlock_irqrestore(&msm_iommu_lock, flags); 549 spin_unlock_irqrestore(&msm_iommu_lock, flags);
555 return ret; 550
551 /* the IOMMU API requires us to return how many bytes were unmapped */
552 len = ret ? 0 : len;
553 return len;
556} 554}
557 555
558static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, 556static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
@@ -684,7 +682,8 @@ static struct iommu_ops msm_iommu_ops = {
684 .map = msm_iommu_map, 682 .map = msm_iommu_map,
685 .unmap = msm_iommu_unmap, 683 .unmap = msm_iommu_unmap,
686 .iova_to_phys = msm_iommu_iova_to_phys, 684 .iova_to_phys = msm_iommu_iova_to_phys,
687 .domain_has_cap = msm_iommu_domain_has_cap 685 .domain_has_cap = msm_iommu_domain_has_cap,
686 .pgsize_bitmap = MSM_IOMMU_PGSIZES,
688}; 687};
689 688
690static int __init get_tex_class(int icp, int ocp, int mt, int nos) 689static int __init get_tex_class(int icp, int ocp, int mt, int nos)
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 8f32b2bf7587..d8edd979d01b 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -33,6 +33,9 @@
33 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ 33 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
34 __i++) 34 __i++)
35 35
36/* bitmap of the page sizes currently supported */
37#define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
38
36/** 39/**
37 * struct omap_iommu_domain - omap iommu domain 40 * struct omap_iommu_domain - omap iommu domain
38 * @pgtable: the page table 41 * @pgtable: the page table
@@ -86,20 +89,24 @@ EXPORT_SYMBOL_GPL(omap_uninstall_iommu_arch);
86 89
87/** 90/**
88 * omap_iommu_save_ctx - Save registers for pm off-mode support 91 * omap_iommu_save_ctx - Save registers for pm off-mode support
89 * @obj: target iommu 92 * @dev: client device
90 **/ 93 **/
91void omap_iommu_save_ctx(struct omap_iommu *obj) 94void omap_iommu_save_ctx(struct device *dev)
92{ 95{
96 struct omap_iommu *obj = dev_to_omap_iommu(dev);
97
93 arch_iommu->save_ctx(obj); 98 arch_iommu->save_ctx(obj);
94} 99}
95EXPORT_SYMBOL_GPL(omap_iommu_save_ctx); 100EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
96 101
97/** 102/**
98 * omap_iommu_restore_ctx - Restore registers for pm off-mode support 103 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
99 * @obj: target iommu 104 * @dev: client device
100 **/ 105 **/
101void omap_iommu_restore_ctx(struct omap_iommu *obj) 106void omap_iommu_restore_ctx(struct device *dev)
102{ 107{
108 struct omap_iommu *obj = dev_to_omap_iommu(dev);
109
103 arch_iommu->restore_ctx(obj); 110 arch_iommu->restore_ctx(obj);
104} 111}
105EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx); 112EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
@@ -820,35 +827,23 @@ static int device_match_by_alias(struct device *dev, void *data)
820} 827}
821 828
822/** 829/**
823 * omap_find_iommu_device() - find an omap iommu device by name
824 * @name: name of the iommu device
825 *
826 * The generic iommu API requires the caller to provide the device
827 * he wishes to attach to a certain iommu domain.
828 *
829 * Drivers generally should not bother with this as it should just
830 * be taken care of by the DMA-API using dev_archdata.
831 *
832 * This function is provided as an interim solution until the latter
833 * materializes, and omap3isp is fully migrated to the DMA-API.
834 */
835struct device *omap_find_iommu_device(const char *name)
836{
837 return driver_find_device(&omap_iommu_driver.driver, NULL,
838 (void *)name,
839 device_match_by_alias);
840}
841EXPORT_SYMBOL_GPL(omap_find_iommu_device);
842
843/**
844 * omap_iommu_attach() - attach iommu device to an iommu domain 830 * omap_iommu_attach() - attach iommu device to an iommu domain
845 * @dev: target omap iommu device 831 * @name: name of target omap iommu device
846 * @iopgd: page table 832 * @iopgd: page table
847 **/ 833 **/
848static struct omap_iommu *omap_iommu_attach(struct device *dev, u32 *iopgd) 834static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
849{ 835{
850 int err = -ENOMEM; 836 int err = -ENOMEM;
851 struct omap_iommu *obj = to_iommu(dev); 837 struct device *dev;
838 struct omap_iommu *obj;
839
840 dev = driver_find_device(&omap_iommu_driver.driver, NULL,
841 (void *)name,
842 device_match_by_alias);
843 if (!dev)
844 return NULL;
845
846 obj = to_iommu(dev);
852 847
853 spin_lock(&obj->iommu_lock); 848 spin_lock(&obj->iommu_lock);
854 849
@@ -1019,12 +1014,11 @@ static void iopte_cachep_ctor(void *iopte)
1019} 1014}
1020 1015
1021static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, 1016static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1022 phys_addr_t pa, int order, int prot) 1017 phys_addr_t pa, size_t bytes, int prot)
1023{ 1018{
1024 struct omap_iommu_domain *omap_domain = domain->priv; 1019 struct omap_iommu_domain *omap_domain = domain->priv;
1025 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1020 struct omap_iommu *oiommu = omap_domain->iommu_dev;
1026 struct device *dev = oiommu->dev; 1021 struct device *dev = oiommu->dev;
1027 size_t bytes = PAGE_SIZE << order;
1028 struct iotlb_entry e; 1022 struct iotlb_entry e;
1029 int omap_pgsz; 1023 int omap_pgsz;
1030 u32 ret, flags; 1024 u32 ret, flags;
@@ -1049,19 +1043,16 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1049 return ret; 1043 return ret;
1050} 1044}
1051 1045
1052static int omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, 1046static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
1053 int order) 1047 size_t size)
1054{ 1048{
1055 struct omap_iommu_domain *omap_domain = domain->priv; 1049 struct omap_iommu_domain *omap_domain = domain->priv;
1056 struct omap_iommu *oiommu = omap_domain->iommu_dev; 1050 struct omap_iommu *oiommu = omap_domain->iommu_dev;
1057 struct device *dev = oiommu->dev; 1051 struct device *dev = oiommu->dev;
1058 size_t unmap_size;
1059
1060 dev_dbg(dev, "unmapping da 0x%lx order %d\n", da, order);
1061 1052
1062 unmap_size = iopgtable_clear_entry(oiommu, da); 1053 dev_dbg(dev, "unmapping da 0x%lx size %u\n", da, size);
1063 1054
1064 return unmap_size ? get_order(unmap_size) : -EINVAL; 1055 return iopgtable_clear_entry(oiommu, da);
1065} 1056}
1066 1057
1067static int 1058static int
@@ -1069,6 +1060,7 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1069{ 1060{
1070 struct omap_iommu_domain *omap_domain = domain->priv; 1061 struct omap_iommu_domain *omap_domain = domain->priv;
1071 struct omap_iommu *oiommu; 1062 struct omap_iommu *oiommu;
1063 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1072 int ret = 0; 1064 int ret = 0;
1073 1065
1074 spin_lock(&omap_domain->lock); 1066 spin_lock(&omap_domain->lock);
@@ -1081,14 +1073,14 @@ omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1081 } 1073 }
1082 1074
1083 /* get a handle to and enable the omap iommu */ 1075 /* get a handle to and enable the omap iommu */
1084 oiommu = omap_iommu_attach(dev, omap_domain->pgtable); 1076 oiommu = omap_iommu_attach(arch_data->name, omap_domain->pgtable);
1085 if (IS_ERR(oiommu)) { 1077 if (IS_ERR(oiommu)) {
1086 ret = PTR_ERR(oiommu); 1078 ret = PTR_ERR(oiommu);
1087 dev_err(dev, "can't get omap iommu: %d\n", ret); 1079 dev_err(dev, "can't get omap iommu: %d\n", ret);
1088 goto out; 1080 goto out;
1089 } 1081 }
1090 1082
1091 omap_domain->iommu_dev = oiommu; 1083 omap_domain->iommu_dev = arch_data->iommu_dev = oiommu;
1092 oiommu->domain = domain; 1084 oiommu->domain = domain;
1093 1085
1094out: 1086out:
@@ -1100,7 +1092,8 @@ static void omap_iommu_detach_dev(struct iommu_domain *domain,
1100 struct device *dev) 1092 struct device *dev)
1101{ 1093{
1102 struct omap_iommu_domain *omap_domain = domain->priv; 1094 struct omap_iommu_domain *omap_domain = domain->priv;
1103 struct omap_iommu *oiommu = to_iommu(dev); 1095 struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
1096 struct omap_iommu *oiommu = dev_to_omap_iommu(dev);
1104 1097
1105 spin_lock(&omap_domain->lock); 1098 spin_lock(&omap_domain->lock);
1106 1099
@@ -1114,7 +1107,7 @@ static void omap_iommu_detach_dev(struct iommu_domain *domain,
1114 1107
1115 omap_iommu_detach(oiommu); 1108 omap_iommu_detach(oiommu);
1116 1109
1117 omap_domain->iommu_dev = NULL; 1110 omap_domain->iommu_dev = arch_data->iommu_dev = NULL;
1118 1111
1119out: 1112out:
1120 spin_unlock(&omap_domain->lock); 1113 spin_unlock(&omap_domain->lock);
@@ -1183,14 +1176,14 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1183 else if (iopte_is_large(*pte)) 1176 else if (iopte_is_large(*pte))
1184 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); 1177 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1185 else 1178 else
1186 dev_err(dev, "bogus pte 0x%x", *pte); 1179 dev_err(dev, "bogus pte 0x%x, da 0x%lx", *pte, da);
1187 } else { 1180 } else {
1188 if (iopgd_is_section(*pgd)) 1181 if (iopgd_is_section(*pgd))
1189 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); 1182 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1190 else if (iopgd_is_super(*pgd)) 1183 else if (iopgd_is_super(*pgd))
1191 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); 1184 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1192 else 1185 else
1193 dev_err(dev, "bogus pgd 0x%x", *pgd); 1186 dev_err(dev, "bogus pgd 0x%x, da 0x%lx", *pgd, da);
1194 } 1187 }
1195 1188
1196 return ret; 1189 return ret;
@@ -1211,6 +1204,7 @@ static struct iommu_ops omap_iommu_ops = {
1211 .unmap = omap_iommu_unmap, 1204 .unmap = omap_iommu_unmap,
1212 .iova_to_phys = omap_iommu_iova_to_phys, 1205 .iova_to_phys = omap_iommu_iova_to_phys,
1213 .domain_has_cap = omap_iommu_domain_has_cap, 1206 .domain_has_cap = omap_iommu_domain_has_cap,
1207 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
1214}; 1208};
1215 1209
1216static int __init omap_iommu_init(void) 1210static int __init omap_iommu_init(void)
diff --git a/drivers/iommu/omap-iovmm.c b/drivers/iommu/omap-iovmm.c
index 46be456fcc00..2e10c3e0a7ae 100644
--- a/drivers/iommu/omap-iovmm.c
+++ b/drivers/iommu/omap-iovmm.c
@@ -231,12 +231,14 @@ static struct iovm_struct *__find_iovm_area(struct omap_iommu *obj,
231 231
232/** 232/**
233 * omap_find_iovm_area - find iovma which includes @da 233 * omap_find_iovm_area - find iovma which includes @da
234 * @dev: client device
234 * @da: iommu device virtual address 235 * @da: iommu device virtual address
235 * 236 *
236 * Find the existing iovma starting at @da 237 * Find the existing iovma starting at @da
237 */ 238 */
238struct iovm_struct *omap_find_iovm_area(struct omap_iommu *obj, u32 da) 239struct iovm_struct *omap_find_iovm_area(struct device *dev, u32 da)
239{ 240{
241 struct omap_iommu *obj = dev_to_omap_iommu(dev);
240 struct iovm_struct *area; 242 struct iovm_struct *area;
241 243
242 mutex_lock(&obj->mmap_lock); 244 mutex_lock(&obj->mmap_lock);
@@ -343,14 +345,15 @@ static void free_iovm_area(struct omap_iommu *obj, struct iovm_struct *area)
343 345
344/** 346/**
345 * omap_da_to_va - convert (d) to (v) 347 * omap_da_to_va - convert (d) to (v)
346 * @obj: objective iommu 348 * @dev: client device
347 * @da: iommu device virtual address 349 * @da: iommu device virtual address
348 * @va: mpu virtual address 350 * @va: mpu virtual address
349 * 351 *
350 * Returns mpu virtual addr which corresponds to a given device virtual addr 352 * Returns mpu virtual addr which corresponds to a given device virtual addr
351 */ 353 */
352void *omap_da_to_va(struct omap_iommu *obj, u32 da) 354void *omap_da_to_va(struct device *dev, u32 da)
353{ 355{
356 struct omap_iommu *obj = dev_to_omap_iommu(dev);
354 void *va = NULL; 357 void *va = NULL;
355 struct iovm_struct *area; 358 struct iovm_struct *area;
356 359
@@ -410,7 +413,6 @@ static int map_iovm_area(struct iommu_domain *domain, struct iovm_struct *new,
410 unsigned int i, j; 413 unsigned int i, j;
411 struct scatterlist *sg; 414 struct scatterlist *sg;
412 u32 da = new->da_start; 415 u32 da = new->da_start;
413 int order;
414 416
415 if (!domain || !sgt) 417 if (!domain || !sgt)
416 return -EINVAL; 418 return -EINVAL;
@@ -429,12 +431,10 @@ static int map_iovm_area(struct iommu_domain *domain, struct iovm_struct *new,
429 if (bytes_to_iopgsz(bytes) < 0) 431 if (bytes_to_iopgsz(bytes) < 0)
430 goto err_out; 432 goto err_out;
431 433
432 order = get_order(bytes);
433
434 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__, 434 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__,
435 i, da, pa, bytes); 435 i, da, pa, bytes);
436 436
437 err = iommu_map(domain, da, pa, order, flags); 437 err = iommu_map(domain, da, pa, bytes, flags);
438 if (err) 438 if (err)
439 goto err_out; 439 goto err_out;
440 440
@@ -449,10 +449,9 @@ err_out:
449 size_t bytes; 449 size_t bytes;
450 450
451 bytes = sg->length + sg->offset; 451 bytes = sg->length + sg->offset;
452 order = get_order(bytes);
453 452
454 /* ignore failures.. we're already handling one */ 453 /* ignore failures.. we're already handling one */
455 iommu_unmap(domain, da, order); 454 iommu_unmap(domain, da, bytes);
456 455
457 da += bytes; 456 da += bytes;
458 } 457 }
@@ -467,7 +466,8 @@ static void unmap_iovm_area(struct iommu_domain *domain, struct omap_iommu *obj,
467 size_t total = area->da_end - area->da_start; 466 size_t total = area->da_end - area->da_start;
468 const struct sg_table *sgt = area->sgt; 467 const struct sg_table *sgt = area->sgt;
469 struct scatterlist *sg; 468 struct scatterlist *sg;
470 int i, err; 469 int i;
470 size_t unmapped;
471 471
472 BUG_ON(!sgtable_ok(sgt)); 472 BUG_ON(!sgtable_ok(sgt));
473 BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE)); 473 BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE));
@@ -475,13 +475,11 @@ static void unmap_iovm_area(struct iommu_domain *domain, struct omap_iommu *obj,
475 start = area->da_start; 475 start = area->da_start;
476 for_each_sg(sgt->sgl, sg, sgt->nents, i) { 476 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
477 size_t bytes; 477 size_t bytes;
478 int order;
479 478
480 bytes = sg->length + sg->offset; 479 bytes = sg->length + sg->offset;
481 order = get_order(bytes);
482 480
483 err = iommu_unmap(domain, start, order); 481 unmapped = iommu_unmap(domain, start, bytes);
484 if (err < 0) 482 if (unmapped < bytes)
485 break; 483 break;
486 484
487 dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n", 485 dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n",
@@ -582,16 +580,18 @@ __iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj,
582 580
583/** 581/**
584 * omap_iommu_vmap - (d)-(p)-(v) address mapper 582 * omap_iommu_vmap - (d)-(p)-(v) address mapper
585 * @obj: objective iommu 583 * @domain: iommu domain
584 * @dev: client device
586 * @sgt: address of scatter gather table 585 * @sgt: address of scatter gather table
587 * @flags: iovma and page property 586 * @flags: iovma and page property
588 * 587 *
589 * Creates 1-n-1 mapping with given @sgt and returns @da. 588 * Creates 1-n-1 mapping with given @sgt and returns @da.
590 * All @sgt element must be io page size aligned. 589 * All @sgt element must be io page size aligned.
591 */ 590 */
592u32 omap_iommu_vmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, 591u32 omap_iommu_vmap(struct iommu_domain *domain, struct device *dev, u32 da,
593 const struct sg_table *sgt, u32 flags) 592 const struct sg_table *sgt, u32 flags)
594{ 593{
594 struct omap_iommu *obj = dev_to_omap_iommu(dev);
595 size_t bytes; 595 size_t bytes;
596 void *va = NULL; 596 void *va = NULL;
597 597
@@ -622,15 +622,17 @@ EXPORT_SYMBOL_GPL(omap_iommu_vmap);
622 622
623/** 623/**
624 * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()' 624 * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()'
625 * @obj: objective iommu 625 * @domain: iommu domain
626 * @dev: client device
626 * @da: iommu device virtual address 627 * @da: iommu device virtual address
627 * 628 *
628 * Free the iommu virtually contiguous memory area starting at 629 * Free the iommu virtually contiguous memory area starting at
629 * @da, which was returned by 'omap_iommu_vmap()'. 630 * @da, which was returned by 'omap_iommu_vmap()'.
630 */ 631 */
631struct sg_table * 632struct sg_table *
632omap_iommu_vunmap(struct iommu_domain *domain, struct omap_iommu *obj, u32 da) 633omap_iommu_vunmap(struct iommu_domain *domain, struct device *dev, u32 da)
633{ 634{
635 struct omap_iommu *obj = dev_to_omap_iommu(dev);
634 struct sg_table *sgt; 636 struct sg_table *sgt;
635 /* 637 /*
636 * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called. 638 * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called.
@@ -647,7 +649,7 @@ EXPORT_SYMBOL_GPL(omap_iommu_vunmap);
647 649
648/** 650/**
649 * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper 651 * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
650 * @obj: objective iommu 652 * @dev: client device
651 * @da: contiguous iommu virtual memory 653 * @da: contiguous iommu virtual memory
652 * @bytes: allocation size 654 * @bytes: allocation size
653 * @flags: iovma and page property 655 * @flags: iovma and page property
@@ -656,9 +658,10 @@ EXPORT_SYMBOL_GPL(omap_iommu_vunmap);
656 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set. 658 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set.
657 */ 659 */
658u32 660u32
659omap_iommu_vmalloc(struct iommu_domain *domain, struct omap_iommu *obj, u32 da, 661omap_iommu_vmalloc(struct iommu_domain *domain, struct device *dev, u32 da,
660 size_t bytes, u32 flags) 662 size_t bytes, u32 flags)
661{ 663{
664 struct omap_iommu *obj = dev_to_omap_iommu(dev);
662 void *va; 665 void *va;
663 struct sg_table *sgt; 666 struct sg_table *sgt;
664 667
@@ -698,15 +701,16 @@ EXPORT_SYMBOL_GPL(omap_iommu_vmalloc);
698 701
699/** 702/**
700 * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()' 703 * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()'
701 * @obj: objective iommu 704 * @dev: client device
702 * @da: iommu device virtual address 705 * @da: iommu device virtual address
703 * 706 *
704 * Frees the iommu virtually continuous memory area starting at 707 * Frees the iommu virtually continuous memory area starting at
705 * @da, as obtained from 'omap_iommu_vmalloc()'. 708 * @da, as obtained from 'omap_iommu_vmalloc()'.
706 */ 709 */
707void omap_iommu_vfree(struct iommu_domain *domain, struct omap_iommu *obj, 710void omap_iommu_vfree(struct iommu_domain *domain, struct device *dev,
708 const u32 da) 711 const u32 da)
709{ 712{
713 struct omap_iommu *obj = dev_to_omap_iommu(dev);
710 struct sg_table *sgt; 714 struct sg_table *sgt;
711 715
712 sgt = unmap_vm_area(domain, obj, da, vfree, 716 sgt = unmap_vm_area(domain, obj, da, vfree,
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-09 12:04:29 -0500