aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2016-08-01 07:25:10 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2016-08-01 07:25:10 -0400
commitdd9671172a06830071c8edb31fb2176f222a2c6e (patch)
treea0c343374f9586a7a2afec1a3a7be745d2939ae1
parent77d9ada23f207ec3d6258985c882f4fb653693f1 (diff)
parentf360d3241f5557f241d55b959e6e65070e77992e (diff)
Merge tag 'iommu-updates-v4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU updates from Joerg Roedel: - big-endian support and preparation for defered probing for the Exynos IOMMU driver - simplifications in iommu-group id handling - support for Mediatek generation one IOMMU hardware - conversion of the AMD IOMMU driver to use the generic IOVA allocator. This driver now also benefits from the recent scalability improvements in the IOVA code. - preparations to use generic DMA mapping code in the Rockchip IOMMU driver - device tree adaption and conversion to use generic page-table code for the MSM IOMMU driver - an iova_to_phys optimization in the ARM-SMMU driver to greatly improve page-table teardown performance with VFIO - various other small fixes and conversions * tag 'iommu-updates-v4.8' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (59 commits) iommu/amd: Initialize dma-ops domains with 3-level page-table iommu/amd: Update Alias-DTE in update_device_table() iommu/vt-d: Return error code in domain_context_mapping_one() iommu/amd: Use container_of to get dma_ops_domain iommu/amd: Flush iova queue before releasing dma_ops_domain iommu/amd: Handle IOMMU_DOMAIN_DMA in ops->domain_free call-back iommu/amd: Use dev_data->domain in get_domain() iommu/amd: Optimize map_sg and unmap_sg iommu/amd: Introduce dir2prot() helper iommu/amd: Implement timeout to flush unmap queues iommu/amd: Implement flush queue iommu/amd: Allow NULL pointer parameter for domain_flush_complete() iommu/amd: Set up data structures for flush queue iommu/amd: Remove align-parameter from __map_single() iommu/amd: Remove other remains of old address allocator iommu/amd: Make use of the generic IOVA allocator iommu/amd: Remove special mapping code for dma_ops path iommu/amd: Pass gfp-flags to iommu_map_page() iommu/amd: Implement apply_dm_region call-back iommu/amd: Create a list of reserved iova addresses ...
Diffstat
-rw-r--r--Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt2
-rw-r--r--Documentation/devicetree/bindings/iommu/mediatek,iommu.txt13
-rw-r--r--Documentation/devicetree/bindings/iommu/msm,iommu-v0.txt64
-rw-r--r--Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt21
-rw-r--r--Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt4
-rw-r--r--MAINTAINERS1
-rw-r--r--drivers/iommu/Kconfig21
-rw-r--r--drivers/iommu/Makefile3
-rw-r--r--drivers/iommu/amd_iommu.c1032
-rw-r--r--drivers/iommu/amd_iommu_types.h1
-rw-r--r--drivers/iommu/amd_iommu_v2.c2
-rw-r--r--drivers/iommu/arm-smmu-v3.c2
-rw-r--r--drivers/iommu/arm-smmu.c28
-rw-r--r--drivers/iommu/dmar.c21
-rw-r--r--drivers/iommu/exynos-iommu.c107
-rw-r--r--drivers/iommu/intel-iommu.c6
-rw-r--r--drivers/iommu/io-pgtable-arm.c2
-rw-r--r--drivers/iommu/iommu.c32
-rw-r--r--drivers/iommu/msm_iommu.c870
-rw-r--r--drivers/iommu/msm_iommu.h73
-rw-r--r--drivers/iommu/msm_iommu_dev.c381
-rw-r--r--drivers/iommu/mtk_iommu.c49
-rw-r--r--drivers/iommu/mtk_iommu.h77
-rw-r--r--drivers/iommu/mtk_iommu_v1.c727
-rw-r--r--drivers/iommu/rockchip-iommu.c181
-rw-r--r--drivers/memory/mtk-smi.c167
-rw-r--r--include/dt-bindings/memory/mt2701-larb-port.h85
-rw-r--r--include/linux/iommu.h3
28 files changed, 2316 insertions, 1659 deletions
diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt
index 947863acc2d4..7b94c88cf2ee 100644
--- a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt
+++ b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt
@@ -1,6 +1,6 @@
1* ARM SMMUv3 Architecture Implementation 1* ARM SMMUv3 Architecture Implementation
2 2
3The SMMUv3 architecture is a significant deparature from previous 3The SMMUv3 architecture is a significant departure from previous
4revisions, replacing the MMIO register interface with in-memory command 4revisions, replacing the MMIO register interface with in-memory command
5and event queues and adding support for the ATS and PRI components of 5and event queues and adding support for the ATS and PRI components of
6the PCIe specification. 6the PCIe specification.
diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt
index cd1b1cd7b5c4..53c20cae309f 100644
--- a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt
+++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt
@@ -1,7 +1,9 @@
1* Mediatek IOMMU Architecture Implementation 1* Mediatek IOMMU Architecture Implementation
2 2
3 Some Mediatek SOCs contain a Multimedia Memory Management Unit (M4U) which 3 Some Mediatek SOCs contain a Multimedia Memory Management Unit (M4U), and
4uses the ARM Short-Descriptor translation table format for address translation. 4this M4U have two generations of HW architecture. Generation one uses flat
5pagetable, and only supports 4K size page mapping. Generation two uses the
6ARM Short-Descriptor translation table format for address translation.
5 7
6 About the M4U Hardware Block Diagram, please check below: 8 About the M4U Hardware Block Diagram, please check below:
7 9
@@ -36,7 +38,9 @@ in each larb. Take a example, There are many ports like MC, PP, VLD in the
36video decode local arbiter, all these ports are according to the video HW. 38video decode local arbiter, all these ports are according to the video HW.
37 39
38Required properties: 40Required properties:
39- compatible : must be "mediatek,mt8173-m4u". 41- compatible : must be one of the following string:
42 "mediatek,mt2701-m4u" for mt2701 which uses generation one m4u HW.
43 "mediatek,mt8173-m4u" for mt8173 which uses generation two m4u HW.
40- reg : m4u register base and size. 44- reg : m4u register base and size.
41- interrupts : the interrupt of m4u. 45- interrupts : the interrupt of m4u.
42- clocks : must contain one entry for each clock-names. 46- clocks : must contain one entry for each clock-names.
@@ -46,7 +50,8 @@ Required properties:
46 according to the local arbiter index, like larb0, larb1, larb2... 50 according to the local arbiter index, like larb0, larb1, larb2...
47- iommu-cells : must be 1. This is the mtk_m4u_id according to the HW. 51- iommu-cells : must be 1. This is the mtk_m4u_id according to the HW.
48 Specifies the mtk_m4u_id as defined in 52 Specifies the mtk_m4u_id as defined in
49 dt-binding/memory/mt8173-larb-port.h. 53 dt-binding/memory/mt2701-larb-port.h for mt2701 and
54 dt-binding/memory/mt8173-larb-port.h for mt8173
50 55
51Example: 56Example:
52 iommu: iommu@10205000 { 57 iommu: iommu@10205000 {
diff --git a/Documentation/devicetree/bindings/iommu/msm,iommu-v0.txt b/Documentation/devicetree/bindings/iommu/msm,iommu-v0.txt
new file mode 100644
index 000000000000..20236385f26e
--- /dev/null
+++ b/Documentation/devicetree/bindings/iommu/msm,iommu-v0.txt
@@ -0,0 +1,64 @@
1* QCOM IOMMU
2
3The MSM IOMMU is an implementation compatible with the ARM VMSA short
4descriptor page tables. It provides address translation for bus masters outside
5of the CPU, each connected to the IOMMU through a port called micro-TLB.
6
7Required Properties:
8
9 - compatible: Must contain "qcom,apq8064-iommu".
10 - reg: Base address and size of the IOMMU registers.
11 - interrupts: Specifiers for the MMU fault interrupts. For instances that
12 support secure mode two interrupts must be specified, for non-secure and
13 secure mode, in that order. For instances that don't support secure mode a
14 single interrupt must be specified.
15 - #iommu-cells: The number of cells needed to specify the stream id. This
16 is always 1.
17 - qcom,ncb: The total number of context banks in the IOMMU.
18 - clocks : List of clocks to be used during SMMU register access. See
19 Documentation/devicetree/bindings/clock/clock-bindings.txt
20 for information about the format. For each clock specified
21 here, there must be a corresponding entry in clock-names
22 (see below).
23
24 - clock-names : List of clock names corresponding to the clocks specified in
25 the "clocks" property (above).
26 Should be "smmu_pclk" for specifying the interface clock
27 required for iommu's register accesses.
28 Should be "smmu_clk" for specifying the functional clock
29 required by iommu for bus accesses.
30
31Each bus master connected to an IOMMU must reference the IOMMU in its device
32node with the following property:
33
34 - iommus: A reference to the IOMMU in multiple cells. The first cell is a
35 phandle to the IOMMU and the second cell is the stream id.
36 A single master device can be connected to more than one iommu
37 and multiple contexts in each of the iommu. So multiple entries
38 are required to list all the iommus and the stream ids that the
39 master is connected to.
40
41Example: mdp iommu and its bus master
42
43 mdp_port0: iommu@7500000 {
44 compatible = "qcom,apq8064-iommu";
45 #iommu-cells = <1>;
46 clock-names =
47 "smmu_pclk",
48 "smmu_clk";
49 clocks =
50 <&mmcc SMMU_AHB_CLK>,
51 <&mmcc MDP_AXI_CLK>;
52 reg = <0x07500000 0x100000>;
53 interrupts =
54 <GIC_SPI 63 0>,
55 <GIC_SPI 64 0>;
56 qcom,ncb = <2>;
57 };
58
59 mdp: qcom,mdp@5100000 {
60 compatible = "qcom,mdp";
61 ...
62 iommus = <&mdp_port0 0
63 &mdp_port0 2>;
64 };
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt
index 06a83ceebba7..aa614b2d7cab 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt
@@ -2,16 +2,31 @@ SMI (Smart Multimedia Interface) Common
2 2
3The hardware block diagram please check bindings/iommu/mediatek,iommu.txt 3The hardware block diagram please check bindings/iommu/mediatek,iommu.txt
4 4
5Mediatek SMI have two generations of HW architecture, mt8173 uses the second
6generation of SMI HW while mt2701 uses the first generation HW of SMI.
7
8There's slight differences between the two SMI, for generation 2, the
9register which control the iommu port is at each larb's register base. But
10for generation 1, the register is at smi ao base(smi always on register
11base). Besides that, the smi async clock should be prepared and enabled for
12SMI generation 1 to transform the smi clock into emi clock domain, but that is
13not needed for SMI generation 2.
14
5Required properties: 15Required properties:
6- compatible : must be "mediatek,mt8173-smi-common" 16- compatible : must be one of :
17 "mediatek,mt2701-smi-common"
18 "mediatek,mt8173-smi-common"
7- reg : the register and size of the SMI block. 19- reg : the register and size of the SMI block.
8- power-domains : a phandle to the power domain of this local arbiter. 20- power-domains : a phandle to the power domain of this local arbiter.
9- clocks : Must contain an entry for each entry in clock-names. 21- clocks : Must contain an entry for each entry in clock-names.
10- clock-names : must contain 2 entries, as follows: 22- clock-names : must contain 3 entries for generation 1 smi HW and 2 entries
23 for generation 2 smi HW as follows:
11 - "apb" : Advanced Peripheral Bus clock, It's the clock for setting 24 - "apb" : Advanced Peripheral Bus clock, It's the clock for setting
12 the register. 25 the register.
13 - "smi" : It's the clock for transfer data and command. 26 - "smi" : It's the clock for transfer data and command.
14 They may be the same if both source clocks are the same. 27 They may be the same if both source clocks are the same.
28 - "async" : asynchronous clock, it help transform the smi clock into the emi
29 clock domain, this clock is only needed by generation 1 smi HW.
15 30
16Example: 31Example:
17 smi_common: smi@14022000 { 32 smi_common: smi@14022000 {
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt
index 55ff3b7e0bb9..21277a56e94c 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt
@@ -3,7 +3,9 @@ SMI (Smart Multimedia Interface) Local Arbiter
3The hardware block diagram please check bindings/iommu/mediatek,iommu.txt 3The hardware block diagram please check bindings/iommu/mediatek,iommu.txt
4 4
5Required properties: 5Required properties:
6- compatible : must be "mediatek,mt8173-smi-larb" 6- compatible : must be one of :
7 "mediatek,mt8173-smi-larb"
8 "mediatek,mt2701-smi-larb"
7- reg : the register and size of this local arbiter. 9- reg : the register and size of this local arbiter.
8- mediatek,smi : a phandle to the smi_common node. 10- mediatek,smi : a phandle to the smi_common node.
9- power-domains : a phandle to the power domain of this local arbiter. 11- power-domains : a phandle to the power domain of this local arbiter.
diff --git a/MAINTAINERS b/MAINTAINERS
index 72f77035c562..efdbb5472314 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6223,6 +6223,7 @@ M: Joerg Roedel <joro@8bytes.org>
6223L: iommu@lists.linux-foundation.org 6223L: iommu@lists.linux-foundation.org
6224T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git 6224T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git
6225S: Maintained 6225S: Maintained
6226F: Documentation/devicetree/bindings/iommu/
6226F: drivers/iommu/ 6227F: drivers/iommu/
6227 6228
6228IP MASQUERADING 6229IP MASQUERADING
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index ad0860383cb3..d432ca828472 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -89,8 +89,8 @@ config MSM_IOMMU
89 bool "MSM IOMMU Support" 89 bool "MSM IOMMU Support"
90 depends on ARM 90 depends on ARM
91 depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST 91 depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST
92 depends on BROKEN
93 select IOMMU_API 92 select IOMMU_API
93 select IOMMU_IO_PGTABLE_ARMV7S
94 help 94 help
95 Support for the IOMMUs found on certain Qualcomm SOCs. 95 Support for the IOMMUs found on certain Qualcomm SOCs.
96 These IOMMUs allow virtualization of the address space used by most 96 These IOMMUs allow virtualization of the address space used by most
@@ -111,6 +111,7 @@ config AMD_IOMMU
111 select PCI_PRI 111 select PCI_PRI
112 select PCI_PASID 112 select PCI_PASID
113 select IOMMU_API 113 select IOMMU_API
114 select IOMMU_IOVA
114 depends on X86_64 && PCI && ACPI 115 depends on X86_64 && PCI && ACPI
115 ---help--- 116 ---help---
116 With this option you can enable support for AMD IOMMU hardware in 117 With this option you can enable support for AMD IOMMU hardware in
@@ -343,4 +344,22 @@ config MTK_IOMMU
343 344
344 If unsure, say N here. 345 If unsure, say N here.
345 346
347config MTK_IOMMU_V1
348 bool "MTK IOMMU Version 1 (M4U gen1) Support"
349 depends on ARM
350 depends on ARCH_MEDIATEK || COMPILE_TEST
351 select ARM_DMA_USE_IOMMU
352 select IOMMU_API
353 select MEMORY
354 select MTK_SMI
355 select COMMON_CLK_MT2701_MMSYS
356 select COMMON_CLK_MT2701_IMGSYS
357 select COMMON_CLK_MT2701_VDECSYS
358 help
359 Support for the M4U on certain Mediatek SoCs. M4U generation 1 HW is
360 Multimedia Memory Managememt Unit. This option enables remapping of
361 DMA memory accesses for the multimedia subsystem.
362
363 if unsure, say N here.
364
346endif # IOMMU_SUPPORT 365endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index c6edb31bf8c6..195f7b997d8e 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -7,7 +7,7 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o
7obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o 7obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
8obj-$(CONFIG_IOMMU_IOVA) += iova.o 8obj-$(CONFIG_IOMMU_IOVA) += iova.o
9obj-$(CONFIG_OF_IOMMU) += of_iommu.o 9obj-$(CONFIG_OF_IOMMU) += of_iommu.o
10obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o 10obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
11obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o 11obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
12obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o 12obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
13obj-$(CONFIG_ARM_SMMU) += arm-smmu.o 13obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
@@ -18,6 +18,7 @@ obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o
18obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o 18obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
19obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o 19obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
20obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o 20obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o
21obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o
21obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o 22obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
22obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o 23obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
23obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o 24obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 634f636393d5..33c177ba93be 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -21,6 +21,7 @@
21#include <linux/pci.h> 21#include <linux/pci.h>
22#include <linux/acpi.h> 22#include <linux/acpi.h>
23#include <linux/amba/bus.h> 23#include <linux/amba/bus.h>
24#include <linux/platform_device.h>
24#include <linux/pci-ats.h> 25#include <linux/pci-ats.h>
25#include <linux/bitmap.h> 26#include <linux/bitmap.h>
26#include <linux/slab.h> 27#include <linux/slab.h>
@@ -38,6 +39,7 @@
38#include <linux/dma-contiguous.h> 39#include <linux/dma-contiguous.h>
39#include <linux/irqdomain.h> 40#include <linux/irqdomain.h>
40#include <linux/percpu.h> 41#include <linux/percpu.h>
42#include <linux/iova.h>
41#include <asm/irq_remapping.h> 43#include <asm/irq_remapping.h>
42#include <asm/io_apic.h> 44#include <asm/io_apic.h>
43#include <asm/apic.h> 45#include <asm/apic.h>
@@ -56,6 +58,17 @@
56 58
57#define LOOP_TIMEOUT 100000 59#define LOOP_TIMEOUT 100000
58 60
61/* IO virtual address start page frame number */
62#define IOVA_START_PFN (1)
63#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
64#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
65
66/* Reserved IOVA ranges */
67#define MSI_RANGE_START (0xfee00000)
68#define MSI_RANGE_END (0xfeefffff)
69#define HT_RANGE_START (0xfd00000000ULL)
70#define HT_RANGE_END (0xffffffffffULL)
71
59/* 72/*
60 * This bitmap is used to advertise the page sizes our hardware support 73 * This bitmap is used to advertise the page sizes our hardware support
61 * to the IOMMU core, which will then use this information to split 74 * to the IOMMU core, which will then use this information to split
@@ -76,6 +89,25 @@ LIST_HEAD(ioapic_map);
76LIST_HEAD(hpet_map); 89LIST_HEAD(hpet_map);
77LIST_HEAD(acpihid_map); 90LIST_HEAD(acpihid_map);
78 91
92#define FLUSH_QUEUE_SIZE 256
93
94struct flush_queue_entry {
95 unsigned long iova_pfn;
96 unsigned long pages;
97 struct dma_ops_domain *dma_dom;
98};
99
100struct flush_queue {
101 spinlock_t lock;
102 unsigned next;
103 struct flush_queue_entry *entries;
104};
105
106DEFINE_PER_CPU(struct flush_queue, flush_queue);
107
108static atomic_t queue_timer_on;
109static struct timer_list queue_timer;
110
79/* 111/*
80 * Domain for untranslated devices - only allocated 112 * Domain for untranslated devices - only allocated
81 * if iommu=pt passed on kernel cmd line. 113 * if iommu=pt passed on kernel cmd line.
@@ -121,44 +153,19 @@ static int protection_domain_init(struct protection_domain *domain);
121static void detach_device(struct device *dev); 153static void detach_device(struct device *dev);
122 154
123/* 155/*
124 * For dynamic growth the aperture size is split into ranges of 128MB of
125 * DMA address space each. This struct represents one such range.
126 */
127struct aperture_range {
128
129 spinlock_t bitmap_lock;
130
131 /* address allocation bitmap */
132 unsigned long *bitmap;
133 unsigned long offset;
134 unsigned long next_bit;
135
136 /*
137 * Array of PTE pages for the aperture. In this array we save all the
138 * leaf pages of the domain page table used for the aperture. This way
139 * we don't need to walk the page table to find a specific PTE. We can
140 * just calculate its address in constant time.
141 */
142 u64 *pte_pages[64];
143};
144
145/*
146 * Data container for a dma_ops specific protection domain 156 * Data container for a dma_ops specific protection domain
147 */ 157 */
148struct dma_ops_domain { 158struct dma_ops_domain {
149 /* generic protection domain information */ 159 /* generic protection domain information */
150 struct protection_domain domain; 160 struct protection_domain domain;
151 161
152 /* size of the aperture for the mappings */ 162 /* IOVA RB-Tree */
153 unsigned long aperture_size; 163 struct iova_domain iovad;
154
155 /* aperture index we start searching for free addresses */
156 u32 __percpu *next_index;
157
158 /* address space relevant data */
159 struct aperture_range *aperture[APERTURE_MAX_RANGES];
160}; 164};
161 165
166static struct iova_domain reserved_iova_ranges;
167static struct lock_class_key reserved_rbtree_key;
168
162/**************************************************************************** 169/****************************************************************************
163 * 170 *
164 * Helper functions 171 * Helper functions
@@ -224,6 +231,12 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom)
224 return container_of(dom, struct protection_domain, domain); 231 return container_of(dom, struct protection_domain, domain);
225} 232}
226 233
234static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domain)
235{
236 BUG_ON(domain->flags != PD_DMA_OPS_MASK);
237 return container_of(domain, struct dma_ops_domain, domain);
238}
239
227static struct iommu_dev_data *alloc_dev_data(u16 devid) 240static struct iommu_dev_data *alloc_dev_data(u16 devid)
228{ 241{
229 struct iommu_dev_data *dev_data; 242 struct iommu_dev_data *dev_data;
@@ -391,43 +404,6 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
391} 404}
392 405
393/* 406/*
394 * This function actually applies the mapping to the page table of the
395 * dma_ops domain.
396 */
397static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
398 struct unity_map_entry *e)
399{
400 u64 addr;
401
402 for (addr = e->address_start; addr < e->address_end;
403 addr += PAGE_SIZE) {
404 if (addr < dma_dom->aperture_size)
405 __set_bit(addr >> PAGE_SHIFT,
406 dma_dom->aperture[0]->bitmap);
407 }
408}
409
410/*
411 * Inits the unity mappings required for a specific device
412 */
413static void init_unity_mappings_for_device(struct device *dev,
414 struct dma_ops_domain *dma_dom)
415{
416 struct unity_map_entry *e;
417 int devid;
418
419 devid = get_device_id(dev);
420 if (devid < 0)
421 return;
422
423 list_for_each_entry(e, &amd_iommu_unity_map, list) {
424 if (!(devid >= e->devid_start && devid <= e->devid_end))
425 continue;
426 alloc_unity_mapping(dma_dom, e);
427 }
428}
429
430/*
431 * This function checks if the driver got a valid device from the caller to 407 * This function checks if the driver got a valid device from the caller to
432 * avoid dereferencing invalid pointers. 408 * avoid dereferencing invalid pointers.
433 */ 409 */
@@ -454,22 +430,12 @@ static bool check_device(struct device *dev)
454 430
455static void init_iommu_group(struct device *dev) 431static void init_iommu_group(struct device *dev)
456{ 432{
457 struct dma_ops_domain *dma_domain;
458 struct iommu_domain *domain;
459 struct iommu_group *group; 433 struct iommu_group *group;
460 434
461 group = iommu_group_get_for_dev(dev); 435 group = iommu_group_get_for_dev(dev);
462 if (IS_ERR(group)) 436 if (IS_ERR(group))
463 return; 437 return;
464 438
465 domain = iommu_group_default_domain(group);
466 if (!domain)
467 goto out;
468
469 dma_domain = to_pdomain(domain)->priv;
470
471 init_unity_mappings_for_device(dev, dma_domain);
472out:
473 iommu_group_put(group); 439 iommu_group_put(group);
474} 440}
475 441
@@ -1220,7 +1186,7 @@ static void domain_flush_complete(struct protection_domain *domain)
1220 int i; 1186 int i;
1221 1187
1222 for (i = 0; i < amd_iommus_present; ++i) { 1188 for (i = 0; i < amd_iommus_present; ++i) {
1223 if (!domain->dev_iommu[i]) 1189 if (domain && !domain->dev_iommu[i])
1224 continue; 1190 continue;
1225 1191
1226 /* 1192 /*
@@ -1397,8 +1363,9 @@ static u64 *fetch_pte(struct protection_domain *domain,
1397static int iommu_map_page(struct protection_domain *dom, 1363static int iommu_map_page(struct protection_domain *dom,
1398 unsigned long bus_addr, 1364 unsigned long bus_addr,
1399 unsigned long phys_addr, 1365 unsigned long phys_addr,
1366 unsigned long page_size,
1400 int prot, 1367 int prot,
1401 unsigned long page_size) 1368 gfp_t gfp)
1402{ 1369{
1403 u64 __pte, *pte; 1370 u64 __pte, *pte;
1404 int i, count; 1371 int i, count;
@@ -1410,7 +1377,7 @@ static int iommu_map_page(struct protection_domain *dom,
1410 return -EINVAL; 1377 return -EINVAL;
1411 1378
1412 count = PAGE_SIZE_PTE_COUNT(page_size); 1379 count = PAGE_SIZE_PTE_COUNT(page_size);
1413 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL); 1380 pte = alloc_pte(dom, bus_addr, page_size, NULL, gfp);
1414 1381
1415 if (!pte) 1382 if (!pte)
1416 return -ENOMEM; 1383 return -ENOMEM;
@@ -1474,320 +1441,37 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
1474/**************************************************************************** 1441/****************************************************************************
1475 * 1442 *
1476 * The next functions belong to the address allocator for the dma_ops 1443 * The next functions belong to the address allocator for the dma_ops
1477 * interface functions. They work like the allocators in the other IOMMU 1444 * interface functions.
1478 * drivers. Its basically a bitmap which marks the allocated pages in
1479 * the aperture. Maybe it could be enhanced in the future to a more
1480 * efficient allocator.
1481 * 1445 *
1482 ****************************************************************************/ 1446 ****************************************************************************/
1483 1447
1484/*
1485 * The address allocator core functions.
1486 *
1487 * called with domain->lock held
1488 */
1489 1448
1490/* 1449static unsigned long dma_ops_alloc_iova(struct device *dev,
1491 * Used to reserve address ranges in the aperture (e.g. for exclusion 1450 struct dma_ops_domain *dma_dom,
1492 * ranges. 1451 unsigned int pages, u64 dma_mask)
1493 */
1494static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1495 unsigned long start_page,
1496 unsigned int pages)
1497{ 1452{
1498 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT; 1453 unsigned long pfn = 0;
1499
1500 if (start_page + pages > last_page)
1501 pages = last_page - start_page;
1502
1503 for (i = start_page; i < start_page + pages; ++i) {
1504 int index = i / APERTURE_RANGE_PAGES;
1505 int page = i % APERTURE_RANGE_PAGES;
1506 __set_bit(page, dom->aperture[index]->bitmap);
1507 }
1508}
1509 1454
1510/* 1455 pages = __roundup_pow_of_two(pages);
1511 * This function is used to add a new aperture range to an existing
1512 * aperture in case of dma_ops domain allocation or address allocation
1513 * failure.
1514 */
1515static int alloc_new_range(struct dma_ops_domain *dma_dom,
1516 bool populate, gfp_t gfp)
1517{
1518 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1519 unsigned long i, old_size, pte_pgsize;
1520 struct aperture_range *range;
1521 struct amd_iommu *iommu;
1522 unsigned long flags;
1523 1456
1524#ifdef CONFIG_IOMMU_STRESS 1457 if (dma_mask > DMA_BIT_MASK(32))
1525 populate = false; 1458 pfn = alloc_iova_fast(&dma_dom->iovad, pages,
1526#endif 1459 IOVA_PFN(DMA_BIT_MASK(32)));
1527 1460
1528 if (index >= APERTURE_MAX_RANGES) 1461 if (!pfn)
1529 return -ENOMEM; 1462 pfn = alloc_iova_fast(&dma_dom->iovad, pages, IOVA_PFN(dma_mask));
1530
1531 range = kzalloc(sizeof(struct aperture_range), gfp);
1532 if (!range)
1533 return -ENOMEM;
1534
1535 range->bitmap = (void *)get_zeroed_page(gfp);
1536 if (!range->bitmap)
1537 goto out_free;
1538
1539 range->offset = dma_dom->aperture_size;
1540
1541 spin_lock_init(&range->bitmap_lock);
1542
1543 if (populate) {
1544 unsigned long address = dma_dom->aperture_size;
1545 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1546 u64 *pte, *pte_page;
1547
1548 for (i = 0; i < num_ptes; ++i) {
1549 pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1550 &pte_page, gfp);
1551 if (!pte)
1552 goto out_free;
1553
1554 range->pte_pages[i] = pte_page;
1555
1556 address += APERTURE_RANGE_SIZE / 64;
1557 }
1558 }
1559 1463
1560 spin_lock_irqsave(&dma_dom->domain.lock, flags); 1464 return (pfn << PAGE_SHIFT);
1561
1562 /* First take the bitmap_lock and then publish the range */
1563 spin_lock(&range->bitmap_lock);
1564
1565 old_size = dma_dom->aperture_size;
1566 dma_dom->aperture[index] = range;
1567 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1568
1569 /* Reserve address range used for MSI messages */
1570 if (old_size < MSI_ADDR_BASE_LO &&
1571 dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1572 unsigned long spage;
1573 int pages;
1574
1575 pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1576 spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1577
1578 dma_ops_reserve_addresses(dma_dom, spage, pages);
1579 }
1580
1581 /* Initialize the exclusion range if necessary */
1582 for_each_iommu(iommu) {
1583 if (iommu->exclusion_start &&
1584 iommu->exclusion_start >= dma_dom->aperture[index]->offset
1585 && iommu->exclusion_start < dma_dom->aperture_size) {
1586 unsigned long startpage;
1587 int pages = iommu_num_pages(iommu->exclusion_start,
1588 iommu->exclusion_length,
1589 PAGE_SIZE);
1590 startpage = iommu->exclusion_start >> PAGE_SHIFT;
1591 dma_ops_reserve_addresses(dma_dom, startpage, pages);
1592 }
1593 }
1594
1595 /*
1596 * Check for areas already mapped as present in the new aperture
1597 * range and mark those pages as reserved in the allocator. Such
1598 * mappings may already exist as a result of requested unity
1599 * mappings for devices.
1600 */
1601 for (i = dma_dom->aperture[index]->offset;
1602 i < dma_dom->aperture_size;
1603 i += pte_pgsize) {
1604 u64 *pte = fetch_pte(&dma_dom->domain, i, &pte_pgsize);
1605 if (!pte || !IOMMU_PTE_PRESENT(*pte))
1606 continue;
1607
1608 dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT,
1609 pte_pgsize >> 12);
1610 }
1611
1612 update_domain(&dma_dom->domain);
1613
1614 spin_unlock(&range->bitmap_lock);
1615
1616 spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
1617
1618 return 0;
1619
1620out_free:
1621 update_domain(&dma_dom->domain);
1622
1623 free_page((unsigned long)range->bitmap);
1624
1625 kfree(range);
1626
1627 return -ENOMEM;
1628} 1465}
1629 1466
1630static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom, 1467static void dma_ops_free_iova(struct dma_ops_domain *dma_dom,
1631 struct aperture_range *range, 1468 unsigned long address,
1632 unsigned long pages, 1469 unsigned int pages)
1633 unsigned long dma_mask,
1634 unsigned long boundary_size,
1635 unsigned long align_mask,
1636 bool trylock)
1637{ 1470{
1638 unsigned long offset, limit, flags; 1471 pages = __roundup_pow_of_two(pages);
1639 dma_addr_t address; 1472 address >>= PAGE_SHIFT;
1640 bool flush = false;
1641
1642 offset = range->offset >> PAGE_SHIFT;
1643 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1644 dma_mask >> PAGE_SHIFT);
1645
1646 if (trylock) {
1647 if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
1648 return -1;
1649 } else {
1650 spin_lock_irqsave(&range->bitmap_lock, flags);
1651 }
1652
1653 address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
1654 pages, offset, boundary_size, align_mask);
1655 if (address == -1) {
1656 /* Nothing found, retry one time */
1657 address = iommu_area_alloc(range->bitmap, limit,
1658 0, pages, offset, boundary_size,
1659 align_mask);
1660 flush = true;
1661 }
1662
1663 if (address != -1)
1664 range->next_bit = address + pages;
1665
1666 spin_unlock_irqrestore(&range->bitmap_lock, flags);
1667
1668 if (flush) {
1669 domain_flush_tlb(&dom->domain);
1670 domain_flush_complete(&dom->domain);
1671 }
1672
1673 return address;
1674}
1675
1676static unsigned long dma_ops_area_alloc(struct device *dev,
1677 struct dma_ops_domain *dom,
1678 unsigned int pages,
1679 unsigned long align_mask,
1680 u64 dma_mask)
1681{
1682 unsigned long boundary_size, mask;
1683 unsigned long address = -1;
1684 bool first = true;
1685 u32 start, i;
1686
1687 preempt_disable();
1688
1689 mask = dma_get_seg_boundary(dev);
1690
1691again:
1692 start = this_cpu_read(*dom->next_index);
1693
1694 /* Sanity check - is it really necessary? */
1695 if (unlikely(start > APERTURE_MAX_RANGES)) {
1696 start = 0;
1697 this_cpu_write(*dom->next_index, 0);
1698 }
1699
1700 boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
1701 1UL << (BITS_PER_LONG - PAGE_SHIFT);
1702
1703 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1704 struct aperture_range *range;
1705 int index;
1706
1707 index = (start + i) % APERTURE_MAX_RANGES;
1708
1709 range = dom->aperture[index];
1710
1711 if (!range || range->offset >= dma_mask)
1712 continue;
1713
1714 address = dma_ops_aperture_alloc(dom, range, pages,
1715 dma_mask, boundary_size,
1716 align_mask, first);
1717 if (address != -1) {
1718 address = range->offset + (address << PAGE_SHIFT);
1719 this_cpu_write(*dom->next_index, index);
1720 break;
1721 }
1722 }
1723
1724 if (address == -1 && first) {
1725 first = false;
1726 goto again;
1727 }
1728
1729 preempt_enable();
1730
1731 return address;
1732}
1733
1734static unsigned long dma_ops_alloc_addresses(struct device *dev,
1735 struct dma_ops_domain *dom,
1736 unsigned int pages,
1737 unsigned long align_mask,
1738 u64 dma_mask)
1739{
1740 unsigned long address = -1;
1741
1742 while (address == -1) {
1743 address = dma_ops_area_alloc(dev, dom, pages,
1744 align_mask, dma_mask);
1745
1746 if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
1747 break;
1748 }
1749
1750 if (unlikely(address == -1))
1751 address = DMA_ERROR_CODE;
1752
1753 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1754
1755 return address;
1756}
1757
1758/*
1759 * The address free function.
1760 *
1761 * called with domain->lock held
1762 */
1763static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1764 unsigned long address,
1765 unsigned int pages)
1766{
1767 unsigned i = address >> APERTURE_RANGE_SHIFT;
1768 struct aperture_range *range = dom->aperture[i];
1769 unsigned long flags;
1770
1771 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1772
1773#ifdef CONFIG_IOMMU_STRESS
1774 if (i < 4)
1775 return;
1776#endif
1777
1778 if (amd_iommu_unmap_flush) {
1779 domain_flush_tlb(&dom->domain);
1780 domain_flush_complete(&dom->domain);
1781 }
1782
1783 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1784
1785 spin_lock_irqsave(&range->bitmap_lock, flags);
1786 if (address + pages > range->next_bit)
1787 range->next_bit = address + pages;
1788 bitmap_clear(range->bitmap, address, pages);
1789 spin_unlock_irqrestore(&range->bitmap_lock, flags);
1790 1473
1474 free_iova_fast(&dma_dom->iovad, address, pages);
1791} 1475}
1792 1476
1793/**************************************************************************** 1477/****************************************************************************
@@ -1961,44 +1645,18 @@ static void free_gcr3_table(struct protection_domain *domain)
1961 */ 1645 */
1962static void dma_ops_domain_free(struct dma_ops_domain *dom) 1646static void dma_ops_domain_free(struct dma_ops_domain *dom)
1963{ 1647{
1964 int i;
1965
1966 if (!dom) 1648 if (!dom)
1967 return; 1649 return;
1968 1650
1969 free_percpu(dom->next_index);
1970
1971 del_domain_from_list(&dom->domain); 1651 del_domain_from_list(&dom->domain);
1972 1652
1973 free_pagetable(&dom->domain); 1653 put_iova_domain(&dom->iovad);
1974 1654
1975 for (i = 0; i < APERTURE_MAX_RANGES; ++i) { 1655 free_pagetable(&dom->domain);
1976 if (!dom->aperture[i])
1977 continue;
1978 free_page((unsigned long)dom->aperture[i]->bitmap);
1979 kfree(dom->aperture[i]);
1980 }
1981 1656
1982 kfree(dom); 1657 kfree(dom);
1983} 1658}
1984 1659
1985static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
1986 int max_apertures)
1987{
1988 int ret, i, apertures;
1989
1990 apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1991 ret = 0;
1992
1993 for (i = apertures; i < max_apertures; ++i) {
1994 ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
1995 if (ret)
1996 break;
1997 }
1998
1999 return ret;
2000}
2001
2002/* 1660/*
2003 * Allocates a new protection domain usable for the dma_ops functions. 1661 * Allocates a new protection domain usable for the dma_ops functions.
2004 * It also initializes the page table and the address allocator data 1662 * It also initializes the page table and the address allocator data
@@ -2007,7 +1665,6 @@ static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
2007static struct dma_ops_domain *dma_ops_domain_alloc(void) 1665static struct dma_ops_domain *dma_ops_domain_alloc(void)
2008{ 1666{
2009 struct dma_ops_domain *dma_dom; 1667 struct dma_ops_domain *dma_dom;
2010 int cpu;
2011 1668
2012 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL); 1669 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
2013 if (!dma_dom) 1670 if (!dma_dom)
@@ -2016,30 +1673,19 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
2016 if (protection_domain_init(&dma_dom->domain)) 1673 if (protection_domain_init(&dma_dom->domain))
2017 goto free_dma_dom; 1674 goto free_dma_dom;
2018 1675
2019 dma_dom->next_index = alloc_percpu(u32); 1676 dma_dom->domain.mode = PAGE_MODE_3_LEVEL;
2020 if (!dma_dom->next_index)
2021 goto free_dma_dom;
2022
2023 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
2024 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); 1677 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2025 dma_dom->domain.flags = PD_DMA_OPS_MASK; 1678 dma_dom->domain.flags = PD_DMA_OPS_MASK;
2026 dma_dom->domain.priv = dma_dom;
2027 if (!dma_dom->domain.pt_root) 1679 if (!dma_dom->domain.pt_root)
2028 goto free_dma_dom; 1680 goto free_dma_dom;
2029 1681
2030 add_domain_to_list(&dma_dom->domain); 1682 init_iova_domain(&dma_dom->iovad, PAGE_SIZE,
1683 IOVA_START_PFN, DMA_32BIT_PFN);
2031 1684
2032 if (alloc_new_range(dma_dom, true, GFP_KERNEL)) 1685 /* Initialize reserved ranges */
2033 goto free_dma_dom; 1686 copy_reserved_iova(&reserved_iova_ranges, &dma_dom->iovad);
2034
2035 /*
2036 * mark the first page as allocated so we never return 0 as
2037 * a valid dma-address. So we can use 0 as error value
2038 */
2039 dma_dom->aperture[0]->bitmap[0] = 1;
2040 1687
2041 for_each_possible_cpu(cpu) 1688 add_domain_to_list(&dma_dom->domain);
2042 *per_cpu_ptr(dma_dom->next_index, cpu) = 0;
2043 1689
2044 return dma_dom; 1690 return dma_dom;
2045 1691
@@ -2482,6 +2128,92 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev)
2482 * 2128 *
2483 *****************************************************************************/ 2129 *****************************************************************************/
2484 2130
2131static void __queue_flush(struct flush_queue *queue)
2132{
2133 struct protection_domain *domain;
2134 unsigned long flags;
2135 int idx;
2136
2137 /* First flush TLB of all known domains */
2138 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
2139 list_for_each_entry(domain, &amd_iommu_pd_list, list)
2140 domain_flush_tlb(domain);
2141 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
2142
2143 /* Wait until flushes have completed */
2144 domain_flush_complete(NULL);
2145
2146 for (idx = 0; idx < queue->next; ++idx) {
2147 struct flush_queue_entry *entry;
2148
2149 entry = queue->entries + idx;
2150
2151 free_iova_fast(&entry->dma_dom->iovad,
2152 entry->iova_pfn,
2153 entry->pages);
2154
2155 /* Not really necessary, just to make sure we catch any bugs */
2156 entry->dma_dom = NULL;
2157 }
2158
2159 queue->next = 0;
2160}
2161
2162static void queue_flush_all(void)
2163{
2164 int cpu;
2165
2166 for_each_possible_cpu(cpu) {
2167 struct flush_queue *queue;
2168 unsigned long flags;
2169
2170 queue = per_cpu_ptr(&flush_queue, cpu);
2171 spin_lock_irqsave(&queue->lock, flags);
2172 if (queue->next > 0)
2173 __queue_flush(queue);
2174 spin_unlock_irqrestore(&queue->lock, flags);
2175 }
2176}
2177
2178static void queue_flush_timeout(unsigned long unsused)
2179{
2180 atomic_set(&queue_timer_on, 0);
2181 queue_flush_all();
2182}
2183
2184static void queue_add(struct dma_ops_domain *dma_dom,
2185 unsigned long address, unsigned long pages)
2186{
2187 struct flush_queue_entry *entry;
2188 struct flush_queue *queue;
2189 unsigned long flags;
2190 int idx;
2191
2192 pages = __roundup_pow_of_two(pages);
2193 address >>= PAGE_SHIFT;
2194
2195 queue = get_cpu_ptr(&flush_queue);
2196 spin_lock_irqsave(&queue->lock, flags);
2197
2198 if (queue->next == FLUSH_QUEUE_SIZE)
2199 __queue_flush(queue);
2200
2201 idx = queue->next++;
2202 entry = queue->entries + idx;
2203
2204 entry->iova_pfn = address;
2205 entry->pages = pages;
2206 entry->dma_dom = dma_dom;
2207
2208 spin_unlock_irqrestore(&queue->lock, flags);
2209
2210 if (atomic_cmpxchg(&queue_timer_on, 0, 1) == 0)
2211 mod_timer(&queue_timer, jiffies + msecs_to_jiffies(10));
2212
2213 put_cpu_ptr(&flush_queue);
2214}
2215
2216
2485/* 2217/*
2486 * In the dma_ops path we only have the struct device. This function 2218 * In the dma_ops path we only have the struct device. This function
2487 * finds the corresponding IOMMU, the protection domain and the 2219 * finds the corresponding IOMMU, the protection domain and the
@@ -2492,16 +2224,11 @@ static struct iommu_group *amd_iommu_device_group(struct device *dev)
2492static struct protection_domain *get_domain(struct device *dev) 2224static struct protection_domain *get_domain(struct device *dev)
2493{ 2225{
2494 struct protection_domain *domain; 2226 struct protection_domain *domain;
2495 struct iommu_domain *io_domain;
2496 2227
2497 if (!check_device(dev)) 2228 if (!check_device(dev))
2498 return ERR_PTR(-EINVAL); 2229 return ERR_PTR(-EINVAL);
2499 2230
2500 io_domain = iommu_get_domain_for_dev(dev); 2231 domain = get_dev_data(dev)->domain;
2501 if (!io_domain)
2502 return NULL;
2503
2504 domain = to_pdomain(io_domain);
2505 if (!dma_ops_domain(domain)) 2232 if (!dma_ops_domain(domain))
2506 return ERR_PTR(-EBUSY); 2233 return ERR_PTR(-EBUSY);
2507 2234
@@ -2512,8 +2239,15 @@ static void update_device_table(struct protection_domain *domain)
2512{ 2239{
2513 struct iommu_dev_data *dev_data; 2240 struct iommu_dev_data *dev_data;
2514 2241
2515 list_for_each_entry(dev_data, &domain->dev_list, list) 2242 list_for_each_entry(dev_data, &domain->dev_list, list) {
2516 set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled); 2243 set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2244
2245 if (dev_data->devid == dev_data->alias)
2246 continue;
2247
2248 /* There is an alias, update device table entry for it */
2249 set_dte_entry(dev_data->alias, domain, dev_data->ats.enabled);
2250 }
2517} 2251}
2518 2252
2519static void update_domain(struct protection_domain *domain) 2253static void update_domain(struct protection_domain *domain)
@@ -2529,94 +2263,17 @@ static void update_domain(struct protection_domain *domain)
2529 domain->updated = false; 2263 domain->updated = false;
2530} 2264}
2531 2265
2532/* 2266static int dir2prot(enum dma_data_direction direction)
2533 * This function fetches the PTE for a given address in the aperture
2534 */
2535static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2536 unsigned long address)
2537{
2538 struct aperture_range *aperture;
2539 u64 *pte, *pte_page;
2540
2541 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2542 if (!aperture)
2543 return NULL;
2544
2545 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2546 if (!pte) {
2547 pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2548 GFP_ATOMIC);
2549 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2550 } else
2551 pte += PM_LEVEL_INDEX(0, address);
2552
2553 update_domain(&dom->domain);
2554
2555 return pte;
2556}
2557
2558/*
2559 * This is the generic map function. It maps one 4kb page at paddr to
2560 * the given address in the DMA address space for the domain.
2561 */
2562static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2563 unsigned long address,
2564 phys_addr_t paddr,
2565 int direction)
2566{ 2267{
2567 u64 *pte, __pte;
2568
2569 WARN_ON(address > dom->aperture_size);
2570
2571 paddr &= PAGE_MASK;
2572
2573 pte = dma_ops_get_pte(dom, address);
2574 if (!pte)
2575 return DMA_ERROR_CODE;
2576
2577 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
2578
2579 if (direction == DMA_TO_DEVICE) 2268 if (direction == DMA_TO_DEVICE)
2580 __pte |= IOMMU_PTE_IR; 2269 return IOMMU_PROT_IR;
2581 else if (direction == DMA_FROM_DEVICE) 2270 else if (direction == DMA_FROM_DEVICE)
2582 __pte |= IOMMU_PTE_IW; 2271 return IOMMU_PROT_IW;
2583 else if (direction == DMA_BIDIRECTIONAL) 2272 else if (direction == DMA_BIDIRECTIONAL)
2584 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW; 2273 return IOMMU_PROT_IW | IOMMU_PROT_IR;
2585 2274 else
2586 WARN_ON_ONCE(*pte); 2275 return 0;
2587
2588 *pte = __pte;
2589
2590 return (dma_addr_t)address;
2591}
2592
2593/*
2594 * The generic unmapping function for on page in the DMA address space.
2595 */
2596static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2597 unsigned long address)
2598{
2599 struct aperture_range *aperture;
2600 u64 *pte;
2601
2602 if (address >= dom->aperture_size)
2603 return;
2604
2605 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2606 if (!aperture)
2607 return;
2608
2609 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2610 if (!pte)
2611 return;
2612
2613 pte += PM_LEVEL_INDEX(0, address);
2614
2615 WARN_ON_ONCE(!*pte);
2616
2617 *pte = 0ULL;
2618} 2276}
2619
2620/* 2277/*
2621 * This function contains common code for mapping of a physically 2278 * This function contains common code for mapping of a physically
2622 * contiguous memory region into DMA address space. It is used by all 2279 * contiguous memory region into DMA address space. It is used by all
@@ -2627,32 +2284,29 @@ static dma_addr_t __map_single(struct device *dev,
2627 struct dma_ops_domain *dma_dom, 2284 struct dma_ops_domain *dma_dom,
2628 phys_addr_t paddr, 2285 phys_addr_t paddr,
2629 size_t size, 2286 size_t size,
2630 int dir, 2287 enum dma_data_direction direction,
2631 bool align,
2632 u64 dma_mask) 2288 u64 dma_mask)
2633{ 2289{
2634 dma_addr_t offset = paddr & ~PAGE_MASK; 2290 dma_addr_t offset = paddr & ~PAGE_MASK;
2635 dma_addr_t address, start, ret; 2291 dma_addr_t address, start, ret;
2636 unsigned int pages; 2292 unsigned int pages;
2637 unsigned long align_mask = 0; 2293 int prot = 0;
2638 int i; 2294 int i;
2639 2295
2640 pages = iommu_num_pages(paddr, size, PAGE_SIZE); 2296 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2641 paddr &= PAGE_MASK; 2297 paddr &= PAGE_MASK;
2642 2298
2643 if (align) 2299 address = dma_ops_alloc_iova(dev, dma_dom, pages, dma_mask);
2644 align_mask = (1UL << get_order(size)) - 1;
2645
2646 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2647 dma_mask);
2648
2649 if (address == DMA_ERROR_CODE) 2300 if (address == DMA_ERROR_CODE)
2650 goto out; 2301 goto out;
2651 2302
2303 prot = dir2prot(direction);
2304
2652 start = address; 2305 start = address;
2653 for (i = 0; i < pages; ++i) { 2306 for (i = 0; i < pages; ++i) {
2654 ret = dma_ops_domain_map(dma_dom, start, paddr, dir); 2307 ret = iommu_map_page(&dma_dom->domain, start, paddr,
2655 if (ret == DMA_ERROR_CODE) 2308 PAGE_SIZE, prot, GFP_ATOMIC);
2309 if (ret)
2656 goto out_unmap; 2310 goto out_unmap;
2657 2311
2658 paddr += PAGE_SIZE; 2312 paddr += PAGE_SIZE;
@@ -2672,10 +2326,13 @@ out_unmap:
2672 2326
2673 for (--i; i >= 0; --i) { 2327 for (--i; i >= 0; --i) {
2674 start -= PAGE_SIZE; 2328 start -= PAGE_SIZE;
2675 dma_ops_domain_unmap(dma_dom, start); 2329 iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);
2676 } 2330 }
2677 2331
2678 dma_ops_free_addresses(dma_dom, address, pages); 2332 domain_flush_tlb(&dma_dom->domain);
2333 domain_flush_complete(&dma_dom->domain);
2334
2335 dma_ops_free_iova(dma_dom, address, pages);
2679 2336
2680 return DMA_ERROR_CODE; 2337 return DMA_ERROR_CODE;
2681} 2338}
@@ -2693,21 +2350,23 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
2693 dma_addr_t i, start; 2350 dma_addr_t i, start;
2694 unsigned int pages; 2351 unsigned int pages;
2695 2352
2696 if ((dma_addr == DMA_ERROR_CODE) ||
2697 (dma_addr + size > dma_dom->aperture_size))
2698 return;
2699
2700 flush_addr = dma_addr; 2353 flush_addr = dma_addr;
2701 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE); 2354 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2702 dma_addr &= PAGE_MASK; 2355 dma_addr &= PAGE_MASK;
2703 start = dma_addr; 2356 start = dma_addr;
2704 2357
2705 for (i = 0; i < pages; ++i) { 2358 for (i = 0; i < pages; ++i) {
2706 dma_ops_domain_unmap(dma_dom, start); 2359 iommu_unmap_page(&dma_dom->domain, start, PAGE_SIZE);
2707 start += PAGE_SIZE; 2360 start += PAGE_SIZE;
2708 } 2361 }
2709 2362
2710 dma_ops_free_addresses(dma_dom, dma_addr, pages); 2363 if (amd_iommu_unmap_flush) {
2364 dma_ops_free_iova(dma_dom, dma_addr, pages);
2365 domain_flush_tlb(&dma_dom->domain);
2366 domain_flush_complete(&dma_dom->domain);
2367 } else {
2368 queue_add(dma_dom, dma_addr, pages);
2369 }
2711} 2370}
2712 2371
2713/* 2372/*
@@ -2720,6 +2379,7 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
2720{ 2379{
2721 phys_addr_t paddr = page_to_phys(page) + offset; 2380 phys_addr_t paddr = page_to_phys(page) + offset;
2722 struct protection_domain *domain; 2381 struct protection_domain *domain;
2382 struct dma_ops_domain *dma_dom;
2723 u64 dma_mask; 2383 u64 dma_mask;
2724 2384
2725 domain = get_domain(dev); 2385 domain = get_domain(dev);
@@ -2729,9 +2389,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
2729 return DMA_ERROR_CODE; 2389 return DMA_ERROR_CODE;
2730 2390
2731 dma_mask = *dev->dma_mask; 2391 dma_mask = *dev->dma_mask;
2392 dma_dom = to_dma_ops_domain(domain);
2732 2393
2733 return __map_single(dev, domain->priv, paddr, size, dir, false, 2394 return __map_single(dev, dma_dom, paddr, size, dir, dma_mask);
2734 dma_mask);
2735} 2395}
2736 2396
2737/* 2397/*
@@ -2741,12 +2401,41 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2741 enum dma_data_direction dir, struct dma_attrs *attrs) 2401 enum dma_data_direction dir, struct dma_attrs *attrs)
2742{ 2402{
2743 struct protection_domain *domain; 2403 struct protection_domain *domain;
2404 struct dma_ops_domain *dma_dom;
2744 2405
2745 domain = get_domain(dev); 2406 domain = get_domain(dev);
2746 if (IS_ERR(domain)) 2407 if (IS_ERR(domain))
2747 return; 2408 return;
2748 2409
2749 __unmap_single(domain->priv, dma_addr, size, dir); 2410 dma_dom = to_dma_ops_domain(domain);
2411
2412 __unmap_single(dma_dom, dma_addr, size, dir);
2413}
2414
2415static int sg_num_pages(struct device *dev,
2416 struct scatterlist *sglist,
2417 int nelems)
2418{
2419 unsigned long mask, boundary_size;
2420 struct scatterlist *s;
2421 int i, npages = 0;
2422
2423 mask = dma_get_seg_boundary(dev);
2424 boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
2425 1UL << (BITS_PER_LONG - PAGE_SHIFT);
2426
2427 for_each_sg(sglist, s, nelems, i) {
2428 int p, n;
2429
2430 s->dma_address = npages << PAGE_SHIFT;
2431 p = npages % boundary_size;
2432 n = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2433 if (p + n > boundary_size)
2434 npages += boundary_size - p;
2435 npages += n;
2436 }
2437
2438 return npages;
2750} 2439}
2751 2440
2752/* 2441/*
@@ -2754,46 +2443,79 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2754 * lists). 2443 * lists).
2755 */ 2444 */
2756static int map_sg(struct device *dev, struct scatterlist *sglist, 2445static int map_sg(struct device *dev, struct scatterlist *sglist,
2757 int nelems, enum dma_data_direction dir, 2446 int nelems, enum dma_data_direction direction,
2758 struct dma_attrs *attrs) 2447 struct dma_attrs *attrs)
2759{ 2448{
2449 int mapped_pages = 0, npages = 0, prot = 0, i;
2760 struct protection_domain *domain; 2450 struct protection_domain *domain;
2761 int i; 2451 struct dma_ops_domain *dma_dom;
2762 struct scatterlist *s; 2452 struct scatterlist *s;
2763 phys_addr_t paddr; 2453 unsigned long address;
2764 int mapped_elems = 0;
2765 u64 dma_mask; 2454 u64 dma_mask;
2766 2455
2767 domain = get_domain(dev); 2456 domain = get_domain(dev);
2768 if (IS_ERR(domain)) 2457 if (IS_ERR(domain))
2769 return 0; 2458 return 0;
2770 2459
2460 dma_dom = to_dma_ops_domain(domain);
2771 dma_mask = *dev->dma_mask; 2461 dma_mask = *dev->dma_mask;
2772 2462
2463 npages = sg_num_pages(dev, sglist, nelems);
2464
2465 address = dma_ops_alloc_iova(dev, dma_dom, npages, dma_mask);
2466 if (address == DMA_ERROR_CODE)
2467 goto out_err;
2468
2469 prot = dir2prot(direction);
2470
2471 /* Map all sg entries */
2773 for_each_sg(sglist, s, nelems, i) { 2472 for_each_sg(sglist, s, nelems, i) {
2774 paddr = sg_phys(s); 2473 int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2775 2474
2776 s->dma_address = __map_single(dev, domain->priv, 2475 for (j = 0; j < pages; ++j) {
2777 paddr, s->length, dir, false, 2476 unsigned long bus_addr, phys_addr;
2778 dma_mask); 2477 int ret;
2779 2478
2780 if (s->dma_address) { 2479 bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
2781 s->dma_length = s->length; 2480 phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT);
2782 mapped_elems++; 2481 ret = iommu_map_page(domain, bus_addr, phys_addr, PAGE_SIZE, prot, GFP_ATOMIC);
2783 } else 2482 if (ret)
2784 goto unmap; 2483 goto out_unmap;
2484
2485 mapped_pages += 1;
2486 }
2785 } 2487 }
2786 2488
2787 return mapped_elems; 2489 /* Everything is mapped - write the right values into s->dma_address */
2490 for_each_sg(sglist, s, nelems, i) {
2491 s->dma_address += address + s->offset;
2492 s->dma_length = s->length;
2493 }
2494
2495 return nelems;
2496
2497out_unmap:
2498 pr_err("%s: IOMMU mapping error in map_sg (io-pages: %d)\n",
2499 dev_name(dev), npages);
2788 2500
2789unmap: 2501 for_each_sg(sglist, s, nelems, i) {
2790 for_each_sg(sglist, s, mapped_elems, i) { 2502 int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE);
2791 if (s->dma_address) 2503
2792 __unmap_single(domain->priv, s->dma_address, 2504 for (j = 0; j < pages; ++j) {
2793 s->dma_length, dir); 2505 unsigned long bus_addr;
2794 s->dma_address = s->dma_length = 0; 2506
2507 bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
2508 iommu_unmap_page(domain, bus_addr, PAGE_SIZE);
2509
2510 if (--mapped_pages)
2511 goto out_free_iova;
2512 }
2795 } 2513 }
2796 2514
2515out_free_iova:
2516 free_iova_fast(&dma_dom->iovad, address, npages);
2517
2518out_err:
2797 return 0; 2519 return 0;
2798} 2520}
2799 2521
@@ -2806,18 +2528,19 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2806 struct dma_attrs *attrs) 2528 struct dma_attrs *attrs)
2807{ 2529{
2808 struct protection_domain *domain; 2530 struct protection_domain *domain;
2809 struct scatterlist *s; 2531 struct dma_ops_domain *dma_dom;
2810 int i; 2532 unsigned long startaddr;
2533 int npages = 2;
2811 2534
2812 domain = get_domain(dev); 2535 domain = get_domain(dev);
2813 if (IS_ERR(domain)) 2536 if (IS_ERR(domain))
2814 return; 2537 return;
2815 2538
2816 for_each_sg(sglist, s, nelems, i) { 2539 startaddr = sg_dma_address(sglist) & PAGE_MASK;
2817 __unmap_single(domain->priv, s->dma_address, 2540 dma_dom = to_dma_ops_domain(domain);
2818 s->dma_length, dir); 2541 npages = sg_num_pages(dev, sglist, nelems);
2819 s->dma_address = s->dma_length = 0; 2542
2820 } 2543 __unmap_single(dma_dom, startaddr, npages << PAGE_SHIFT, dir);
2821} 2544}
2822 2545
2823/* 2546/*
@@ -2829,6 +2552,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
2829{ 2552{
2830 u64 dma_mask = dev->coherent_dma_mask; 2553 u64 dma_mask = dev->coherent_dma_mask;
2831 struct protection_domain *domain; 2554 struct protection_domain *domain;
2555 struct dma_ops_domain *dma_dom;
2832 struct page *page; 2556 struct page *page;
2833 2557
2834 domain = get_domain(dev); 2558 domain = get_domain(dev);
@@ -2839,6 +2563,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
2839 } else if (IS_ERR(domain)) 2563 } else if (IS_ERR(domain))
2840 return NULL; 2564 return NULL;
2841 2565
2566 dma_dom = to_dma_ops_domain(domain);
2842 size = PAGE_ALIGN(size); 2567 size = PAGE_ALIGN(size);
2843 dma_mask = dev->coherent_dma_mask; 2568 dma_mask = dev->coherent_dma_mask;
2844 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); 2569 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
@@ -2858,8 +2583,8 @@ static void *alloc_coherent(struct device *dev, size_t size,
2858 if (!dma_mask) 2583 if (!dma_mask)
2859 dma_mask = *dev->dma_mask; 2584 dma_mask = *dev->dma_mask;
2860 2585
2861 *dma_addr = __map_single(dev, domain->priv, page_to_phys(page), 2586 *dma_addr = __map_single(dev, dma_dom, page_to_phys(page),
2862 size, DMA_BIDIRECTIONAL, true, dma_mask); 2587 size, DMA_BIDIRECTIONAL, dma_mask);
2863 2588
2864 if (*dma_addr == DMA_ERROR_CODE) 2589 if (*dma_addr == DMA_ERROR_CODE)
2865 goto out_free; 2590 goto out_free;
@@ -2882,6 +2607,7 @@ static void free_coherent(struct device *dev, size_t size,
2882 struct dma_attrs *attrs) 2607 struct dma_attrs *attrs)
2883{ 2608{
2884 struct protection_domain *domain; 2609 struct protection_domain *domain;
2610 struct dma_ops_domain *dma_dom;
2885 struct page *page; 2611 struct page *page;
2886 2612
2887 page = virt_to_page(virt_addr); 2613 page = virt_to_page(virt_addr);
@@ -2891,7 +2617,9 @@ static void free_coherent(struct device *dev, size_t size,
2891 if (IS_ERR(domain)) 2617 if (IS_ERR(domain))
2892 goto free_mem; 2618 goto free_mem;
2893 2619
2894 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); 2620 dma_dom = to_dma_ops_domain(domain);
2621
2622 __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);
2895 2623
2896free_mem: 2624free_mem:
2897 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) 2625 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
@@ -2907,48 +2635,92 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2907 return check_device(dev); 2635 return check_device(dev);
2908} 2636}
2909 2637
2910static int set_dma_mask(struct device *dev, u64 mask) 2638static struct dma_map_ops amd_iommu_dma_ops = {
2639 .alloc = alloc_coherent,
2640 .free = free_coherent,
2641 .map_page = map_page,
2642 .unmap_page = unmap_page,
2643 .map_sg = map_sg,
2644 .unmap_sg = unmap_sg,
2645 .dma_supported = amd_iommu_dma_supported,
2646};
2647
2648static int init_reserved_iova_ranges(void)
2911{ 2649{
2912 struct protection_domain *domain; 2650 struct pci_dev *pdev = NULL;
2913 int max_apertures = 1; 2651 struct iova *val;
2914 2652
2915 domain = get_domain(dev); 2653 init_iova_domain(&reserved_iova_ranges, PAGE_SIZE,
2916 if (IS_ERR(domain)) 2654 IOVA_START_PFN, DMA_32BIT_PFN);
2917 return PTR_ERR(domain);
2918 2655
2919 if (mask == DMA_BIT_MASK(64)) 2656 lockdep_set_class(&reserved_iova_ranges.iova_rbtree_lock,
2920 max_apertures = 8; 2657 &reserved_rbtree_key);
2921 else if (mask > DMA_BIT_MASK(32)) 2658
2922 max_apertures = 4; 2659 /* MSI memory range */
2660 val = reserve_iova(&reserved_iova_ranges,
2661 IOVA_PFN(MSI_RANGE_START), IOVA_PFN(MSI_RANGE_END));
2662 if (!val) {
2663 pr_err("Reserving MSI range failed\n");
2664 return -ENOMEM;
2665 }
2666
2667 /* HT memory range */
2668 val = reserve_iova(&reserved_iova_ranges,
2669 IOVA_PFN(HT_RANGE_START), IOVA_PFN(HT_RANGE_END));
2670 if (!val) {
2671 pr_err("Reserving HT range failed\n");
2672 return -ENOMEM;
2673 }
2923 2674
2924 /* 2675 /*
2925 * To prevent lock contention it doesn't make sense to allocate more 2676 * Memory used for PCI resources
2926 * apertures than online cpus 2677 * FIXME: Check whether we can reserve the PCI-hole completly
2927 */ 2678 */
2928 if (max_apertures > num_online_cpus()) 2679 for_each_pci_dev(pdev) {
2929 max_apertures = num_online_cpus(); 2680 int i;
2681
2682 for (i = 0; i < PCI_NUM_RESOURCES; ++i) {
2683 struct resource *r = &pdev->resource[i];
2684
2685 if (!(r->flags & IORESOURCE_MEM))
2686 continue;
2930 2687
2931 if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures)) 2688 val = reserve_iova(&reserved_iova_ranges,
2932 dev_err(dev, "Can't allocate %d iommu apertures\n", 2689 IOVA_PFN(r->start),
2933 max_apertures); 2690 IOVA_PFN(r->end));
2691 if (!val) {
2692 pr_err("Reserve pci-resource range failed\n");
2693 return -ENOMEM;
2694 }
2695 }
2696 }
2934 2697
2935 return 0; 2698 return 0;
2936} 2699}
2937 2700
2938static struct dma_map_ops amd_iommu_dma_ops = {
2939 .alloc = alloc_coherent,
2940 .free = free_coherent,
2941 .map_page = map_page,
2942 .unmap_page = unmap_page,
2943 .map_sg = map_sg,
2944 .unmap_sg = unmap_sg,
2945 .dma_supported = amd_iommu_dma_supported,
2946 .set_dma_mask = set_dma_mask,
2947};
2948
2949int __init amd_iommu_init_api(void) 2701int __init amd_iommu_init_api(void)
2950{ 2702{
2951 int err = 0; 2703 int ret, cpu, err = 0;
2704
2705 ret = iova_cache_get();
2706 if (ret)
2707 return ret;
2708
2709 ret = init_reserved_iova_ranges();
2710 if (ret)
2711 return ret;
2712
2713 for_each_possible_cpu(cpu) {
2714 struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
2715
2716 queue->entries = kzalloc(FLUSH_QUEUE_SIZE *
2717 sizeof(*queue->entries),
2718 GFP_KERNEL);
2719 if (!queue->entries)
2720 goto out_put_iova;
2721
2722 spin_lock_init(&queue->lock);
2723 }
2952 2724
2953 err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); 2725 err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
2954 if (err) 2726 if (err)
@@ -2958,11 +2730,26 @@ int __init amd_iommu_init_api(void)
2958 if (err) 2730 if (err)
2959 return err; 2731 return err;
2960#endif 2732#endif
2733 err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops);
2734 if (err)
2735 return err;
2961 return 0; 2736 return 0;
2737
2738out_put_iova:
2739 for_each_possible_cpu(cpu) {
2740 struct flush_queue *queue = per_cpu_ptr(&flush_queue, cpu);
2741
2742 kfree(queue->entries);
2743 }
2744
2745 return -ENOMEM;
2962} 2746}
2963 2747
2964int __init amd_iommu_init_dma_ops(void) 2748int __init amd_iommu_init_dma_ops(void)
2965{ 2749{
2750 setup_timer(&queue_timer, queue_flush_timeout, 0);
2751 atomic_set(&queue_timer_on, 0);
2752
2966 swiotlb = iommu_pass_through ? 1 : 0; 2753 swiotlb = iommu_pass_through ? 1 : 0;
2967 iommu_detected = 1; 2754 iommu_detected = 1;
2968 2755
@@ -2981,6 +2768,7 @@ int __init amd_iommu_init_dma_ops(void)
2981 pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); 2768 pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
2982 2769
2983 return 0; 2770 return 0;
2771
2984} 2772}
2985 2773
2986/***************************************************************************** 2774/*****************************************************************************
@@ -3103,9 +2891,7 @@ static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
3103static void amd_iommu_domain_free(struct iommu_domain *dom) 2891static void amd_iommu_domain_free(struct iommu_domain *dom)
3104{ 2892{
3105 struct protection_domain *domain; 2893 struct protection_domain *domain;
3106 2894 struct dma_ops_domain *dma_dom;
3107 if (!dom)
3108 return;
3109 2895
3110 domain = to_pdomain(dom); 2896 domain = to_pdomain(dom);
3111 2897
@@ -3114,13 +2900,31 @@ static void amd_iommu_domain_free(struct iommu_domain *dom)
3114 2900
3115 BUG_ON(domain->dev_cnt != 0); 2901 BUG_ON(domain->dev_cnt != 0);
3116 2902
3117 if (domain->mode != PAGE_MODE_NONE) 2903 if (!dom)
3118 free_pagetable(domain); 2904 return;
2905
2906 switch (dom->type) {
2907 case IOMMU_DOMAIN_DMA:
2908 /*
2909 * First make sure the domain is no longer referenced from the
2910 * flush queue
2911 */
2912 queue_flush_all();
3119 2913
3120 if (domain->flags & PD_IOMMUV2_MASK) 2914 /* Now release the domain */
3121 free_gcr3_table(domain); 2915 dma_dom = to_dma_ops_domain(domain);
2916 dma_ops_domain_free(dma_dom);
2917 break;
2918 default:
2919 if (domain->mode != PAGE_MODE_NONE)
2920 free_pagetable(domain);
2921
2922 if (domain->flags & PD_IOMMUV2_MASK)
2923 free_gcr3_table(domain);
3122 2924
3123 protection_domain_free(domain); 2925 protection_domain_free(domain);
2926 break;
2927 }
3124} 2928}
3125 2929
3126static void amd_iommu_detach_device(struct iommu_domain *dom, 2930static void amd_iommu_detach_device(struct iommu_domain *dom,
@@ -3190,7 +2994,7 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
3190 prot |= IOMMU_PROT_IW; 2994 prot |= IOMMU_PROT_IW;
3191 2995
3192 mutex_lock(&domain->api_lock); 2996 mutex_lock(&domain->api_lock);
3193 ret = iommu_map_page(domain, iova, paddr, prot, page_size); 2997 ret = iommu_map_page(domain, iova, paddr, page_size, prot, GFP_KERNEL);
3194 mutex_unlock(&domain->api_lock); 2998 mutex_unlock(&domain->api_lock);
3195 2999
3196 return ret; 3000 return ret;
@@ -3292,6 +3096,19 @@ static void amd_iommu_put_dm_regions(struct device *dev,
3292 kfree(entry); 3096 kfree(entry);
3293} 3097}
3294 3098
3099static void amd_iommu_apply_dm_region(struct device *dev,
3100 struct iommu_domain *domain,
3101 struct iommu_dm_region *region)
3102{
3103 struct dma_ops_domain *dma_dom = to_dma_ops_domain(to_pdomain(domain));
3104 unsigned long start, end;
3105
3106 start = IOVA_PFN(region->start);
3107 end = IOVA_PFN(region->start + region->length);
3108
3109 WARN_ON_ONCE(reserve_iova(&dma_dom->iovad, start, end) == NULL);
3110}
3111
3295static const struct iommu_ops amd_iommu_ops = { 3112static const struct iommu_ops amd_iommu_ops = {
3296 .capable = amd_iommu_capable, 3113 .capable = amd_iommu_capable,
3297 .domain_alloc = amd_iommu_domain_alloc, 3114 .domain_alloc = amd_iommu_domain_alloc,
@@ -3307,6 +3124,7 @@ static const struct iommu_ops amd_iommu_ops = {
3307 .device_group = amd_iommu_device_group, 3124 .device_group = amd_iommu_device_group,
3308 .get_dm_regions = amd_iommu_get_dm_regions, 3125 .get_dm_regions = amd_iommu_get_dm_regions,
3309 .put_dm_regions = amd_iommu_put_dm_regions, 3126 .put_dm_regions = amd_iommu_put_dm_regions,
3127 .apply_dm_region = amd_iommu_apply_dm_region,
3310 .pgsize_bitmap = AMD_IOMMU_PGSIZES, 3128 .pgsize_bitmap = AMD_IOMMU_PGSIZES,
3311}; 3129};
3312 3130
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index 590956ac704e..caf5e3822715 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -421,7 +421,6 @@ struct protection_domain {
421 bool updated; /* complete domain flush required */ 421 bool updated; /* complete domain flush required */
422 unsigned dev_cnt; /* devices assigned to this domain */ 422 unsigned dev_cnt; /* devices assigned to this domain */
423 unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ 423 unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */
424 void *priv; /* private data */
425}; 424};
426 425
427/* 426/*
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index fbdaf81ae925..594849a3a9be 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -960,7 +960,7 @@ static int __init amd_iommu_v2_init(void)
960 spin_lock_init(&state_lock); 960 spin_lock_init(&state_lock);
961 961
962 ret = -ENOMEM; 962 ret = -ENOMEM;
963 iommu_wq = create_workqueue("amd_iommu_v2"); 963 iommu_wq = alloc_workqueue("amd_iommu_v2", WQ_MEM_RECLAIM, 0);
964 if (iommu_wq == NULL) 964 if (iommu_wq == NULL)
965 goto out; 965 goto out;
966 966
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 5f6b3bcab078..ce801170d5f2 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2687,6 +2687,8 @@ static int __init arm_smmu_init(void)
2687 if (ret) 2687 if (ret)
2688 return ret; 2688 return ret;
2689 2689
2690 pci_request_acs();
2691
2690 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops); 2692 return bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2691} 2693}
2692 2694
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 9345a3fcb706..4f49fe29f202 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -987,8 +987,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
987 * handler seeing a half-initialised domain state. 987 * handler seeing a half-initialised domain state.
988 */ 988 */
989 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 989 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
990 ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, 990 ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
991 "arm-smmu-context-fault", domain); 991 IRQF_SHARED, "arm-smmu-context-fault", domain);
992 if (ret < 0) { 992 if (ret < 0) {
993 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 993 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
994 cfg->irptndx, irq); 994 cfg->irptndx, irq);
@@ -1028,7 +1028,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
1028 1028
1029 if (cfg->irptndx != INVALID_IRPTNDX) { 1029 if (cfg->irptndx != INVALID_IRPTNDX) {
1030 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 1030 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1031 free_irq(irq, domain); 1031 devm_free_irq(smmu->dev, irq, domain);
1032 } 1032 }
1033 1033
1034 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 1034 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
@@ -1986,15 +1986,15 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1986 } 1986 }
1987 1987
1988 for (i = 0; i < smmu->num_global_irqs; ++i) { 1988 for (i = 0; i < smmu->num_global_irqs; ++i) {
1989 err = request_irq(smmu->irqs[i], 1989 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1990 arm_smmu_global_fault, 1990 arm_smmu_global_fault,
1991 IRQF_SHARED, 1991 IRQF_SHARED,
1992 "arm-smmu global fault", 1992 "arm-smmu global fault",
1993 smmu); 1993 smmu);
1994 if (err) { 1994 if (err) {
1995 dev_err(dev, "failed to request global IRQ %d (%u)\n", 1995 dev_err(dev, "failed to request global IRQ %d (%u)\n",
1996 i, smmu->irqs[i]); 1996 i, smmu->irqs[i]);
1997 goto out_free_irqs; 1997 goto out_put_masters;
1998 } 1998 }
1999 } 1999 }
2000 2000
@@ -2006,10 +2006,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
2006 arm_smmu_device_reset(smmu); 2006 arm_smmu_device_reset(smmu);
2007 return 0; 2007 return 0;
2008 2008
2009out_free_irqs:
2010 while (i--)
2011 free_irq(smmu->irqs[i], smmu);
2012
2013out_put_masters: 2009out_put_masters:
2014 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { 2010 for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {
2015 struct arm_smmu_master *master 2011 struct arm_smmu_master *master
@@ -2050,7 +2046,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
2050 dev_err(dev, "removing device with active domains!\n"); 2046 dev_err(dev, "removing device with active domains!\n");
2051 2047
2052 for (i = 0; i < smmu->num_global_irqs; ++i) 2048 for (i = 0; i < smmu->num_global_irqs; ++i)
2053 free_irq(smmu->irqs[i], smmu); 2049 devm_free_irq(smmu->dev, smmu->irqs[i], smmu);
2054 2050
2055 /* Turn the thing off */ 2051 /* Turn the thing off */
2056 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); 2052 writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
@@ -2096,8 +2092,10 @@ static int __init arm_smmu_init(void)
2096#endif 2092#endif
2097 2093
2098#ifdef CONFIG_PCI 2094#ifdef CONFIG_PCI
2099 if (!iommu_present(&pci_bus_type)) 2095 if (!iommu_present(&pci_bus_type)) {
2096 pci_request_acs();
2100 bus_set_iommu(&pci_bus_type, &arm_smmu_ops); 2097 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2098 }
2101#endif 2099#endif
2102 2100
2103 return 0; 2101 return 0;
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 7330a66e2b7e..58470f5ced04 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -241,8 +241,20 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
241 if (!dmar_match_pci_path(info, scope->bus, path, level)) 241 if (!dmar_match_pci_path(info, scope->bus, path, level))
242 continue; 242 continue;
243 243
244 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^ 244 /*
245 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { 245 * We expect devices with endpoint scope to have normal PCI
246 * headers, and devices with bridge scope to have bridge PCI
247 * headers. However PCI NTB devices may be listed in the
248 * DMAR table with bridge scope, even though they have a
249 * normal PCI header. NTB devices are identified by class
250 * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
251 * for this special case.
252 */
253 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
254 info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
255 (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
256 (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
257 info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
246 pr_warn("Device scope type does not match for %s\n", 258 pr_warn("Device scope type does not match for %s\n",
247 pci_name(info->dev)); 259 pci_name(info->dev));
248 return -EINVAL; 260 return -EINVAL;
@@ -1155,8 +1167,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index)
1155 (unsigned long long)qi->desc[index].high); 1167 (unsigned long long)qi->desc[index].high);
1156 memcpy(&qi->desc[index], &qi->desc[wait_index], 1168 memcpy(&qi->desc[index], &qi->desc[wait_index],
1157 sizeof(struct qi_desc)); 1169 sizeof(struct qi_desc));
1158 __iommu_flush_cache(iommu, &qi->desc[index],
1159 sizeof(struct qi_desc));
1160 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); 1170 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1161 return -EINVAL; 1171 return -EINVAL;
1162 } 1172 }
@@ -1231,9 +1241,6 @@ restart:
1231 1241
1232 hw[wait_index] = wait_desc; 1242 hw[wait_index] = wait_desc;
1233 1243
1234 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1235 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1236
1237 qi->free_head = (qi->free_head + 2) % QI_LENGTH; 1244 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1238 qi->free_cnt -= 2; 1245 qi->free_cnt -= 2;
1239 1246
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 5ecc86cb74c8..33dcc29ec200 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -54,6 +54,10 @@ typedef u32 sysmmu_pte_t;
54#define lv2ent_small(pent) ((*(pent) & 2) == 2) 54#define lv2ent_small(pent) ((*(pent) & 2) == 2)
55#define lv2ent_large(pent) ((*(pent) & 3) == 1) 55#define lv2ent_large(pent) ((*(pent) & 3) == 1)
56 56
57#ifdef CONFIG_BIG_ENDIAN
58#warning "revisit driver if we can enable big-endian ptes"
59#endif
60
57/* 61/*
58 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces 62 * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces
59 * v5.0 introduced support for 36bit physical address space by shifting 63 * v5.0 introduced support for 36bit physical address space by shifting
@@ -322,14 +326,27 @@ static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd)
322 __sysmmu_tlb_invalidate(data); 326 __sysmmu_tlb_invalidate(data);
323} 327}
324 328
329static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data)
330{
331 BUG_ON(clk_prepare_enable(data->clk_master));
332 BUG_ON(clk_prepare_enable(data->clk));
333 BUG_ON(clk_prepare_enable(data->pclk));
334 BUG_ON(clk_prepare_enable(data->aclk));
335}
336
337static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data)
338{
339 clk_disable_unprepare(data->aclk);
340 clk_disable_unprepare(data->pclk);
341 clk_disable_unprepare(data->clk);
342 clk_disable_unprepare(data->clk_master);
343}
344
325static void __sysmmu_get_version(struct sysmmu_drvdata *data) 345static void __sysmmu_get_version(struct sysmmu_drvdata *data)
326{ 346{
327 u32 ver; 347 u32 ver;
328 348
329 clk_enable(data->clk_master); 349 __sysmmu_enable_clocks(data);
330 clk_enable(data->clk);
331 clk_enable(data->pclk);
332 clk_enable(data->aclk);
333 350
334 ver = readl(data->sfrbase + REG_MMU_VERSION); 351 ver = readl(data->sfrbase + REG_MMU_VERSION);
335 352
@@ -342,10 +359,7 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data)
342 dev_dbg(data->sysmmu, "hardware version: %d.%d\n", 359 dev_dbg(data->sysmmu, "hardware version: %d.%d\n",
343 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); 360 MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version));
344 361
345 clk_disable(data->aclk); 362 __sysmmu_disable_clocks(data);
346 clk_disable(data->pclk);
347 clk_disable(data->clk);
348 clk_disable(data->clk_master);
349} 363}
350 364
351static void show_fault_information(struct sysmmu_drvdata *data, 365static void show_fault_information(struct sysmmu_drvdata *data,
@@ -427,10 +441,7 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
427 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); 441 writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
428 writel(0, data->sfrbase + REG_MMU_CFG); 442 writel(0, data->sfrbase + REG_MMU_CFG);
429 443
430 clk_disable(data->aclk); 444 __sysmmu_disable_clocks(data);
431 clk_disable(data->pclk);
432 clk_disable(data->clk);
433 clk_disable(data->clk_master);
434} 445}
435 446
436static bool __sysmmu_disable(struct sysmmu_drvdata *data) 447static bool __sysmmu_disable(struct sysmmu_drvdata *data)
@@ -475,10 +486,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data)
475 486
476static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) 487static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
477{ 488{
478 clk_enable(data->clk_master); 489 __sysmmu_enable_clocks(data);
479 clk_enable(data->clk);
480 clk_enable(data->pclk);
481 clk_enable(data->aclk);
482 490
483 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); 491 writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
484 492
@@ -488,6 +496,12 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
488 496
489 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); 497 writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
490 498
499 /*
500 * SYSMMU driver keeps master's clock enabled only for the short
501 * time, while accessing the registers. For performing address
502 * translation during DMA transaction it relies on the client
503 * driver to enable it.
504 */
491 clk_disable(data->clk_master); 505 clk_disable(data->clk_master);
492} 506}
493 507
@@ -524,16 +538,15 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
524{ 538{
525 unsigned long flags; 539 unsigned long flags;
526 540
527 clk_enable(data->clk_master);
528 541
529 spin_lock_irqsave(&data->lock, flags); 542 spin_lock_irqsave(&data->lock, flags);
530 if (is_sysmmu_active(data)) { 543 if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) {
531 if (data->version >= MAKE_MMU_VER(3, 3)) 544 clk_enable(data->clk_master);
532 __sysmmu_tlb_invalidate_entry(data, iova, 1); 545 __sysmmu_tlb_invalidate_entry(data, iova, 1);
546 clk_disable(data->clk_master);
533 } 547 }
534 spin_unlock_irqrestore(&data->lock, flags); 548 spin_unlock_irqrestore(&data->lock, flags);
535 549
536 clk_disable(data->clk_master);
537} 550}
538 551
539static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, 552static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
@@ -572,6 +585,8 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
572 spin_unlock_irqrestore(&data->lock, flags); 585 spin_unlock_irqrestore(&data->lock, flags);
573} 586}
574 587
588static struct iommu_ops exynos_iommu_ops;
589
575static int __init exynos_sysmmu_probe(struct platform_device *pdev) 590static int __init exynos_sysmmu_probe(struct platform_device *pdev)
576{ 591{
577 int irq, ret; 592 int irq, ret;
@@ -602,37 +617,22 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
602 } 617 }
603 618
604 data->clk = devm_clk_get(dev, "sysmmu"); 619 data->clk = devm_clk_get(dev, "sysmmu");
605 if (!IS_ERR(data->clk)) { 620 if (PTR_ERR(data->clk) == -ENOENT)
606 ret = clk_prepare(data->clk);
607 if (ret) {
608 dev_err(dev, "Failed to prepare clk\n");
609 return ret;
610 }
611 } else {
612 data->clk = NULL; 621 data->clk = NULL;
613 } 622 else if (IS_ERR(data->clk))
623 return PTR_ERR(data->clk);
614 624
615 data->aclk = devm_clk_get(dev, "aclk"); 625 data->aclk = devm_clk_get(dev, "aclk");
616 if (!IS_ERR(data->aclk)) { 626 if (PTR_ERR(data->aclk) == -ENOENT)
617 ret = clk_prepare(data->aclk);
618 if (ret) {
619 dev_err(dev, "Failed to prepare aclk\n");
620 return ret;
621 }
622 } else {
623 data->aclk = NULL; 627 data->aclk = NULL;
624 } 628 else if (IS_ERR(data->aclk))
629 return PTR_ERR(data->aclk);
625 630
626 data->pclk = devm_clk_get(dev, "pclk"); 631 data->pclk = devm_clk_get(dev, "pclk");
627 if (!IS_ERR(data->pclk)) { 632 if (PTR_ERR(data->pclk) == -ENOENT)
628 ret = clk_prepare(data->pclk);
629 if (ret) {
630 dev_err(dev, "Failed to prepare pclk\n");
631 return ret;
632 }
633 } else {
634 data->pclk = NULL; 633 data->pclk = NULL;
635 } 634 else if (IS_ERR(data->pclk))
635 return PTR_ERR(data->pclk);
636 636
637 if (!data->clk && (!data->aclk || !data->pclk)) { 637 if (!data->clk && (!data->aclk || !data->pclk)) {
638 dev_err(dev, "Failed to get device clock(s)!\n"); 638 dev_err(dev, "Failed to get device clock(s)!\n");
@@ -640,15 +640,10 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
640 } 640 }
641 641
642 data->clk_master = devm_clk_get(dev, "master"); 642 data->clk_master = devm_clk_get(dev, "master");
643 if (!IS_ERR(data->clk_master)) { 643 if (PTR_ERR(data->clk_master) == -ENOENT)
644 ret = clk_prepare(data->clk_master);
645 if (ret) {
646 dev_err(dev, "Failed to prepare master's clk\n");
647 return ret;
648 }
649 } else {
650 data->clk_master = NULL; 644 data->clk_master = NULL;
651 } 645 else if (IS_ERR(data->clk_master))
646 return PTR_ERR(data->clk_master);
652 647
653 data->sysmmu = dev; 648 data->sysmmu = dev;
654 spin_lock_init(&data->lock); 649 spin_lock_init(&data->lock);
@@ -665,6 +660,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
665 660
666 pm_runtime_enable(dev); 661 pm_runtime_enable(dev);
667 662
663 of_iommu_set_ops(dev->of_node, &exynos_iommu_ops);
664
668 return 0; 665 return 0;
669} 666}
670 667
@@ -709,6 +706,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = {
709 .name = "exynos-sysmmu", 706 .name = "exynos-sysmmu",
710 .of_match_table = sysmmu_of_match, 707 .of_match_table = sysmmu_of_match,
711 .pm = &sysmmu_pm_ops, 708 .pm = &sysmmu_pm_ops,
709 .suppress_bind_attrs = true,
712 } 710 }
713}; 711};
714 712
@@ -716,7 +714,7 @@ static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val)
716{ 714{
717 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), 715 dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent),
718 DMA_TO_DEVICE); 716 DMA_TO_DEVICE);
719 *ent = val; 717 *ent = cpu_to_le32(val);
720 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), 718 dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent),
721 DMA_TO_DEVICE); 719 DMA_TO_DEVICE);
722} 720}
@@ -1357,7 +1355,6 @@ static int __init exynos_iommu_of_setup(struct device_node *np)
1357 if (!dma_dev) 1355 if (!dma_dev)
1358 dma_dev = &pdev->dev; 1356 dma_dev = &pdev->dev;
1359 1357
1360 of_iommu_set_ops(np, &exynos_iommu_ops);
1361 return 0; 1358 return 0;
1362} 1359}
1363 1360
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 4b9040bb2f1c..afbaa2c69a59 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -1672,7 +1672,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
1672 return -ENOMEM; 1672 return -ENOMEM;
1673 } 1673 }
1674 1674
1675 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **); 1675 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1676 iommu->domains = kzalloc(size, GFP_KERNEL); 1676 iommu->domains = kzalloc(size, GFP_KERNEL);
1677 1677
1678 if (iommu->domains) { 1678 if (iommu->domains) {
@@ -1737,7 +1737,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu)
1737static void free_dmar_iommu(struct intel_iommu *iommu) 1737static void free_dmar_iommu(struct intel_iommu *iommu)
1738{ 1738{
1739 if ((iommu->domains) && (iommu->domain_ids)) { 1739 if ((iommu->domains) && (iommu->domain_ids)) {
1740 int elems = (cap_ndoms(iommu->cap) >> 8) + 1; 1740 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1741 int i; 1741 int i;
1742 1742
1743 for (i = 0; i < elems; i++) 1743 for (i = 0; i < elems; i++)
@@ -2076,7 +2076,7 @@ out_unlock:
2076 spin_unlock(&iommu->lock); 2076 spin_unlock(&iommu->lock);
2077 spin_unlock_irqrestore(&device_domain_lock, flags); 2077 spin_unlock_irqrestore(&device_domain_lock, flags);
2078 2078
2079 return 0; 2079 return ret;
2080} 2080}
2081 2081
2082struct domain_context_mapping_data { 2082struct domain_context_mapping_data {
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index a1ed1b73fed4..f5c90e1366ce 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -576,7 +576,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
576 return 0; 576 return 0;
577 577
578found_translation: 578found_translation:
579 iova &= (ARM_LPAE_GRANULE(data) - 1); 579 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);
580 return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; 580 return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova;
581} 581}
582 582
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3000051f48b4..b06d93594436 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -34,8 +34,7 @@
34#include <trace/events/iommu.h> 34#include <trace/events/iommu.h>
35 35
36static struct kset *iommu_group_kset; 36static struct kset *iommu_group_kset;
37static struct ida iommu_group_ida; 37static DEFINE_IDA(iommu_group_ida);
38static struct mutex iommu_group_mutex;
39 38
40struct iommu_callback_data { 39struct iommu_callback_data {
41 const struct iommu_ops *ops; 40 const struct iommu_ops *ops;
@@ -144,9 +143,7 @@ static void iommu_group_release(struct kobject *kobj)
144 if (group->iommu_data_release) 143 if (group->iommu_data_release)
145 group->iommu_data_release(group->iommu_data); 144 group->iommu_data_release(group->iommu_data);
146 145
147 mutex_lock(&iommu_group_mutex); 146 ida_simple_remove(&iommu_group_ida, group->id);
148 ida_remove(&iommu_group_ida, group->id);
149 mutex_unlock(&iommu_group_mutex);
150 147
151 if (group->default_domain) 148 if (group->default_domain)
152 iommu_domain_free(group->default_domain); 149 iommu_domain_free(group->default_domain);
@@ -186,26 +183,17 @@ struct iommu_group *iommu_group_alloc(void)
186 INIT_LIST_HEAD(&group->devices); 183 INIT_LIST_HEAD(&group->devices);
187 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); 184 BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier);
188 185
189 mutex_lock(&iommu_group_mutex); 186 ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL);
190 187 if (ret < 0) {
191again:
192 if (unlikely(0 == ida_pre_get(&iommu_group_ida, GFP_KERNEL))) {
193 kfree(group); 188 kfree(group);
194 mutex_unlock(&iommu_group_mutex); 189 return ERR_PTR(ret);
195 return ERR_PTR(-ENOMEM);
196 } 190 }
197 191 group->id = ret;
198 if (-EAGAIN == ida_get_new(&iommu_group_ida, &group->id))
199 goto again;
200
201 mutex_unlock(&iommu_group_mutex);
202 192
203 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, 193 ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,
204 NULL, "%d", group->id); 194 NULL, "%d", group->id);
205 if (ret) { 195 if (ret) {
206 mutex_lock(&iommu_group_mutex); 196 ida_simple_remove(&iommu_group_ida, group->id);
207 ida_remove(&iommu_group_ida, group->id);
208 mutex_unlock(&iommu_group_mutex);
209 kfree(group); 197 kfree(group);
210 return ERR_PTR(ret); 198 return ERR_PTR(ret);
211 } 199 }
@@ -348,6 +336,9 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group,
348 list_for_each_entry(entry, &mappings, list) { 336 list_for_each_entry(entry, &mappings, list) {
349 dma_addr_t start, end, addr; 337 dma_addr_t start, end, addr;
350 338
339 if (domain->ops->apply_dm_region)
340 domain->ops->apply_dm_region(dev, domain, entry);
341
351 start = ALIGN(entry->start, pg_size); 342 start = ALIGN(entry->start, pg_size);
352 end = ALIGN(entry->start + entry->length, pg_size); 343 end = ALIGN(entry->start + entry->length, pg_size);
353 344
@@ -1483,9 +1474,6 @@ static int __init iommu_init(void)
1483{ 1474{
1484 iommu_group_kset = kset_create_and_add("iommu_groups", 1475 iommu_group_kset = kset_create_and_add("iommu_groups",
1485 NULL, kernel_kobj); 1476 NULL, kernel_kobj);
1486 ida_init(&iommu_group_ida);
1487 mutex_init(&iommu_group_mutex);
1488
1489 BUG_ON(!iommu_group_kset); 1477 BUG_ON(!iommu_group_kset);
1490 1478
1491 return 0; 1479 return 0;
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index e321fa517a45..b09692bb5b0a 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -27,32 +27,35 @@
27#include <linux/slab.h> 27#include <linux/slab.h>
28#include <linux/iommu.h> 28#include <linux/iommu.h>
29#include <linux/clk.h> 29#include <linux/clk.h>
30#include <linux/err.h>
31#include <linux/of_iommu.h>
30 32
31#include <asm/cacheflush.h> 33#include <asm/cacheflush.h>
32#include <asm/sizes.h> 34#include <asm/sizes.h>
33 35
34#include "msm_iommu_hw-8xxx.h" 36#include "msm_iommu_hw-8xxx.h"
35#include "msm_iommu.h" 37#include "msm_iommu.h"
38#include "io-pgtable.h"
36 39
37#define MRC(reg, processor, op1, crn, crm, op2) \ 40#define MRC(reg, processor, op1, crn, crm, op2) \
38__asm__ __volatile__ ( \ 41__asm__ __volatile__ ( \
39" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ 42" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \
40: "=r" (reg)) 43: "=r" (reg))
41 44
42#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
43#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
44
45/* bitmap of the page sizes currently supported */ 45/* bitmap of the page sizes currently supported */
46#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) 46#define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
47 47
48static int msm_iommu_tex_class[4];
49
50DEFINE_SPINLOCK(msm_iommu_lock); 48DEFINE_SPINLOCK(msm_iommu_lock);
49static LIST_HEAD(qcom_iommu_devices);
50static struct iommu_ops msm_iommu_ops;
51 51
52struct msm_priv { 52struct msm_priv {
53 unsigned long *pgtable;
54 struct list_head list_attached; 53 struct list_head list_attached;
55 struct iommu_domain domain; 54 struct iommu_domain domain;
55 struct io_pgtable_cfg cfg;
56 struct io_pgtable_ops *iop;
57 struct device *dev;
58 spinlock_t pgtlock; /* pagetable lock */
56}; 59};
57 60
58static struct msm_priv *to_msm_priv(struct iommu_domain *dom) 61static struct msm_priv *to_msm_priv(struct iommu_domain *dom)
@@ -60,67 +63,183 @@ static struct msm_priv *to_msm_priv(struct iommu_domain *dom)
60 return container_of(dom, struct msm_priv, domain); 63 return container_of(dom, struct msm_priv, domain);
61} 64}
62 65
63static int __enable_clocks(struct msm_iommu_drvdata *drvdata) 66static int __enable_clocks(struct msm_iommu_dev *iommu)
64{ 67{
65 int ret; 68 int ret;
66 69
67 ret = clk_enable(drvdata->pclk); 70 ret = clk_enable(iommu->pclk);
68 if (ret) 71 if (ret)
69 goto fail; 72 goto fail;
70 73
71 if (drvdata->clk) { 74 if (iommu->clk) {
72 ret = clk_enable(drvdata->clk); 75 ret = clk_enable(iommu->clk);
73 if (ret) 76 if (ret)
74 clk_disable(drvdata->pclk); 77 clk_disable(iommu->pclk);
75 } 78 }
76fail: 79fail:
77 return ret; 80 return ret;
78} 81}
79 82
80static void __disable_clocks(struct msm_iommu_drvdata *drvdata) 83static void __disable_clocks(struct msm_iommu_dev *iommu)
81{ 84{
82 clk_disable(drvdata->clk); 85 if (iommu->clk)
83 clk_disable(drvdata->pclk); 86 clk_disable(iommu->clk);
87 clk_disable(iommu->pclk);
84} 88}
85 89
86static int __flush_iotlb(struct iommu_domain *domain) 90static void msm_iommu_reset(void __iomem *base, int ncb)
87{ 91{
88 struct msm_priv *priv = to_msm_priv(domain); 92 int ctx;
89 struct msm_iommu_drvdata *iommu_drvdata; 93
90 struct msm_iommu_ctx_drvdata *ctx_drvdata; 94 SET_RPUE(base, 0);
91 int ret = 0; 95 SET_RPUEIE(base, 0);
92#ifndef CONFIG_IOMMU_PGTABLES_L2 96 SET_ESRRESTORE(base, 0);
93 unsigned long *fl_table = priv->pgtable; 97 SET_TBE(base, 0);
94 int i; 98 SET_CR(base, 0);
95 99 SET_SPDMBE(base, 0);
96 if (!list_empty(&priv->list_attached)) { 100 SET_TESTBUSCR(base, 0);
97 dmac_flush_range(fl_table, fl_table + SZ_16K); 101 SET_TLBRSW(base, 0);
98 102 SET_GLOBAL_TLBIALL(base, 0);
99 for (i = 0; i < NUM_FL_PTE; i++) 103 SET_RPU_ACR(base, 0);
100 if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { 104 SET_TLBLKCRWE(base, 1);
101 void *sl_table = __va(fl_table[i] & 105
102 FL_BASE_MASK); 106 for (ctx = 0; ctx < ncb; ctx++) {
103 dmac_flush_range(sl_table, sl_table + SZ_4K); 107 SET_BPRCOSH(base, ctx, 0);
104 } 108 SET_BPRCISH(base, ctx, 0);
109 SET_BPRCNSH(base, ctx, 0);
110 SET_BPSHCFG(base, ctx, 0);
111 SET_BPMTCFG(base, ctx, 0);
112 SET_ACTLR(base, ctx, 0);
113 SET_SCTLR(base, ctx, 0);
114 SET_FSRRESTORE(base, ctx, 0);
115 SET_TTBR0(base, ctx, 0);
116 SET_TTBR1(base, ctx, 0);
117 SET_TTBCR(base, ctx, 0);
118 SET_BFBCR(base, ctx, 0);
119 SET_PAR(base, ctx, 0);
120 SET_FAR(base, ctx, 0);
121 SET_CTX_TLBIALL(base, ctx, 0);
122 SET_TLBFLPTER(base, ctx, 0);
123 SET_TLBSLPTER(base, ctx, 0);
124 SET_TLBLKCR(base, ctx, 0);
125 SET_CONTEXTIDR(base, ctx, 0);
105 } 126 }
106#endif 127}
107 128
108 list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { 129static void __flush_iotlb(void *cookie)
130{
131 struct msm_priv *priv = cookie;
132 struct msm_iommu_dev *iommu = NULL;
133 struct msm_iommu_ctx_dev *master;
134 int ret = 0;
135
136 list_for_each_entry(iommu, &priv->list_attached, dom_node) {
137 ret = __enable_clocks(iommu);
138 if (ret)
139 goto fail;
109 140
110 BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent); 141 list_for_each_entry(master, &iommu->ctx_list, list)
142 SET_CTX_TLBIALL(iommu->base, master->num, 0);
111 143
112 iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); 144 __disable_clocks(iommu);
113 BUG_ON(!iommu_drvdata); 145 }
146fail:
147 return;
148}
114 149
115 ret = __enable_clocks(iommu_drvdata); 150static void __flush_iotlb_range(unsigned long iova, size_t size,
151 size_t granule, bool leaf, void *cookie)
152{
153 struct msm_priv *priv = cookie;
154 struct msm_iommu_dev *iommu = NULL;
155 struct msm_iommu_ctx_dev *master;
156 int ret = 0;
157 int temp_size;
158
159 list_for_each_entry(iommu, &priv->list_attached, dom_node) {
160 ret = __enable_clocks(iommu);
116 if (ret) 161 if (ret)
117 goto fail; 162 goto fail;
118 163
119 SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); 164 list_for_each_entry(master, &iommu->ctx_list, list) {
120 __disable_clocks(iommu_drvdata); 165 temp_size = size;
166 do {
167 iova &= TLBIVA_VA;
168 iova |= GET_CONTEXTIDR_ASID(iommu->base,
169 master->num);
170 SET_TLBIVA(iommu->base, master->num, iova);
171 iova += granule;
172 } while (temp_size -= granule);
173 }
174
175 __disable_clocks(iommu);
121 } 176 }
177
122fail: 178fail:
123 return ret; 179 return;
180}
181
182static void __flush_iotlb_sync(void *cookie)
183{
184 /*
185 * Nothing is needed here, the barrier to guarantee
186 * completion of the tlb sync operation is implicitly
187 * taken care when the iommu client does a writel before
188 * kick starting the other master.
189 */
190}
191
192static const struct iommu_gather_ops msm_iommu_gather_ops = {
193 .tlb_flush_all = __flush_iotlb,
194 .tlb_add_flush = __flush_iotlb_range,
195 .tlb_sync = __flush_iotlb_sync,
196};
197
198static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end)
199{
200 int idx;
201
202 do {
203 idx = find_next_zero_bit(map, end, start);
204 if (idx == end)
205 return -ENOSPC;
206 } while (test_and_set_bit(idx, map));
207
208 return idx;
209}
210
211static void msm_iommu_free_ctx(unsigned long *map, int idx)
212{
213 clear_bit(idx, map);
214}
215
216static void config_mids(struct msm_iommu_dev *iommu,
217 struct msm_iommu_ctx_dev *master)
218{
219 int mid, ctx, i;
220
221 for (i = 0; i < master->num_mids; i++) {
222 mid = master->mids[i];
223 ctx = master->num;
224
225 SET_M2VCBR_N(iommu->base, mid, 0);
226 SET_CBACR_N(iommu->base, ctx, 0);
227
228 /* Set VMID = 0 */
229 SET_VMID(iommu->base, mid, 0);
230
231 /* Set the context number for that MID to this context */
232 SET_CBNDX(iommu->base, mid, ctx);
233
234 /* Set MID associated with this context bank to 0*/
235 SET_CBVMID(iommu->base, ctx, 0);
236
237 /* Set the ASID for TLB tagging for this context */
238 SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx);
239
240 /* Set security bit override to be Non-secure */
241 SET_NSCFG(iommu->base, mid, 3);
242 }
124} 243}
125 244
126static void __reset_context(void __iomem *base, int ctx) 245static void __reset_context(void __iomem *base, int ctx)
@@ -143,15 +262,17 @@ static void __reset_context(void __iomem *base, int ctx)
143 SET_TLBFLPTER(base, ctx, 0); 262 SET_TLBFLPTER(base, ctx, 0);
144 SET_TLBSLPTER(base, ctx, 0); 263 SET_TLBSLPTER(base, ctx, 0);
145 SET_TLBLKCR(base, ctx, 0); 264 SET_TLBLKCR(base, ctx, 0);
146 SET_PRRR(base, ctx, 0);
147 SET_NMRR(base, ctx, 0);
148} 265}
149 266
150static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) 267static void __program_context(void __iomem *base, int ctx,
268 struct msm_priv *priv)
151{ 269{
152 unsigned int prrr, nmrr;
153 __reset_context(base, ctx); 270 __reset_context(base, ctx);
154 271
272 /* Turn on TEX Remap */
273 SET_TRE(base, ctx, 1);
274 SET_AFE(base, ctx, 1);
275
155 /* Set up HTW mode */ 276 /* Set up HTW mode */
156 /* TLB miss configuration: perform HTW on miss */ 277 /* TLB miss configuration: perform HTW on miss */
157 SET_TLBMCFG(base, ctx, 0x3); 278 SET_TLBMCFG(base, ctx, 0x3);
@@ -159,8 +280,13 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
159 /* V2P configuration: HTW for access */ 280 /* V2P configuration: HTW for access */
160 SET_V2PCFG(base, ctx, 0x3); 281 SET_V2PCFG(base, ctx, 0x3);
161 282
162 SET_TTBCR(base, ctx, 0); 283 SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr);
163 SET_TTBR0_PA(base, ctx, (pgtable >> 14)); 284 SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]);
285 SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]);
286
287 /* Set prrr and nmrr */
288 SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr);
289 SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr);
164 290
165 /* Invalidate the TLB for this context */ 291 /* Invalidate the TLB for this context */
166 SET_CTX_TLBIALL(base, ctx, 0); 292 SET_CTX_TLBIALL(base, ctx, 0);
@@ -179,38 +305,9 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable)
179 SET_RCOSH(base, ctx, 1); 305 SET_RCOSH(base, ctx, 1);
180 SET_RCNSH(base, ctx, 1); 306 SET_RCNSH(base, ctx, 1);
181 307
182 /* Turn on TEX Remap */
183 SET_TRE(base, ctx, 1);
184
185 /* Set TEX remap attributes */
186 RCP15_PRRR(prrr);
187 RCP15_NMRR(nmrr);
188 SET_PRRR(base, ctx, prrr);
189 SET_NMRR(base, ctx, nmrr);
190
191 /* Turn on BFB prefetch */ 308 /* Turn on BFB prefetch */
192 SET_BFBDFE(base, ctx, 1); 309 SET_BFBDFE(base, ctx, 1);
193 310
194#ifdef CONFIG_IOMMU_PGTABLES_L2
195 /* Configure page tables as inner-cacheable and shareable to reduce
196 * the TLB miss penalty.
197 */
198 SET_TTBR0_SH(base, ctx, 1);
199 SET_TTBR1_SH(base, ctx, 1);
200
201 SET_TTBR0_NOS(base, ctx, 1);
202 SET_TTBR1_NOS(base, ctx, 1);
203
204 SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */
205 SET_TTBR0_IRGNL(base, ctx, 1);
206
207 SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */
208 SET_TTBR1_IRGNL(base, ctx, 1);
209
210 SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */
211 SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */
212#endif
213
214 /* Enable the MMU */ 311 /* Enable the MMU */
215 SET_M(base, ctx, 1); 312 SET_M(base, ctx, 1);
216} 313}
@@ -227,13 +324,6 @@ static struct iommu_domain *msm_iommu_domain_alloc(unsigned type)
227 goto fail_nomem; 324 goto fail_nomem;
228 325
229 INIT_LIST_HEAD(&priv->list_attached); 326 INIT_LIST_HEAD(&priv->list_attached);
230 priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL,
231 get_order(SZ_16K));
232
233 if (!priv->pgtable)
234 goto fail_nomem;
235
236 memset(priv->pgtable, 0, SZ_16K);
237 327
238 priv->domain.geometry.aperture_start = 0; 328 priv->domain.geometry.aperture_start = 0;
239 priv->domain.geometry.aperture_end = (1ULL << 32) - 1; 329 priv->domain.geometry.aperture_end = (1ULL << 32) - 1;
@@ -250,304 +340,137 @@ static void msm_iommu_domain_free(struct iommu_domain *domain)
250{ 340{
251 struct msm_priv *priv; 341 struct msm_priv *priv;
252 unsigned long flags; 342 unsigned long flags;
253 unsigned long *fl_table;
254 int i;
255 343
256 spin_lock_irqsave(&msm_iommu_lock, flags); 344 spin_lock_irqsave(&msm_iommu_lock, flags);
257 priv = to_msm_priv(domain); 345 priv = to_msm_priv(domain);
346 kfree(priv);
347 spin_unlock_irqrestore(&msm_iommu_lock, flags);
348}
258 349
259 fl_table = priv->pgtable; 350static int msm_iommu_domain_config(struct msm_priv *priv)
260 351{
261 for (i = 0; i < NUM_FL_PTE; i++) 352 spin_lock_init(&priv->pgtlock);
262 if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) 353
263 free_page((unsigned long) __va(((fl_table[i]) & 354 priv->cfg = (struct io_pgtable_cfg) {
264 FL_BASE_MASK))); 355 .quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP,
356 .pgsize_bitmap = msm_iommu_ops.pgsize_bitmap,
357 .ias = 32,
358 .oas = 32,
359 .tlb = &msm_iommu_gather_ops,
360 .iommu_dev = priv->dev,
361 };
362
363 priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv);
364 if (!priv->iop) {
365 dev_err(priv->dev, "Failed to allocate pgtable\n");
366 return -EINVAL;
367 }
265 368
266 free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); 369 msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap;
267 priv->pgtable = NULL;
268 370
269 kfree(priv); 371 return 0;
270 spin_unlock_irqrestore(&msm_iommu_lock, flags);
271} 372}
272 373
273static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) 374static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
274{ 375{
275 struct msm_priv *priv;
276 struct msm_iommu_ctx_dev *ctx_dev;
277 struct msm_iommu_drvdata *iommu_drvdata;
278 struct msm_iommu_ctx_drvdata *ctx_drvdata;
279 struct msm_iommu_ctx_drvdata *tmp_drvdata;
280 int ret = 0; 376 int ret = 0;
281 unsigned long flags; 377 unsigned long flags;
378 struct msm_iommu_dev *iommu;
379 struct msm_priv *priv = to_msm_priv(domain);
380 struct msm_iommu_ctx_dev *master;
282 381
283 spin_lock_irqsave(&msm_iommu_lock, flags); 382 priv->dev = dev;
284 383 msm_iommu_domain_config(priv);
285 priv = to_msm_priv(domain);
286
287 if (!dev) {
288 ret = -EINVAL;
289 goto fail;
290 }
291
292 iommu_drvdata = dev_get_drvdata(dev->parent);
293 ctx_drvdata = dev_get_drvdata(dev);
294 ctx_dev = dev->platform_data;
295
296 if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) {
297 ret = -EINVAL;
298 goto fail;
299 }
300
301 if (!list_empty(&ctx_drvdata->attached_elm)) {
302 ret = -EBUSY;
303 goto fail;
304 }
305 384
306 list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) 385 spin_lock_irqsave(&msm_iommu_lock, flags);
307 if (tmp_drvdata == ctx_drvdata) { 386 list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) {
308 ret = -EBUSY; 387 master = list_first_entry(&iommu->ctx_list,
309 goto fail; 388 struct msm_iommu_ctx_dev,
389 list);
390 if (master->of_node == dev->of_node) {
391 ret = __enable_clocks(iommu);
392 if (ret)
393 goto fail;
394
395 list_for_each_entry(master, &iommu->ctx_list, list) {
396 if (master->num) {
397 dev_err(dev, "domain already attached");
398 ret = -EEXIST;
399 goto fail;
400 }
401 master->num =
402 msm_iommu_alloc_ctx(iommu->context_map,
403 0, iommu->ncb);
404 if (IS_ERR_VALUE(master->num)) {
405 ret = -ENODEV;
406 goto fail;
407 }
408 config_mids(iommu, master);
409 __program_context(iommu->base, master->num,
410 priv);
411 }
412 __disable_clocks(iommu);
413 list_add(&iommu->dom_node, &priv->list_attached);
310 } 414 }
311 415 }
312 ret = __enable_clocks(iommu_drvdata);
313 if (ret)
314 goto fail;
315
316 __program_context(iommu_drvdata->base, ctx_dev->num,
317 __pa(priv->pgtable));
318
319 __disable_clocks(iommu_drvdata);
320 list_add(&(ctx_drvdata->attached_elm), &priv->list_attached);
321 ret = __flush_iotlb(domain);
322 416
323fail: 417fail:
324 spin_unlock_irqrestore(&msm_iommu_lock, flags); 418 spin_unlock_irqrestore(&msm_iommu_lock, flags);
419
325 return ret; 420 return ret;
326} 421}
327 422
328static void msm_iommu_detach_dev(struct iommu_domain *domain, 423static void msm_iommu_detach_dev(struct iommu_domain *domain,
329 struct device *dev) 424 struct device *dev)
330{ 425{
331 struct msm_priv *priv; 426 struct msm_priv *priv = to_msm_priv(domain);
332 struct msm_iommu_ctx_dev *ctx_dev;
333 struct msm_iommu_drvdata *iommu_drvdata;
334 struct msm_iommu_ctx_drvdata *ctx_drvdata;
335 unsigned long flags; 427 unsigned long flags;
428 struct msm_iommu_dev *iommu;
429 struct msm_iommu_ctx_dev *master;
336 int ret; 430 int ret;
337 431
338 spin_lock_irqsave(&msm_iommu_lock, flags); 432 free_io_pgtable_ops(priv->iop);
339 priv = to_msm_priv(domain);
340
341 if (!dev)
342 goto fail;
343
344 iommu_drvdata = dev_get_drvdata(dev->parent);
345 ctx_drvdata = dev_get_drvdata(dev);
346 ctx_dev = dev->platform_data;
347
348 if (!iommu_drvdata || !ctx_drvdata || !ctx_dev)
349 goto fail;
350
351 ret = __flush_iotlb(domain);
352 if (ret)
353 goto fail;
354
355 ret = __enable_clocks(iommu_drvdata);
356 if (ret)
357 goto fail;
358 433
359 __reset_context(iommu_drvdata->base, ctx_dev->num); 434 spin_lock_irqsave(&msm_iommu_lock, flags);
360 __disable_clocks(iommu_drvdata); 435 list_for_each_entry(iommu, &priv->list_attached, dom_node) {
361 list_del_init(&ctx_drvdata->attached_elm); 436 ret = __enable_clocks(iommu);
437 if (ret)
438 goto fail;
362 439
440 list_for_each_entry(master, &iommu->ctx_list, list) {
441 msm_iommu_free_ctx(iommu->context_map, master->num);
442 __reset_context(iommu->base, master->num);
443 }
444 __disable_clocks(iommu);
445 }
363fail: 446fail:
364 spin_unlock_irqrestore(&msm_iommu_lock, flags); 447 spin_unlock_irqrestore(&msm_iommu_lock, flags);
365} 448}
366 449
367static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, 450static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
368 phys_addr_t pa, size_t len, int prot) 451 phys_addr_t pa, size_t len, int prot)
369{ 452{
370 struct msm_priv *priv; 453 struct msm_priv *priv = to_msm_priv(domain);
371 unsigned long flags; 454 unsigned long flags;
372 unsigned long *fl_table; 455 int ret;
373 unsigned long *fl_pte;
374 unsigned long fl_offset;
375 unsigned long *sl_table;
376 unsigned long *sl_pte;
377 unsigned long sl_offset;
378 unsigned int pgprot;
379 int ret = 0, tex, sh;
380
381 spin_lock_irqsave(&msm_iommu_lock, flags);
382
383 sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0;
384 tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK];
385
386 if (tex < 0 || tex > NUM_TEX_CLASS - 1) {
387 ret = -EINVAL;
388 goto fail;
389 }
390
391 priv = to_msm_priv(domain);
392
393 fl_table = priv->pgtable;
394
395 if (len != SZ_16M && len != SZ_1M &&
396 len != SZ_64K && len != SZ_4K) {
397 pr_debug("Bad size: %d\n", len);
398 ret = -EINVAL;
399 goto fail;
400 }
401
402 if (!fl_table) {
403 pr_debug("Null page table\n");
404 ret = -EINVAL;
405 goto fail;
406 }
407
408 if (len == SZ_16M || len == SZ_1M) {
409 pgprot = sh ? FL_SHARED : 0;
410 pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0;
411 pgprot |= tex & 0x02 ? FL_CACHEABLE : 0;
412 pgprot |= tex & 0x04 ? FL_TEX0 : 0;
413 } else {
414 pgprot = sh ? SL_SHARED : 0;
415 pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0;
416 pgprot |= tex & 0x02 ? SL_CACHEABLE : 0;
417 pgprot |= tex & 0x04 ? SL_TEX0 : 0;
418 }
419
420 fl_offset = FL_OFFSET(va); /* Upper 12 bits */
421 fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
422
423 if (len == SZ_16M) {
424 int i = 0;
425 for (i = 0; i < 16; i++)
426 *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION |
427 FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT |
428 FL_SHARED | FL_NG | pgprot;
429 }
430
431 if (len == SZ_1M)
432 *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG |
433 FL_TYPE_SECT | FL_SHARED | pgprot;
434
435 /* Need a 2nd level table */
436 if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) {
437 unsigned long *sl;
438 sl = (unsigned long *) __get_free_pages(GFP_ATOMIC,
439 get_order(SZ_4K));
440
441 if (!sl) {
442 pr_debug("Could not allocate second level table\n");
443 ret = -ENOMEM;
444 goto fail;
445 }
446
447 memset(sl, 0, SZ_4K);
448 *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE);
449 }
450
451 sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
452 sl_offset = SL_OFFSET(va);
453 sl_pte = sl_table + sl_offset;
454
455
456 if (len == SZ_4K)
457 *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG |
458 SL_SHARED | SL_TYPE_SMALL | pgprot;
459
460 if (len == SZ_64K) {
461 int i;
462 456
463 for (i = 0; i < 16; i++) 457 spin_lock_irqsave(&priv->pgtlock, flags);
464 *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | 458 ret = priv->iop->map(priv->iop, iova, pa, len, prot);
465 SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; 459 spin_unlock_irqrestore(&priv->pgtlock, flags);
466 }
467 460
468 ret = __flush_iotlb(domain);
469fail:
470 spin_unlock_irqrestore(&msm_iommu_lock, flags);
471 return ret; 461 return ret;
472} 462}
473 463
474static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, 464static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
475 size_t len) 465 size_t len)
476{ 466{
477 struct msm_priv *priv; 467 struct msm_priv *priv = to_msm_priv(domain);
478 unsigned long flags; 468 unsigned long flags;
479 unsigned long *fl_table;
480 unsigned long *fl_pte;
481 unsigned long fl_offset;
482 unsigned long *sl_table;
483 unsigned long *sl_pte;
484 unsigned long sl_offset;
485 int i, ret = 0;
486
487 spin_lock_irqsave(&msm_iommu_lock, flags);
488
489 priv = to_msm_priv(domain);
490 469
491 fl_table = priv->pgtable; 470 spin_lock_irqsave(&priv->pgtlock, flags);
471 len = priv->iop->unmap(priv->iop, iova, len);
472 spin_unlock_irqrestore(&priv->pgtlock, flags);
492 473
493 if (len != SZ_16M && len != SZ_1M &&
494 len != SZ_64K && len != SZ_4K) {
495 pr_debug("Bad length: %d\n", len);
496 goto fail;
497 }
498
499 if (!fl_table) {
500 pr_debug("Null page table\n");
501 goto fail;
502 }
503
504 fl_offset = FL_OFFSET(va); /* Upper 12 bits */
505 fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */
506
507 if (*fl_pte == 0) {
508 pr_debug("First level PTE is 0\n");
509 goto fail;
510 }
511
512 /* Unmap supersection */
513 if (len == SZ_16M)
514 for (i = 0; i < 16; i++)
515 *(fl_pte+i) = 0;
516
517 if (len == SZ_1M)
518 *fl_pte = 0;
519
520 sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK));
521 sl_offset = SL_OFFSET(va);
522 sl_pte = sl_table + sl_offset;
523
524 if (len == SZ_64K) {
525 for (i = 0; i < 16; i++)
526 *(sl_pte+i) = 0;
527 }
528
529 if (len == SZ_4K)
530 *sl_pte = 0;
531
532 if (len == SZ_4K || len == SZ_64K) {
533 int used = 0;
534
535 for (i = 0; i < NUM_SL_PTE; i++)
536 if (sl_table[i])
537 used = 1;
538 if (!used) {
539 free_page((unsigned long)sl_table);
540 *fl_pte = 0;
541 }
542 }
543
544 ret = __flush_iotlb(domain);
545
546fail:
547 spin_unlock_irqrestore(&msm_iommu_lock, flags);
548
549 /* the IOMMU API requires us to return how many bytes were unmapped */
550 len = ret ? 0 : len;
551 return len; 474 return len;
552} 475}
553 476
@@ -555,47 +478,46 @@ static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain,
555 dma_addr_t va) 478 dma_addr_t va)
556{ 479{
557 struct msm_priv *priv; 480 struct msm_priv *priv;
558 struct msm_iommu_drvdata *iommu_drvdata; 481 struct msm_iommu_dev *iommu;
559 struct msm_iommu_ctx_drvdata *ctx_drvdata; 482 struct msm_iommu_ctx_dev *master;
560 unsigned int par; 483 unsigned int par;
561 unsigned long flags; 484 unsigned long flags;
562 void __iomem *base;
563 phys_addr_t ret = 0; 485 phys_addr_t ret = 0;
564 int ctx;
565 486
566 spin_lock_irqsave(&msm_iommu_lock, flags); 487 spin_lock_irqsave(&msm_iommu_lock, flags);
567 488
568 priv = to_msm_priv(domain); 489 priv = to_msm_priv(domain);
569 if (list_empty(&priv->list_attached)) 490 iommu = list_first_entry(&priv->list_attached,
570 goto fail; 491 struct msm_iommu_dev, dom_node);
571 492
572 ctx_drvdata = list_entry(priv->list_attached.next, 493 if (list_empty(&iommu->ctx_list))
573 struct msm_iommu_ctx_drvdata, attached_elm); 494 goto fail;
574 iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent);
575 495
576 base = iommu_drvdata->base; 496 master = list_first_entry(&iommu->ctx_list,
577 ctx = ctx_drvdata->num; 497 struct msm_iommu_ctx_dev, list);
498 if (!master)
499 goto fail;
578 500
579 ret = __enable_clocks(iommu_drvdata); 501 ret = __enable_clocks(iommu);
580 if (ret) 502 if (ret)
581 goto fail; 503 goto fail;
582 504
583 /* Invalidate context TLB */ 505 /* Invalidate context TLB */
584 SET_CTX_TLBIALL(base, ctx, 0); 506 SET_CTX_TLBIALL(iommu->base, master->num, 0);
585 SET_V2PPR(base, ctx, va & V2Pxx_VA); 507 SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA);
586 508
587 par = GET_PAR(base, ctx); 509 par = GET_PAR(iommu->base, master->num);
588 510
589 /* We are dealing with a supersection */ 511 /* We are dealing with a supersection */
590 if (GET_NOFAULT_SS(base, ctx)) 512 if (GET_NOFAULT_SS(iommu->base, master->num))
591 ret = (par & 0xFF000000) | (va & 0x00FFFFFF); 513 ret = (par & 0xFF000000) | (va & 0x00FFFFFF);
592 else /* Upper 20 bits from PAR, lower 12 from VA */ 514 else /* Upper 20 bits from PAR, lower 12 from VA */
593 ret = (par & 0xFFFFF000) | (va & 0x00000FFF); 515 ret = (par & 0xFFFFF000) | (va & 0x00000FFF);
594 516
595 if (GET_FAULT(base, ctx)) 517 if (GET_FAULT(iommu->base, master->num))
596 ret = 0; 518 ret = 0;
597 519
598 __disable_clocks(iommu_drvdata); 520 __disable_clocks(iommu);
599fail: 521fail:
600 spin_unlock_irqrestore(&msm_iommu_lock, flags); 522 spin_unlock_irqrestore(&msm_iommu_lock, flags);
601 return ret; 523 return ret;
@@ -629,49 +551,92 @@ static void print_ctx_regs(void __iomem *base, int ctx)
629 GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); 551 GET_TTBR0(base, ctx), GET_TTBR1(base, ctx));
630 pr_err("SCTLR = %08x ACTLR = %08x\n", 552 pr_err("SCTLR = %08x ACTLR = %08x\n",
631 GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); 553 GET_SCTLR(base, ctx), GET_ACTLR(base, ctx));
632 pr_err("PRRR = %08x NMRR = %08x\n", 554}
633 GET_PRRR(base, ctx), GET_NMRR(base, ctx)); 555
556static void insert_iommu_master(struct device *dev,
557 struct msm_iommu_dev **iommu,
558 struct of_phandle_args *spec)
559{
560 struct msm_iommu_ctx_dev *master = dev->archdata.iommu;
561 int sid;
562
563 if (list_empty(&(*iommu)->ctx_list)) {
564 master = kzalloc(sizeof(*master), GFP_ATOMIC);
565 master->of_node = dev->of_node;
566 list_add(&master->list, &(*iommu)->ctx_list);
567 dev->archdata.iommu = master;
568 }
569
570 for (sid = 0; sid < master->num_mids; sid++)
571 if (master->mids[sid] == spec->args[0]) {
572 dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n",
573 sid);
574 return;
575 }
576
577 master->mids[master->num_mids++] = spec->args[0];
578}
579
580static int qcom_iommu_of_xlate(struct device *dev,
581 struct of_phandle_args *spec)
582{
583 struct msm_iommu_dev *iommu;
584 unsigned long flags;
585 int ret = 0;
586
587 spin_lock_irqsave(&msm_iommu_lock, flags);
588 list_for_each_entry(iommu, &qcom_iommu_devices, dev_node)
589 if (iommu->dev->of_node == spec->np)
590 break;
591
592 if (!iommu || iommu->dev->of_node != spec->np) {
593 ret = -ENODEV;
594 goto fail;
595 }
596
597 insert_iommu_master(dev, &iommu, spec);
598fail:
599 spin_unlock_irqrestore(&msm_iommu_lock, flags);
600
601 return ret;
634} 602}
635 603
636irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) 604irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id)
637{ 605{
638 struct msm_iommu_drvdata *drvdata = dev_id; 606 struct msm_iommu_dev *iommu = dev_id;
639 void __iomem *base;
640 unsigned int fsr; 607 unsigned int fsr;
641 int i, ret; 608 int i, ret;
642 609
643 spin_lock(&msm_iommu_lock); 610 spin_lock(&msm_iommu_lock);
644 611
645 if (!drvdata) { 612 if (!iommu) {
646 pr_err("Invalid device ID in context interrupt handler\n"); 613 pr_err("Invalid device ID in context interrupt handler\n");
647 goto fail; 614 goto fail;
648 } 615 }
649 616
650 base = drvdata->base;
651
652 pr_err("Unexpected IOMMU page fault!\n"); 617 pr_err("Unexpected IOMMU page fault!\n");
653 pr_err("base = %08x\n", (unsigned int) base); 618 pr_err("base = %08x\n", (unsigned int)iommu->base);
654 619
655 ret = __enable_clocks(drvdata); 620 ret = __enable_clocks(iommu);
656 if (ret) 621 if (ret)
657 goto fail; 622 goto fail;
658 623
659 for (i = 0; i < drvdata->ncb; i++) { 624 for (i = 0; i < iommu->ncb; i++) {
660 fsr = GET_FSR(base, i); 625 fsr = GET_FSR(iommu->base, i);
661 if (fsr) { 626 if (fsr) {
662 pr_err("Fault occurred in context %d.\n", i); 627 pr_err("Fault occurred in context %d.\n", i);
663 pr_err("Interesting registers:\n"); 628 pr_err("Interesting registers:\n");
664 print_ctx_regs(base, i); 629 print_ctx_regs(iommu->base, i);
665 SET_FSR(base, i, 0x4000000F); 630 SET_FSR(iommu->base, i, 0x4000000F);
666 } 631 }
667 } 632 }
668 __disable_clocks(drvdata); 633 __disable_clocks(iommu);
669fail: 634fail:
670 spin_unlock(&msm_iommu_lock); 635 spin_unlock(&msm_iommu_lock);
671 return 0; 636 return 0;
672} 637}
673 638
674static const struct iommu_ops msm_iommu_ops = { 639static struct iommu_ops msm_iommu_ops = {
675 .capable = msm_iommu_capable, 640 .capable = msm_iommu_capable,
676 .domain_alloc = msm_iommu_domain_alloc, 641 .domain_alloc = msm_iommu_domain_alloc,
677 .domain_free = msm_iommu_domain_free, 642 .domain_free = msm_iommu_domain_free,
@@ -682,54 +647,163 @@ static const struct iommu_ops msm_iommu_ops = {
682 .map_sg = default_iommu_map_sg, 647 .map_sg = default_iommu_map_sg,
683 .iova_to_phys = msm_iommu_iova_to_phys, 648 .iova_to_phys = msm_iommu_iova_to_phys,
684 .pgsize_bitmap = MSM_IOMMU_PGSIZES, 649 .pgsize_bitmap = MSM_IOMMU_PGSIZES,
650 .of_xlate = qcom_iommu_of_xlate,
685}; 651};
686 652
687static int __init get_tex_class(int icp, int ocp, int mt, int nos) 653static int msm_iommu_probe(struct platform_device *pdev)
688{ 654{
689 int i = 0; 655 struct resource *r;
690 unsigned int prrr = 0; 656 struct msm_iommu_dev *iommu;
691 unsigned int nmrr = 0; 657 int ret, par, val;
692 int c_icp, c_ocp, c_mt, c_nos; 658
693 659 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
694 RCP15_PRRR(prrr); 660 if (!iommu)
695 RCP15_NMRR(nmrr); 661 return -ENODEV;
696 662
697 for (i = 0; i < NUM_TEX_CLASS; i++) { 663 iommu->dev = &pdev->dev;
698 c_nos = PRRR_NOS(prrr, i); 664 INIT_LIST_HEAD(&iommu->ctx_list);
699 c_mt = PRRR_MT(prrr, i); 665
700 c_icp = NMRR_ICP(nmrr, i); 666 iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk");
701 c_ocp = NMRR_OCP(nmrr, i); 667 if (IS_ERR(iommu->pclk)) {
702 668 dev_err(iommu->dev, "could not get smmu_pclk\n");
703 if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) 669 return PTR_ERR(iommu->pclk);
704 return i; 670 }
671
672 ret = clk_prepare(iommu->pclk);
673 if (ret) {
674 dev_err(iommu->dev, "could not prepare smmu_pclk\n");
675 return ret;
676 }
677
678 iommu->clk = devm_clk_get(iommu->dev, "iommu_clk");
679 if (IS_ERR(iommu->clk)) {
680 dev_err(iommu->dev, "could not get iommu_clk\n");
681 clk_unprepare(iommu->pclk);
682 return PTR_ERR(iommu->clk);
683 }
684
685 ret = clk_prepare(iommu->clk);
686 if (ret) {
687 dev_err(iommu->dev, "could not prepare iommu_clk\n");
688 clk_unprepare(iommu->pclk);
689 return ret;
690 }
691
692 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
693 iommu->base = devm_ioremap_resource(iommu->dev, r);
694 if (IS_ERR(iommu->base)) {
695 dev_err(iommu->dev, "could not get iommu base\n");
696 ret = PTR_ERR(iommu->base);
697 goto fail;
705 } 698 }
706 699
707 return -ENODEV; 700 iommu->irq = platform_get_irq(pdev, 0);
701 if (iommu->irq < 0) {
702 dev_err(iommu->dev, "could not get iommu irq\n");
703 ret = -ENODEV;
704 goto fail;
705 }
706
707 ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val);
708 if (ret) {
709 dev_err(iommu->dev, "could not get ncb\n");
710 goto fail;
711 }
712 iommu->ncb = val;
713
714 msm_iommu_reset(iommu->base, iommu->ncb);
715 SET_M(iommu->base, 0, 1);
716 SET_PAR(iommu->base, 0, 0);
717 SET_V2PCFG(iommu->base, 0, 1);
718 SET_V2PPR(iommu->base, 0, 0);
719 par = GET_PAR(iommu->base, 0);
720 SET_V2PCFG(iommu->base, 0, 0);
721 SET_M(iommu->base, 0, 0);
722
723 if (!par) {
724 pr_err("Invalid PAR value detected\n");
725 ret = -ENODEV;
726 goto fail;
727 }
728
729 ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL,
730 msm_iommu_fault_handler,
731 IRQF_ONESHOT | IRQF_SHARED,
732 "msm_iommu_secure_irpt_handler",
733 iommu);
734 if (ret) {
735 pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret);
736 goto fail;
737 }
738
739 list_add(&iommu->dev_node, &qcom_iommu_devices);
740 of_iommu_set_ops(pdev->dev.of_node, &msm_iommu_ops);
741
742 pr_info("device mapped at %p, irq %d with %d ctx banks\n",
743 iommu->base, iommu->irq, iommu->ncb);
744
745 return ret;
746fail:
747 clk_unprepare(iommu->clk);
748 clk_unprepare(iommu->pclk);
749 return ret;
750}
751
752static const struct of_device_id msm_iommu_dt_match[] = {
753 { .compatible = "qcom,apq8064-iommu" },
754 {}
755};
756
757static int msm_iommu_remove(struct platform_device *pdev)
758{
759 struct msm_iommu_dev *iommu = platform_get_drvdata(pdev);
760
761 clk_unprepare(iommu->clk);
762 clk_unprepare(iommu->pclk);
763 return 0;
708} 764}
709 765
710static void __init setup_iommu_tex_classes(void) 766static struct platform_driver msm_iommu_driver = {
767 .driver = {
768 .name = "msm_iommu",
769 .of_match_table = msm_iommu_dt_match,
770 },
771 .probe = msm_iommu_probe,
772 .remove = msm_iommu_remove,
773};
774
775static int __init msm_iommu_driver_init(void)
711{ 776{
712 msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = 777 int ret;
713 get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1);
714 778
715 msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = 779 ret = platform_driver_register(&msm_iommu_driver);
716 get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); 780 if (ret != 0)
781 pr_err("Failed to register IOMMU driver\n");
717 782
718 msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = 783 return ret;
719 get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); 784}
720 785
721 msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = 786static void __exit msm_iommu_driver_exit(void)
722 get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); 787{
788 platform_driver_unregister(&msm_iommu_driver);
723} 789}
724 790
791subsys_initcall(msm_iommu_driver_init);
792module_exit(msm_iommu_driver_exit);
793
725static int __init msm_iommu_init(void) 794static int __init msm_iommu_init(void)
726{ 795{
727 setup_iommu_tex_classes();
728 bus_set_iommu(&platform_bus_type, &msm_iommu_ops); 796 bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
729 return 0; 797 return 0;
730} 798}
731 799
732subsys_initcall(msm_iommu_init); 800static int __init msm_iommu_of_setup(struct device_node *np)
801{
802 msm_iommu_init();
803 return 0;
804}
805
806IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu", msm_iommu_of_setup);
733 807
734MODULE_LICENSE("GPL v2"); 808MODULE_LICENSE("GPL v2");
735MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); 809MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/msm_iommu.h b/drivers/iommu/msm_iommu.h
index 5c7c955e6d25..4ca25d50d679 100644
--- a/drivers/iommu/msm_iommu.h
+++ b/drivers/iommu/msm_iommu.h
@@ -42,74 +42,53 @@
42 */ 42 */
43#define MAX_NUM_MIDS 32 43#define MAX_NUM_MIDS 32
44 44
45/* Maximum number of context banks that can be present in IOMMU */
46#define IOMMU_MAX_CBS 128
47
45/** 48/**
46 * struct msm_iommu_dev - a single IOMMU hardware instance 49 * struct msm_iommu_dev - a single IOMMU hardware instance
47 * name Human-readable name given to this IOMMU HW instance
48 * ncb Number of context banks present on this IOMMU HW instance 50 * ncb Number of context banks present on this IOMMU HW instance
51 * dev: IOMMU device
52 * irq: Interrupt number
53 * clk: The bus clock for this IOMMU hardware instance
54 * pclk: The clock for the IOMMU bus interconnect
55 * dev_node: list head in qcom_iommu_device_list
56 * dom_node: list head for domain
57 * ctx_list: list of 'struct msm_iommu_ctx_dev'
58 * context_map: Bitmap to track allocated context banks
49 */ 59 */
50struct msm_iommu_dev { 60struct msm_iommu_dev {
51 const char *name; 61 void __iomem *base;
52 int ncb; 62 int ncb;
63 struct device *dev;
64 int irq;
65 struct clk *clk;
66 struct clk *pclk;
67 struct list_head dev_node;
68 struct list_head dom_node;
69 struct list_head ctx_list;
70 DECLARE_BITMAP(context_map, IOMMU_MAX_CBS);
53}; 71};
54 72
55/** 73/**
56 * struct msm_iommu_ctx_dev - an IOMMU context bank instance 74 * struct msm_iommu_ctx_dev - an IOMMU context bank instance
57 * name Human-readable name given to this context bank 75 * of_node node ptr of client device
58 * num Index of this context bank within the hardware 76 * num Index of this context bank within the hardware
59 * mids List of Machine IDs that are to be mapped into this context 77 * mids List of Machine IDs that are to be mapped into this context
60 * bank, terminated by -1. The MID is a set of signals on the 78 * bank, terminated by -1. The MID is a set of signals on the
61 * AXI bus that identifies the function associated with a specific 79 * AXI bus that identifies the function associated with a specific
62 * memory request. (See ARM spec). 80 * memory request. (See ARM spec).
81 * num_mids Total number of mids
82 * node list head in ctx_list
63 */ 83 */
64struct msm_iommu_ctx_dev { 84struct msm_iommu_ctx_dev {
65 const char *name; 85 struct device_node *of_node;
66 int num; 86 int num;
67 int mids[MAX_NUM_MIDS]; 87 int mids[MAX_NUM_MIDS];
88 int num_mids;
89 struct list_head list;
68}; 90};
69 91
70
71/**
72 * struct msm_iommu_drvdata - A single IOMMU hardware instance
73 * @base: IOMMU config port base address (VA)
74 * @ncb The number of contexts on this IOMMU
75 * @irq: Interrupt number
76 * @clk: The bus clock for this IOMMU hardware instance
77 * @pclk: The clock for the IOMMU bus interconnect
78 *
79 * A msm_iommu_drvdata holds the global driver data about a single piece
80 * of an IOMMU hardware instance.
81 */
82struct msm_iommu_drvdata {
83 void __iomem *base;
84 int irq;
85 int ncb;
86 struct clk *clk;
87 struct clk *pclk;
88};
89
90/**
91 * struct msm_iommu_ctx_drvdata - an IOMMU context bank instance
92 * @num: Hardware context number of this context
93 * @pdev: Platform device associated wit this HW instance
94 * @attached_elm: List element for domains to track which devices are
95 * attached to them
96 *
97 * A msm_iommu_ctx_drvdata holds the driver data for a single context bank
98 * within each IOMMU hardware instance
99 */
100struct msm_iommu_ctx_drvdata {
101 int num;
102 struct platform_device *pdev;
103 struct list_head attached_elm;
104};
105
106/*
107 * Look up an IOMMU context device by its context name. NULL if none found.
108 * Useful for testing and drivers that do not yet fully have IOMMU stuff in
109 * their platform devices.
110 */
111struct device *msm_iommu_get_ctx(const char *ctx_name);
112
113/* 92/*
114 * Interrupt handler for the IOMMU context fault interrupt. Hooking the 93 * Interrupt handler for the IOMMU context fault interrupt. Hooking the
115 * interrupt is not supported in the API yet, but this will print an error 94 * interrupt is not supported in the API yet, but this will print an error
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
deleted file mode 100644
index 4b09e815accf..000000000000
--- a/drivers/iommu/msm_iommu_dev.c
+++ /dev/null
@@ -1,381 +0,0 @@
1/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/platform_device.h>
23#include <linux/io.h>
24#include <linux/clk.h>
25#include <linux/iommu.h>
26#include <linux/interrupt.h>
27#include <linux/err.h>
28#include <linux/slab.h>
29
30#include "msm_iommu_hw-8xxx.h"
31#include "msm_iommu.h"
32
33struct iommu_ctx_iter_data {
34 /* input */
35 const char *name;
36
37 /* output */
38 struct device *dev;
39};
40
41static struct platform_device *msm_iommu_root_dev;
42
43static int each_iommu_ctx(struct device *dev, void *data)
44{
45 struct iommu_ctx_iter_data *res = data;
46 struct msm_iommu_ctx_dev *c = dev->platform_data;
47
48 if (!res || !c || !c->name || !res->name)
49 return -EINVAL;
50
51 if (!strcmp(res->name, c->name)) {
52 res->dev = dev;
53 return 1;
54 }
55 return 0;
56}
57
58static int each_iommu(struct device *dev, void *data)
59{
60 return device_for_each_child(dev, data, each_iommu_ctx);
61}
62
63struct device *msm_iommu_get_ctx(const char *ctx_name)
64{
65 struct iommu_ctx_iter_data r;
66 int found;
67
68 if (!msm_iommu_root_dev) {
69 pr_err("No root IOMMU device.\n");
70 goto fail;
71 }
72
73 r.name = ctx_name;
74 found = device_for_each_child(&msm_iommu_root_dev->dev, &r, each_iommu);
75
76 if (!found) {
77 pr_err("Could not find context <%s>\n", ctx_name);
78 goto fail;
79 }
80
81 return r.dev;
82fail:
83 return NULL;
84}
85EXPORT_SYMBOL(msm_iommu_get_ctx);
86
87static void msm_iommu_reset(void __iomem *base, int ncb)
88{
89 int ctx;
90
91 SET_RPUE(base, 0);
92 SET_RPUEIE(base, 0);
93 SET_ESRRESTORE(base, 0);
94 SET_TBE(base, 0);
95 SET_CR(base, 0);
96 SET_SPDMBE(base, 0);
97 SET_TESTBUSCR(base, 0);
98 SET_TLBRSW(base, 0);
99 SET_GLOBAL_TLBIALL(base, 0);
100 SET_RPU_ACR(base, 0);
101 SET_TLBLKCRWE(base, 1);
102
103 for (ctx = 0; ctx < ncb; ctx++) {
104 SET_BPRCOSH(base, ctx, 0);
105 SET_BPRCISH(base, ctx, 0);
106 SET_BPRCNSH(base, ctx, 0);
107 SET_BPSHCFG(base, ctx, 0);
108 SET_BPMTCFG(base, ctx, 0);
109 SET_ACTLR(base, ctx, 0);
110 SET_SCTLR(base, ctx, 0);
111 SET_FSRRESTORE(base, ctx, 0);
112 SET_TTBR0(base, ctx, 0);
113 SET_TTBR1(base, ctx, 0);
114 SET_TTBCR(base, ctx, 0);
115 SET_BFBCR(base, ctx, 0);
116 SET_PAR(base, ctx, 0);
117 SET_FAR(base, ctx, 0);
118 SET_CTX_TLBIALL(base, ctx, 0);
119 SET_TLBFLPTER(base, ctx, 0);
120 SET_TLBSLPTER(base, ctx, 0);
121 SET_TLBLKCR(base, ctx, 0);
122 SET_PRRR(base, ctx, 0);
123 SET_NMRR(base, ctx, 0);
124 SET_CONTEXTIDR(base, ctx, 0);
125 }
126}
127
128static int msm_iommu_probe(struct platform_device *pdev)
129{
130 struct resource *r;
131 struct clk *iommu_clk;
132 struct clk *iommu_pclk;
133 struct msm_iommu_drvdata *drvdata;
134 struct msm_iommu_dev *iommu_dev = dev_get_platdata(&pdev->dev);
135 void __iomem *regs_base;
136 int ret, irq, par;
137
138 if (pdev->id == -1) {
139 msm_iommu_root_dev = pdev;
140 return 0;
141 }
142
143 drvdata = kzalloc(sizeof(*drvdata), GFP_KERNEL);
144
145 if (!drvdata) {
146 ret = -ENOMEM;
147 goto fail;
148 }
149
150 if (!iommu_dev) {
151 ret = -ENODEV;
152 goto fail;
153 }
154
155 iommu_pclk = clk_get(NULL, "smmu_pclk");
156 if (IS_ERR(iommu_pclk)) {
157 ret = -ENODEV;
158 goto fail;
159 }
160
161 ret = clk_prepare_enable(iommu_pclk);
162 if (ret)
163 goto fail_enable;
164
165 iommu_clk = clk_get(&pdev->dev, "iommu_clk");
166
167 if (!IS_ERR(iommu_clk)) {
168 if (clk_get_rate(iommu_clk) == 0)
169 clk_set_rate(iommu_clk, 1);
170
171 ret = clk_prepare_enable(iommu_clk);
172 if (ret) {
173 clk_put(iommu_clk);
174 goto fail_pclk;
175 }
176 } else
177 iommu_clk = NULL;
178
179 r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase");
180 regs_base = devm_ioremap_resource(&pdev->dev, r);
181 if (IS_ERR(regs_base)) {
182 ret = PTR_ERR(regs_base);
183 goto fail_clk;
184 }
185
186 irq = platform_get_irq_byname(pdev, "secure_irq");
187 if (irq < 0) {
188 ret = -ENODEV;
189 goto fail_clk;
190 }
191
192 msm_iommu_reset(regs_base, iommu_dev->ncb);
193
194 SET_M(regs_base, 0, 1);
195 SET_PAR(regs_base, 0, 0);
196 SET_V2PCFG(regs_base, 0, 1);
197 SET_V2PPR(regs_base, 0, 0);
198 par = GET_PAR(regs_base, 0);
199 SET_V2PCFG(regs_base, 0, 0);
200 SET_M(regs_base, 0, 0);
201
202 if (!par) {
203 pr_err("%s: Invalid PAR value detected\n", iommu_dev->name);
204 ret = -ENODEV;
205 goto fail_clk;
206 }
207
208 ret = request_irq(irq, msm_iommu_fault_handler, 0,
209 "msm_iommu_secure_irpt_handler", drvdata);
210 if (ret) {
211 pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
212 goto fail_clk;
213 }
214
215
216 drvdata->pclk = iommu_pclk;
217 drvdata->clk = iommu_clk;
218 drvdata->base = regs_base;
219 drvdata->irq = irq;
220 drvdata->ncb = iommu_dev->ncb;
221
222 pr_info("device %s mapped at %p, irq %d with %d ctx banks\n",
223 iommu_dev->name, regs_base, irq, iommu_dev->ncb);
224
225 platform_set_drvdata(pdev, drvdata);
226
227 clk_disable(iommu_clk);
228
229 clk_disable(iommu_pclk);
230
231 return 0;
232fail_clk:
233 if (iommu_clk) {
234 clk_disable(iommu_clk);
235 clk_put(iommu_clk);
236 }
237fail_pclk:
238 clk_disable_unprepare(iommu_pclk);
239fail_enable:
240 clk_put(iommu_pclk);
241fail:
242 kfree(drvdata);
243 return ret;
244}
245
246static int msm_iommu_remove(struct platform_device *pdev)
247{
248 struct msm_iommu_drvdata *drv = NULL;
249
250 drv = platform_get_drvdata(pdev);
251 if (drv) {
252 if (drv->clk) {
253 clk_unprepare(drv->clk);
254 clk_put(drv->clk);
255 }
256 clk_unprepare(drv->pclk);
257 clk_put(drv->pclk);
258 memset(drv, 0, sizeof(*drv));
259 kfree(drv);
260 }
261 return 0;
262}
263
264static int msm_iommu_ctx_probe(struct platform_device *pdev)
265{
266 struct msm_iommu_ctx_dev *c = dev_get_platdata(&pdev->dev);
267 struct msm_iommu_drvdata *drvdata;
268 struct msm_iommu_ctx_drvdata *ctx_drvdata;
269 int i, ret;
270
271 if (!c || !pdev->dev.parent)
272 return -EINVAL;
273
274 drvdata = dev_get_drvdata(pdev->dev.parent);
275 if (!drvdata)
276 return -ENODEV;
277
278 ctx_drvdata = kzalloc(sizeof(*ctx_drvdata), GFP_KERNEL);
279 if (!ctx_drvdata)
280 return -ENOMEM;
281
282 ctx_drvdata->num = c->num;
283 ctx_drvdata->pdev = pdev;
284
285 INIT_LIST_HEAD(&ctx_drvdata->attached_elm);
286 platform_set_drvdata(pdev, ctx_drvdata);
287
288 ret = clk_prepare_enable(drvdata->pclk);
289 if (ret)
290 goto fail;
291
292 if (drvdata->clk) {
293 ret = clk_prepare_enable(drvdata->clk);
294 if (ret) {
295 clk_disable_unprepare(drvdata->pclk);
296 goto fail;
297 }
298 }
299
300 /* Program the M2V tables for this context */
301 for (i = 0; i < MAX_NUM_MIDS; i++) {
302 int mid = c->mids[i];
303 if (mid == -1)
304 break;
305
306 SET_M2VCBR_N(drvdata->base, mid, 0);
307 SET_CBACR_N(drvdata->base, c->num, 0);
308
309 /* Set VMID = 0 */
310 SET_VMID(drvdata->base, mid, 0);
311
312 /* Set the context number for that MID to this context */
313 SET_CBNDX(drvdata->base, mid, c->num);
314
315 /* Set MID associated with this context bank to 0*/
316 SET_CBVMID(drvdata->base, c->num, 0);
317
318 /* Set the ASID for TLB tagging for this context */
319 SET_CONTEXTIDR_ASID(drvdata->base, c->num, c->num);
320
321 /* Set security bit override to be Non-secure */
322 SET_NSCFG(drvdata->base, mid, 3);
323 }
324
325 clk_disable(drvdata->clk);
326 clk_disable(drvdata->pclk);
327
328 dev_info(&pdev->dev, "context %s using bank %d\n", c->name, c->num);
329 return 0;
330fail:
331 kfree(ctx_drvdata);
332 return ret;
333}
334
335static int msm_iommu_ctx_remove(struct platform_device *pdev)
336{
337 struct msm_iommu_ctx_drvdata *drv = NULL;
338 drv = platform_get_drvdata(pdev);
339 if (drv) {
340 memset(drv, 0, sizeof(struct msm_iommu_ctx_drvdata));
341 kfree(drv);
342 }
343 return 0;
344}
345
346static struct platform_driver msm_iommu_driver = {
347 .driver = {
348 .name = "msm_iommu",
349 },
350 .probe = msm_iommu_probe,
351 .remove = msm_iommu_remove,
352};
353
354static struct platform_driver msm_iommu_ctx_driver = {
355 .driver = {
356 .name = "msm_iommu_ctx",
357 },
358 .probe = msm_iommu_ctx_probe,
359 .remove = msm_iommu_ctx_remove,
360};
361
362static struct platform_driver * const drivers[] = {
363 &msm_iommu_driver,
364 &msm_iommu_ctx_driver,
365};
366
367static int __init msm_iommu_driver_init(void)
368{
369 return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
370}
371
372static void __exit msm_iommu_driver_exit(void)
373{
374 platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
375}
376
377subsys_initcall(msm_iommu_driver_init);
378module_exit(msm_iommu_driver_exit);
379
380MODULE_LICENSE("GPL v2");
381MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index c3043d8754e3..b12c12d74c33 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -34,7 +34,7 @@
34#include <dt-bindings/memory/mt8173-larb-port.h> 34#include <dt-bindings/memory/mt8173-larb-port.h>
35#include <soc/mediatek/smi.h> 35#include <soc/mediatek/smi.h>
36 36
37#include "io-pgtable.h" 37#include "mtk_iommu.h"
38 38
39#define REG_MMU_PT_BASE_ADDR 0x000 39#define REG_MMU_PT_BASE_ADDR 0x000
40 40
@@ -93,20 +93,6 @@
93 93
94#define MTK_PROTECT_PA_ALIGN 128 94#define MTK_PROTECT_PA_ALIGN 128
95 95
96struct mtk_iommu_suspend_reg {
97 u32 standard_axi_mode;
98 u32 dcm_dis;
99 u32 ctrl_reg;
100 u32 int_control0;
101 u32 int_main_control;
102};
103
104struct mtk_iommu_client_priv {
105 struct list_head client;
106 unsigned int mtk_m4u_id;
107 struct device *m4udev;
108};
109
110struct mtk_iommu_domain { 96struct mtk_iommu_domain {
111 spinlock_t pgtlock; /* lock for page table */ 97 spinlock_t pgtlock; /* lock for page table */
112 98
@@ -116,19 +102,6 @@ struct mtk_iommu_domain {
116 struct iommu_domain domain; 102 struct iommu_domain domain;
117}; 103};
118 104
119struct mtk_iommu_data {
120 void __iomem *base;
121 int irq;
122 struct device *dev;
123 struct clk *bclk;
124 phys_addr_t protect_base; /* protect memory base */
125 struct mtk_iommu_suspend_reg reg;
126 struct mtk_iommu_domain *m4u_dom;
127 struct iommu_group *m4u_group;
128 struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */
129 bool enable_4GB;
130};
131
132static struct iommu_ops mtk_iommu_ops; 105static struct iommu_ops mtk_iommu_ops;
133 106
134static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) 107static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
@@ -455,7 +428,6 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
455 if (!dev->archdata.iommu) { 428 if (!dev->archdata.iommu) {
456 /* Get the m4u device */ 429 /* Get the m4u device */
457 m4updev = of_find_device_by_node(args->np); 430 m4updev = of_find_device_by_node(args->np);
458 of_node_put(args->np);
459 if (WARN_ON(!m4updev)) 431 if (WARN_ON(!m4updev))
460 return -EINVAL; 432 return -EINVAL;
461 433
@@ -552,25 +524,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
552 return 0; 524 return 0;
553} 525}
554 526
555static int compare_of(struct device *dev, void *data)
556{
557 return dev->of_node == data;
558}
559
560static int mtk_iommu_bind(struct device *dev)
561{
562 struct mtk_iommu_data *data = dev_get_drvdata(dev);
563
564 return component_bind_all(dev, &data->smi_imu);
565}
566
567static void mtk_iommu_unbind(struct device *dev)
568{
569 struct mtk_iommu_data *data = dev_get_drvdata(dev);
570
571 component_unbind_all(dev, &data->smi_imu);
572}
573
574static const struct component_master_ops mtk_iommu_com_ops = { 527static const struct component_master_ops mtk_iommu_com_ops = {
575 .bind = mtk_iommu_bind, 528 .bind = mtk_iommu_bind,
576 .unbind = mtk_iommu_unbind, 529 .unbind = mtk_iommu_unbind,
diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h
new file mode 100644
index 000000000000..9ed0a8462ccf
--- /dev/null
+++ b/drivers/iommu/mtk_iommu.h
@@ -0,0 +1,77 @@
1/*
2 * Copyright (c) 2015-2016 MediaTek Inc.
3 * Author: Honghui Zhang <honghui.zhang@mediatek.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
14
15#ifndef _MTK_IOMMU_H_
16#define _MTK_IOMMU_H_
17
18#include <linux/clk.h>
19#include <linux/component.h>
20#include <linux/device.h>
21#include <linux/io.h>
22#include <linux/iommu.h>
23#include <linux/list.h>
24#include <linux/spinlock.h>
25#include <soc/mediatek/smi.h>
26
27#include "io-pgtable.h"
28
29struct mtk_iommu_suspend_reg {
30 u32 standard_axi_mode;
31 u32 dcm_dis;
32 u32 ctrl_reg;
33 u32 int_control0;
34 u32 int_main_control;
35};
36
37struct mtk_iommu_client_priv {
38 struct list_head client;
39 unsigned int mtk_m4u_id;
40 struct device *m4udev;
41};
42
43struct mtk_iommu_domain;
44
45struct mtk_iommu_data {
46 void __iomem *base;
47 int irq;
48 struct device *dev;
49 struct clk *bclk;
50 phys_addr_t protect_base; /* protect memory base */
51 struct mtk_iommu_suspend_reg reg;
52 struct mtk_iommu_domain *m4u_dom;
53 struct iommu_group *m4u_group;
54 struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */
55 bool enable_4GB;
56};
57
58static int compare_of(struct device *dev, void *data)
59{
60 return dev->of_node == data;
61}
62
63static int mtk_iommu_bind(struct device *dev)
64{
65 struct mtk_iommu_data *data = dev_get_drvdata(dev);
66
67 return component_bind_all(dev, &data->smi_imu);
68}
69
70static void mtk_iommu_unbind(struct device *dev)
71{
72 struct mtk_iommu_data *data = dev_get_drvdata(dev);
73
74 component_unbind_all(dev, &data->smi_imu);
75}
76
77#endif
diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
new file mode 100644
index 000000000000..b8aeb0768483
--- /dev/null
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -0,0 +1,727 @@
1/*
2 * Copyright (c) 2015-2016 MediaTek Inc.
3 * Author: Honghui Zhang <honghui.zhang@mediatek.com>
4 *
5 * Based on driver/iommu/mtk_iommu.c
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 */
16#include <linux/bootmem.h>
17#include <linux/bug.h>
18#include <linux/clk.h>
19#include <linux/component.h>
20#include <linux/device.h>
21#include <linux/dma-iommu.h>
22#include <linux/err.h>
23#include <linux/interrupt.h>
24#include <linux/io.h>
25#include <linux/iommu.h>
26#include <linux/iopoll.h>
27#include <linux/kmemleak.h>
28#include <linux/list.h>
29#include <linux/of_address.h>
30#include <linux/of_iommu.h>
31#include <linux/of_irq.h>
32#include <linux/of_platform.h>
33#include <linux/platform_device.h>
34#include <linux/slab.h>
35#include <linux/spinlock.h>
36#include <asm/barrier.h>
37#include <asm/dma-iommu.h>
38#include <linux/module.h>
39#include <dt-bindings/memory/mt2701-larb-port.h>
40#include <soc/mediatek/smi.h>
41#include "mtk_iommu.h"
42
43#define REG_MMU_PT_BASE_ADDR 0x000
44
45#define F_ALL_INVLD 0x2
46#define F_MMU_INV_RANGE 0x1
47#define F_INVLD_EN0 BIT(0)
48#define F_INVLD_EN1 BIT(1)
49
50#define F_MMU_FAULT_VA_MSK 0xfffff000
51#define MTK_PROTECT_PA_ALIGN 128
52
53#define REG_MMU_CTRL_REG 0x210
54#define F_MMU_CTRL_COHERENT_EN BIT(8)
55#define REG_MMU_IVRP_PADDR 0x214
56#define REG_MMU_INT_CONTROL 0x220
57#define F_INT_TRANSLATION_FAULT BIT(0)
58#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1)
59#define F_INT_INVALID_PA_FAULT BIT(2)
60#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3)
61#define F_INT_TABLE_WALK_FAULT BIT(4)
62#define F_INT_TLB_MISS_FAULT BIT(5)
63#define F_INT_PFH_DMA_FIFO_OVERFLOW BIT(6)
64#define F_INT_MISS_DMA_FIFO_OVERFLOW BIT(7)
65
66#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5)
67#define F_INT_CLR_BIT BIT(12)
68
69#define REG_MMU_FAULT_ST 0x224
70#define REG_MMU_FAULT_VA 0x228
71#define REG_MMU_INVLD_PA 0x22C
72#define REG_MMU_INT_ID 0x388
73#define REG_MMU_INVALIDATE 0x5c0
74#define REG_MMU_INVLD_START_A 0x5c4
75#define REG_MMU_INVLD_END_A 0x5c8
76
77#define REG_MMU_INV_SEL 0x5d8
78#define REG_MMU_STANDARD_AXI_MODE 0x5e8
79
80#define REG_MMU_DCM 0x5f0
81#define F_MMU_DCM_ON BIT(1)
82#define REG_MMU_CPE_DONE 0x60c
83#define F_DESC_VALID 0x2
84#define F_DESC_NONSEC BIT(3)
85#define MT2701_M4U_TF_LARB(TF) (6 - (((TF) >> 13) & 0x7))
86#define MT2701_M4U_TF_PORT(TF) (((TF) >> 8) & 0xF)
87/* MTK generation one iommu HW only support 4K size mapping */
88#define MT2701_IOMMU_PAGE_SHIFT 12
89#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT)
90
91/*
92 * MTK m4u support 4GB iova address space, and only support 4K page
93 * mapping. So the pagetable size should be exactly as 4M.
94 */
95#define M2701_IOMMU_PGT_SIZE SZ_4M
96
97struct mtk_iommu_domain {
98 spinlock_t pgtlock; /* lock for page table */
99 struct iommu_domain domain;
100 u32 *pgt_va;
101 dma_addr_t pgt_pa;
102 struct mtk_iommu_data *data;
103};
104
105static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
106{
107 return container_of(dom, struct mtk_iommu_domain, domain);
108}
109
110static const int mt2701_m4u_in_larb[] = {
111 LARB0_PORT_OFFSET, LARB1_PORT_OFFSET,
112 LARB2_PORT_OFFSET, LARB3_PORT_OFFSET
113};
114
115static inline int mt2701_m4u_to_larb(int id)
116{
117 int i;
118
119 for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--)
120 if ((id) >= mt2701_m4u_in_larb[i])
121 return i;
122
123 return 0;
124}
125
126static inline int mt2701_m4u_to_port(int id)
127{
128 int larb = mt2701_m4u_to_larb(id);
129
130 return id - mt2701_m4u_in_larb[larb];
131}
132
133static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data)
134{
135 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
136 data->base + REG_MMU_INV_SEL);
137 writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);
138 wmb(); /* Make sure the tlb flush all done */
139}
140
141static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data,
142 unsigned long iova, size_t size)
143{
144 int ret;
145 u32 tmp;
146
147 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
148 data->base + REG_MMU_INV_SEL);
149 writel_relaxed(iova & F_MMU_FAULT_VA_MSK,
150 data->base + REG_MMU_INVLD_START_A);
151 writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK,
152 data->base + REG_MMU_INVLD_END_A);
153 writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);
154
155 ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,
156 tmp, tmp != 0, 10, 100000);
157 if (ret) {
158 dev_warn(data->dev,
159 "Partial TLB flush timed out, falling back to full flush\n");
160 mtk_iommu_tlb_flush_all(data);
161 }
162 /* Clear the CPE status */
163 writel_relaxed(0, data->base + REG_MMU_CPE_DONE);
164}
165
166static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
167{
168 struct mtk_iommu_data *data = dev_id;
169 struct mtk_iommu_domain *dom = data->m4u_dom;
170 u32 int_state, regval, fault_iova, fault_pa;
171 unsigned int fault_larb, fault_port;
172
173 /* Read error information from registers */
174 int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST);
175 fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
176
177 fault_iova &= F_MMU_FAULT_VA_MSK;
178 fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
179 regval = readl_relaxed(data->base + REG_MMU_INT_ID);
180 fault_larb = MT2701_M4U_TF_LARB(regval);
181 fault_port = MT2701_M4U_TF_PORT(regval);
182
183 /*
184 * MTK v1 iommu HW could not determine whether the fault is read or
185 * write fault, report as read fault.
186 */
187 if (report_iommu_fault(&dom->domain, data->dev, fault_iova,
188 IOMMU_FAULT_READ))
189 dev_err_ratelimited(data->dev,
190 "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n",
191 int_state, fault_iova, fault_pa,
192 fault_larb, fault_port);
193
194 /* Interrupt clear */
195 regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL);
196 regval |= F_INT_CLR_BIT;
197 writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);
198
199 mtk_iommu_tlb_flush_all(data);
200
201 return IRQ_HANDLED;
202}
203
204static void mtk_iommu_config(struct mtk_iommu_data *data,
205 struct device *dev, bool enable)
206{
207 struct mtk_iommu_client_priv *head, *cur, *next;
208 struct mtk_smi_larb_iommu *larb_mmu;
209 unsigned int larbid, portid;
210
211 head = dev->archdata.iommu;
212 list_for_each_entry_safe(cur, next, &head->client, client) {
213 larbid = mt2701_m4u_to_larb(cur->mtk_m4u_id);
214 portid = mt2701_m4u_to_port(cur->mtk_m4u_id);
215 larb_mmu = &data->smi_imu.larb_imu[larbid];
216
217 dev_dbg(dev, "%s iommu port: %d\n",
218 enable ? "enable" : "disable", portid);
219
220 if (enable)
221 larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);
222 else
223 larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);
224 }
225}
226
227static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data)
228{
229 struct mtk_iommu_domain *dom = data->m4u_dom;
230
231 spin_lock_init(&dom->pgtlock);
232
233 dom->pgt_va = dma_zalloc_coherent(data->dev,
234 M2701_IOMMU_PGT_SIZE,
235 &dom->pgt_pa, GFP_KERNEL);
236 if (!dom->pgt_va)
237 return -ENOMEM;
238
239 writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR);
240
241 dom->data = data;
242
243 return 0;
244}
245
246static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
247{
248 struct mtk_iommu_domain *dom;
249
250 if (type != IOMMU_DOMAIN_UNMANAGED)
251 return NULL;
252
253 dom = kzalloc(sizeof(*dom), GFP_KERNEL);
254 if (!dom)
255 return NULL;
256
257 return &dom->domain;
258}
259
260static void mtk_iommu_domain_free(struct iommu_domain *domain)
261{
262 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
263 struct mtk_iommu_data *data = dom->data;
264
265 dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE,
266 dom->pgt_va, dom->pgt_pa);
267 kfree(to_mtk_domain(domain));
268}
269
270static int mtk_iommu_attach_device(struct iommu_domain *domain,
271 struct device *dev)
272{
273 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
274 struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
275 struct mtk_iommu_data *data;
276 int ret;
277
278 if (!priv)
279 return -ENODEV;
280
281 data = dev_get_drvdata(priv->m4udev);
282 if (!data->m4u_dom) {
283 data->m4u_dom = dom;
284 ret = mtk_iommu_domain_finalise(data);
285 if (ret) {
286 data->m4u_dom = NULL;
287 return ret;
288 }
289 }
290
291 mtk_iommu_config(data, dev, true);
292 return 0;
293}
294
295static void mtk_iommu_detach_device(struct iommu_domain *domain,
296 struct device *dev)
297{
298 struct mtk_iommu_client_priv *priv = dev->archdata.iommu;
299 struct mtk_iommu_data *data;
300
301 if (!priv)
302 return;
303
304 data = dev_get_drvdata(priv->m4udev);
305 mtk_iommu_config(data, dev, false);
306}
307
308static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
309 phys_addr_t paddr, size_t size, int prot)
310{
311 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
312 unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
313 unsigned long flags;
314 unsigned int i;
315 u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
316 u32 pabase = (u32)paddr;
317 int map_size = 0;
318
319 spin_lock_irqsave(&dom->pgtlock, flags);
320 for (i = 0; i < page_num; i++) {
321 if (pgt_base_iova[i]) {
322 memset(pgt_base_iova, 0, i * sizeof(u32));
323 break;
324 }
325 pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC;
326 pabase += MT2701_IOMMU_PAGE_SIZE;
327 map_size += MT2701_IOMMU_PAGE_SIZE;
328 }
329
330 spin_unlock_irqrestore(&dom->pgtlock, flags);
331
332 mtk_iommu_tlb_flush_range(dom->data, iova, size);
333
334 return map_size == size ? 0 : -EEXIST;
335}
336
337static size_t mtk_iommu_unmap(struct iommu_domain *domain,
338 unsigned long iova, size_t size)
339{
340 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
341 unsigned long flags;
342 u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT);
343 unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;
344
345 spin_lock_irqsave(&dom->pgtlock, flags);
346 memset(pgt_base_iova, 0, page_num * sizeof(u32));
347 spin_unlock_irqrestore(&dom->pgtlock, flags);
348
349 mtk_iommu_tlb_flush_range(dom->data, iova, size);
350
351 return size;
352}
353
354static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
355 dma_addr_t iova)
356{
357 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
358 unsigned long flags;
359 phys_addr_t pa;
360
361 spin_lock_irqsave(&dom->pgtlock, flags);
362 pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT));
363 pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1));
364 spin_unlock_irqrestore(&dom->pgtlock, flags);
365
366 return pa;
367}
368
369/*
370 * MTK generation one iommu HW only support one iommu domain, and all the client
371 * sharing the same iova address space.
372 */
373static int mtk_iommu_create_mapping(struct device *dev,
374 struct of_phandle_args *args)
375{
376 struct mtk_iommu_client_priv *head, *priv, *next;
377 struct platform_device *m4updev;
378 struct dma_iommu_mapping *mtk_mapping;
379 struct device *m4udev;
380 int ret;
381
382 if (args->args_count != 1) {
383 dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",
384 args->args_count);
385 return -EINVAL;
386 }
387
388 if (!dev->archdata.iommu) {
389 /* Get the m4u device */
390 m4updev = of_find_device_by_node(args->np);
391 if (WARN_ON(!m4updev))
392 return -EINVAL;
393
394 head = kzalloc(sizeof(*head), GFP_KERNEL);
395 if (!head)
396 return -ENOMEM;
397
398 dev->archdata.iommu = head;
399 INIT_LIST_HEAD(&head->client);
400 head->m4udev = &m4updev->dev;
401 } else {
402 head = dev->archdata.iommu;
403 }
404
405 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
406 if (!priv) {
407 ret = -ENOMEM;
408 goto err_free_mem;
409 }
410 priv->mtk_m4u_id = args->args[0];
411 list_add_tail(&priv->client, &head->client);
412
413 m4udev = head->m4udev;
414 mtk_mapping = m4udev->archdata.iommu;
415 if (!mtk_mapping) {
416 /* MTK iommu support 4GB iova address space. */
417 mtk_mapping = arm_iommu_create_mapping(&platform_bus_type,
418 0, 1ULL << 32);
419 if (IS_ERR(mtk_mapping)) {
420 ret = PTR_ERR(mtk_mapping);
421 goto err_free_mem;
422 }
423 m4udev->archdata.iommu = mtk_mapping;
424 }
425
426 ret = arm_iommu_attach_device(dev, mtk_mapping);
427 if (ret)
428 goto err_release_mapping;
429
430 return 0;
431
432err_release_mapping:
433 arm_iommu_release_mapping(mtk_mapping);
434 m4udev->archdata.iommu = NULL;
435err_free_mem:
436 list_for_each_entry_safe(priv, next, &head->client, client)
437 kfree(priv);
438 kfree(head);
439 dev->archdata.iommu = NULL;
440 return ret;
441}
442
443static int mtk_iommu_add_device(struct device *dev)
444{
445 struct iommu_group *group;
446 struct of_phandle_args iommu_spec;
447 struct of_phandle_iterator it;
448 int err;
449
450 of_for_each_phandle(&it, err, dev->of_node, "iommus",
451 "#iommu-cells", 0) {
452 int count = of_phandle_iterator_args(&it, iommu_spec.args,
453 MAX_PHANDLE_ARGS);
454 iommu_spec.np = of_node_get(it.node);
455 iommu_spec.args_count = count;
456
457 mtk_iommu_create_mapping(dev, &iommu_spec);
458 of_node_put(iommu_spec.np);
459 }
460
461 if (!dev->archdata.iommu) /* Not a iommu client device */
462 return -ENODEV;
463
464 group = iommu_group_get_for_dev(dev);
465 if (IS_ERR(group))
466 return PTR_ERR(group);
467
468 iommu_group_put(group);
469 return 0;
470}
471
472static void mtk_iommu_remove_device(struct device *dev)
473{
474 struct mtk_iommu_client_priv *head, *cur, *next;
475
476 head = dev->archdata.iommu;
477 if (!head)
478 return;
479
480 list_for_each_entry_safe(cur, next, &head->client, client) {
481 list_del(&cur->client);
482 kfree(cur);
483 }
484 kfree(head);
485 dev->archdata.iommu = NULL;
486
487 iommu_group_remove_device(dev);
488}
489
490static struct iommu_group *mtk_iommu_device_group(struct device *dev)
491{
492 struct mtk_iommu_data *data;
493 struct mtk_iommu_client_priv *priv;
494
495 priv = dev->archdata.iommu;
496 if (!priv)
497 return ERR_PTR(-ENODEV);
498
499 /* All the client devices are in the same m4u iommu-group */
500 data = dev_get_drvdata(priv->m4udev);
501 if (!data->m4u_group) {
502 data->m4u_group = iommu_group_alloc();
503 if (IS_ERR(data->m4u_group))
504 dev_err(dev, "Failed to allocate M4U IOMMU group\n");
505 }
506 return data->m4u_group;
507}
508
509static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
510{
511 u32 regval;
512 int ret;
513
514 ret = clk_prepare_enable(data->bclk);
515 if (ret) {
516 dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);
517 return ret;
518 }
519
520 regval = F_MMU_CTRL_COHERENT_EN | F_MMU_TF_PROTECT_SEL(2);
521 writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
522
523 regval = F_INT_TRANSLATION_FAULT |
524 F_INT_MAIN_MULTI_HIT_FAULT |
525 F_INT_INVALID_PA_FAULT |
526 F_INT_ENTRY_REPLACEMENT_FAULT |
527 F_INT_TABLE_WALK_FAULT |
528 F_INT_TLB_MISS_FAULT |
529 F_INT_PFH_DMA_FIFO_OVERFLOW |
530 F_INT_MISS_DMA_FIFO_OVERFLOW;
531 writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);
532
533 /* protect memory,hw will write here while translation fault */
534 writel_relaxed(data->protect_base,
535 data->base + REG_MMU_IVRP_PADDR);
536
537 writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM);
538
539 if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
540 dev_name(data->dev), (void *)data)) {
541 writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);
542 clk_disable_unprepare(data->bclk);
543 dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);
544 return -ENODEV;
545 }
546
547 return 0;
548}
549
550static struct iommu_ops mtk_iommu_ops = {
551 .domain_alloc = mtk_iommu_domain_alloc,
552 .domain_free = mtk_iommu_domain_free,
553 .attach_dev = mtk_iommu_attach_device,
554 .detach_dev = mtk_iommu_detach_device,
555 .map = mtk_iommu_map,
556 .unmap = mtk_iommu_unmap,
557 .map_sg = default_iommu_map_sg,
558 .iova_to_phys = mtk_iommu_iova_to_phys,
559 .add_device = mtk_iommu_add_device,
560 .remove_device = mtk_iommu_remove_device,
561 .device_group = mtk_iommu_device_group,
562 .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT,
563};
564
565static const struct of_device_id mtk_iommu_of_ids[] = {
566 { .compatible = "mediatek,mt2701-m4u", },
567 {}
568};
569
570static const struct component_master_ops mtk_iommu_com_ops = {
571 .bind = mtk_iommu_bind,
572 .unbind = mtk_iommu_unbind,
573};
574
575static int mtk_iommu_probe(struct platform_device *pdev)
576{
577 struct mtk_iommu_data *data;
578 struct device *dev = &pdev->dev;
579 struct resource *res;
580 struct component_match *match = NULL;
581 struct of_phandle_args larb_spec;
582 struct of_phandle_iterator it;
583 void *protect;
584 int larb_nr, ret, err;
585
586 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
587 if (!data)
588 return -ENOMEM;
589
590 data->dev = dev;
591
592 /* Protect memory. HW will access here while translation fault.*/
593 protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2,
594 GFP_KERNEL | GFP_DMA);
595 if (!protect)
596 return -ENOMEM;
597 data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
598
599 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
600 data->base = devm_ioremap_resource(dev, res);
601 if (IS_ERR(data->base))
602 return PTR_ERR(data->base);
603
604 data->irq = platform_get_irq(pdev, 0);
605 if (data->irq < 0)
606 return data->irq;
607
608 data->bclk = devm_clk_get(dev, "bclk");
609 if (IS_ERR(data->bclk))
610 return PTR_ERR(data->bclk);
611
612 larb_nr = 0;
613 of_for_each_phandle(&it, err, dev->of_node,
614 "mediatek,larbs", NULL, 0) {
615 struct platform_device *plarbdev;
616 int count = of_phandle_iterator_args(&it, larb_spec.args,
617 MAX_PHANDLE_ARGS);
618
619 if (count)
620 continue;
621
622 larb_spec.np = of_node_get(it.node);
623 if (!of_device_is_available(larb_spec.np))
624 continue;
625
626 plarbdev = of_find_device_by_node(larb_spec.np);
627 of_node_put(larb_spec.np);
628 if (!plarbdev) {
629 plarbdev = of_platform_device_create(
630 larb_spec.np, NULL,
631 platform_bus_type.dev_root);
632 if (!plarbdev)
633 return -EPROBE_DEFER;
634 }
635
636 data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev;
637 component_match_add(dev, &match, compare_of, larb_spec.np);
638 larb_nr++;
639 }
640
641 data->smi_imu.larb_nr = larb_nr;
642
643 platform_set_drvdata(pdev, data);
644
645 ret = mtk_iommu_hw_init(data);
646 if (ret)
647 return ret;
648
649 if (!iommu_present(&platform_bus_type))
650 bus_set_iommu(&platform_bus_type, &mtk_iommu_ops);
651
652 return component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
653}
654
655static int mtk_iommu_remove(struct platform_device *pdev)
656{
657 struct mtk_iommu_data *data = platform_get_drvdata(pdev);
658
659 if (iommu_present(&platform_bus_type))
660 bus_set_iommu(&platform_bus_type, NULL);
661
662 clk_disable_unprepare(data->bclk);
663 devm_free_irq(&pdev->dev, data->irq, data);
664 component_master_del(&pdev->dev, &mtk_iommu_com_ops);
665 return 0;
666}
667
668static int __maybe_unused mtk_iommu_suspend(struct device *dev)
669{
670 struct mtk_iommu_data *data = dev_get_drvdata(dev);
671 struct mtk_iommu_suspend_reg *reg = &data->reg;
672 void __iomem *base = data->base;
673
674 reg->standard_axi_mode = readl_relaxed(base +
675 REG_MMU_STANDARD_AXI_MODE);
676 reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM);
677 reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
678 reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL);
679 return 0;
680}
681
682static int __maybe_unused mtk_iommu_resume(struct device *dev)
683{
684 struct mtk_iommu_data *data = dev_get_drvdata(dev);
685 struct mtk_iommu_suspend_reg *reg = &data->reg;
686 void __iomem *base = data->base;
687
688 writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR);
689 writel_relaxed(reg->standard_axi_mode,
690 base + REG_MMU_STANDARD_AXI_MODE);
691 writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM);
692 writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);
693 writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL);
694 writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR);
695 return 0;
696}
697
698static const struct dev_pm_ops mtk_iommu_pm_ops = {
699 SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
700};
701
702static struct platform_driver mtk_iommu_driver = {
703 .probe = mtk_iommu_probe,
704 .remove = mtk_iommu_remove,
705 .driver = {
706 .name = "mtk-iommu",
707 .of_match_table = mtk_iommu_of_ids,
708 .pm = &mtk_iommu_pm_ops,
709 }
710};
711
712static int __init m4u_init(void)
713{
714 return platform_driver_register(&mtk_iommu_driver);
715}
716
717static void __exit m4u_exit(void)
718{
719 return platform_driver_unregister(&mtk_iommu_driver);
720}
721
722subsys_initcall(m4u_init);
723module_exit(m4u_exit);
724
725MODULE_DESCRIPTION("IOMMU API for MTK architected m4u v1 implementations");
726MODULE_AUTHOR("Honghui Zhang <honghui.zhang@mediatek.com>");
727MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index 25b4627cb57f..9afcbf79f0b0 100644
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -4,11 +4,10 @@
4 * published by the Free Software Foundation. 4 * published by the Free Software Foundation.
5 */ 5 */
6 6
7#include <asm/cacheflush.h>
8#include <asm/pgtable.h>
9#include <linux/compiler.h> 7#include <linux/compiler.h>
10#include <linux/delay.h> 8#include <linux/delay.h>
11#include <linux/device.h> 9#include <linux/device.h>
10#include <linux/dma-iommu.h>
12#include <linux/errno.h> 11#include <linux/errno.h>
13#include <linux/interrupt.h> 12#include <linux/interrupt.h>
14#include <linux/io.h> 13#include <linux/io.h>
@@ -77,7 +76,9 @@
77 76
78struct rk_iommu_domain { 77struct rk_iommu_domain {
79 struct list_head iommus; 78 struct list_head iommus;
79 struct platform_device *pdev;
80 u32 *dt; /* page directory table */ 80 u32 *dt; /* page directory table */
81 dma_addr_t dt_dma;
81 spinlock_t iommus_lock; /* lock for iommus list */ 82 spinlock_t iommus_lock; /* lock for iommus list */
82 spinlock_t dt_lock; /* lock for modifying page directory table */ 83 spinlock_t dt_lock; /* lock for modifying page directory table */
83 84
@@ -93,14 +94,12 @@ struct rk_iommu {
93 struct iommu_domain *domain; /* domain to which iommu is attached */ 94 struct iommu_domain *domain; /* domain to which iommu is attached */
94}; 95};
95 96
96static inline void rk_table_flush(u32 *va, unsigned int count) 97static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma,
98 unsigned int count)
97{ 99{
98 phys_addr_t pa_start = virt_to_phys(va); 100 size_t size = count * sizeof(u32); /* count of u32 entry */
99 phys_addr_t pa_end = virt_to_phys(va + count);
100 size_t size = pa_end - pa_start;
101 101
102 __cpuc_flush_dcache_area(va, size); 102 dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE);
103 outer_flush_range(pa_start, pa_end);
104} 103}
105 104
106static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) 105static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
@@ -183,10 +182,9 @@ static inline bool rk_dte_is_pt_valid(u32 dte)
183 return dte & RK_DTE_PT_VALID; 182 return dte & RK_DTE_PT_VALID;
184} 183}
185 184
186static u32 rk_mk_dte(u32 *pt) 185static inline u32 rk_mk_dte(dma_addr_t pt_dma)
187{ 186{
188 phys_addr_t pt_phys = virt_to_phys(pt); 187 return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID;
189 return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID;
190} 188}
191 189
192/* 190/*
@@ -603,13 +601,16 @@ static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain,
603static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, 601static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
604 dma_addr_t iova) 602 dma_addr_t iova)
605{ 603{
604 struct device *dev = &rk_domain->pdev->dev;
606 u32 *page_table, *dte_addr; 605 u32 *page_table, *dte_addr;
607 u32 dte; 606 u32 dte_index, dte;
608 phys_addr_t pt_phys; 607 phys_addr_t pt_phys;
608 dma_addr_t pt_dma;
609 609
610 assert_spin_locked(&rk_domain->dt_lock); 610 assert_spin_locked(&rk_domain->dt_lock);
611 611
612 dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; 612 dte_index = rk_iova_dte_index(iova);
613 dte_addr = &rk_domain->dt[dte_index];
613 dte = *dte_addr; 614 dte = *dte_addr;
614 if (rk_dte_is_pt_valid(dte)) 615 if (rk_dte_is_pt_valid(dte))
615 goto done; 616 goto done;
@@ -618,19 +619,27 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
618 if (!page_table) 619 if (!page_table)
619 return ERR_PTR(-ENOMEM); 620 return ERR_PTR(-ENOMEM);
620 621
621 dte = rk_mk_dte(page_table); 622 pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE);
622 *dte_addr = dte; 623 if (dma_mapping_error(dev, pt_dma)) {
624 dev_err(dev, "DMA mapping error while allocating page table\n");
625 free_page((unsigned long)page_table);
626 return ERR_PTR(-ENOMEM);
627 }
623 628
624 rk_table_flush(page_table, NUM_PT_ENTRIES); 629 dte = rk_mk_dte(pt_dma);
625 rk_table_flush(dte_addr, 1); 630 *dte_addr = dte;
626 631
632 rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES);
633 rk_table_flush(rk_domain,
634 rk_domain->dt_dma + dte_index * sizeof(u32), 1);
627done: 635done:
628 pt_phys = rk_dte_pt_address(dte); 636 pt_phys = rk_dte_pt_address(dte);
629 return (u32 *)phys_to_virt(pt_phys); 637 return (u32 *)phys_to_virt(pt_phys);
630} 638}
631 639
632static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, 640static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,
633 u32 *pte_addr, dma_addr_t iova, size_t size) 641 u32 *pte_addr, dma_addr_t pte_dma,
642 size_t size)
634{ 643{
635 unsigned int pte_count; 644 unsigned int pte_count;
636 unsigned int pte_total = size / SPAGE_SIZE; 645 unsigned int pte_total = size / SPAGE_SIZE;
@@ -645,14 +654,14 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,
645 pte_addr[pte_count] = rk_mk_pte_invalid(pte); 654 pte_addr[pte_count] = rk_mk_pte_invalid(pte);
646 } 655 }
647 656
648 rk_table_flush(pte_addr, pte_count); 657 rk_table_flush(rk_domain, pte_dma, pte_count);
649 658
650 return pte_count * SPAGE_SIZE; 659 return pte_count * SPAGE_SIZE;
651} 660}
652 661
653static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, 662static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
654 dma_addr_t iova, phys_addr_t paddr, size_t size, 663 dma_addr_t pte_dma, dma_addr_t iova,
655 int prot) 664 phys_addr_t paddr, size_t size, int prot)
656{ 665{
657 unsigned int pte_count; 666 unsigned int pte_count;
658 unsigned int pte_total = size / SPAGE_SIZE; 667 unsigned int pte_total = size / SPAGE_SIZE;
@@ -671,7 +680,7 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
671 paddr += SPAGE_SIZE; 680 paddr += SPAGE_SIZE;
672 } 681 }
673 682
674 rk_table_flush(pte_addr, pte_count); 683 rk_table_flush(rk_domain, pte_dma, pte_total);
675 684
676 /* 685 /*
677 * Zap the first and last iova to evict from iotlb any previously 686 * Zap the first and last iova to evict from iotlb any previously
@@ -684,7 +693,8 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
684 return 0; 693 return 0;
685unwind: 694unwind:
686 /* Unmap the range of iovas that we just mapped */ 695 /* Unmap the range of iovas that we just mapped */
687 rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); 696 rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma,
697 pte_count * SPAGE_SIZE);
688 698
689 iova += pte_count * SPAGE_SIZE; 699 iova += pte_count * SPAGE_SIZE;
690 page_phys = rk_pte_page_address(pte_addr[pte_count]); 700 page_phys = rk_pte_page_address(pte_addr[pte_count]);
@@ -699,8 +709,9 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
699{ 709{
700 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 710 struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
701 unsigned long flags; 711 unsigned long flags;
702 dma_addr_t iova = (dma_addr_t)_iova; 712 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
703 u32 *page_table, *pte_addr; 713 u32 *page_table, *pte_addr;
714 u32 dte_index, pte_index;
704 int ret; 715 int ret;
705 716
706 spin_lock_irqsave(&rk_domain->dt_lock, flags); 717 spin_lock_irqsave(&rk_domain->dt_lock, flags);
@@ -718,8 +729,13 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
718 return PTR_ERR(page_table); 729 return PTR_ERR(page_table);
719 } 730 }
720 731
721 pte_addr = &page_table[rk_iova_pte_index(iova)]; 732 dte_index = rk_domain->dt[rk_iova_dte_index(iova)];
722 ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); 733 pte_index = rk_iova_pte_index(iova);
734 pte_addr = &page_table[pte_index];
735 pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32);
736 ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova,
737 paddr, size, prot);
738
723 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 739 spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
724 740
725 return ret; 741 return ret;
@@ -730,7 +746,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
730{ 746{
731 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 747 struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
732 unsigned long flags; 748 unsigned long flags;
733 dma_addr_t iova = (dma_addr_t)_iova; 749 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
734 phys_addr_t pt_phys; 750 phys_addr_t pt_phys;
735 u32 dte; 751 u32 dte;
736 u32 *pte_addr; 752 u32 *pte_addr;
@@ -754,7 +770,8 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
754 770
755 pt_phys = rk_dte_pt_address(dte); 771 pt_phys = rk_dte_pt_address(dte);
756 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); 772 pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
757 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); 773 pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
774 unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size);
758 775
759 spin_unlock_irqrestore(&rk_domain->dt_lock, flags); 776 spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
760 777
@@ -787,7 +804,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
787 struct rk_iommu_domain *rk_domain = to_rk_domain(domain); 804 struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
788 unsigned long flags; 805 unsigned long flags;
789 int ret, i; 806 int ret, i;
790 phys_addr_t dte_addr;
791 807
792 /* 808 /*
793 * Allow 'virtual devices' (e.g., drm) to attach to domain. 809 * Allow 'virtual devices' (e.g., drm) to attach to domain.
@@ -807,14 +823,14 @@ static int rk_iommu_attach_device(struct iommu_domain *domain,
807 823
808 iommu->domain = domain; 824 iommu->domain = domain;
809 825
810 ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, 826 ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq,
811 IRQF_SHARED, dev_name(dev), iommu); 827 IRQF_SHARED, dev_name(dev), iommu);
812 if (ret) 828 if (ret)
813 return ret; 829 return ret;
814 830
815 dte_addr = virt_to_phys(rk_domain->dt);
816 for (i = 0; i < iommu->num_mmu; i++) { 831 for (i = 0; i < iommu->num_mmu; i++) {
817 rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr); 832 rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
833 rk_domain->dt_dma);
818 rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); 834 rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
819 rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); 835 rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
820 } 836 }
@@ -860,7 +876,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
860 } 876 }
861 rk_iommu_disable_stall(iommu); 877 rk_iommu_disable_stall(iommu);
862 878
863 devm_free_irq(dev, iommu->irq, iommu); 879 devm_free_irq(iommu->dev, iommu->irq, iommu);
864 880
865 iommu->domain = NULL; 881 iommu->domain = NULL;
866 882
@@ -870,14 +886,30 @@ static void rk_iommu_detach_device(struct iommu_domain *domain,
870static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) 886static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
871{ 887{
872 struct rk_iommu_domain *rk_domain; 888 struct rk_iommu_domain *rk_domain;
889 struct platform_device *pdev;
890 struct device *iommu_dev;
873 891
874 if (type != IOMMU_DOMAIN_UNMANAGED) 892 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
875 return NULL; 893 return NULL;
876 894
877 rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); 895 /* Register a pdev per domain, so DMA API can base on this *dev
878 if (!rk_domain) 896 * even some virtual master doesn't have an iommu slave
897 */
898 pdev = platform_device_register_simple("rk_iommu_domain",
899 PLATFORM_DEVID_AUTO, NULL, 0);
900 if (IS_ERR(pdev))
879 return NULL; 901 return NULL;
880 902
903 rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL);
904 if (!rk_domain)
905 goto err_unreg_pdev;
906
907 rk_domain->pdev = pdev;
908
909 if (type == IOMMU_DOMAIN_DMA &&
910 iommu_get_dma_cookie(&rk_domain->domain))
911 goto err_unreg_pdev;
912
881 /* 913 /*
882 * rk32xx iommus use a 2 level pagetable. 914 * rk32xx iommus use a 2 level pagetable.
883 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. 915 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
@@ -885,18 +917,36 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
885 */ 917 */
886 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); 918 rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
887 if (!rk_domain->dt) 919 if (!rk_domain->dt)
888 goto err_dt; 920 goto err_put_cookie;
921
922 iommu_dev = &pdev->dev;
923 rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt,
924 SPAGE_SIZE, DMA_TO_DEVICE);
925 if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) {
926 dev_err(iommu_dev, "DMA map error for DT\n");
927 goto err_free_dt;
928 }
889 929
890 rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); 930 rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES);
891 931
892 spin_lock_init(&rk_domain->iommus_lock); 932 spin_lock_init(&rk_domain->iommus_lock);
893 spin_lock_init(&rk_domain->dt_lock); 933 spin_lock_init(&rk_domain->dt_lock);
894 INIT_LIST_HEAD(&rk_domain->iommus); 934 INIT_LIST_HEAD(&rk_domain->iommus);
895 935
936 rk_domain->domain.geometry.aperture_start = 0;
937 rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
938 rk_domain->domain.geometry.force_aperture = true;
939
896 return &rk_domain->domain; 940 return &rk_domain->domain;
897 941
898err_dt: 942err_free_dt:
899 kfree(rk_domain); 943 free_page((unsigned long)rk_domain->dt);
944err_put_cookie:
945 if (type == IOMMU_DOMAIN_DMA)
946 iommu_put_dma_cookie(&rk_domain->domain);
947err_unreg_pdev:
948 platform_device_unregister(pdev);
949
900 return NULL; 950 return NULL;
901} 951}
902 952
@@ -912,12 +962,20 @@ static void rk_iommu_domain_free(struct iommu_domain *domain)
912 if (rk_dte_is_pt_valid(dte)) { 962 if (rk_dte_is_pt_valid(dte)) {
913 phys_addr_t pt_phys = rk_dte_pt_address(dte); 963 phys_addr_t pt_phys = rk_dte_pt_address(dte);
914 u32 *page_table = phys_to_virt(pt_phys); 964 u32 *page_table = phys_to_virt(pt_phys);
965 dma_unmap_single(&rk_domain->pdev->dev, pt_phys,
966 SPAGE_SIZE, DMA_TO_DEVICE);
915 free_page((unsigned long)page_table); 967 free_page((unsigned long)page_table);
916 } 968 }
917 } 969 }
918 970
971 dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma,
972 SPAGE_SIZE, DMA_TO_DEVICE);
919 free_page((unsigned long)rk_domain->dt); 973 free_page((unsigned long)rk_domain->dt);
920 kfree(rk_domain); 974
975 if (domain->type == IOMMU_DOMAIN_DMA)
976 iommu_put_dma_cookie(&rk_domain->domain);
977
978 platform_device_unregister(rk_domain->pdev);
921} 979}
922 980
923static bool rk_iommu_is_dev_iommu_master(struct device *dev) 981static bool rk_iommu_is_dev_iommu_master(struct device *dev)
@@ -1022,17 +1080,43 @@ static const struct iommu_ops rk_iommu_ops = {
1022 .detach_dev = rk_iommu_detach_device, 1080 .detach_dev = rk_iommu_detach_device,
1023 .map = rk_iommu_map, 1081 .map = rk_iommu_map,
1024 .unmap = rk_iommu_unmap, 1082 .unmap = rk_iommu_unmap,
1083 .map_sg = default_iommu_map_sg,
1025 .add_device = rk_iommu_add_device, 1084 .add_device = rk_iommu_add_device,
1026 .remove_device = rk_iommu_remove_device, 1085 .remove_device = rk_iommu_remove_device,
1027 .iova_to_phys = rk_iommu_iova_to_phys, 1086 .iova_to_phys = rk_iommu_iova_to_phys,
1028 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, 1087 .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
1029}; 1088};
1030 1089
1090static int rk_iommu_domain_probe(struct platform_device *pdev)
1091{
1092 struct device *dev = &pdev->dev;
1093
1094 dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL);
1095 if (!dev->dma_parms)
1096 return -ENOMEM;
1097
1098 /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */
1099 arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false);
1100
1101 dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
1102 dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
1103
1104 return 0;
1105}
1106
1107static struct platform_driver rk_iommu_domain_driver = {
1108 .probe = rk_iommu_domain_probe,
1109 .driver = {
1110 .name = "rk_iommu_domain",
1111 },
1112};
1113
1031static int rk_iommu_probe(struct platform_device *pdev) 1114static int rk_iommu_probe(struct platform_device *pdev)
1032{ 1115{
1033 struct device *dev = &pdev->dev; 1116 struct device *dev = &pdev->dev;
1034 struct rk_iommu *iommu; 1117 struct rk_iommu *iommu;
1035 struct resource *res; 1118 struct resource *res;
1119 int num_res = pdev->num_resources;
1036 int i; 1120 int i;
1037 1121
1038 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); 1122 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
@@ -1042,12 +1126,13 @@ static int rk_iommu_probe(struct platform_device *pdev)
1042 platform_set_drvdata(pdev, iommu); 1126 platform_set_drvdata(pdev, iommu);
1043 iommu->dev = dev; 1127 iommu->dev = dev;
1044 iommu->num_mmu = 0; 1128 iommu->num_mmu = 0;
1045 iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * iommu->num_mmu, 1129
1130 iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res,
1046 GFP_KERNEL); 1131 GFP_KERNEL);
1047 if (!iommu->bases) 1132 if (!iommu->bases)
1048 return -ENOMEM; 1133 return -ENOMEM;
1049 1134
1050 for (i = 0; i < pdev->num_resources; i++) { 1135 for (i = 0; i < num_res; i++) {
1051 res = platform_get_resource(pdev, IORESOURCE_MEM, i); 1136 res = platform_get_resource(pdev, IORESOURCE_MEM, i);
1052 if (!res) 1137 if (!res)
1053 continue; 1138 continue;
@@ -1103,11 +1188,19 @@ static int __init rk_iommu_init(void)
1103 if (ret) 1188 if (ret)
1104 return ret; 1189 return ret;
1105 1190
1106 return platform_driver_register(&rk_iommu_driver); 1191 ret = platform_driver_register(&rk_iommu_domain_driver);
1192 if (ret)
1193 return ret;
1194
1195 ret = platform_driver_register(&rk_iommu_driver);
1196 if (ret)
1197 platform_driver_unregister(&rk_iommu_domain_driver);
1198 return ret;
1107} 1199}
1108static void __exit rk_iommu_exit(void) 1200static void __exit rk_iommu_exit(void)
1109{ 1201{
1110 platform_driver_unregister(&rk_iommu_driver); 1202 platform_driver_unregister(&rk_iommu_driver);
1203 platform_driver_unregister(&rk_iommu_domain_driver);
1111} 1204}
1112 1205
1113subsys_initcall(rk_iommu_init); 1206subsys_initcall(rk_iommu_init);
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index f6b57579185a..4afbc412f959 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -21,19 +21,50 @@
21#include <linux/platform_device.h> 21#include <linux/platform_device.h>
22#include <linux/pm_runtime.h> 22#include <linux/pm_runtime.h>
23#include <soc/mediatek/smi.h> 23#include <soc/mediatek/smi.h>
24#include <dt-bindings/memory/mt2701-larb-port.h>
24 25
25#define SMI_LARB_MMU_EN 0xf00 26#define SMI_LARB_MMU_EN 0xf00
27#define REG_SMI_SECUR_CON_BASE 0x5c0
28
29/* every register control 8 port, register offset 0x4 */
30#define REG_SMI_SECUR_CON_OFFSET(id) (((id) >> 3) << 2)
31#define REG_SMI_SECUR_CON_ADDR(id) \
32 (REG_SMI_SECUR_CON_BASE + REG_SMI_SECUR_CON_OFFSET(id))
33
34/*
35 * every port have 4 bit to control, bit[port + 3] control virtual or physical,
36 * bit[port + 2 : port + 1] control the domain, bit[port] control the security
37 * or non-security.
38 */
39#define SMI_SECUR_CON_VAL_MSK(id) (~(0xf << (((id) & 0x7) << 2)))
40#define SMI_SECUR_CON_VAL_VIRT(id) BIT((((id) & 0x7) << 2) + 3)
41/* mt2701 domain should be set to 3 */
42#define SMI_SECUR_CON_VAL_DOMAIN(id) (0x3 << ((((id) & 0x7) << 2) + 1))
43
44struct mtk_smi_larb_gen {
45 int port_in_larb[MTK_LARB_NR_MAX + 1];
46 void (*config_port)(struct device *);
47};
26 48
27struct mtk_smi { 49struct mtk_smi {
28 struct device *dev; 50 struct device *dev;
29 struct clk *clk_apb, *clk_smi; 51 struct clk *clk_apb, *clk_smi;
52 struct clk *clk_async; /*only needed by mt2701*/
53 void __iomem *smi_ao_base;
30}; 54};
31 55
32struct mtk_smi_larb { /* larb: local arbiter */ 56struct mtk_smi_larb { /* larb: local arbiter */
33 struct mtk_smi smi; 57 struct mtk_smi smi;
34 void __iomem *base; 58 void __iomem *base;
35 struct device *smi_common_dev; 59 struct device *smi_common_dev;
36 u32 *mmu; 60 const struct mtk_smi_larb_gen *larb_gen;
61 int larbid;
62 u32 *mmu;
63};
64
65enum mtk_smi_gen {
66 MTK_SMI_GEN1,
67 MTK_SMI_GEN2
37}; 68};
38 69
39static int mtk_smi_enable(const struct mtk_smi *smi) 70static int mtk_smi_enable(const struct mtk_smi *smi)
@@ -71,6 +102,7 @@ static void mtk_smi_disable(const struct mtk_smi *smi)
71int mtk_smi_larb_get(struct device *larbdev) 102int mtk_smi_larb_get(struct device *larbdev)
72{ 103{
73 struct mtk_smi_larb *larb = dev_get_drvdata(larbdev); 104 struct mtk_smi_larb *larb = dev_get_drvdata(larbdev);
105 const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen;
74 struct mtk_smi *common = dev_get_drvdata(larb->smi_common_dev); 106 struct mtk_smi *common = dev_get_drvdata(larb->smi_common_dev);
75 int ret; 107 int ret;
76 108
@@ -87,7 +119,7 @@ int mtk_smi_larb_get(struct device *larbdev)
87 } 119 }
88 120
89 /* Configure the iommu info for this larb */ 121 /* Configure the iommu info for this larb */
90 writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN); 122 larb_gen->config_port(larbdev);
91 123
92 return 0; 124 return 0;
93} 125}
@@ -126,6 +158,45 @@ mtk_smi_larb_bind(struct device *dev, struct device *master, void *data)
126 return -ENODEV; 158 return -ENODEV;
127} 159}
128 160
161static void mtk_smi_larb_config_port(struct device *dev)
162{
163 struct mtk_smi_larb *larb = dev_get_drvdata(dev);
164
165 writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN);
166}
167
168
169static void mtk_smi_larb_config_port_gen1(struct device *dev)
170{
171 struct mtk_smi_larb *larb = dev_get_drvdata(dev);
172 const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen;
173 struct mtk_smi *common = dev_get_drvdata(larb->smi_common_dev);
174 int i, m4u_port_id, larb_port_num;
175 u32 sec_con_val, reg_val;
176
177 m4u_port_id = larb_gen->port_in_larb[larb->larbid];
178 larb_port_num = larb_gen->port_in_larb[larb->larbid + 1]
179 - larb_gen->port_in_larb[larb->larbid];
180
181 for (i = 0; i < larb_port_num; i++, m4u_port_id++) {
182 if (*larb->mmu & BIT(i)) {
183 /* bit[port + 3] controls the virtual or physical */
184 sec_con_val = SMI_SECUR_CON_VAL_VIRT(m4u_port_id);
185 } else {
186 /* do not need to enable m4u for this port */
187 continue;
188 }
189 reg_val = readl(common->smi_ao_base
190 + REG_SMI_SECUR_CON_ADDR(m4u_port_id));
191 reg_val &= SMI_SECUR_CON_VAL_MSK(m4u_port_id);
192 reg_val |= sec_con_val;
193 reg_val |= SMI_SECUR_CON_VAL_DOMAIN(m4u_port_id);
194 writel(reg_val,
195 common->smi_ao_base
196 + REG_SMI_SECUR_CON_ADDR(m4u_port_id));
197 }
198}
199
129static void 200static void
130mtk_smi_larb_unbind(struct device *dev, struct device *master, void *data) 201mtk_smi_larb_unbind(struct device *dev, struct device *master, void *data)
131{ 202{
@@ -137,6 +208,31 @@ static const struct component_ops mtk_smi_larb_component_ops = {
137 .unbind = mtk_smi_larb_unbind, 208 .unbind = mtk_smi_larb_unbind,
138}; 209};
139 210
211static const struct mtk_smi_larb_gen mtk_smi_larb_mt8173 = {
212 /* mt8173 do not need the port in larb */
213 .config_port = mtk_smi_larb_config_port,
214};
215
216static const struct mtk_smi_larb_gen mtk_smi_larb_mt2701 = {
217 .port_in_larb = {
218 LARB0_PORT_OFFSET, LARB1_PORT_OFFSET,
219 LARB2_PORT_OFFSET, LARB3_PORT_OFFSET
220 },
221 .config_port = mtk_smi_larb_config_port_gen1,
222};
223
224static const struct of_device_id mtk_smi_larb_of_ids[] = {
225 {
226 .compatible = "mediatek,mt8173-smi-larb",
227 .data = &mtk_smi_larb_mt8173
228 },
229 {
230 .compatible = "mediatek,mt2701-smi-larb",
231 .data = &mtk_smi_larb_mt2701
232 },
233 {}
234};
235
140static int mtk_smi_larb_probe(struct platform_device *pdev) 236static int mtk_smi_larb_probe(struct platform_device *pdev)
141{ 237{
142 struct mtk_smi_larb *larb; 238 struct mtk_smi_larb *larb;
@@ -144,14 +240,20 @@ static int mtk_smi_larb_probe(struct platform_device *pdev)
144 struct device *dev = &pdev->dev; 240 struct device *dev = &pdev->dev;
145 struct device_node *smi_node; 241 struct device_node *smi_node;
146 struct platform_device *smi_pdev; 242 struct platform_device *smi_pdev;
243 const struct of_device_id *of_id;
147 244
148 if (!dev->pm_domain) 245 if (!dev->pm_domain)
149 return -EPROBE_DEFER; 246 return -EPROBE_DEFER;
150 247
248 of_id = of_match_node(mtk_smi_larb_of_ids, pdev->dev.of_node);
249 if (!of_id)
250 return -EINVAL;
251
151 larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL); 252 larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL);
152 if (!larb) 253 if (!larb)
153 return -ENOMEM; 254 return -ENOMEM;
154 255
256 larb->larb_gen = of_id->data;
155 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 257 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
156 larb->base = devm_ioremap_resource(dev, res); 258 larb->base = devm_ioremap_resource(dev, res);
157 if (IS_ERR(larb->base)) 259 if (IS_ERR(larb->base))
@@ -191,24 +293,34 @@ static int mtk_smi_larb_remove(struct platform_device *pdev)
191 return 0; 293 return 0;
192} 294}
193 295
194static const struct of_device_id mtk_smi_larb_of_ids[] = {
195 { .compatible = "mediatek,mt8173-smi-larb",},
196 {}
197};
198
199static struct platform_driver mtk_smi_larb_driver = { 296static struct platform_driver mtk_smi_larb_driver = {
200 .probe = mtk_smi_larb_probe, 297 .probe = mtk_smi_larb_probe,
201 .remove = mtk_smi_larb_remove, 298 .remove = mtk_smi_larb_remove,
202 .driver = { 299 .driver = {
203 .name = "mtk-smi-larb", 300 .name = "mtk-smi-larb",
204 .of_match_table = mtk_smi_larb_of_ids, 301 .of_match_table = mtk_smi_larb_of_ids,
205 } 302 }
206}; 303};
207 304
305static const struct of_device_id mtk_smi_common_of_ids[] = {
306 {
307 .compatible = "mediatek,mt8173-smi-common",
308 .data = (void *)MTK_SMI_GEN2
309 },
310 {
311 .compatible = "mediatek,mt2701-smi-common",
312 .data = (void *)MTK_SMI_GEN1
313 },
314 {}
315};
316
208static int mtk_smi_common_probe(struct platform_device *pdev) 317static int mtk_smi_common_probe(struct platform_device *pdev)
209{ 318{
210 struct device *dev = &pdev->dev; 319 struct device *dev = &pdev->dev;
211 struct mtk_smi *common; 320 struct mtk_smi *common;
321 struct resource *res;
322 const struct of_device_id *of_id;
323 enum mtk_smi_gen smi_gen;
212 324
213 if (!dev->pm_domain) 325 if (!dev->pm_domain)
214 return -EPROBE_DEFER; 326 return -EPROBE_DEFER;
@@ -226,6 +338,29 @@ static int mtk_smi_common_probe(struct platform_device *pdev)
226 if (IS_ERR(common->clk_smi)) 338 if (IS_ERR(common->clk_smi))
227 return PTR_ERR(common->clk_smi); 339 return PTR_ERR(common->clk_smi);
228 340
341 of_id = of_match_node(mtk_smi_common_of_ids, pdev->dev.of_node);
342 if (!of_id)
343 return -EINVAL;
344
345 /*
346 * for mtk smi gen 1, we need to get the ao(always on) base to config
347 * m4u port, and we need to enable the aync clock for transform the smi
348 * clock into emi clock domain, but for mtk smi gen2, there's no smi ao
349 * base.
350 */
351 smi_gen = (enum mtk_smi_gen)of_id->data;
352 if (smi_gen == MTK_SMI_GEN1) {
353 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
354 common->smi_ao_base = devm_ioremap_resource(dev, res);
355 if (IS_ERR(common->smi_ao_base))
356 return PTR_ERR(common->smi_ao_base);
357
358 common->clk_async = devm_clk_get(dev, "async");
359 if (IS_ERR(common->clk_async))
360 return PTR_ERR(common->clk_async);
361
362 clk_prepare_enable(common->clk_async);
363 }
229 pm_runtime_enable(dev); 364 pm_runtime_enable(dev);
230 platform_set_drvdata(pdev, common); 365 platform_set_drvdata(pdev, common);
231 return 0; 366 return 0;
@@ -237,11 +372,6 @@ static int mtk_smi_common_remove(struct platform_device *pdev)
237 return 0; 372 return 0;
238} 373}
239 374
240static const struct of_device_id mtk_smi_common_of_ids[] = {
241 { .compatible = "mediatek,mt8173-smi-common", },
242 {}
243};
244
245static struct platform_driver mtk_smi_common_driver = { 375static struct platform_driver mtk_smi_common_driver = {
246 .probe = mtk_smi_common_probe, 376 .probe = mtk_smi_common_probe,
247 .remove = mtk_smi_common_remove, 377 .remove = mtk_smi_common_remove,
@@ -272,4 +402,5 @@ err_unreg_smi:
272 platform_driver_unregister(&mtk_smi_common_driver); 402 platform_driver_unregister(&mtk_smi_common_driver);
273 return ret; 403 return ret;
274} 404}
405
275subsys_initcall(mtk_smi_init); 406subsys_initcall(mtk_smi_init);
diff --git a/include/dt-bindings/memory/mt2701-larb-port.h b/include/dt-bindings/memory/mt2701-larb-port.h
new file mode 100644
index 000000000000..78f66786da91
--- /dev/null
+++ b/include/dt-bindings/memory/mt2701-larb-port.h
@@ -0,0 +1,85 @@
1/*
2 * Copyright (c) 2015 MediaTek Inc.
3 * Author: Honghui Zhang <honghui.zhang@mediatek.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
14
15#ifndef _MT2701_LARB_PORT_H_
16#define _MT2701_LARB_PORT_H_
17
18/*
19 * Mediatek m4u generation 1 such as mt2701 has flat m4u port numbers,
20 * the first port's id for larb[N] would be the last port's id of larb[N - 1]
21 * plus one while larb[0]'s first port number is 0. The definition of
22 * MT2701_M4U_ID_LARBx is following HW register spec.
23 * But m4u generation 2 like mt8173 have different port number, it use fixed
24 * offset for each larb, the first port's id for larb[N] would be (N * 32).
25 */
26#define LARB0_PORT_OFFSET 0
27#define LARB1_PORT_OFFSET 11
28#define LARB2_PORT_OFFSET 21
29#define LARB3_PORT_OFFSET 43
30
31#define MT2701_M4U_ID_LARB0(port) ((port) + LARB0_PORT_OFFSET)
32#define MT2701_M4U_ID_LARB1(port) ((port) + LARB1_PORT_OFFSET)
33#define MT2701_M4U_ID_LARB2(port) ((port) + LARB2_PORT_OFFSET)
34
35/* Port define for larb0 */
36#define MT2701_M4U_PORT_DISP_OVL_0 MT2701_M4U_ID_LARB0(0)
37#define MT2701_M4U_PORT_DISP_RDMA1 MT2701_M4U_ID_LARB0(1)
38#define MT2701_M4U_PORT_DISP_RDMA MT2701_M4U_ID_LARB0(2)
39#define MT2701_M4U_PORT_DISP_WDMA MT2701_M4U_ID_LARB0(3)
40#define MT2701_M4U_PORT_MM_CMDQ MT2701_M4U_ID_LARB0(4)
41#define MT2701_M4U_PORT_MDP_RDMA MT2701_M4U_ID_LARB0(5)
42#define MT2701_M4U_PORT_MDP_WDMA MT2701_M4U_ID_LARB0(6)
43#define MT2701_M4U_PORT_MDP_ROTO MT2701_M4U_ID_LARB0(7)
44#define MT2701_M4U_PORT_MDP_ROTCO MT2701_M4U_ID_LARB0(8)
45#define MT2701_M4U_PORT_MDP_ROTVO MT2701_M4U_ID_LARB0(9)
46#define MT2701_M4U_PORT_MDP_RDMA1 MT2701_M4U_ID_LARB0(10)
47
48/* Port define for larb1 */
49#define MT2701_M4U_PORT_VDEC_MC_EXT MT2701_M4U_ID_LARB1(0)
50#define MT2701_M4U_PORT_VDEC_PP_EXT MT2701_M4U_ID_LARB1(1)
51#define MT2701_M4U_PORT_VDEC_PPWRAP_EXT MT2701_M4U_ID_LARB1(2)
52#define MT2701_M4U_PORT_VDEC_AVC_MV_EXT MT2701_M4U_ID_LARB1(3)
53#define MT2701_M4U_PORT_VDEC_PRED_RD_EXT MT2701_M4U_ID_LARB1(4)
54#define MT2701_M4U_PORT_VDEC_PRED_WR_EXT MT2701_M4U_ID_LARB1(5)
55#define MT2701_M4U_PORT_VDEC_VLD_EXT MT2701_M4U_ID_LARB1(6)
56#define MT2701_M4U_PORT_VDEC_VLD2_EXT MT2701_M4U_ID_LARB1(7)
57#define MT2701_M4U_PORT_VDEC_TILE_EXT MT2701_M4U_ID_LARB1(8)
58#define MT2701_M4U_PORT_VDEC_IMG_RESZ_EXT MT2701_M4U_ID_LARB1(9)
59
60/* Port define for larb2 */
61#define MT2701_M4U_PORT_VENC_RCPU MT2701_M4U_ID_LARB2(0)
62#define MT2701_M4U_PORT_VENC_REC_FRM MT2701_M4U_ID_LARB2(1)
63#define MT2701_M4U_PORT_VENC_BSDMA MT2701_M4U_ID_LARB2(2)
64#define MT2701_M4U_PORT_JPGENC_RDMA MT2701_M4U_ID_LARB2(3)
65#define MT2701_M4U_PORT_VENC_LT_RCPU MT2701_M4U_ID_LARB2(4)
66#define MT2701_M4U_PORT_VENC_LT_REC_FRM MT2701_M4U_ID_LARB2(5)
67#define MT2701_M4U_PORT_VENC_LT_BSDMA MT2701_M4U_ID_LARB2(6)
68#define MT2701_M4U_PORT_JPGDEC_BSDMA MT2701_M4U_ID_LARB2(7)
69#define MT2701_M4U_PORT_VENC_SV_COMV MT2701_M4U_ID_LARB2(8)
70#define MT2701_M4U_PORT_VENC_RD_COMV MT2701_M4U_ID_LARB2(9)
71#define MT2701_M4U_PORT_JPGENC_BSDMA MT2701_M4U_ID_LARB2(10)
72#define MT2701_M4U_PORT_VENC_CUR_LUMA MT2701_M4U_ID_LARB2(11)
73#define MT2701_M4U_PORT_VENC_CUR_CHROMA MT2701_M4U_ID_LARB2(12)
74#define MT2701_M4U_PORT_VENC_REF_LUMA MT2701_M4U_ID_LARB2(13)
75#define MT2701_M4U_PORT_VENC_REF_CHROMA MT2701_M4U_ID_LARB2(14)
76#define MT2701_M4U_PORT_IMG_RESZ MT2701_M4U_ID_LARB2(15)
77#define MT2701_M4U_PORT_VENC_LT_SV_COMV MT2701_M4U_ID_LARB2(16)
78#define MT2701_M4U_PORT_VENC_LT_RD_COMV MT2701_M4U_ID_LARB2(17)
79#define MT2701_M4U_PORT_VENC_LT_CUR_LUMA MT2701_M4U_ID_LARB2(18)
80#define MT2701_M4U_PORT_VENC_LT_CUR_CHROMA MT2701_M4U_ID_LARB2(19)
81#define MT2701_M4U_PORT_VENC_LT_REF_LUMA MT2701_M4U_ID_LARB2(20)
82#define MT2701_M4U_PORT_VENC_LT_REF_CHROMA MT2701_M4U_ID_LARB2(21)
83#define MT2701_M4U_PORT_JPGDEC_WDMA MT2701_M4U_ID_LARB2(22)
84
85#endif
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 664683aedcce..a35fb8b42e1a 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -152,6 +152,7 @@ struct iommu_dm_region {
152 * @domain_set_attr: Change domain attributes 152 * @domain_set_attr: Change domain attributes
153 * @get_dm_regions: Request list of direct mapping requirements for a device 153 * @get_dm_regions: Request list of direct mapping requirements for a device
154 * @put_dm_regions: Free list of direct mapping requirements for a device 154 * @put_dm_regions: Free list of direct mapping requirements for a device
155 * @apply_dm_region: Temporary helper call-back for iova reserved ranges
155 * @domain_window_enable: Configure and enable a particular window for a domain 156 * @domain_window_enable: Configure and enable a particular window for a domain
156 * @domain_window_disable: Disable a particular window for a domain 157 * @domain_window_disable: Disable a particular window for a domain
157 * @domain_set_windows: Set the number of windows for a domain 158 * @domain_set_windows: Set the number of windows for a domain
@@ -186,6 +187,8 @@ struct iommu_ops {
186 /* Request/Free a list of direct mapping requirements for a device */ 187 /* Request/Free a list of direct mapping requirements for a device */
187 void (*get_dm_regions)(struct device *dev, struct list_head *list); 188 void (*get_dm_regions)(struct device *dev, struct list_head *list);
188 void (*put_dm_regions)(struct device *dev, struct list_head *list); 189 void (*put_dm_regions)(struct device *dev, struct list_head *list);
190 void (*apply_dm_region)(struct device *dev, struct iommu_domain *domain,
191 struct iommu_dm_region *region);
189 192
190 /* Window handling functions */ 193 /* Window handling functions */
191 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, 194 int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr,
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-24 08:07:12 -0400