aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-06-06 14:48:46 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-06-06 14:48:46 -0400
commit2732ea9e8528ec3d065e04ecd357f61eec4869a8 (patch)
tree64ed2014fb448caf250ff6b09c894a42d7d6d6c0
parent75bcc84445bccf40a6215ff6fc6dd91978b1490e (diff)
parentc0981b863a31a1891aa2719957983f4297770f87 (diff)
Merge tag 'iommu-updates-v3.16' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu into next
Pull IOMMU updates from Joerg Roedel: "The changes include: - a new IOMMU driver for ARM Renesas SOCs - updates and fixes for the ARM Exynos driver to bring it closer to a usable state again - convert the AMD IOMMUv2 driver to use the mmu_notifier->release call-back instead of the task_exit notifier - random other fixes and minor improvements to a number of other IOMMU drivers" * tag 'iommu-updates-v3.16' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (54 commits) iommu/msm: Use devm_ioremap_resource to simplify code iommu/amd: Fix recently introduced compile warnings arm/ipmmu-vmsa: Fix compile error iommu/exynos: Fix checkpatch warning iommu/exynos: Fix trivial typo iommu/exynos: Remove invalid symbol dependency iommu: fsl_pamu.c: Fix for possible null pointer dereference iommu/amd: Remove duplicate checking code iommu/amd: Handle parallel invalidate_range_start/end calls correctly iommu/amd: Remove IOMMUv2 pasid_state_list iommu/amd: Implement mmu_notifier_release call-back iommu/amd: Convert IOMMUv2 state_table into state_list iommu/amd: Don't access IOMMUv2 state_table directly iommu/ipmmu-vmsa: Support clearing mappings iommu/ipmmu-vmsa: Remove stage 2 PTE bits definitions iommu/ipmmu-vmsa: Support 2MB mappings iommu/ipmmu-vmsa: Rewrite page table management iommu/ipmmu-vmsa: PMD is never folded, PUD always is iommu/ipmmu-vmsa: Set the PTE contiguous hint bit when possible iommu/ipmmu-vmsa: Define driver-specific page directory sizes ...
Diffstat
-rw-r--r--Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt70
-rw-r--r--drivers/iommu/Kconfig26
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/amd_iommu.c8
-rw-r--r--drivers/iommu/amd_iommu_v2.c184
-rw-r--r--drivers/iommu/arm-smmu.c4
-rw-r--r--drivers/iommu/exynos-iommu.c1033
-rw-r--r--drivers/iommu/fsl_pamu.c3
-rw-r--r--drivers/iommu/ipmmu-vmsa.c1255
-rw-r--r--drivers/iommu/msm_iommu_dev.c38
-rw-r--r--drivers/iommu/omap-iommu.c31
-rw-r--r--drivers/iommu/omap-iopgtable.h3
-rw-r--r--drivers/iommu/shmobile-ipmmu.c20
-rw-r--r--include/linux/platform_data/ipmmu-vmsa.h24
14 files changed, 2087 insertions, 613 deletions
diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt
new file mode 100644
index 000000000000..6fa4c737af23
--- /dev/null
+++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt
@@ -0,0 +1,70 @@
1Samsung Exynos IOMMU H/W, System MMU (System Memory Management Unit)
2
3Samsung's Exynos architecture contains System MMUs that enables scattered
4physical memory chunks visible as a contiguous region to DMA-capable peripheral
5devices like MFC, FIMC, FIMD, GScaler, FIMC-IS and so forth.
6
7System MMU is an IOMMU and supports identical translation table format to
8ARMv7 translation tables with minimum set of page properties including access
9permissions, shareability and security protection. In addition, System MMU has
10another capabilities like L2 TLB or block-fetch buffers to minimize translation
11latency.
12
13System MMUs are in many to one relation with peripheral devices, i.e. single
14peripheral device might have multiple System MMUs (usually one for each bus
15master), but one System MMU can handle transactions from only one peripheral
16device. The relation between a System MMU and the peripheral device needs to be
17defined in device node of the peripheral device.
18
19MFC in all Exynos SoCs and FIMD, M2M Scalers and G2D in Exynos5420 has 2 System
20MMUs.
21* MFC has one System MMU on its left and right bus.
22* FIMD in Exynos5420 has one System MMU for window 0 and 4, the other system MMU
23 for window 1, 2 and 3.
24* M2M Scalers and G2D in Exynos5420 has one System MMU on the read channel and
25 the other System MMU on the write channel.
26The drivers must consider how to handle those System MMUs. One of the idea is
27to implement child devices or sub-devices which are the client devices of the
28System MMU.
29
30Note:
31The current DT binding for the Exynos System MMU is incomplete.
32The following properties can be removed or changed, if found incompatible with
33the "Generic IOMMU Binding" support for attaching devices to the IOMMU.
34
35Required properties:
36- compatible: Should be "samsung,exynos-sysmmu"
37- reg: A tuple of base address and size of System MMU registers.
38- interrupt-parent: The phandle of the interrupt controller of System MMU
39- interrupts: An interrupt specifier for interrupt signal of System MMU,
40 according to the format defined by a particular interrupt
41 controller.
42- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock.
43 Optional "master" if the clock to the System MMU is gated by
44 another gate clock other than "sysmmu".
45 Exynos4 SoCs, there needs no "master" clock.
46 Exynos5 SoCs, some System MMUs must have "master" clocks.
47- clocks: Required if the System MMU is needed to gate its clock.
48- samsung,power-domain: Required if the System MMU is needed to gate its power.
49 Please refer to the following document:
50 Documentation/devicetree/bindings/arm/exynos/power_domain.txt
51
52Examples:
53 gsc_0: gsc@13e00000 {
54 compatible = "samsung,exynos5-gsc";
55 reg = <0x13e00000 0x1000>;
56 interrupts = <0 85 0>;
57 samsung,power-domain = <&pd_gsc>;
58 clocks = <&clock CLK_GSCL0>;
59 clock-names = "gscl";
60 };
61
62 sysmmu_gsc0: sysmmu@13E80000 {
63 compatible = "samsung,exynos-sysmmu";
64 reg = <0x13E80000 0x1000>;
65 interrupt-parent = <&combiner>;
66 interrupts = <2 0>;
67 clock-names = "sysmmu", "master";
68 clocks = <&clock CLK_SMMU_GSCL0>, <&clock CLK_GSCL0>;
69 samsung,power-domain = <&pd_gsc>;
70 };
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index df56e4c74a7e..d260605e6d5f 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -178,13 +178,13 @@ config TEGRA_IOMMU_SMMU
178 178
179config EXYNOS_IOMMU 179config EXYNOS_IOMMU
180 bool "Exynos IOMMU Support" 180 bool "Exynos IOMMU Support"
181 depends on ARCH_EXYNOS && EXYNOS_DEV_SYSMMU 181 depends on ARCH_EXYNOS
182 select IOMMU_API 182 select IOMMU_API
183 help 183 help
184 Support for the IOMMU(System MMU) of Samsung Exynos application 184 Support for the IOMMU (System MMU) of Samsung Exynos application
185 processor family. This enables H/W multimedia accellerators to see 185 processor family. This enables H/W multimedia accelerators to see
186 non-linear physical memory chunks as a linear memory in their 186 non-linear physical memory chunks as linear memory in their
187 address spaces 187 address space.
188 188
189 If unsure, say N here. 189 If unsure, say N here.
190 190
@@ -193,9 +193,9 @@ config EXYNOS_IOMMU_DEBUG
193 depends on EXYNOS_IOMMU 193 depends on EXYNOS_IOMMU
194 help 194 help
195 Select this to see the detailed log message that shows what 195 Select this to see the detailed log message that shows what
196 happens in the IOMMU driver 196 happens in the IOMMU driver.
197 197
198 Say N unless you need kernel log message for IOMMU debugging 198 Say N unless you need kernel log message for IOMMU debugging.
199 199
200config SHMOBILE_IPMMU 200config SHMOBILE_IPMMU
201 bool 201 bool
@@ -272,6 +272,18 @@ config SHMOBILE_IOMMU_L1SIZE
272 default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB 272 default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB
273 default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB 273 default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB
274 274
275config IPMMU_VMSA
276 bool "Renesas VMSA-compatible IPMMU"
277 depends on ARM_LPAE
278 depends on ARCH_SHMOBILE || COMPILE_TEST
279 select IOMMU_API
280 select ARM_DMA_USE_IOMMU
281 help
282 Support for the Renesas VMSA-compatible IPMMU Renesas found in the
283 R-Mobile APE6 and R-Car H2/M2 SoCs.
284
285 If unsure, say N.
286
275config SPAPR_TCE_IOMMU 287config SPAPR_TCE_IOMMU
276 bool "sPAPR TCE IOMMU Support" 288 bool "sPAPR TCE IOMMU Support"
277 depends on PPC_POWERNV || PPC_PSERIES 289 depends on PPC_POWERNV || PPC_PSERIES
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 5d58bf16e9e3..8893bad048e0 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
7obj-$(CONFIG_ARM_SMMU) += arm-smmu.o 7obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
8obj-$(CONFIG_DMAR_TABLE) += dmar.o 8obj-$(CONFIG_DMAR_TABLE) += dmar.o
9obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o 9obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o
10obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o
10obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o 11obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o
11obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o 12obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
12obj-$(CONFIG_OMAP_IOMMU) += omap-iommu2.o 13obj-$(CONFIG_OMAP_IOMMU) += omap-iommu2.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 57068e8035b5..4aec6a29e316 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3499,8 +3499,6 @@ int __init amd_iommu_init_passthrough(void)
3499{ 3499{
3500 struct iommu_dev_data *dev_data; 3500 struct iommu_dev_data *dev_data;
3501 struct pci_dev *dev = NULL; 3501 struct pci_dev *dev = NULL;
3502 struct amd_iommu *iommu;
3503 u16 devid;
3504 int ret; 3502 int ret;
3505 3503
3506 ret = alloc_passthrough_domain(); 3504 ret = alloc_passthrough_domain();
@@ -3514,12 +3512,6 @@ int __init amd_iommu_init_passthrough(void)
3514 dev_data = get_dev_data(&dev->dev); 3512 dev_data = get_dev_data(&dev->dev);
3515 dev_data->passthrough = true; 3513 dev_data->passthrough = true;
3516 3514
3517 devid = get_device_id(&dev->dev);
3518
3519 iommu = amd_iommu_rlookup_table[devid];
3520 if (!iommu)
3521 continue;
3522
3523 attach_device(&dev->dev, pt_domain); 3515 attach_device(&dev->dev, pt_domain);
3524 } 3516 }
3525 3517
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 203b2e6a91cf..d4daa05efe60 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -45,6 +45,8 @@ struct pri_queue {
45struct pasid_state { 45struct pasid_state {
46 struct list_head list; /* For global state-list */ 46 struct list_head list; /* For global state-list */
47 atomic_t count; /* Reference count */ 47 atomic_t count; /* Reference count */
48 atomic_t mmu_notifier_count; /* Counting nested mmu_notifier
49 calls */
48 struct task_struct *task; /* Task bound to this PASID */ 50 struct task_struct *task; /* Task bound to this PASID */
49 struct mm_struct *mm; /* mm_struct for the faults */ 51 struct mm_struct *mm; /* mm_struct for the faults */
50 struct mmu_notifier mn; /* mmu_otifier handle */ 52 struct mmu_notifier mn; /* mmu_otifier handle */
@@ -56,6 +58,8 @@ struct pasid_state {
56}; 58};
57 59
58struct device_state { 60struct device_state {
61 struct list_head list;
62 u16 devid;
59 atomic_t count; 63 atomic_t count;
60 struct pci_dev *pdev; 64 struct pci_dev *pdev;
61 struct pasid_state **states; 65 struct pasid_state **states;
@@ -81,13 +85,9 @@ struct fault {
81 u16 flags; 85 u16 flags;
82}; 86};
83 87
84static struct device_state **state_table; 88static LIST_HEAD(state_list);
85static spinlock_t state_lock; 89static spinlock_t state_lock;
86 90
87/* List and lock for all pasid_states */
88static LIST_HEAD(pasid_state_list);
89static DEFINE_SPINLOCK(ps_lock);
90
91static struct workqueue_struct *iommu_wq; 91static struct workqueue_struct *iommu_wq;
92 92
93/* 93/*
@@ -99,7 +99,6 @@ static u64 *empty_page_table;
99 99
100static void free_pasid_states(struct device_state *dev_state); 100static void free_pasid_states(struct device_state *dev_state);
101static void unbind_pasid(struct device_state *dev_state, int pasid); 101static void unbind_pasid(struct device_state *dev_state, int pasid);
102static int task_exit(struct notifier_block *nb, unsigned long e, void *data);
103 102
104static u16 device_id(struct pci_dev *pdev) 103static u16 device_id(struct pci_dev *pdev)
105{ 104{
@@ -111,13 +110,25 @@ static u16 device_id(struct pci_dev *pdev)
111 return devid; 110 return devid;
112} 111}
113 112
113static struct device_state *__get_device_state(u16 devid)
114{
115 struct device_state *dev_state;
116
117 list_for_each_entry(dev_state, &state_list, list) {
118 if (dev_state->devid == devid)
119 return dev_state;
120 }
121
122 return NULL;
123}
124
114static struct device_state *get_device_state(u16 devid) 125static struct device_state *get_device_state(u16 devid)
115{ 126{
116 struct device_state *dev_state; 127 struct device_state *dev_state;
117 unsigned long flags; 128 unsigned long flags;
118 129
119 spin_lock_irqsave(&state_lock, flags); 130 spin_lock_irqsave(&state_lock, flags);
120 dev_state = state_table[devid]; 131 dev_state = __get_device_state(devid);
121 if (dev_state != NULL) 132 if (dev_state != NULL)
122 atomic_inc(&dev_state->count); 133 atomic_inc(&dev_state->count);
123 spin_unlock_irqrestore(&state_lock, flags); 134 spin_unlock_irqrestore(&state_lock, flags);
@@ -158,29 +169,6 @@ static void put_device_state_wait(struct device_state *dev_state)
158 free_device_state(dev_state); 169 free_device_state(dev_state);
159} 170}
160 171
161static struct notifier_block profile_nb = {
162 .notifier_call = task_exit,
163};
164
165static void link_pasid_state(struct pasid_state *pasid_state)
166{
167 spin_lock(&ps_lock);
168 list_add_tail(&pasid_state->list, &pasid_state_list);
169 spin_unlock(&ps_lock);
170}
171
172static void __unlink_pasid_state(struct pasid_state *pasid_state)
173{
174 list_del(&pasid_state->list);
175}
176
177static void unlink_pasid_state(struct pasid_state *pasid_state)
178{
179 spin_lock(&ps_lock);
180 __unlink_pasid_state(pasid_state);
181 spin_unlock(&ps_lock);
182}
183
184/* Must be called under dev_state->lock */ 172/* Must be called under dev_state->lock */
185static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state, 173static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,
186 int pasid, bool alloc) 174 int pasid, bool alloc)
@@ -337,7 +325,6 @@ static void unbind_pasid(struct device_state *dev_state, int pasid)
337 if (pasid_state == NULL) 325 if (pasid_state == NULL)
338 return; 326 return;
339 327
340 unlink_pasid_state(pasid_state);
341 __unbind_pasid(pasid_state); 328 __unbind_pasid(pasid_state);
342 put_pasid_state_wait(pasid_state); /* Reference taken in this function */ 329 put_pasid_state_wait(pasid_state); /* Reference taken in this function */
343} 330}
@@ -379,7 +366,12 @@ static void free_pasid_states(struct device_state *dev_state)
379 continue; 366 continue;
380 367
381 put_pasid_state(pasid_state); 368 put_pasid_state(pasid_state);
382 unbind_pasid(dev_state, i); 369
370 /*
371 * This will call the mn_release function and
372 * unbind the PASID
373 */
374 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
383 } 375 }
384 376
385 if (dev_state->pasid_levels == 2) 377 if (dev_state->pasid_levels == 2)
@@ -443,8 +435,11 @@ static void mn_invalidate_range_start(struct mmu_notifier *mn,
443 pasid_state = mn_to_state(mn); 435 pasid_state = mn_to_state(mn);
444 dev_state = pasid_state->device_state; 436 dev_state = pasid_state->device_state;
445 437
446 amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid, 438 if (atomic_add_return(1, &pasid_state->mmu_notifier_count) == 1) {
447 __pa(empty_page_table)); 439 amd_iommu_domain_set_gcr3(dev_state->domain,
440 pasid_state->pasid,
441 __pa(empty_page_table));
442 }
448} 443}
449 444
450static void mn_invalidate_range_end(struct mmu_notifier *mn, 445static void mn_invalidate_range_end(struct mmu_notifier *mn,
@@ -457,11 +452,31 @@ static void mn_invalidate_range_end(struct mmu_notifier *mn,
457 pasid_state = mn_to_state(mn); 452 pasid_state = mn_to_state(mn);
458 dev_state = pasid_state->device_state; 453 dev_state = pasid_state->device_state;
459 454
460 amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid, 455 if (atomic_dec_and_test(&pasid_state->mmu_notifier_count)) {
461 __pa(pasid_state->mm->pgd)); 456 amd_iommu_domain_set_gcr3(dev_state->domain,
457 pasid_state->pasid,
458 __pa(pasid_state->mm->pgd));
459 }
460}
461
462static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
463{
464 struct pasid_state *pasid_state;
465 struct device_state *dev_state;
466
467 might_sleep();
468
469 pasid_state = mn_to_state(mn);
470 dev_state = pasid_state->device_state;
471
472 if (pasid_state->device_state->inv_ctx_cb)
473 dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);
474
475 unbind_pasid(dev_state, pasid_state->pasid);
462} 476}
463 477
464static struct mmu_notifier_ops iommu_mn = { 478static struct mmu_notifier_ops iommu_mn = {
479 .release = mn_release,
465 .clear_flush_young = mn_clear_flush_young, 480 .clear_flush_young = mn_clear_flush_young,
466 .change_pte = mn_change_pte, 481 .change_pte = mn_change_pte,
467 .invalidate_page = mn_invalidate_page, 482 .invalidate_page = mn_invalidate_page,
@@ -606,53 +621,6 @@ static struct notifier_block ppr_nb = {
606 .notifier_call = ppr_notifier, 621 .notifier_call = ppr_notifier,
607}; 622};
608 623
609static int task_exit(struct notifier_block *nb, unsigned long e, void *data)
610{
611 struct pasid_state *pasid_state;
612 struct task_struct *task;
613
614 task = data;
615
616 /*
617 * Using this notifier is a hack - but there is no other choice
618 * at the moment. What I really want is a sleeping notifier that
619 * is called when an MM goes down. But such a notifier doesn't
620 * exist yet. The notifier needs to sleep because it has to make
621 * sure that the device does not use the PASID and the address
622 * space anymore before it is destroyed. This includes waiting
623 * for pending PRI requests to pass the workqueue. The
624 * MMU-Notifiers would be a good fit, but they use RCU and so
625 * they are not allowed to sleep. Lets see how we can solve this
626 * in a more intelligent way in the future.
627 */
628again:
629 spin_lock(&ps_lock);
630 list_for_each_entry(pasid_state, &pasid_state_list, list) {
631 struct device_state *dev_state;
632 int pasid;
633
634 if (pasid_state->task != task)
635 continue;
636
637 /* Drop Lock and unbind */
638 spin_unlock(&ps_lock);
639
640 dev_state = pasid_state->device_state;
641 pasid = pasid_state->pasid;
642
643 if (pasid_state->device_state->inv_ctx_cb)
644 dev_state->inv_ctx_cb(dev_state->pdev, pasid);
645
646 unbind_pasid(dev_state, pasid);
647
648 /* Task may be in the list multiple times */
649 goto again;
650 }
651 spin_unlock(&ps_lock);
652
653 return NOTIFY_OK;
654}
655
656int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid, 624int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
657 struct task_struct *task) 625 struct task_struct *task)
658{ 626{
@@ -682,6 +650,7 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
682 goto out; 650 goto out;
683 651
684 atomic_set(&pasid_state->count, 1); 652 atomic_set(&pasid_state->count, 1);
653 atomic_set(&pasid_state->mmu_notifier_count, 0);
685 init_waitqueue_head(&pasid_state->wq); 654 init_waitqueue_head(&pasid_state->wq);
686 spin_lock_init(&pasid_state->lock); 655 spin_lock_init(&pasid_state->lock);
687 656
@@ -705,8 +674,6 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid,
705 if (ret) 674 if (ret)
706 goto out_clear_state; 675 goto out_clear_state;
707 676
708 link_pasid_state(pasid_state);
709
710 return 0; 677 return 0;
711 678
712out_clear_state: 679out_clear_state:
@@ -727,6 +694,7 @@ EXPORT_SYMBOL(amd_iommu_bind_pasid);
727 694
728void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid) 695void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
729{ 696{
697 struct pasid_state *pasid_state;
730 struct device_state *dev_state; 698 struct device_state *dev_state;
731 u16 devid; 699 u16 devid;
732 700
@@ -743,7 +711,17 @@ void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid)
743 if (pasid < 0 || pasid >= dev_state->max_pasids) 711 if (pasid < 0 || pasid >= dev_state->max_pasids)
744 goto out; 712 goto out;
745 713
746 unbind_pasid(dev_state, pasid); 714 pasid_state = get_pasid_state(dev_state, pasid);
715 if (pasid_state == NULL)
716 goto out;
717 /*
718 * Drop reference taken here. We are safe because we still hold
719 * the reference taken in the amd_iommu_bind_pasid function.
720 */
721 put_pasid_state(pasid_state);
722
723 /* This will call the mn_release function and unbind the PASID */
724 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);
747 725
748out: 726out:
749 put_device_state(dev_state); 727 put_device_state(dev_state);
@@ -773,7 +751,8 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
773 751
774 spin_lock_init(&dev_state->lock); 752 spin_lock_init(&dev_state->lock);
775 init_waitqueue_head(&dev_state->wq); 753 init_waitqueue_head(&dev_state->wq);
776 dev_state->pdev = pdev; 754 dev_state->pdev = pdev;
755 dev_state->devid = devid;
777 756
778 tmp = pasids; 757 tmp = pasids;
779 for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9) 758 for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9)
@@ -803,13 +782,13 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids)
803 782
804 spin_lock_irqsave(&state_lock, flags); 783 spin_lock_irqsave(&state_lock, flags);
805 784
806 if (state_table[devid] != NULL) { 785 if (__get_device_state(devid) != NULL) {
807 spin_unlock_irqrestore(&state_lock, flags); 786 spin_unlock_irqrestore(&state_lock, flags);
808 ret = -EBUSY; 787 ret = -EBUSY;
809 goto out_free_domain; 788 goto out_free_domain;
810 } 789 }
811 790
812 state_table[devid] = dev_state; 791 list_add_tail(&dev_state->list, &state_list);
813 792
814 spin_unlock_irqrestore(&state_lock, flags); 793 spin_unlock_irqrestore(&state_lock, flags);
815 794
@@ -841,13 +820,13 @@ void amd_iommu_free_device(struct pci_dev *pdev)
841 820
842 spin_lock_irqsave(&state_lock, flags); 821 spin_lock_irqsave(&state_lock, flags);
843 822
844 dev_state = state_table[devid]; 823 dev_state = __get_device_state(devid);
845 if (dev_state == NULL) { 824 if (dev_state == NULL) {
846 spin_unlock_irqrestore(&state_lock, flags); 825 spin_unlock_irqrestore(&state_lock, flags);
847 return; 826 return;
848 } 827 }
849 828
850 state_table[devid] = NULL; 829 list_del(&dev_state->list);
851 830
852 spin_unlock_irqrestore(&state_lock, flags); 831 spin_unlock_irqrestore(&state_lock, flags);
853 832
@@ -874,7 +853,7 @@ int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev,
874 spin_lock_irqsave(&state_lock, flags); 853 spin_lock_irqsave(&state_lock, flags);
875 854
876 ret = -EINVAL; 855 ret = -EINVAL;
877 dev_state = state_table[devid]; 856 dev_state = __get_device_state(devid);
878 if (dev_state == NULL) 857 if (dev_state == NULL)
879 goto out_unlock; 858 goto out_unlock;
880 859
@@ -905,7 +884,7 @@ int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev,
905 spin_lock_irqsave(&state_lock, flags); 884 spin_lock_irqsave(&state_lock, flags);
906 885
907 ret = -EINVAL; 886 ret = -EINVAL;
908 dev_state = state_table[devid]; 887 dev_state = __get_device_state(devid);
909 if (dev_state == NULL) 888 if (dev_state == NULL)
910 goto out_unlock; 889 goto out_unlock;
911 890
@@ -922,7 +901,6 @@ EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb);
922 901
923static int __init amd_iommu_v2_init(void) 902static int __init amd_iommu_v2_init(void)
924{ 903{
925 size_t state_table_size;
926 int ret; 904 int ret;
927 905
928 pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n"); 906 pr_info("AMD IOMMUv2 driver by Joerg Roedel <joerg.roedel@amd.com>\n");
@@ -938,16 +916,10 @@ static int __init amd_iommu_v2_init(void)
938 916
939 spin_lock_init(&state_lock); 917 spin_lock_init(&state_lock);
940 918
941 state_table_size = MAX_DEVICES * sizeof(struct device_state *);
942 state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
943 get_order(state_table_size));
944 if (state_table == NULL)
945 return -ENOMEM;
946
947 ret = -ENOMEM; 919 ret = -ENOMEM;
948 iommu_wq = create_workqueue("amd_iommu_v2"); 920 iommu_wq = create_workqueue("amd_iommu_v2");
949 if (iommu_wq == NULL) 921 if (iommu_wq == NULL)
950 goto out_free; 922 goto out;
951 923
952 ret = -ENOMEM; 924 ret = -ENOMEM;
953 empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL); 925 empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL);
@@ -955,29 +927,24 @@ static int __init amd_iommu_v2_init(void)
955 goto out_destroy_wq; 927 goto out_destroy_wq;
956 928
957 amd_iommu_register_ppr_notifier(&ppr_nb); 929 amd_iommu_register_ppr_notifier(&ppr_nb);
958 profile_event_register(PROFILE_TASK_EXIT, &profile_nb);
959 930
960 return 0; 931 return 0;
961 932
962out_destroy_wq: 933out_destroy_wq:
963 destroy_workqueue(iommu_wq); 934 destroy_workqueue(iommu_wq);
964 935
965out_free: 936out:
966 free_pages((unsigned long)state_table, get_order(state_table_size));
967
968 return ret; 937 return ret;
969} 938}
970 939
971static void __exit amd_iommu_v2_exit(void) 940static void __exit amd_iommu_v2_exit(void)
972{ 941{
973 struct device_state *dev_state; 942 struct device_state *dev_state;
974 size_t state_table_size;
975 int i; 943 int i;
976 944
977 if (!amd_iommu_v2_supported()) 945 if (!amd_iommu_v2_supported())
978 return; 946 return;
979 947
980 profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb);
981 amd_iommu_unregister_ppr_notifier(&ppr_nb); 948 amd_iommu_unregister_ppr_notifier(&ppr_nb);
982 949
983 flush_workqueue(iommu_wq); 950 flush_workqueue(iommu_wq);
@@ -1000,9 +967,6 @@ static void __exit amd_iommu_v2_exit(void)
1000 967
1001 destroy_workqueue(iommu_wq); 968 destroy_workqueue(iommu_wq);
1002 969
1003 state_table_size = MAX_DEVICES * sizeof(struct device_state *);
1004 free_pages((unsigned long)state_table, get_order(state_table_size));
1005
1006 free_page((unsigned long)empty_page_table); 970 free_page((unsigned long)empty_page_table);
1007} 971}
1008 972
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 647c3c7fd742..1599354e974d 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1167,7 +1167,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1167 for (i = 0; i < master->num_streamids; ++i) { 1167 for (i = 0; i < master->num_streamids; ++i) {
1168 u32 idx, s2cr; 1168 u32 idx, s2cr;
1169 idx = master->smrs ? master->smrs[i].idx : master->streamids[i]; 1169 idx = master->smrs ? master->smrs[i].idx : master->streamids[i];
1170 s2cr = (S2CR_TYPE_TRANS << S2CR_TYPE_SHIFT) | 1170 s2cr = S2CR_TYPE_TRANS |
1171 (smmu_domain->root_cfg.cbndx << S2CR_CBNDX_SHIFT); 1171 (smmu_domain->root_cfg.cbndx << S2CR_CBNDX_SHIFT);
1172 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); 1172 writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx));
1173 } 1173 }
@@ -1804,7 +1804,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1804 * allocation (PTRS_PER_PGD). 1804 * allocation (PTRS_PER_PGD).
1805 */ 1805 */
1806#ifdef CONFIG_64BIT 1806#ifdef CONFIG_64BIT
1807 smmu->s1_output_size = min(39UL, size); 1807 smmu->s1_output_size = min((unsigned long)VA_BITS, size);
1808#else 1808#else
1809 smmu->s1_output_size = min(32UL, size); 1809 smmu->s1_output_size = min(32UL, size);
1810#endif 1810#endif
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 685f9263cfee..99054d2c040d 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -29,7 +29,8 @@
29#include <asm/cacheflush.h> 29#include <asm/cacheflush.h>
30#include <asm/pgtable.h> 30#include <asm/pgtable.h>
31 31
32#include <mach/sysmmu.h> 32typedef u32 sysmmu_iova_t;
33typedef u32 sysmmu_pte_t;
33 34
34/* We does not consider super section mapping (16MB) */ 35/* We does not consider super section mapping (16MB) */
35#define SECT_ORDER 20 36#define SECT_ORDER 20
@@ -44,28 +45,44 @@
44#define LPAGE_MASK (~(LPAGE_SIZE - 1)) 45#define LPAGE_MASK (~(LPAGE_SIZE - 1))
45#define SPAGE_MASK (~(SPAGE_SIZE - 1)) 46#define SPAGE_MASK (~(SPAGE_SIZE - 1))
46 47
47#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) 48#define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \
48#define lv1ent_page(sent) ((*(sent) & 3) == 1) 49 ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
50#define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK)
51#define lv1ent_page_zero(sent) ((*(sent) & 3) == 1)
52#define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \
53 ((*(sent) & 3) == 1))
49#define lv1ent_section(sent) ((*(sent) & 3) == 2) 54#define lv1ent_section(sent) ((*(sent) & 3) == 2)
50 55
51#define lv2ent_fault(pent) ((*(pent) & 3) == 0) 56#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
52#define lv2ent_small(pent) ((*(pent) & 2) == 2) 57#define lv2ent_small(pent) ((*(pent) & 2) == 2)
53#define lv2ent_large(pent) ((*(pent) & 3) == 1) 58#define lv2ent_large(pent) ((*(pent) & 3) == 1)
54 59
60static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size)
61{
62 return iova & (size - 1);
63}
64
55#define section_phys(sent) (*(sent) & SECT_MASK) 65#define section_phys(sent) (*(sent) & SECT_MASK)
56#define section_offs(iova) ((iova) & 0xFFFFF) 66#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE)
57#define lpage_phys(pent) (*(pent) & LPAGE_MASK) 67#define lpage_phys(pent) (*(pent) & LPAGE_MASK)
58#define lpage_offs(iova) ((iova) & 0xFFFF) 68#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE)
59#define spage_phys(pent) (*(pent) & SPAGE_MASK) 69#define spage_phys(pent) (*(pent) & SPAGE_MASK)
60#define spage_offs(iova) ((iova) & 0xFFF) 70#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE)
61
62#define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
63#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
64 71
65#define NUM_LV1ENTRIES 4096 72#define NUM_LV1ENTRIES 4096
66#define NUM_LV2ENTRIES 256 73#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
74
75static u32 lv1ent_offset(sysmmu_iova_t iova)
76{
77 return iova >> SECT_ORDER;
78}
79
80static u32 lv2ent_offset(sysmmu_iova_t iova)
81{
82 return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1);
83}
67 84
68#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long)) 85#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
69 86
70#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) 87#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
71 88
@@ -80,6 +97,13 @@
80#define CTRL_BLOCK 0x7 97#define CTRL_BLOCK 0x7
81#define CTRL_DISABLE 0x0 98#define CTRL_DISABLE 0x0
82 99
100#define CFG_LRU 0x1
101#define CFG_QOS(n) ((n & 0xF) << 7)
102#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
103#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */
104#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */
105#define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */
106
83#define REG_MMU_CTRL 0x000 107#define REG_MMU_CTRL 0x000
84#define REG_MMU_CFG 0x004 108#define REG_MMU_CFG 0x004
85#define REG_MMU_STATUS 0x008 109#define REG_MMU_STATUS 0x008
@@ -96,19 +120,32 @@
96 120
97#define REG_MMU_VERSION 0x034 121#define REG_MMU_VERSION 0x034
98 122
123#define MMU_MAJ_VER(val) ((val) >> 7)
124#define MMU_MIN_VER(val) ((val) & 0x7F)
125#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
126
127#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F))
128
99#define REG_PB0_SADDR 0x04C 129#define REG_PB0_SADDR 0x04C
100#define REG_PB0_EADDR 0x050 130#define REG_PB0_EADDR 0x050
101#define REG_PB1_SADDR 0x054 131#define REG_PB1_SADDR 0x054
102#define REG_PB1_EADDR 0x058 132#define REG_PB1_EADDR 0x058
103 133
104static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova) 134#define has_sysmmu(dev) (dev->archdata.iommu != NULL)
135
136static struct kmem_cache *lv2table_kmem_cache;
137static sysmmu_pte_t *zero_lv2_table;
138#define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table))
139
140static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova)
105{ 141{
106 return pgtable + lv1ent_offset(iova); 142 return pgtable + lv1ent_offset(iova);
107} 143}
108 144
109static unsigned long *page_entry(unsigned long *sent, unsigned long iova) 145static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova)
110{ 146{
111 return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova); 147 return (sysmmu_pte_t *)phys_to_virt(
148 lv2table_base(sent)) + lv2ent_offset(iova);
112} 149}
113 150
114enum exynos_sysmmu_inttype { 151enum exynos_sysmmu_inttype {
@@ -124,16 +161,6 @@ enum exynos_sysmmu_inttype {
124 SYSMMU_FAULTS_NUM 161 SYSMMU_FAULTS_NUM
125}; 162};
126 163
127/*
128 * @itype: type of fault.
129 * @pgtable_base: the physical address of page table base. This is 0 if @itype
130 * is SYSMMU_BUSERROR.
131 * @fault_addr: the device (virtual) address that the System MMU tried to
132 * translated. This is 0 if @itype is SYSMMU_BUSERROR.
133 */
134typedef int (*sysmmu_fault_handler_t)(enum exynos_sysmmu_inttype itype,
135 unsigned long pgtable_base, unsigned long fault_addr);
136
137static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { 164static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
138 REG_PAGE_FAULT_ADDR, 165 REG_PAGE_FAULT_ADDR,
139 REG_AR_FAULT_ADDR, 166 REG_AR_FAULT_ADDR,
@@ -157,27 +184,34 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
157 "UNKNOWN FAULT" 184 "UNKNOWN FAULT"
158}; 185};
159 186
187/* attached to dev.archdata.iommu of the master device */
188struct exynos_iommu_owner {
189 struct list_head client; /* entry of exynos_iommu_domain.clients */
190 struct device *dev;
191 struct device *sysmmu;
192 struct iommu_domain *domain;
193 void *vmm_data; /* IO virtual memory manager's data */
194 spinlock_t lock; /* Lock to preserve consistency of System MMU */
195};
196
160struct exynos_iommu_domain { 197struct exynos_iommu_domain {
161 struct list_head clients; /* list of sysmmu_drvdata.node */ 198 struct list_head clients; /* list of sysmmu_drvdata.node */
162 unsigned long *pgtable; /* lv1 page table, 16KB */ 199 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
163 short *lv2entcnt; /* free lv2 entry counter for each section */ 200 short *lv2entcnt; /* free lv2 entry counter for each section */
164 spinlock_t lock; /* lock for this structure */ 201 spinlock_t lock; /* lock for this structure */
165 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ 202 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
166}; 203};
167 204
168struct sysmmu_drvdata { 205struct sysmmu_drvdata {
169 struct list_head node; /* entry of exynos_iommu_domain.clients */
170 struct device *sysmmu; /* System MMU's device descriptor */ 206 struct device *sysmmu; /* System MMU's device descriptor */
171 struct device *dev; /* Owner of system MMU */ 207 struct device *master; /* Owner of system MMU */
172 char *dbgname; 208 void __iomem *sfrbase;
173 int nsfrs; 209 struct clk *clk;
174 void __iomem **sfrbases; 210 struct clk *clk_master;
175 struct clk *clk[2];
176 int activations; 211 int activations;
177 rwlock_t lock; 212 spinlock_t lock;
178 struct iommu_domain *domain; 213 struct iommu_domain *domain;
179 sysmmu_fault_handler_t fault_handler; 214 phys_addr_t pgtable;
180 unsigned long pgtable;
181}; 215};
182 216
183static bool set_sysmmu_active(struct sysmmu_drvdata *data) 217static bool set_sysmmu_active(struct sysmmu_drvdata *data)
@@ -204,6 +238,11 @@ static void sysmmu_unblock(void __iomem *sfrbase)
204 __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); 238 __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
205} 239}
206 240
241static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data)
242{
243 return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
244}
245
207static bool sysmmu_block(void __iomem *sfrbase) 246static bool sysmmu_block(void __iomem *sfrbase)
208{ 247{
209 int i = 120; 248 int i = 120;
@@ -226,429 +265,428 @@ static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
226} 265}
227 266
228static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, 267static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
229 unsigned long iova) 268 sysmmu_iova_t iova, unsigned int num_inv)
230{ 269{
231 __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY); 270 unsigned int i;
271
272 for (i = 0; i < num_inv; i++) {
273 __raw_writel((iova & SPAGE_MASK) | 1,
274 sfrbase + REG_MMU_FLUSH_ENTRY);
275 iova += SPAGE_SIZE;
276 }
232} 277}
233 278
234static void __sysmmu_set_ptbase(void __iomem *sfrbase, 279static void __sysmmu_set_ptbase(void __iomem *sfrbase,
235 unsigned long pgd) 280 phys_addr_t pgd)
236{ 281{
237 __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
238 __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); 282 __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
239 283
240 __sysmmu_tlb_invalidate(sfrbase); 284 __sysmmu_tlb_invalidate(sfrbase);
241} 285}
242 286
243static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base, 287static void show_fault_information(const char *name,
244 unsigned long size, int idx) 288 enum exynos_sysmmu_inttype itype,
289 phys_addr_t pgtable_base, sysmmu_iova_t fault_addr)
245{ 290{
246 __raw_writel(base, sfrbase + REG_PB0_SADDR + idx * 8); 291 sysmmu_pte_t *ent;
247 __raw_writel(size - 1 + base, sfrbase + REG_PB0_EADDR + idx * 8);
248}
249
250static void __set_fault_handler(struct sysmmu_drvdata *data,
251 sysmmu_fault_handler_t handler)
252{
253 unsigned long flags;
254
255 write_lock_irqsave(&data->lock, flags);
256 data->fault_handler = handler;
257 write_unlock_irqrestore(&data->lock, flags);
258}
259
260void exynos_sysmmu_set_fault_handler(struct device *dev,
261 sysmmu_fault_handler_t handler)
262{
263 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
264
265 __set_fault_handler(data, handler);
266}
267
268static int default_fault_handler(enum exynos_sysmmu_inttype itype,
269 unsigned long pgtable_base, unsigned long fault_addr)
270{
271 unsigned long *ent;
272 292
273 if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) 293 if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
274 itype = SYSMMU_FAULT_UNKNOWN; 294 itype = SYSMMU_FAULT_UNKNOWN;
275 295
276 pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n", 296 pr_err("%s occurred at %#x by %s(Page table base: %pa)\n",
277 sysmmu_fault_name[itype], fault_addr, pgtable_base); 297 sysmmu_fault_name[itype], fault_addr, name, &pgtable_base);
278 298
279 ent = section_entry(__va(pgtable_base), fault_addr); 299 ent = section_entry(phys_to_virt(pgtable_base), fault_addr);
280 pr_err("\tLv1 entry: 0x%lx\n", *ent); 300 pr_err("\tLv1 entry: %#x\n", *ent);
281 301
282 if (lv1ent_page(ent)) { 302 if (lv1ent_page(ent)) {
283 ent = page_entry(ent, fault_addr); 303 ent = page_entry(ent, fault_addr);
284 pr_err("\t Lv2 entry: 0x%lx\n", *ent); 304 pr_err("\t Lv2 entry: %#x\n", *ent);
285 } 305 }
286
287 pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
288
289 BUG();
290
291 return 0;
292} 306}
293 307
294static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) 308static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
295{ 309{
296 /* SYSMMU is in blocked when interrupt occurred. */ 310 /* SYSMMU is in blocked when interrupt occurred. */
297 struct sysmmu_drvdata *data = dev_id; 311 struct sysmmu_drvdata *data = dev_id;
298 struct resource *irqres;
299 struct platform_device *pdev;
300 enum exynos_sysmmu_inttype itype; 312 enum exynos_sysmmu_inttype itype;
301 unsigned long addr = -1; 313 sysmmu_iova_t addr = -1;
302 314 int ret = -ENOSYS;
303 int i, ret = -ENOSYS;
304
305 read_lock(&data->lock);
306 315
307 WARN_ON(!is_sysmmu_active(data)); 316 WARN_ON(!is_sysmmu_active(data));
308 317
309 pdev = to_platform_device(data->sysmmu); 318 spin_lock(&data->lock);
310 for (i = 0; i < (pdev->num_resources / 2); i++) { 319
311 irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i); 320 if (!IS_ERR(data->clk_master))
312 if (irqres && ((int)irqres->start == irq)) 321 clk_enable(data->clk_master);
313 break;
314 }
315 322
316 if (i == pdev->num_resources) { 323 itype = (enum exynos_sysmmu_inttype)
324 __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS));
325 if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
317 itype = SYSMMU_FAULT_UNKNOWN; 326 itype = SYSMMU_FAULT_UNKNOWN;
327 else
328 addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]);
329
330 if (itype == SYSMMU_FAULT_UNKNOWN) {
331 pr_err("%s: Fault is not occurred by System MMU '%s'!\n",
332 __func__, dev_name(data->sysmmu));
333 pr_err("%s: Please check if IRQ is correctly configured.\n",
334 __func__);
335 BUG();
318 } else { 336 } else {
319 itype = (enum exynos_sysmmu_inttype) 337 unsigned int base =
320 __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS)); 338 __raw_readl(data->sfrbase + REG_PT_BASE_ADDR);
321 if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) 339 show_fault_information(dev_name(data->sysmmu),
322 itype = SYSMMU_FAULT_UNKNOWN; 340 itype, base, addr);
323 else 341 if (data->domain)
324 addr = __raw_readl( 342 ret = report_iommu_fault(data->domain,
325 data->sfrbases[i] + fault_reg_offset[itype]); 343 data->master, addr, itype);
326 } 344 }
327 345
328 if (data->domain) 346 /* fault is not recovered by fault handler */
329 ret = report_iommu_fault(data->domain, data->dev, 347 BUG_ON(ret != 0);
330 addr, itype);
331 348
332 if ((ret == -ENOSYS) && data->fault_handler) { 349 __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR);
333 unsigned long base = data->pgtable;
334 if (itype != SYSMMU_FAULT_UNKNOWN)
335 base = __raw_readl(
336 data->sfrbases[i] + REG_PT_BASE_ADDR);
337 ret = data->fault_handler(itype, base, addr);
338 }
339 350
340 if (!ret && (itype != SYSMMU_FAULT_UNKNOWN)) 351 sysmmu_unblock(data->sfrbase);
341 __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
342 else
343 dev_dbg(data->sysmmu, "(%s) %s is not handled.\n",
344 data->dbgname, sysmmu_fault_name[itype]);
345 352
346 if (itype != SYSMMU_FAULT_UNKNOWN) 353 if (!IS_ERR(data->clk_master))
347 sysmmu_unblock(data->sfrbases[i]); 354 clk_disable(data->clk_master);
348 355
349 read_unlock(&data->lock); 356 spin_unlock(&data->lock);
350 357
351 return IRQ_HANDLED; 358 return IRQ_HANDLED;
352} 359}
353 360
354static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) 361static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
355{ 362{
363 if (!IS_ERR(data->clk_master))
364 clk_enable(data->clk_master);
365
366 __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
367 __raw_writel(0, data->sfrbase + REG_MMU_CFG);
368
369 clk_disable(data->clk);
370 if (!IS_ERR(data->clk_master))
371 clk_disable(data->clk_master);
372}
373
374static bool __sysmmu_disable(struct sysmmu_drvdata *data)
375{
376 bool disabled;
356 unsigned long flags; 377 unsigned long flags;
357 bool disabled = false;
358 int i;
359 378
360 write_lock_irqsave(&data->lock, flags); 379 spin_lock_irqsave(&data->lock, flags);
361 380
362 if (!set_sysmmu_inactive(data)) 381 disabled = set_sysmmu_inactive(data);
363 goto finish;
364 382
365 for (i = 0; i < data->nsfrs; i++) 383 if (disabled) {
366 __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL); 384 data->pgtable = 0;
385 data->domain = NULL;
367 386
368 if (data->clk[1]) 387 __sysmmu_disable_nocount(data);
369 clk_disable(data->clk[1]);
370 if (data->clk[0])
371 clk_disable(data->clk[0]);
372 388
373 disabled = true; 389 dev_dbg(data->sysmmu, "Disabled\n");
374 data->pgtable = 0; 390 } else {
375 data->domain = NULL; 391 dev_dbg(data->sysmmu, "%d times left to disable\n",
376finish: 392 data->activations);
377 write_unlock_irqrestore(&data->lock, flags); 393 }
378 394
379 if (disabled) 395 spin_unlock_irqrestore(&data->lock, flags);
380 dev_dbg(data->sysmmu, "(%s) Disabled\n", data->dbgname);
381 else
382 dev_dbg(data->sysmmu, "(%s) %d times left to be disabled\n",
383 data->dbgname, data->activations);
384 396
385 return disabled; 397 return disabled;
386} 398}
387 399
388/* __exynos_sysmmu_enable: Enables System MMU 400static void __sysmmu_init_config(struct sysmmu_drvdata *data)
389 *
390 * returns -error if an error occurred and System MMU is not enabled,
391 * 0 if the System MMU has been just enabled and 1 if System MMU was already
392 * enabled before.
393 */
394static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data,
395 unsigned long pgtable, struct iommu_domain *domain)
396{ 401{
397 int i, ret = 0; 402 unsigned int cfg = CFG_LRU | CFG_QOS(15);
398 unsigned long flags; 403 unsigned int ver;
404
405 ver = __raw_sysmmu_version(data);
406 if (MMU_MAJ_VER(ver) == 3) {
407 if (MMU_MIN_VER(ver) >= 2) {
408 cfg |= CFG_FLPDCACHE;
409 if (MMU_MIN_VER(ver) == 3) {
410 cfg |= CFG_ACGEN;
411 cfg &= ~CFG_LRU;
412 } else {
413 cfg |= CFG_SYSSEL;
414 }
415 }
416 }
399 417
400 write_lock_irqsave(&data->lock, flags); 418 __raw_writel(cfg, data->sfrbase + REG_MMU_CFG);
419}
401 420
402 if (!set_sysmmu_active(data)) { 421static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
403 if (WARN_ON(pgtable != data->pgtable)) { 422{
404 ret = -EBUSY; 423 if (!IS_ERR(data->clk_master))
405 set_sysmmu_inactive(data); 424 clk_enable(data->clk_master);
406 } else { 425 clk_enable(data->clk);
407 ret = 1;
408 }
409 426
410 dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname); 427 __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
411 goto finish;
412 }
413 428
414 if (data->clk[0]) 429 __sysmmu_init_config(data);
415 clk_enable(data->clk[0]);
416 if (data->clk[1])
417 clk_enable(data->clk[1]);
418 430
419 data->pgtable = pgtable; 431 __sysmmu_set_ptbase(data->sfrbase, data->pgtable);
420 432
421 for (i = 0; i < data->nsfrs; i++) { 433 __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
422 __sysmmu_set_ptbase(data->sfrbases[i], pgtable);
423 434
424 if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) { 435 if (!IS_ERR(data->clk_master))
425 /* System MMU version is 3.x */ 436 clk_disable(data->clk_master);
426 __raw_writel((1 << 12) | (2 << 28), 437}
427 data->sfrbases[i] + REG_MMU_CFG); 438
428 __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0); 439static int __sysmmu_enable(struct sysmmu_drvdata *data,
429 __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1); 440 phys_addr_t pgtable, struct iommu_domain *domain)
430 } 441{
442 int ret = 0;
443 unsigned long flags;
444
445 spin_lock_irqsave(&data->lock, flags);
446 if (set_sysmmu_active(data)) {
447 data->pgtable = pgtable;
448 data->domain = domain;
449
450 __sysmmu_enable_nocount(data);
451
452 dev_dbg(data->sysmmu, "Enabled\n");
453 } else {
454 ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
431 455
432 __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL); 456 dev_dbg(data->sysmmu, "already enabled\n");
433 } 457 }
434 458
435 data->domain = domain; 459 if (WARN_ON(ret < 0))
460 set_sysmmu_inactive(data); /* decrement count */
436 461
437 dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname); 462 spin_unlock_irqrestore(&data->lock, flags);
438finish:
439 write_unlock_irqrestore(&data->lock, flags);
440 463
441 return ret; 464 return ret;
442} 465}
443 466
444int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) 467/* __exynos_sysmmu_enable: Enables System MMU
468 *
469 * returns -error if an error occurred and System MMU is not enabled,
470 * 0 if the System MMU has been just enabled and 1 if System MMU was already
471 * enabled before.
472 */
473static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable,
474 struct iommu_domain *domain)
445{ 475{
446 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 476 int ret = 0;
447 int ret; 477 unsigned long flags;
478 struct exynos_iommu_owner *owner = dev->archdata.iommu;
479 struct sysmmu_drvdata *data;
448 480
449 BUG_ON(!memblock_is_memory(pgtable)); 481 BUG_ON(!has_sysmmu(dev));
450 482
451 ret = pm_runtime_get_sync(data->sysmmu); 483 spin_lock_irqsave(&owner->lock, flags);
452 if (ret < 0) {
453 dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname);
454 return ret;
455 }
456 484
457 ret = __exynos_sysmmu_enable(data, pgtable, NULL); 485 data = dev_get_drvdata(owner->sysmmu);
458 if (WARN_ON(ret < 0)) { 486
459 pm_runtime_put(data->sysmmu); 487 ret = __sysmmu_enable(data, pgtable, domain);
460 dev_err(data->sysmmu, 488 if (ret >= 0)
461 "(%s) Already enabled with page table %#lx\n", 489 data->master = dev;
462 data->dbgname, data->pgtable); 490
463 } else { 491 spin_unlock_irqrestore(&owner->lock, flags);
464 data->dev = dev;
465 }
466 492
467 return ret; 493 return ret;
468} 494}
469 495
496int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable)
497{
498 BUG_ON(!memblock_is_memory(pgtable));
499
500 return __exynos_sysmmu_enable(dev, pgtable, NULL);
501}
502
470static bool exynos_sysmmu_disable(struct device *dev) 503static bool exynos_sysmmu_disable(struct device *dev)
471{ 504{
472 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 505 unsigned long flags;
473 bool disabled; 506 bool disabled = true;
507 struct exynos_iommu_owner *owner = dev->archdata.iommu;
508 struct sysmmu_drvdata *data;
509
510 BUG_ON(!has_sysmmu(dev));
511
512 spin_lock_irqsave(&owner->lock, flags);
513
514 data = dev_get_drvdata(owner->sysmmu);
515
516 disabled = __sysmmu_disable(data);
517 if (disabled)
518 data->master = NULL;
474 519
475 disabled = __exynos_sysmmu_disable(data); 520 spin_unlock_irqrestore(&owner->lock, flags);
476 pm_runtime_put(data->sysmmu);
477 521
478 return disabled; 522 return disabled;
479} 523}
480 524
481static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova) 525static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
526 sysmmu_iova_t iova)
527{
528 if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3))
529 __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);
530}
531
532static void sysmmu_tlb_invalidate_flpdcache(struct device *dev,
533 sysmmu_iova_t iova)
482{ 534{
483 unsigned long flags; 535 unsigned long flags;
484 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 536 struct exynos_iommu_owner *owner = dev->archdata.iommu;
537 struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu);
538
539 if (!IS_ERR(data->clk_master))
540 clk_enable(data->clk_master);
485 541
486 read_lock_irqsave(&data->lock, flags); 542 spin_lock_irqsave(&data->lock, flags);
543 if (is_sysmmu_active(data))
544 __sysmmu_tlb_invalidate_flpdcache(data, iova);
545 spin_unlock_irqrestore(&data->lock, flags);
487 546
547 if (!IS_ERR(data->clk_master))
548 clk_disable(data->clk_master);
549}
550
551static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova,
552 size_t size)
553{
554 struct exynos_iommu_owner *owner = dev->archdata.iommu;
555 unsigned long flags;
556 struct sysmmu_drvdata *data;
557
558 data = dev_get_drvdata(owner->sysmmu);
559
560 spin_lock_irqsave(&data->lock, flags);
488 if (is_sysmmu_active(data)) { 561 if (is_sysmmu_active(data)) {
489 int i; 562 unsigned int num_inv = 1;
490 for (i = 0; i < data->nsfrs; i++) { 563
491 if (sysmmu_block(data->sfrbases[i])) { 564 if (!IS_ERR(data->clk_master))
492 __sysmmu_tlb_invalidate_entry( 565 clk_enable(data->clk_master);
493 data->sfrbases[i], iova); 566
494 sysmmu_unblock(data->sfrbases[i]); 567 /*
495 } 568 * L2TLB invalidation required
569 * 4KB page: 1 invalidation
570 * 64KB page: 16 invalidation
571 * 1MB page: 64 invalidation
572 * because it is set-associative TLB
573 * with 8-way and 64 sets.
574 * 1MB page can be cached in one of all sets.
575 * 64KB page can be one of 16 consecutive sets.
576 */
577 if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2)
578 num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
579
580 if (sysmmu_block(data->sfrbase)) {
581 __sysmmu_tlb_invalidate_entry(
582 data->sfrbase, iova, num_inv);
583 sysmmu_unblock(data->sfrbase);
496 } 584 }
585 if (!IS_ERR(data->clk_master))
586 clk_disable(data->clk_master);
497 } else { 587 } else {
498 dev_dbg(data->sysmmu, 588 dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n",
499 "(%s) Disabled. Skipping invalidating TLB.\n", 589 iova);
500 data->dbgname);
501 } 590 }
502 591 spin_unlock_irqrestore(&data->lock, flags);
503 read_unlock_irqrestore(&data->lock, flags);
504} 592}
505 593
506void exynos_sysmmu_tlb_invalidate(struct device *dev) 594void exynos_sysmmu_tlb_invalidate(struct device *dev)
507{ 595{
596 struct exynos_iommu_owner *owner = dev->archdata.iommu;
508 unsigned long flags; 597 unsigned long flags;
509 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 598 struct sysmmu_drvdata *data;
510 599
511 read_lock_irqsave(&data->lock, flags); 600 data = dev_get_drvdata(owner->sysmmu);
512 601
602 spin_lock_irqsave(&data->lock, flags);
513 if (is_sysmmu_active(data)) { 603 if (is_sysmmu_active(data)) {
514 int i; 604 if (!IS_ERR(data->clk_master))
515 for (i = 0; i < data->nsfrs; i++) { 605 clk_enable(data->clk_master);
516 if (sysmmu_block(data->sfrbases[i])) { 606 if (sysmmu_block(data->sfrbase)) {
517 __sysmmu_tlb_invalidate(data->sfrbases[i]); 607 __sysmmu_tlb_invalidate(data->sfrbase);
518 sysmmu_unblock(data->sfrbases[i]); 608 sysmmu_unblock(data->sfrbase);
519 }
520 } 609 }
610 if (!IS_ERR(data->clk_master))
611 clk_disable(data->clk_master);
521 } else { 612 } else {
522 dev_dbg(data->sysmmu, 613 dev_dbg(dev, "disabled. Skipping TLB invalidation\n");
523 "(%s) Disabled. Skipping invalidating TLB.\n",
524 data->dbgname);
525 } 614 }
526 615 spin_unlock_irqrestore(&data->lock, flags);
527 read_unlock_irqrestore(&data->lock, flags);
528} 616}
529 617
530static int exynos_sysmmu_probe(struct platform_device *pdev) 618static int __init exynos_sysmmu_probe(struct platform_device *pdev)
531{ 619{
532 int i, ret; 620 int irq, ret;
533 struct device *dev; 621 struct device *dev = &pdev->dev;
534 struct sysmmu_drvdata *data; 622 struct sysmmu_drvdata *data;
623 struct resource *res;
535 624
536 dev = &pdev->dev; 625 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
626 if (!data)
627 return -ENOMEM;
537 628
538 data = kzalloc(sizeof(*data), GFP_KERNEL); 629 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
539 if (!data) { 630 data->sfrbase = devm_ioremap_resource(dev, res);
540 dev_dbg(dev, "Not enough memory\n"); 631 if (IS_ERR(data->sfrbase))
541 ret = -ENOMEM; 632 return PTR_ERR(data->sfrbase);
542 goto err_alloc;
543 }
544 633
545 dev_set_drvdata(dev, data); 634 irq = platform_get_irq(pdev, 0);
546 data->nsfrs = pdev->num_resources / 2; 635 if (irq <= 0) {
547 data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs, 636 dev_err(dev, "Unable to find IRQ resource\n");
548 GFP_KERNEL); 637 return irq;
549 if (data->sfrbases == NULL) {
550 dev_dbg(dev, "Not enough memory\n");
551 ret = -ENOMEM;
552 goto err_init;
553 } 638 }
554 639
555 for (i = 0; i < data->nsfrs; i++) { 640 ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0,
556 struct resource *res; 641 dev_name(dev), data);
557 res = platform_get_resource(pdev, IORESOURCE_MEM, i); 642 if (ret) {
558 if (!res) { 643 dev_err(dev, "Unabled to register handler of irq %d\n", irq);
559 dev_dbg(dev, "Unable to find IOMEM region\n"); 644 return ret;
560 ret = -ENOENT;
561 goto err_res;
562 }
563
564 data->sfrbases[i] = ioremap(res->start, resource_size(res));
565 if (!data->sfrbases[i]) {
566 dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
567 res->start);
568 ret = -ENOENT;
569 goto err_res;
570 }
571 } 645 }
572 646
573 for (i = 0; i < data->nsfrs; i++) { 647 data->clk = devm_clk_get(dev, "sysmmu");
574 ret = platform_get_irq(pdev, i); 648 if (IS_ERR(data->clk)) {
575 if (ret <= 0) { 649 dev_err(dev, "Failed to get clock!\n");
576 dev_dbg(dev, "Unable to find IRQ resource\n"); 650 return PTR_ERR(data->clk);
577 goto err_irq; 651 } else {
578 } 652 ret = clk_prepare(data->clk);
579
580 ret = request_irq(ret, exynos_sysmmu_irq, 0,
581 dev_name(dev), data);
582 if (ret) { 653 if (ret) {
583 dev_dbg(dev, "Unabled to register interrupt handler\n"); 654 dev_err(dev, "Failed to prepare clk\n");
584 goto err_irq; 655 return ret;
585 } 656 }
586 } 657 }
587 658
588 if (dev_get_platdata(dev)) { 659 data->clk_master = devm_clk_get(dev, "master");
589 char *deli, *beg; 660 if (!IS_ERR(data->clk_master)) {
590 struct sysmmu_platform_data *platdata = dev_get_platdata(dev); 661 ret = clk_prepare(data->clk_master);
591 662 if (ret) {
592 beg = platdata->clockname; 663 clk_unprepare(data->clk);
593 664 dev_err(dev, "Failed to prepare master's clk\n");
594 for (deli = beg; (*deli != '\0') && (*deli != ','); deli++) 665 return ret;
595 /* NOTHING */;
596
597 if (*deli == '\0')
598 deli = NULL;
599 else
600 *deli = '\0';
601
602 data->clk[0] = clk_get(dev, beg);
603 if (IS_ERR(data->clk[0])) {
604 data->clk[0] = NULL;
605 dev_dbg(dev, "No clock descriptor registered\n");
606 }
607
608 if (data->clk[0] && deli) {
609 *deli = ',';
610 data->clk[1] = clk_get(dev, deli + 1);
611 if (IS_ERR(data->clk[1]))
612 data->clk[1] = NULL;
613 } 666 }
614
615 data->dbgname = platdata->dbgname;
616 } 667 }
617 668
618 data->sysmmu = dev; 669 data->sysmmu = dev;
619 rwlock_init(&data->lock); 670 spin_lock_init(&data->lock);
620 INIT_LIST_HEAD(&data->node);
621 671
622 __set_fault_handler(data, &default_fault_handler); 672 platform_set_drvdata(pdev, data);
623 673
624 if (dev->parent) 674 pm_runtime_enable(dev);
625 pm_runtime_enable(dev);
626 675
627 dev_dbg(dev, "(%s) Initialized\n", data->dbgname);
628 return 0; 676 return 0;
629err_irq:
630 while (i-- > 0) {
631 int irq;
632
633 irq = platform_get_irq(pdev, i);
634 free_irq(irq, data);
635 }
636err_res:
637 while (data->nsfrs-- > 0)
638 iounmap(data->sfrbases[data->nsfrs]);
639 kfree(data->sfrbases);
640err_init:
641 kfree(data);
642err_alloc:
643 dev_err(dev, "Failed to initialize\n");
644 return ret;
645} 677}
646 678
647static struct platform_driver exynos_sysmmu_driver = { 679static const struct of_device_id sysmmu_of_match[] __initconst = {
648 .probe = exynos_sysmmu_probe, 680 { .compatible = "samsung,exynos-sysmmu", },
649 .driver = { 681 { },
682};
683
684static struct platform_driver exynos_sysmmu_driver __refdata = {
685 .probe = exynos_sysmmu_probe,
686 .driver = {
650 .owner = THIS_MODULE, 687 .owner = THIS_MODULE,
651 .name = "exynos-sysmmu", 688 .name = "exynos-sysmmu",
689 .of_match_table = sysmmu_of_match,
652 } 690 }
653}; 691};
654 692
@@ -662,21 +700,32 @@ static inline void pgtable_flush(void *vastart, void *vaend)
662static int exynos_iommu_domain_init(struct iommu_domain *domain) 700static int exynos_iommu_domain_init(struct iommu_domain *domain)
663{ 701{
664 struct exynos_iommu_domain *priv; 702 struct exynos_iommu_domain *priv;
703 int i;
665 704
666 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 705 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
667 if (!priv) 706 if (!priv)
668 return -ENOMEM; 707 return -ENOMEM;
669 708
670 priv->pgtable = (unsigned long *)__get_free_pages( 709 priv->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
671 GFP_KERNEL | __GFP_ZERO, 2);
672 if (!priv->pgtable) 710 if (!priv->pgtable)
673 goto err_pgtable; 711 goto err_pgtable;
674 712
675 priv->lv2entcnt = (short *)__get_free_pages( 713 priv->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
676 GFP_KERNEL | __GFP_ZERO, 1);
677 if (!priv->lv2entcnt) 714 if (!priv->lv2entcnt)
678 goto err_counter; 715 goto err_counter;
679 716
717 /* w/a of System MMU v3.3 to prevent caching 1MiB mapping */
718 for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
719 priv->pgtable[i + 0] = ZERO_LV2LINK;
720 priv->pgtable[i + 1] = ZERO_LV2LINK;
721 priv->pgtable[i + 2] = ZERO_LV2LINK;
722 priv->pgtable[i + 3] = ZERO_LV2LINK;
723 priv->pgtable[i + 4] = ZERO_LV2LINK;
724 priv->pgtable[i + 5] = ZERO_LV2LINK;
725 priv->pgtable[i + 6] = ZERO_LV2LINK;
726 priv->pgtable[i + 7] = ZERO_LV2LINK;
727 }
728
680 pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES); 729 pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
681 730
682 spin_lock_init(&priv->lock); 731 spin_lock_init(&priv->lock);
@@ -700,7 +749,7 @@ err_pgtable:
700static void exynos_iommu_domain_destroy(struct iommu_domain *domain) 749static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
701{ 750{
702 struct exynos_iommu_domain *priv = domain->priv; 751 struct exynos_iommu_domain *priv = domain->priv;
703 struct sysmmu_drvdata *data; 752 struct exynos_iommu_owner *owner;
704 unsigned long flags; 753 unsigned long flags;
705 int i; 754 int i;
706 755
@@ -708,16 +757,20 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
708 757
709 spin_lock_irqsave(&priv->lock, flags); 758 spin_lock_irqsave(&priv->lock, flags);
710 759
711 list_for_each_entry(data, &priv->clients, node) { 760 list_for_each_entry(owner, &priv->clients, client) {
712 while (!exynos_sysmmu_disable(data->dev)) 761 while (!exynos_sysmmu_disable(owner->dev))
713 ; /* until System MMU is actually disabled */ 762 ; /* until System MMU is actually disabled */
714 } 763 }
715 764
765 while (!list_empty(&priv->clients))
766 list_del_init(priv->clients.next);
767
716 spin_unlock_irqrestore(&priv->lock, flags); 768 spin_unlock_irqrestore(&priv->lock, flags);
717 769
718 for (i = 0; i < NUM_LV1ENTRIES; i++) 770 for (i = 0; i < NUM_LV1ENTRIES; i++)
719 if (lv1ent_page(priv->pgtable + i)) 771 if (lv1ent_page(priv->pgtable + i))
720 kfree(__va(lv2table_base(priv->pgtable + i))); 772 kmem_cache_free(lv2table_kmem_cache,
773 phys_to_virt(lv2table_base(priv->pgtable + i)));
721 774
722 free_pages((unsigned long)priv->pgtable, 2); 775 free_pages((unsigned long)priv->pgtable, 2);
723 free_pages((unsigned long)priv->lv2entcnt, 1); 776 free_pages((unsigned long)priv->lv2entcnt, 1);
@@ -728,114 +781,134 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
728static int exynos_iommu_attach_device(struct iommu_domain *domain, 781static int exynos_iommu_attach_device(struct iommu_domain *domain,
729 struct device *dev) 782 struct device *dev)
730{ 783{
731 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 784 struct exynos_iommu_owner *owner = dev->archdata.iommu;
732 struct exynos_iommu_domain *priv = domain->priv; 785 struct exynos_iommu_domain *priv = domain->priv;
786 phys_addr_t pagetable = virt_to_phys(priv->pgtable);
733 unsigned long flags; 787 unsigned long flags;
734 int ret; 788 int ret;
735 789
736 ret = pm_runtime_get_sync(data->sysmmu);
737 if (ret < 0)
738 return ret;
739
740 ret = 0;
741
742 spin_lock_irqsave(&priv->lock, flags); 790 spin_lock_irqsave(&priv->lock, flags);
743 791
744 ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain); 792 ret = __exynos_sysmmu_enable(dev, pagetable, domain);
745
746 if (ret == 0) { 793 if (ret == 0) {
747 /* 'data->node' must not be appeared in priv->clients */ 794 list_add_tail(&owner->client, &priv->clients);
748 BUG_ON(!list_empty(&data->node)); 795 owner->domain = domain;
749 data->dev = dev;
750 list_add_tail(&data->node, &priv->clients);
751 } 796 }
752 797
753 spin_unlock_irqrestore(&priv->lock, flags); 798 spin_unlock_irqrestore(&priv->lock, flags);
754 799
755 if (ret < 0) { 800 if (ret < 0) {
756 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n", 801 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
757 __func__, __pa(priv->pgtable)); 802 __func__, &pagetable);
758 pm_runtime_put(data->sysmmu); 803 return ret;
759 } else if (ret > 0) {
760 dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
761 __func__, __pa(priv->pgtable));
762 } else {
763 dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n",
764 __func__, __pa(priv->pgtable));
765 } 804 }
766 805
806 dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n",
807 __func__, &pagetable, (ret == 0) ? "" : ", again");
808
767 return ret; 809 return ret;
768} 810}
769 811
770static void exynos_iommu_detach_device(struct iommu_domain *domain, 812static void exynos_iommu_detach_device(struct iommu_domain *domain,
771 struct device *dev) 813 struct device *dev)
772{ 814{
773 struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); 815 struct exynos_iommu_owner *owner;
774 struct exynos_iommu_domain *priv = domain->priv; 816 struct exynos_iommu_domain *priv = domain->priv;
775 struct list_head *pos; 817 phys_addr_t pagetable = virt_to_phys(priv->pgtable);
776 unsigned long flags; 818 unsigned long flags;
777 bool found = false;
778 819
779 spin_lock_irqsave(&priv->lock, flags); 820 spin_lock_irqsave(&priv->lock, flags);
780 821
781 list_for_each(pos, &priv->clients) { 822 list_for_each_entry(owner, &priv->clients, client) {
782 if (list_entry(pos, struct sysmmu_drvdata, node) == data) { 823 if (owner == dev->archdata.iommu) {
783 found = true; 824 if (exynos_sysmmu_disable(dev)) {
825 list_del_init(&owner->client);
826 owner->domain = NULL;
827 }
784 break; 828 break;
785 } 829 }
786 } 830 }
787 831
788 if (!found)
789 goto finish;
790
791 if (__exynos_sysmmu_disable(data)) {
792 dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n",
793 __func__, __pa(priv->pgtable));
794 list_del_init(&data->node);
795
796 } else {
797 dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed",
798 __func__, __pa(priv->pgtable));
799 }
800
801finish:
802 spin_unlock_irqrestore(&priv->lock, flags); 832 spin_unlock_irqrestore(&priv->lock, flags);
803 833
804 if (found) 834 if (owner == dev->archdata.iommu)
805 pm_runtime_put(data->sysmmu); 835 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
836 __func__, &pagetable);
837 else
838 dev_err(dev, "%s: No IOMMU is attached\n", __func__);
806} 839}
807 840
808static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, 841static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv,
809 short *pgcounter) 842 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
810{ 843{
844 if (lv1ent_section(sent)) {
845 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova);
846 return ERR_PTR(-EADDRINUSE);
847 }
848
811 if (lv1ent_fault(sent)) { 849 if (lv1ent_fault(sent)) {
812 unsigned long *pent; 850 sysmmu_pte_t *pent;
851 bool need_flush_flpd_cache = lv1ent_zero(sent);
813 852
814 pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC); 853 pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC);
815 BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1)); 854 BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1));
816 if (!pent) 855 if (!pent)
817 return NULL; 856 return ERR_PTR(-ENOMEM);
818 857
819 *sent = mk_lv1ent_page(__pa(pent)); 858 *sent = mk_lv1ent_page(virt_to_phys(pent));
820 *pgcounter = NUM_LV2ENTRIES; 859 *pgcounter = NUM_LV2ENTRIES;
821 pgtable_flush(pent, pent + NUM_LV2ENTRIES); 860 pgtable_flush(pent, pent + NUM_LV2ENTRIES);
822 pgtable_flush(sent, sent + 1); 861 pgtable_flush(sent, sent + 1);
862
863 /*
864 * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache
865 * may caches the address of zero_l2_table. This function
866 * replaces the zero_l2_table with new L2 page table to write
867 * valid mappings.
868 * Accessing the valid area may cause page fault since FLPD
869 * cache may still caches zero_l2_table for the valid area
870 * instead of new L2 page table that have the mapping
871 * information of the valid area
872 * Thus any replacement of zero_l2_table with other valid L2
873 * page table must involve FLPD cache invalidation for System
874 * MMU v3.3.
875 * FLPD cache invalidation is performed with TLB invalidation
876 * by VPN without blocking. It is safe to invalidate TLB without
877 * blocking because the target address of TLB invalidation is
878 * not currently mapped.
879 */
880 if (need_flush_flpd_cache) {
881 struct exynos_iommu_owner *owner;
882
883 spin_lock(&priv->lock);
884 list_for_each_entry(owner, &priv->clients, client)
885 sysmmu_tlb_invalidate_flpdcache(
886 owner->dev, iova);
887 spin_unlock(&priv->lock);
888 }
823 } 889 }
824 890
825 return page_entry(sent, iova); 891 return page_entry(sent, iova);
826} 892}
827 893
828static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt) 894static int lv1set_section(struct exynos_iommu_domain *priv,
895 sysmmu_pte_t *sent, sysmmu_iova_t iova,
896 phys_addr_t paddr, short *pgcnt)
829{ 897{
830 if (lv1ent_section(sent)) 898 if (lv1ent_section(sent)) {
899 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
900 iova);
831 return -EADDRINUSE; 901 return -EADDRINUSE;
902 }
832 903
833 if (lv1ent_page(sent)) { 904 if (lv1ent_page(sent)) {
834 if (*pgcnt != NUM_LV2ENTRIES) 905 if (*pgcnt != NUM_LV2ENTRIES) {
906 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
907 iova);
835 return -EADDRINUSE; 908 return -EADDRINUSE;
909 }
836 910
837 kfree(page_entry(sent, 0)); 911 kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0));
838
839 *pgcnt = 0; 912 *pgcnt = 0;
840 } 913 }
841 914
@@ -843,14 +916,26 @@ static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
843 916
844 pgtable_flush(sent, sent + 1); 917 pgtable_flush(sent, sent + 1);
845 918
919 spin_lock(&priv->lock);
920 if (lv1ent_page_zero(sent)) {
921 struct exynos_iommu_owner *owner;
922 /*
923 * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
924 * entry by speculative prefetch of SLPD which has no mapping.
925 */
926 list_for_each_entry(owner, &priv->clients, client)
927 sysmmu_tlb_invalidate_flpdcache(owner->dev, iova);
928 }
929 spin_unlock(&priv->lock);
930
846 return 0; 931 return 0;
847} 932}
848 933
849static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, 934static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
850 short *pgcnt) 935 short *pgcnt)
851{ 936{
852 if (size == SPAGE_SIZE) { 937 if (size == SPAGE_SIZE) {
853 if (!lv2ent_fault(pent)) 938 if (WARN_ON(!lv2ent_fault(pent)))
854 return -EADDRINUSE; 939 return -EADDRINUSE;
855 940
856 *pent = mk_lv2ent_spage(paddr); 941 *pent = mk_lv2ent_spage(paddr);
@@ -858,9 +943,11 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
858 *pgcnt -= 1; 943 *pgcnt -= 1;
859 } else { /* size == LPAGE_SIZE */ 944 } else { /* size == LPAGE_SIZE */
860 int i; 945 int i;
946
861 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { 947 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
862 if (!lv2ent_fault(pent)) { 948 if (WARN_ON(!lv2ent_fault(pent))) {
863 memset(pent, 0, sizeof(*pent) * i); 949 if (i > 0)
950 memset(pent - i, 0, sizeof(*pent) * i);
864 return -EADDRINUSE; 951 return -EADDRINUSE;
865 } 952 }
866 953
@@ -873,11 +960,38 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
873 return 0; 960 return 0;
874} 961}
875 962
876static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, 963/*
964 * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
965 *
966 * System MMU v3.x have an advanced logic to improve address translation
967 * performance with caching more page table entries by a page table walk.
968 * However, the logic has a bug that caching fault page table entries and System
969 * MMU reports page fault if the cached fault entry is hit even though the fault
970 * entry is updated to a valid entry after the entry is cached.
971 * To prevent caching fault page table entries which may be updated to valid
972 * entries later, the virtual memory manager should care about the w/a about the
973 * problem. The followings describe w/a.
974 *
975 * Any two consecutive I/O virtual address regions must have a hole of 128KiB
976 * in maximum to prevent misbehavior of System MMU 3.x. (w/a of h/w bug)
977 *
978 * Precisely, any start address of I/O virtual region must be aligned by
979 * the following sizes for System MMU v3.1 and v3.2.
980 * System MMU v3.1: 128KiB
981 * System MMU v3.2: 256KiB
982 *
983 * Because System MMU v3.3 caches page table entries more aggressively, it needs
984 * more w/a.
985 * - Any two consecutive I/O virtual regions must be have a hole of larger size
986 * than or equal size to 128KiB.
987 * - Start address of an I/O virtual region must be aligned by 128KiB.
988 */
989static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova,
877 phys_addr_t paddr, size_t size, int prot) 990 phys_addr_t paddr, size_t size, int prot)
878{ 991{
879 struct exynos_iommu_domain *priv = domain->priv; 992 struct exynos_iommu_domain *priv = domain->priv;
880 unsigned long *entry; 993 sysmmu_pte_t *entry;
994 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
881 unsigned long flags; 995 unsigned long flags;
882 int ret = -ENOMEM; 996 int ret = -ENOMEM;
883 997
@@ -888,38 +1002,52 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
888 entry = section_entry(priv->pgtable, iova); 1002 entry = section_entry(priv->pgtable, iova);
889 1003
890 if (size == SECT_SIZE) { 1004 if (size == SECT_SIZE) {
891 ret = lv1set_section(entry, paddr, 1005 ret = lv1set_section(priv, entry, iova, paddr,
892 &priv->lv2entcnt[lv1ent_offset(iova)]); 1006 &priv->lv2entcnt[lv1ent_offset(iova)]);
893 } else { 1007 } else {
894 unsigned long *pent; 1008 sysmmu_pte_t *pent;
895 1009
896 pent = alloc_lv2entry(entry, iova, 1010 pent = alloc_lv2entry(priv, entry, iova,
897 &priv->lv2entcnt[lv1ent_offset(iova)]); 1011 &priv->lv2entcnt[lv1ent_offset(iova)]);
898 1012
899 if (!pent) 1013 if (IS_ERR(pent))
900 ret = -ENOMEM; 1014 ret = PTR_ERR(pent);
901 else 1015 else
902 ret = lv2set_page(pent, paddr, size, 1016 ret = lv2set_page(pent, paddr, size,
903 &priv->lv2entcnt[lv1ent_offset(iova)]); 1017 &priv->lv2entcnt[lv1ent_offset(iova)]);
904 } 1018 }
905 1019
906 if (ret) { 1020 if (ret)
907 pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n", 1021 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
908 __func__, iova, size); 1022 __func__, ret, size, iova);
909 }
910 1023
911 spin_unlock_irqrestore(&priv->pgtablelock, flags); 1024 spin_unlock_irqrestore(&priv->pgtablelock, flags);
912 1025
913 return ret; 1026 return ret;
914} 1027}
915 1028
1029static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv,
1030 sysmmu_iova_t iova, size_t size)
1031{
1032 struct exynos_iommu_owner *owner;
1033 unsigned long flags;
1034
1035 spin_lock_irqsave(&priv->lock, flags);
1036
1037 list_for_each_entry(owner, &priv->clients, client)
1038 sysmmu_tlb_invalidate_entry(owner->dev, iova, size);
1039
1040 spin_unlock_irqrestore(&priv->lock, flags);
1041}
1042
916static size_t exynos_iommu_unmap(struct iommu_domain *domain, 1043static size_t exynos_iommu_unmap(struct iommu_domain *domain,
917 unsigned long iova, size_t size) 1044 unsigned long l_iova, size_t size)
918{ 1045{
919 struct exynos_iommu_domain *priv = domain->priv; 1046 struct exynos_iommu_domain *priv = domain->priv;
920 struct sysmmu_drvdata *data; 1047 sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
1048 sysmmu_pte_t *ent;
1049 size_t err_pgsize;
921 unsigned long flags; 1050 unsigned long flags;
922 unsigned long *ent;
923 1051
924 BUG_ON(priv->pgtable == NULL); 1052 BUG_ON(priv->pgtable == NULL);
925 1053
@@ -928,9 +1056,12 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
928 ent = section_entry(priv->pgtable, iova); 1056 ent = section_entry(priv->pgtable, iova);
929 1057
930 if (lv1ent_section(ent)) { 1058 if (lv1ent_section(ent)) {
931 BUG_ON(size < SECT_SIZE); 1059 if (WARN_ON(size < SECT_SIZE)) {
1060 err_pgsize = SECT_SIZE;
1061 goto err;
1062 }
932 1063
933 *ent = 0; 1064 *ent = ZERO_LV2LINK; /* w/a for h/w bug in Sysmem MMU v3.3 */
934 pgtable_flush(ent, ent + 1); 1065 pgtable_flush(ent, ent + 1);
935 size = SECT_SIZE; 1066 size = SECT_SIZE;
936 goto done; 1067 goto done;
@@ -954,34 +1085,42 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain,
954 if (lv2ent_small(ent)) { 1085 if (lv2ent_small(ent)) {
955 *ent = 0; 1086 *ent = 0;
956 size = SPAGE_SIZE; 1087 size = SPAGE_SIZE;
1088 pgtable_flush(ent, ent + 1);
957 priv->lv2entcnt[lv1ent_offset(iova)] += 1; 1089 priv->lv2entcnt[lv1ent_offset(iova)] += 1;
958 goto done; 1090 goto done;
959 } 1091 }
960 1092
961 /* lv1ent_large(ent) == true here */ 1093 /* lv1ent_large(ent) == true here */
962 BUG_ON(size < LPAGE_SIZE); 1094 if (WARN_ON(size < LPAGE_SIZE)) {
1095 err_pgsize = LPAGE_SIZE;
1096 goto err;
1097 }
963 1098
964 memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); 1099 memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
1100 pgtable_flush(ent, ent + SPAGES_PER_LPAGE);
965 1101
966 size = LPAGE_SIZE; 1102 size = LPAGE_SIZE;
967 priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; 1103 priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
968done: 1104done:
969 spin_unlock_irqrestore(&priv->pgtablelock, flags); 1105 spin_unlock_irqrestore(&priv->pgtablelock, flags);
970 1106
971 spin_lock_irqsave(&priv->lock, flags); 1107 exynos_iommu_tlb_invalidate_entry(priv, iova, size);
972 list_for_each_entry(data, &priv->clients, node)
973 sysmmu_tlb_invalidate_entry(data->dev, iova);
974 spin_unlock_irqrestore(&priv->lock, flags);
975
976 1108
977 return size; 1109 return size;
1110err:
1111 spin_unlock_irqrestore(&priv->pgtablelock, flags);
1112
1113 pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
1114 __func__, size, iova, err_pgsize);
1115
1116 return 0;
978} 1117}
979 1118
980static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, 1119static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
981 dma_addr_t iova) 1120 dma_addr_t iova)
982{ 1121{
983 struct exynos_iommu_domain *priv = domain->priv; 1122 struct exynos_iommu_domain *priv = domain->priv;
984 unsigned long *entry; 1123 sysmmu_pte_t *entry;
985 unsigned long flags; 1124 unsigned long flags;
986 phys_addr_t phys = 0; 1125 phys_addr_t phys = 0;
987 1126
@@ -1005,6 +1144,32 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
1005 return phys; 1144 return phys;
1006} 1145}
1007 1146
1147static int exynos_iommu_add_device(struct device *dev)
1148{
1149 struct iommu_group *group;
1150 int ret;
1151
1152 group = iommu_group_get(dev);
1153
1154 if (!group) {
1155 group = iommu_group_alloc();
1156 if (IS_ERR(group)) {
1157 dev_err(dev, "Failed to allocate IOMMU group\n");
1158 return PTR_ERR(group);
1159 }
1160 }
1161
1162 ret = iommu_group_add_device(group, dev);
1163 iommu_group_put(group);
1164
1165 return ret;
1166}
1167
1168static void exynos_iommu_remove_device(struct device *dev)
1169{
1170 iommu_group_remove_device(dev);
1171}
1172
1008static struct iommu_ops exynos_iommu_ops = { 1173static struct iommu_ops exynos_iommu_ops = {
1009 .domain_init = exynos_iommu_domain_init, 1174 .domain_init = exynos_iommu_domain_init,
1010 .domain_destroy = exynos_iommu_domain_destroy, 1175 .domain_destroy = exynos_iommu_domain_destroy,
@@ -1013,6 +1178,8 @@ static struct iommu_ops exynos_iommu_ops = {
1013 .map = exynos_iommu_map, 1178 .map = exynos_iommu_map,
1014 .unmap = exynos_iommu_unmap, 1179 .unmap = exynos_iommu_unmap,
1015 .iova_to_phys = exynos_iommu_iova_to_phys, 1180 .iova_to_phys = exynos_iommu_iova_to_phys,
1181 .add_device = exynos_iommu_add_device,
1182 .remove_device = exynos_iommu_remove_device,
1016 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, 1183 .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
1017}; 1184};
1018 1185
@@ -1020,11 +1187,41 @@ static int __init exynos_iommu_init(void)
1020{ 1187{
1021 int ret; 1188 int ret;
1022 1189
1190 lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
1191 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
1192 if (!lv2table_kmem_cache) {
1193 pr_err("%s: Failed to create kmem cache\n", __func__);
1194 return -ENOMEM;
1195 }
1196
1023 ret = platform_driver_register(&exynos_sysmmu_driver); 1197 ret = platform_driver_register(&exynos_sysmmu_driver);
1198 if (ret) {
1199 pr_err("%s: Failed to register driver\n", __func__);
1200 goto err_reg_driver;
1201 }
1024 1202
1025 if (ret == 0) 1203 zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
1026 bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); 1204 if (zero_lv2_table == NULL) {
1205 pr_err("%s: Failed to allocate zero level2 page table\n",
1206 __func__);
1207 ret = -ENOMEM;
1208 goto err_zero_lv2;
1209 }
1027 1210
1211 ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
1212 if (ret) {
1213 pr_err("%s: Failed to register exynos-iommu driver.\n",
1214 __func__);
1215 goto err_set_iommu;
1216 }
1217
1218 return 0;
1219err_set_iommu:
1220 kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
1221err_zero_lv2:
1222 platform_driver_unregister(&exynos_sysmmu_driver);
1223err_reg_driver:
1224 kmem_cache_destroy(lv2table_kmem_cache);
1028 return ret; 1225 return ret;
1029} 1226}
1030subsys_initcall(exynos_iommu_init); 1227subsys_initcall(exynos_iommu_init);
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index cba0498eb011..b99dd88e31b9 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -592,8 +592,7 @@ found_cpu_node:
592 /* advance to next node in cache hierarchy */ 592 /* advance to next node in cache hierarchy */
593 node = of_find_node_by_phandle(*prop); 593 node = of_find_node_by_phandle(*prop);
594 if (!node) { 594 if (!node) {
595 pr_debug("Invalid node for cache hierarchy %s\n", 595 pr_debug("Invalid node for cache hierarchy\n");
596 node->full_name);
597 return ~(u32)0; 596 return ~(u32)0;
598 } 597 }
599 } 598 }
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
new file mode 100644
index 000000000000..53cde086e83b
--- /dev/null
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -0,0 +1,1255 @@
1/*
2 * IPMMU VMSA
3 *
4 * Copyright (C) 2014 Renesas Electronics Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 */
10
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/err.h>
14#include <linux/export.h>
15#include <linux/interrupt.h>
16#include <linux/io.h>
17#include <linux/iommu.h>
18#include <linux/module.h>
19#include <linux/platform_data/ipmmu-vmsa.h>
20#include <linux/platform_device.h>
21#include <linux/sizes.h>
22#include <linux/slab.h>
23
24#include <asm/dma-iommu.h>
25#include <asm/pgalloc.h>
26
27struct ipmmu_vmsa_device {
28 struct device *dev;
29 void __iomem *base;
30 struct list_head list;
31
32 const struct ipmmu_vmsa_platform_data *pdata;
33 unsigned int num_utlbs;
34
35 struct dma_iommu_mapping *mapping;
36};
37
38struct ipmmu_vmsa_domain {
39 struct ipmmu_vmsa_device *mmu;
40 struct iommu_domain *io_domain;
41
42 unsigned int context_id;
43 spinlock_t lock; /* Protects mappings */
44 pgd_t *pgd;
45};
46
47struct ipmmu_vmsa_archdata {
48 struct ipmmu_vmsa_device *mmu;
49 unsigned int utlb;
50};
51
52static DEFINE_SPINLOCK(ipmmu_devices_lock);
53static LIST_HEAD(ipmmu_devices);
54
55#define TLB_LOOP_TIMEOUT 100 /* 100us */
56
57/* -----------------------------------------------------------------------------
58 * Registers Definition
59 */
60
61#define IM_CTX_SIZE 0x40
62
63#define IMCTR 0x0000
64#define IMCTR_TRE (1 << 17)
65#define IMCTR_AFE (1 << 16)
66#define IMCTR_RTSEL_MASK (3 << 4)
67#define IMCTR_RTSEL_SHIFT 4
68#define IMCTR_TREN (1 << 3)
69#define IMCTR_INTEN (1 << 2)
70#define IMCTR_FLUSH (1 << 1)
71#define IMCTR_MMUEN (1 << 0)
72
73#define IMCAAR 0x0004
74
75#define IMTTBCR 0x0008
76#define IMTTBCR_EAE (1 << 31)
77#define IMTTBCR_PMB (1 << 30)
78#define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
79#define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
80#define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
81#define IMTTBCR_SH1_MASK (3 << 28)
82#define IMTTBCR_ORGN1_NC (0 << 26)
83#define IMTTBCR_ORGN1_WB_WA (1 << 26)
84#define IMTTBCR_ORGN1_WT (2 << 26)
85#define IMTTBCR_ORGN1_WB (3 << 26)
86#define IMTTBCR_ORGN1_MASK (3 << 26)
87#define IMTTBCR_IRGN1_NC (0 << 24)
88#define IMTTBCR_IRGN1_WB_WA (1 << 24)
89#define IMTTBCR_IRGN1_WT (2 << 24)
90#define IMTTBCR_IRGN1_WB (3 << 24)
91#define IMTTBCR_IRGN1_MASK (3 << 24)
92#define IMTTBCR_TSZ1_MASK (7 << 16)
93#define IMTTBCR_TSZ1_SHIFT 16
94#define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
95#define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
96#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
97#define IMTTBCR_SH0_MASK (3 << 12)
98#define IMTTBCR_ORGN0_NC (0 << 10)
99#define IMTTBCR_ORGN0_WB_WA (1 << 10)
100#define IMTTBCR_ORGN0_WT (2 << 10)
101#define IMTTBCR_ORGN0_WB (3 << 10)
102#define IMTTBCR_ORGN0_MASK (3 << 10)
103#define IMTTBCR_IRGN0_NC (0 << 8)
104#define IMTTBCR_IRGN0_WB_WA (1 << 8)
105#define IMTTBCR_IRGN0_WT (2 << 8)
106#define IMTTBCR_IRGN0_WB (3 << 8)
107#define IMTTBCR_IRGN0_MASK (3 << 8)
108#define IMTTBCR_SL0_LVL_2 (0 << 4)
109#define IMTTBCR_SL0_LVL_1 (1 << 4)
110#define IMTTBCR_TSZ0_MASK (7 << 0)
111#define IMTTBCR_TSZ0_SHIFT O
112
113#define IMBUSCR 0x000c
114#define IMBUSCR_DVM (1 << 2)
115#define IMBUSCR_BUSSEL_SYS (0 << 0)
116#define IMBUSCR_BUSSEL_CCI (1 << 0)
117#define IMBUSCR_BUSSEL_IMCAAR (2 << 0)
118#define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0)
119#define IMBUSCR_BUSSEL_MASK (3 << 0)
120
121#define IMTTLBR0 0x0010
122#define IMTTUBR0 0x0014
123#define IMTTLBR1 0x0018
124#define IMTTUBR1 0x001c
125
126#define IMSTR 0x0020
127#define IMSTR_ERRLVL_MASK (3 << 12)
128#define IMSTR_ERRLVL_SHIFT 12
129#define IMSTR_ERRCODE_TLB_FORMAT (1 << 8)
130#define IMSTR_ERRCODE_ACCESS_PERM (4 << 8)
131#define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8)
132#define IMSTR_ERRCODE_MASK (7 << 8)
133#define IMSTR_MHIT (1 << 4)
134#define IMSTR_ABORT (1 << 2)
135#define IMSTR_PF (1 << 1)
136#define IMSTR_TF (1 << 0)
137
138#define IMMAIR0 0x0028
139#define IMMAIR1 0x002c
140#define IMMAIR_ATTR_MASK 0xff
141#define IMMAIR_ATTR_DEVICE 0x04
142#define IMMAIR_ATTR_NC 0x44
143#define IMMAIR_ATTR_WBRWA 0xff
144#define IMMAIR_ATTR_SHIFT(n) ((n) << 3)
145#define IMMAIR_ATTR_IDX_NC 0
146#define IMMAIR_ATTR_IDX_WBRWA 1
147#define IMMAIR_ATTR_IDX_DEV 2
148
149#define IMEAR 0x0030
150
151#define IMPCTR 0x0200
152#define IMPSTR 0x0208
153#define IMPEAR 0x020c
154#define IMPMBA(n) (0x0280 + ((n) * 4))
155#define IMPMBD(n) (0x02c0 + ((n) * 4))
156
157#define IMUCTR(n) (0x0300 + ((n) * 16))
158#define IMUCTR_FIXADDEN (1 << 31)
159#define IMUCTR_FIXADD_MASK (0xff << 16)
160#define IMUCTR_FIXADD_SHIFT 16
161#define IMUCTR_TTSEL_MMU(n) ((n) << 4)
162#define IMUCTR_TTSEL_PMB (8 << 4)
163#define IMUCTR_TTSEL_MASK (15 << 4)
164#define IMUCTR_FLUSH (1 << 1)
165#define IMUCTR_MMUEN (1 << 0)
166
167#define IMUASID(n) (0x0308 + ((n) * 16))
168#define IMUASID_ASID8_MASK (0xff << 8)
169#define IMUASID_ASID8_SHIFT 8
170#define IMUASID_ASID0_MASK (0xff << 0)
171#define IMUASID_ASID0_SHIFT 0
172
173/* -----------------------------------------------------------------------------
174 * Page Table Bits
175 */
176
177/*
178 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access,
179 * Long-descriptor format" that the NStable bit being set in a table descriptor
180 * will result in the NStable and NS bits of all child entries being ignored and
181 * considered as being set. The IPMMU seems not to comply with this, as it
182 * generates a secure access page fault if any of the NStable and NS bits isn't
183 * set when running in non-secure mode.
184 */
185#ifndef PMD_NSTABLE
186#define PMD_NSTABLE (_AT(pmdval_t, 1) << 63)
187#endif
188
189#define ARM_VMSA_PTE_XN (((pteval_t)3) << 53)
190#define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52)
191#define ARM_VMSA_PTE_AF (((pteval_t)1) << 10)
192#define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8)
193#define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8)
194#define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8)
195#define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8)
196#define ARM_VMSA_PTE_NS (((pteval_t)1) << 5)
197#define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0)
198
199/* Stage-1 PTE */
200#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11)
201#define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6)
202#define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6)
203#define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6)
204#define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2)
205#define ARM_VMSA_PTE_ATTRINDX_SHIFT 2
206
207#define ARM_VMSA_PTE_ATTRS_MASK \
208 (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \
209 ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \
210 ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK)
211
212#define ARM_VMSA_PTE_CONT_ENTRIES 16
213#define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES)
214
215#define IPMMU_PTRS_PER_PTE 512
216#define IPMMU_PTRS_PER_PMD 512
217#define IPMMU_PTRS_PER_PGD 4
218
219/* -----------------------------------------------------------------------------
220 * Read/Write Access
221 */
222
223static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
224{
225 return ioread32(mmu->base + offset);
226}
227
228static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
229 u32 data)
230{
231 iowrite32(data, mmu->base + offset);
232}
233
234static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg)
235{
236 return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg);
237}
238
239static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg,
240 u32 data)
241{
242 ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data);
243}
244
245/* -----------------------------------------------------------------------------
246 * TLB and microTLB Management
247 */
248
249/* Wait for any pending TLB invalidations to complete */
250static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
251{
252 unsigned int count = 0;
253
254 while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) {
255 cpu_relax();
256 if (++count == TLB_LOOP_TIMEOUT) {
257 dev_err_ratelimited(domain->mmu->dev,
258 "TLB sync timed out -- MMU may be deadlocked\n");
259 return;
260 }
261 udelay(1);
262 }
263}
264
265static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
266{
267 u32 reg;
268
269 reg = ipmmu_ctx_read(domain, IMCTR);
270 reg |= IMCTR_FLUSH;
271 ipmmu_ctx_write(domain, IMCTR, reg);
272
273 ipmmu_tlb_sync(domain);
274}
275
276/*
277 * Enable MMU translation for the microTLB.
278 */
279static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
280 unsigned int utlb)
281{
282 struct ipmmu_vmsa_device *mmu = domain->mmu;
283
284 /*
285 * TODO: Reference-count the microTLB as several bus masters can be
286 * connected to the same microTLB.
287 */
288
289 /* TODO: What should we set the ASID to ? */
290 ipmmu_write(mmu, IMUASID(utlb), 0);
291 /* TODO: Do we need to flush the microTLB ? */
292 ipmmu_write(mmu, IMUCTR(utlb),
293 IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH |
294 IMUCTR_MMUEN);
295}
296
297/*
298 * Disable MMU translation for the microTLB.
299 */
300static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
301 unsigned int utlb)
302{
303 struct ipmmu_vmsa_device *mmu = domain->mmu;
304
305 ipmmu_write(mmu, IMUCTR(utlb), 0);
306}
307
308static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr,
309 size_t size)
310{
311 unsigned long offset = (unsigned long)addr & ~PAGE_MASK;
312
313 /*
314 * TODO: Add support for coherent walk through CCI with DVM and remove
315 * cache handling.
316 */
317 dma_map_page(mmu->dev, virt_to_page(addr), offset, size, DMA_TO_DEVICE);
318}
319
320/* -----------------------------------------------------------------------------
321 * Domain/Context Management
322 */
323
324static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
325{
326 phys_addr_t ttbr;
327 u32 reg;
328
329 /*
330 * TODO: When adding support for multiple contexts, find an unused
331 * context.
332 */
333 domain->context_id = 0;
334
335 /* TTBR0 */
336 ipmmu_flush_pgtable(domain->mmu, domain->pgd,
337 IPMMU_PTRS_PER_PGD * sizeof(*domain->pgd));
338 ttbr = __pa(domain->pgd);
339 ipmmu_ctx_write(domain, IMTTLBR0, ttbr);
340 ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32);
341
342 /*
343 * TTBCR
344 * We use long descriptors with inner-shareable WBWA tables and allocate
345 * the whole 32-bit VA space to TTBR0.
346 */
347 ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE |
348 IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
349 IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1);
350
351 /*
352 * MAIR0
353 * We need three attributes only, non-cacheable, write-back read/write
354 * allocate and device memory.
355 */
356 reg = (IMMAIR_ATTR_NC << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC))
357 | (IMMAIR_ATTR_WBRWA << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA))
358 | (IMMAIR_ATTR_DEVICE << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV));
359 ipmmu_ctx_write(domain, IMMAIR0, reg);
360
361 /* IMBUSCR */
362 ipmmu_ctx_write(domain, IMBUSCR,
363 ipmmu_ctx_read(domain, IMBUSCR) &
364 ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
365
366 /*
367 * IMSTR
368 * Clear all interrupt flags.
369 */
370 ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR));
371
372 /*
373 * IMCTR
374 * Enable the MMU and interrupt generation. The long-descriptor
375 * translation table format doesn't use TEX remapping. Don't enable AF
376 * software management as we have no use for it. Flush the TLB as
377 * required when modifying the context registers.
378 */
379 ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
380
381 return 0;
382}
383
384static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
385{
386 /*
387 * Disable the context. Flush the TLB as required when modifying the
388 * context registers.
389 *
390 * TODO: Is TLB flush really needed ?
391 */
392 ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH);
393 ipmmu_tlb_sync(domain);
394}
395
396/* -----------------------------------------------------------------------------
397 * Fault Handling
398 */
399
400static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
401{
402 const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
403 struct ipmmu_vmsa_device *mmu = domain->mmu;
404 u32 status;
405 u32 iova;
406
407 status = ipmmu_ctx_read(domain, IMSTR);
408 if (!(status & err_mask))
409 return IRQ_NONE;
410
411 iova = ipmmu_ctx_read(domain, IMEAR);
412
413 /*
414 * Clear the error status flags. Unlike traditional interrupt flag
415 * registers that must be cleared by writing 1, this status register
416 * seems to require 0. The error address register must be read before,
417 * otherwise its value will be 0.
418 */
419 ipmmu_ctx_write(domain, IMSTR, 0);
420
421 /* Log fatal errors. */
422 if (status & IMSTR_MHIT)
423 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n",
424 iova);
425 if (status & IMSTR_ABORT)
426 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n",
427 iova);
428
429 if (!(status & (IMSTR_PF | IMSTR_TF)))
430 return IRQ_NONE;
431
432 /*
433 * Try to handle page faults and translation faults.
434 *
435 * TODO: We need to look up the faulty device based on the I/O VA. Use
436 * the IOMMU device for now.
437 */
438 if (!report_iommu_fault(domain->io_domain, mmu->dev, iova, 0))
439 return IRQ_HANDLED;
440
441 dev_err_ratelimited(mmu->dev,
442 "Unhandled fault: status 0x%08x iova 0x%08x\n",
443 status, iova);
444
445 return IRQ_HANDLED;
446}
447
448static irqreturn_t ipmmu_irq(int irq, void *dev)
449{
450 struct ipmmu_vmsa_device *mmu = dev;
451 struct iommu_domain *io_domain;
452 struct ipmmu_vmsa_domain *domain;
453
454 if (!mmu->mapping)
455 return IRQ_NONE;
456
457 io_domain = mmu->mapping->domain;
458 domain = io_domain->priv;
459
460 return ipmmu_domain_irq(domain);
461}
462
463/* -----------------------------------------------------------------------------
464 * Page Table Management
465 */
466
467#define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
468
469static void ipmmu_free_ptes(pmd_t *pmd)
470{
471 pgtable_t table = pmd_pgtable(*pmd);
472 __free_page(table);
473}
474
475static void ipmmu_free_pmds(pud_t *pud)
476{
477 pmd_t *pmd = pmd_offset(pud, 0);
478 pgtable_t table;
479 unsigned int i;
480
481 for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) {
482 if (!pmd_table(*pmd))
483 continue;
484
485 ipmmu_free_ptes(pmd);
486 pmd++;
487 }
488
489 table = pud_pgtable(*pud);
490 __free_page(table);
491}
492
493static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain)
494{
495 pgd_t *pgd, *pgd_base = domain->pgd;
496 unsigned int i;
497
498 /*
499 * Recursively free the page tables for this domain. We don't care about
500 * speculative TLB filling, because the TLB will be nuked next time this
501 * context bank is re-allocated and no devices currently map to these
502 * tables.
503 */
504 pgd = pgd_base;
505 for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) {
506 if (pgd_none(*pgd))
507 continue;
508 ipmmu_free_pmds((pud_t *)pgd);
509 pgd++;
510 }
511
512 kfree(pgd_base);
513}
514
515/*
516 * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte)
517 * functions as they would flush the CPU TLB.
518 */
519
520static pte_t *ipmmu_alloc_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
521 unsigned long iova)
522{
523 pte_t *pte;
524
525 if (!pmd_none(*pmd))
526 return pte_offset_kernel(pmd, iova);
527
528 pte = (pte_t *)get_zeroed_page(GFP_ATOMIC);
529 if (!pte)
530 return NULL;
531
532 ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE);
533 *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE);
534 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
535
536 return pte + pte_index(iova);
537}
538
539static pmd_t *ipmmu_alloc_pmd(struct ipmmu_vmsa_device *mmu, pgd_t *pgd,
540 unsigned long iova)
541{
542 pud_t *pud = (pud_t *)pgd;
543 pmd_t *pmd;
544
545 if (!pud_none(*pud))
546 return pmd_offset(pud, iova);
547
548 pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC);
549 if (!pmd)
550 return NULL;
551
552 ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE);
553 *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE);
554 ipmmu_flush_pgtable(mmu, pud, sizeof(*pud));
555
556 return pmd + pmd_index(iova);
557}
558
559static u64 ipmmu_page_prot(unsigned int prot, u64 type)
560{
561 u64 pgprot = ARM_VMSA_PTE_XN | ARM_VMSA_PTE_nG | ARM_VMSA_PTE_AF
562 | ARM_VMSA_PTE_SH_IS | ARM_VMSA_PTE_AP_UNPRIV
563 | ARM_VMSA_PTE_NS | type;
564
565 if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
566 pgprot |= ARM_VMSA_PTE_AP_RDONLY;
567
568 if (prot & IOMMU_CACHE)
569 pgprot |= IMMAIR_ATTR_IDX_WBRWA << ARM_VMSA_PTE_ATTRINDX_SHIFT;
570
571 if (prot & IOMMU_EXEC)
572 pgprot &= ~ARM_VMSA_PTE_XN;
573 else if (!(prot & (IOMMU_READ | IOMMU_WRITE)))
574 /* If no access create a faulting entry to avoid TLB fills. */
575 pgprot &= ~ARM_VMSA_PTE_PAGE;
576
577 return pgprot;
578}
579
580static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
581 unsigned long iova, unsigned long pfn,
582 size_t size, int prot)
583{
584 pteval_t pteval = ipmmu_page_prot(prot, ARM_VMSA_PTE_PAGE);
585 unsigned int num_ptes = 1;
586 pte_t *pte, *start;
587 unsigned int i;
588
589 pte = ipmmu_alloc_pte(mmu, pmd, iova);
590 if (!pte)
591 return -ENOMEM;
592
593 start = pte;
594
595 /*
596 * Install the page table entries. We can be called both for a single
597 * page or for a block of 16 physically contiguous pages. In the latter
598 * case set the PTE contiguous hint.
599 */
600 if (size == SZ_64K) {
601 pteval |= ARM_VMSA_PTE_CONT;
602 num_ptes = ARM_VMSA_PTE_CONT_ENTRIES;
603 }
604
605 for (i = num_ptes; i; --i)
606 *pte++ = pfn_pte(pfn++, __pgprot(pteval));
607
608 ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * num_ptes);
609
610 return 0;
611}
612
613static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd,
614 unsigned long iova, unsigned long pfn,
615 int prot)
616{
617 pmdval_t pmdval = ipmmu_page_prot(prot, PMD_TYPE_SECT);
618
619 *pmd = pfn_pmd(pfn, __pgprot(pmdval));
620 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
621
622 return 0;
623}
624
625static int ipmmu_create_mapping(struct ipmmu_vmsa_domain *domain,
626 unsigned long iova, phys_addr_t paddr,
627 size_t size, int prot)
628{
629 struct ipmmu_vmsa_device *mmu = domain->mmu;
630 pgd_t *pgd = domain->pgd;
631 unsigned long flags;
632 unsigned long pfn;
633 pmd_t *pmd;
634 int ret;
635
636 if (!pgd)
637 return -EINVAL;
638
639 if (size & ~PAGE_MASK)
640 return -EINVAL;
641
642 if (paddr & ~((1ULL << 40) - 1))
643 return -ERANGE;
644
645 pfn = __phys_to_pfn(paddr);
646 pgd += pgd_index(iova);
647
648 /* Update the page tables. */
649 spin_lock_irqsave(&domain->lock, flags);
650
651 pmd = ipmmu_alloc_pmd(mmu, pgd, iova);
652 if (!pmd) {
653 ret = -ENOMEM;
654 goto done;
655 }
656
657 switch (size) {
658 case SZ_2M:
659 ret = ipmmu_alloc_init_pmd(mmu, pmd, iova, pfn, prot);
660 break;
661 case SZ_64K:
662 case SZ_4K:
663 ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot);
664 break;
665 default:
666 ret = -EINVAL;
667 break;
668 }
669
670done:
671 spin_unlock_irqrestore(&domain->lock, flags);
672
673 if (!ret)
674 ipmmu_tlb_invalidate(domain);
675
676 return ret;
677}
678
679static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud)
680{
681 /* Free the page table. */
682 pgtable_t table = pud_pgtable(*pud);
683 __free_page(table);
684
685 /* Clear the PUD. */
686 *pud = __pud(0);
687 ipmmu_flush_pgtable(mmu, pud, sizeof(*pud));
688}
689
690static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud,
691 pmd_t *pmd)
692{
693 unsigned int i;
694
695 /* Free the page table. */
696 if (pmd_table(*pmd)) {
697 pgtable_t table = pmd_pgtable(*pmd);
698 __free_page(table);
699 }
700
701 /* Clear the PMD. */
702 *pmd = __pmd(0);
703 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
704
705 /* Check whether the PUD is still needed. */
706 pmd = pmd_offset(pud, 0);
707 for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) {
708 if (!pmd_none(pmd[i]))
709 return;
710 }
711
712 /* Clear the parent PUD. */
713 ipmmu_clear_pud(mmu, pud);
714}
715
716static void ipmmu_clear_pte(struct ipmmu_vmsa_device *mmu, pud_t *pud,
717 pmd_t *pmd, pte_t *pte, unsigned int num_ptes)
718{
719 unsigned int i;
720
721 /* Clear the PTE. */
722 for (i = num_ptes; i; --i)
723 pte[i-1] = __pte(0);
724
725 ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * num_ptes);
726
727 /* Check whether the PMD is still needed. */
728 pte = pte_offset_kernel(pmd, 0);
729 for (i = 0; i < IPMMU_PTRS_PER_PTE; ++i) {
730 if (!pte_none(pte[i]))
731 return;
732 }
733
734 /* Clear the parent PMD. */
735 ipmmu_clear_pmd(mmu, pud, pmd);
736}
737
738static int ipmmu_split_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd)
739{
740 pte_t *pte, *start;
741 pteval_t pteval;
742 unsigned long pfn;
743 unsigned int i;
744
745 pte = (pte_t *)get_zeroed_page(GFP_ATOMIC);
746 if (!pte)
747 return -ENOMEM;
748
749 /* Copy the PMD attributes. */
750 pteval = (pmd_val(*pmd) & ARM_VMSA_PTE_ATTRS_MASK)
751 | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_PAGE;
752
753 pfn = pmd_pfn(*pmd);
754 start = pte;
755
756 for (i = IPMMU_PTRS_PER_PTE; i; --i)
757 *pte++ = pfn_pte(pfn++, __pgprot(pteval));
758
759 ipmmu_flush_pgtable(mmu, start, PAGE_SIZE);
760 *pmd = __pmd(__pa(start) | PMD_NSTABLE | PMD_TYPE_TABLE);
761 ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd));
762
763 return 0;
764}
765
766static void ipmmu_split_pte(struct ipmmu_vmsa_device *mmu, pte_t *pte)
767{
768 unsigned int i;
769
770 for (i = ARM_VMSA_PTE_CONT_ENTRIES; i; --i)
771 pte[i-1] = __pte(pte_val(*pte) & ~ARM_VMSA_PTE_CONT);
772
773 ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * ARM_VMSA_PTE_CONT_ENTRIES);
774}
775
776static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain,
777 unsigned long iova, size_t size)
778{
779 struct ipmmu_vmsa_device *mmu = domain->mmu;
780 unsigned long flags;
781 pgd_t *pgd = domain->pgd;
782 pud_t *pud;
783 pmd_t *pmd;
784 pte_t *pte;
785 int ret = 0;
786
787 if (!pgd)
788 return -EINVAL;
789
790 if (size & ~PAGE_MASK)
791 return -EINVAL;
792
793 pgd += pgd_index(iova);
794 pud = (pud_t *)pgd;
795
796 spin_lock_irqsave(&domain->lock, flags);
797
798 /* If there's no PUD or PMD we're done. */
799 if (pud_none(*pud))
800 goto done;
801
802 pmd = pmd_offset(pud, iova);
803 if (pmd_none(*pmd))
804 goto done;
805
806 /*
807 * When freeing a 2MB block just clear the PMD. In the unlikely case the
808 * block is mapped as individual pages this will free the corresponding
809 * PTE page table.
810 */
811 if (size == SZ_2M) {
812 ipmmu_clear_pmd(mmu, pud, pmd);
813 goto done;
814 }
815
816 /*
817 * If the PMD has been mapped as a section remap it as pages to allow
818 * freeing individual pages.
819 */
820 if (pmd_sect(*pmd))
821 ipmmu_split_pmd(mmu, pmd);
822
823 pte = pte_offset_kernel(pmd, iova);
824
825 /*
826 * When freeing a 64kB block just clear the PTE entries. We don't have
827 * to care about the contiguous hint of the surrounding entries.
828 */
829 if (size == SZ_64K) {
830 ipmmu_clear_pte(mmu, pud, pmd, pte, ARM_VMSA_PTE_CONT_ENTRIES);
831 goto done;
832 }
833
834 /*
835 * If the PTE has been mapped with the contiguous hint set remap it and
836 * its surrounding PTEs to allow unmapping a single page.
837 */
838 if (pte_val(*pte) & ARM_VMSA_PTE_CONT)
839 ipmmu_split_pte(mmu, pte);
840
841 /* Clear the PTE. */
842 ipmmu_clear_pte(mmu, pud, pmd, pte, 1);
843
844done:
845 spin_unlock_irqrestore(&domain->lock, flags);
846
847 if (ret)
848 ipmmu_tlb_invalidate(domain);
849
850 return 0;
851}
852
853/* -----------------------------------------------------------------------------
854 * IOMMU Operations
855 */
856
857static int ipmmu_domain_init(struct iommu_domain *io_domain)
858{
859 struct ipmmu_vmsa_domain *domain;
860
861 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
862 if (!domain)
863 return -ENOMEM;
864
865 spin_lock_init(&domain->lock);
866
867 domain->pgd = kzalloc(IPMMU_PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
868 if (!domain->pgd) {
869 kfree(domain);
870 return -ENOMEM;
871 }
872
873 io_domain->priv = domain;
874 domain->io_domain = io_domain;
875
876 return 0;
877}
878
879static void ipmmu_domain_destroy(struct iommu_domain *io_domain)
880{
881 struct ipmmu_vmsa_domain *domain = io_domain->priv;
882
883 /*
884 * Free the domain resources. We assume that all devices have already
885 * been detached.
886 */
887 ipmmu_domain_destroy_context(domain);
888 ipmmu_free_pgtables(domain);
889 kfree(domain);
890}
891
892static int ipmmu_attach_device(struct iommu_domain *io_domain,
893 struct device *dev)
894{
895 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
896 struct ipmmu_vmsa_device *mmu = archdata->mmu;
897 struct ipmmu_vmsa_domain *domain = io_domain->priv;
898 unsigned long flags;
899 int ret = 0;
900
901 if (!mmu) {
902 dev_err(dev, "Cannot attach to IPMMU\n");
903 return -ENXIO;
904 }
905
906 spin_lock_irqsave(&domain->lock, flags);
907
908 if (!domain->mmu) {
909 /* The domain hasn't been used yet, initialize it. */
910 domain->mmu = mmu;
911 ret = ipmmu_domain_init_context(domain);
912 } else if (domain->mmu != mmu) {
913 /*
914 * Something is wrong, we can't attach two devices using
915 * different IOMMUs to the same domain.
916 */
917 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
918 dev_name(mmu->dev), dev_name(domain->mmu->dev));
919 ret = -EINVAL;
920 }
921
922 spin_unlock_irqrestore(&domain->lock, flags);
923
924 if (ret < 0)
925 return ret;
926
927 ipmmu_utlb_enable(domain, archdata->utlb);
928
929 return 0;
930}
931
932static void ipmmu_detach_device(struct iommu_domain *io_domain,
933 struct device *dev)
934{
935 struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
936 struct ipmmu_vmsa_domain *domain = io_domain->priv;
937
938 ipmmu_utlb_disable(domain, archdata->utlb);
939
940 /*
941 * TODO: Optimize by disabling the context when no device is attached.
942 */
943}
944
945static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
946 phys_addr_t paddr, size_t size, int prot)
947{
948 struct ipmmu_vmsa_domain *domain = io_domain->priv;
949
950 if (!domain)
951 return -ENODEV;
952
953 return ipmmu_create_mapping(domain, iova, paddr, size, prot);
954}
955
956static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
957 size_t size)
958{
959 struct ipmmu_vmsa_domain *domain = io_domain->priv;
960 int ret;
961
962 ret = ipmmu_clear_mapping(domain, iova, size);
963 return ret ? 0 : size;
964}
965
966static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
967 dma_addr_t iova)
968{
969 struct ipmmu_vmsa_domain *domain = io_domain->priv;
970 pgd_t pgd;
971 pud_t pud;
972 pmd_t pmd;
973 pte_t pte;
974
975 /* TODO: Is locking needed ? */
976
977 if (!domain->pgd)
978 return 0;
979
980 pgd = *(domain->pgd + pgd_index(iova));
981 if (pgd_none(pgd))
982 return 0;
983
984 pud = *pud_offset(&pgd, iova);
985 if (pud_none(pud))
986 return 0;
987
988 pmd = *pmd_offset(&pud, iova);
989 if (pmd_none(pmd))
990 return 0;
991
992 if (pmd_sect(pmd))
993 return __pfn_to_phys(pmd_pfn(pmd)) | (iova & ~PMD_MASK);
994
995 pte = *(pmd_page_vaddr(pmd) + pte_index(iova));
996 if (pte_none(pte))
997 return 0;
998
999 return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK);
1000}
1001
1002static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev)
1003{
1004 const struct ipmmu_vmsa_master *master = mmu->pdata->masters;
1005 const char *devname = dev_name(dev);
1006 unsigned int i;
1007
1008 for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) {
1009 if (strcmp(master->name, devname) == 0)
1010 return master->utlb;
1011 }
1012
1013 return -1;
1014}
1015
1016static int ipmmu_add_device(struct device *dev)
1017{
1018 struct ipmmu_vmsa_archdata *archdata;
1019 struct ipmmu_vmsa_device *mmu;
1020 struct iommu_group *group;
1021 int utlb = -1;
1022 int ret;
1023
1024 if (dev->archdata.iommu) {
1025 dev_warn(dev, "IOMMU driver already assigned to device %s\n",
1026 dev_name(dev));
1027 return -EINVAL;
1028 }
1029
1030 /* Find the master corresponding to the device. */
1031 spin_lock(&ipmmu_devices_lock);
1032
1033 list_for_each_entry(mmu, &ipmmu_devices, list) {
1034 utlb = ipmmu_find_utlb(mmu, dev);
1035 if (utlb >= 0) {
1036 /*
1037 * TODO Take a reference to the MMU to protect
1038 * against device removal.
1039 */
1040 break;
1041 }
1042 }
1043
1044 spin_unlock(&ipmmu_devices_lock);
1045
1046 if (utlb < 0)
1047 return -ENODEV;
1048
1049 if (utlb >= mmu->num_utlbs)
1050 return -EINVAL;
1051
1052 /* Create a device group and add the device to it. */
1053 group = iommu_group_alloc();
1054 if (IS_ERR(group)) {
1055 dev_err(dev, "Failed to allocate IOMMU group\n");
1056 return PTR_ERR(group);
1057 }
1058
1059 ret = iommu_group_add_device(group, dev);
1060 iommu_group_put(group);
1061
1062 if (ret < 0) {
1063 dev_err(dev, "Failed to add device to IPMMU group\n");
1064 return ret;
1065 }
1066
1067 archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
1068 if (!archdata) {
1069 ret = -ENOMEM;
1070 goto error;
1071 }
1072
1073 archdata->mmu = mmu;
1074 archdata->utlb = utlb;
1075 dev->archdata.iommu = archdata;
1076
1077 /*
1078 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
1079 * VAs. This will allocate a corresponding IOMMU domain.
1080 *
1081 * TODO:
1082 * - Create one mapping per context (TLB).
1083 * - Make the mapping size configurable ? We currently use a 2GB mapping
1084 * at a 1GB offset to ensure that NULL VAs will fault.
1085 */
1086 if (!mmu->mapping) {
1087 struct dma_iommu_mapping *mapping;
1088
1089 mapping = arm_iommu_create_mapping(&platform_bus_type,
1090 SZ_1G, SZ_2G);
1091 if (IS_ERR(mapping)) {
1092 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
1093 return PTR_ERR(mapping);
1094 }
1095
1096 mmu->mapping = mapping;
1097 }
1098
1099 /* Attach the ARM VA mapping to the device. */
1100 ret = arm_iommu_attach_device(dev, mmu->mapping);
1101 if (ret < 0) {
1102 dev_err(dev, "Failed to attach device to VA mapping\n");
1103 goto error;
1104 }
1105
1106 return 0;
1107
1108error:
1109 kfree(dev->archdata.iommu);
1110 dev->archdata.iommu = NULL;
1111 iommu_group_remove_device(dev);
1112 return ret;
1113}
1114
1115static void ipmmu_remove_device(struct device *dev)
1116{
1117 arm_iommu_detach_device(dev);
1118 iommu_group_remove_device(dev);
1119 kfree(dev->archdata.iommu);
1120 dev->archdata.iommu = NULL;
1121}
1122
1123static struct iommu_ops ipmmu_ops = {
1124 .domain_init = ipmmu_domain_init,
1125 .domain_destroy = ipmmu_domain_destroy,
1126 .attach_dev = ipmmu_attach_device,
1127 .detach_dev = ipmmu_detach_device,
1128 .map = ipmmu_map,
1129 .unmap = ipmmu_unmap,
1130 .iova_to_phys = ipmmu_iova_to_phys,
1131 .add_device = ipmmu_add_device,
1132 .remove_device = ipmmu_remove_device,
1133 .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K,
1134};
1135
1136/* -----------------------------------------------------------------------------
1137 * Probe/remove and init
1138 */
1139
1140static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
1141{
1142 unsigned int i;
1143
1144 /* Disable all contexts. */
1145 for (i = 0; i < 4; ++i)
1146 ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0);
1147}
1148
1149static int ipmmu_probe(struct platform_device *pdev)
1150{
1151 struct ipmmu_vmsa_device *mmu;
1152 struct resource *res;
1153 int irq;
1154 int ret;
1155
1156 if (!pdev->dev.platform_data) {
1157 dev_err(&pdev->dev, "missing platform data\n");
1158 return -EINVAL;
1159 }
1160
1161 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
1162 if (!mmu) {
1163 dev_err(&pdev->dev, "cannot allocate device data\n");
1164 return -ENOMEM;
1165 }
1166
1167 mmu->dev = &pdev->dev;
1168 mmu->pdata = pdev->dev.platform_data;
1169 mmu->num_utlbs = 32;
1170
1171 /* Map I/O memory and request IRQ. */
1172 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1173 mmu->base = devm_ioremap_resource(&pdev->dev, res);
1174 if (IS_ERR(mmu->base))
1175 return PTR_ERR(mmu->base);
1176
1177 irq = platform_get_irq(pdev, 0);
1178 if (irq < 0) {
1179 dev_err(&pdev->dev, "no IRQ found\n");
1180 return irq;
1181 }
1182
1183 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
1184 dev_name(&pdev->dev), mmu);
1185 if (ret < 0) {
1186 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
1187 return irq;
1188 }
1189
1190 ipmmu_device_reset(mmu);
1191
1192 /*
1193 * We can't create the ARM mapping here as it requires the bus to have
1194 * an IOMMU, which only happens when bus_set_iommu() is called in
1195 * ipmmu_init() after the probe function returns.
1196 */
1197
1198 spin_lock(&ipmmu_devices_lock);
1199 list_add(&mmu->list, &ipmmu_devices);
1200 spin_unlock(&ipmmu_devices_lock);
1201
1202 platform_set_drvdata(pdev, mmu);
1203
1204 return 0;
1205}
1206
1207static int ipmmu_remove(struct platform_device *pdev)
1208{
1209 struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
1210
1211 spin_lock(&ipmmu_devices_lock);
1212 list_del(&mmu->list);
1213 spin_unlock(&ipmmu_devices_lock);
1214
1215 arm_iommu_release_mapping(mmu->mapping);
1216
1217 ipmmu_device_reset(mmu);
1218
1219 return 0;
1220}
1221
1222static struct platform_driver ipmmu_driver = {
1223 .driver = {
1224 .owner = THIS_MODULE,
1225 .name = "ipmmu-vmsa",
1226 },
1227 .probe = ipmmu_probe,
1228 .remove = ipmmu_remove,
1229};
1230
1231static int __init ipmmu_init(void)
1232{
1233 int ret;
1234
1235 ret = platform_driver_register(&ipmmu_driver);
1236 if (ret < 0)
1237 return ret;
1238
1239 if (!iommu_present(&platform_bus_type))
1240 bus_set_iommu(&platform_bus_type, &ipmmu_ops);
1241
1242 return 0;
1243}
1244
1245static void __exit ipmmu_exit(void)
1246{
1247 return platform_driver_unregister(&ipmmu_driver);
1248}
1249
1250subsys_initcall(ipmmu_init);
1251module_exit(ipmmu_exit);
1252
1253MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
1254MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
1255MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
index 08ba4972da9d..61def7cb5263 100644
--- a/drivers/iommu/msm_iommu_dev.c
+++ b/drivers/iommu/msm_iommu_dev.c
@@ -127,13 +127,12 @@ static void msm_iommu_reset(void __iomem *base, int ncb)
127 127
128static int msm_iommu_probe(struct platform_device *pdev) 128static int msm_iommu_probe(struct platform_device *pdev)
129{ 129{
130 struct resource *r, *r2; 130 struct resource *r;
131 struct clk *iommu_clk; 131 struct clk *iommu_clk;
132 struct clk *iommu_pclk; 132 struct clk *iommu_pclk;
133 struct msm_iommu_drvdata *drvdata; 133 struct msm_iommu_drvdata *drvdata;
134 struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data; 134 struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data;
135 void __iomem *regs_base; 135 void __iomem *regs_base;
136 resource_size_t len;
137 int ret, irq, par; 136 int ret, irq, par;
138 137
139 if (pdev->id == -1) { 138 if (pdev->id == -1) {
@@ -178,35 +177,16 @@ static int msm_iommu_probe(struct platform_device *pdev)
178 iommu_clk = NULL; 177 iommu_clk = NULL;
179 178
180 r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase"); 179 r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase");
181 180 regs_base = devm_ioremap_resource(&pdev->dev, r);
182 if (!r) { 181 if (IS_ERR(regs_base)) {
183 ret = -ENODEV; 182 ret = PTR_ERR(regs_base);
184 goto fail_clk;
185 }
186
187 len = resource_size(r);
188
189 r2 = request_mem_region(r->start, len, r->name);
190 if (!r2) {
191 pr_err("Could not request memory region: start=%p, len=%d\n",
192 (void *) r->start, len);
193 ret = -EBUSY;
194 goto fail_clk; 183 goto fail_clk;
195 } 184 }
196 185
197 regs_base = ioremap(r2->start, len);
198
199 if (!regs_base) {
200 pr_err("Could not ioremap: start=%p, len=%d\n",
201 (void *) r2->start, len);
202 ret = -EBUSY;
203 goto fail_mem;
204 }
205
206 irq = platform_get_irq_byname(pdev, "secure_irq"); 186 irq = platform_get_irq_byname(pdev, "secure_irq");
207 if (irq < 0) { 187 if (irq < 0) {
208 ret = -ENODEV; 188 ret = -ENODEV;
209 goto fail_io; 189 goto fail_clk;
210 } 190 }
211 191
212 msm_iommu_reset(regs_base, iommu_dev->ncb); 192 msm_iommu_reset(regs_base, iommu_dev->ncb);
@@ -222,14 +202,14 @@ static int msm_iommu_probe(struct platform_device *pdev)
222 if (!par) { 202 if (!par) {
223 pr_err("%s: Invalid PAR value detected\n", iommu_dev->name); 203 pr_err("%s: Invalid PAR value detected\n", iommu_dev->name);
224 ret = -ENODEV; 204 ret = -ENODEV;
225 goto fail_io; 205 goto fail_clk;
226 } 206 }
227 207
228 ret = request_irq(irq, msm_iommu_fault_handler, 0, 208 ret = request_irq(irq, msm_iommu_fault_handler, 0,
229 "msm_iommu_secure_irpt_handler", drvdata); 209 "msm_iommu_secure_irpt_handler", drvdata);
230 if (ret) { 210 if (ret) {
231 pr_err("Request IRQ %d failed with ret=%d\n", irq, ret); 211 pr_err("Request IRQ %d failed with ret=%d\n", irq, ret);
232 goto fail_io; 212 goto fail_clk;
233 } 213 }
234 214
235 215
@@ -250,10 +230,6 @@ static int msm_iommu_probe(struct platform_device *pdev)
250 clk_disable(iommu_pclk); 230 clk_disable(iommu_pclk);
251 231
252 return 0; 232 return 0;
253fail_io:
254 iounmap(regs_base);
255fail_mem:
256 release_mem_region(r->start, len);
257fail_clk: 233fail_clk:
258 if (iommu_clk) { 234 if (iommu_clk) {
259 clk_disable(iommu_clk); 235 clk_disable(iommu_clk);
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 7fcbfc498fa9..895af06a667f 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -34,6 +34,9 @@
34#include "omap-iopgtable.h" 34#include "omap-iopgtable.h"
35#include "omap-iommu.h" 35#include "omap-iommu.h"
36 36
37#define to_iommu(dev) \
38 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
39
37#define for_each_iotlb_cr(obj, n, __i, cr) \ 40#define for_each_iotlb_cr(obj, n, __i, cr) \
38 for (__i = 0; \ 41 for (__i = 0; \
39 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ 42 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
@@ -391,6 +394,7 @@ static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
391 __func__, start, da, bytes); 394 __func__, start, da, bytes);
392 iotlb_load_cr(obj, &cr); 395 iotlb_load_cr(obj, &cr);
393 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); 396 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
397 break;
394 } 398 }
395 } 399 }
396 pm_runtime_put_sync(obj->dev); 400 pm_runtime_put_sync(obj->dev);
@@ -1037,19 +1041,18 @@ static void iopte_cachep_ctor(void *iopte)
1037 clean_dcache_area(iopte, IOPTE_TABLE_SIZE); 1041 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1038} 1042}
1039 1043
1040static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, 1044static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
1041 u32 flags)
1042{ 1045{
1043 memset(e, 0, sizeof(*e)); 1046 memset(e, 0, sizeof(*e));
1044 1047
1045 e->da = da; 1048 e->da = da;
1046 e->pa = pa; 1049 e->pa = pa;
1047 e->valid = 1; 1050 e->valid = MMU_CAM_V;
1048 /* FIXME: add OMAP1 support */ 1051 /* FIXME: add OMAP1 support */
1049 e->pgsz = flags & MMU_CAM_PGSZ_MASK; 1052 e->pgsz = pgsz;
1050 e->endian = flags & MMU_RAM_ENDIAN_MASK; 1053 e->endian = MMU_RAM_ENDIAN_LITTLE;
1051 e->elsz = flags & MMU_RAM_ELSZ_MASK; 1054 e->elsz = MMU_RAM_ELSZ_8;
1052 e->mixed = flags & MMU_RAM_MIXED_MASK; 1055 e->mixed = 0;
1053 1056
1054 return iopgsz_to_bytes(e->pgsz); 1057 return iopgsz_to_bytes(e->pgsz);
1055} 1058}
@@ -1062,9 +1065,8 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1062 struct device *dev = oiommu->dev; 1065 struct device *dev = oiommu->dev;
1063 struct iotlb_entry e; 1066 struct iotlb_entry e;
1064 int omap_pgsz; 1067 int omap_pgsz;
1065 u32 ret, flags; 1068 u32 ret;
1066 1069
1067 /* we only support mapping a single iommu page for now */
1068 omap_pgsz = bytes_to_iopgsz(bytes); 1070 omap_pgsz = bytes_to_iopgsz(bytes);
1069 if (omap_pgsz < 0) { 1071 if (omap_pgsz < 0) {
1070 dev_err(dev, "invalid size to map: %d\n", bytes); 1072 dev_err(dev, "invalid size to map: %d\n", bytes);
@@ -1073,9 +1075,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1073 1075
1074 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); 1076 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
1075 1077
1076 flags = omap_pgsz | prot; 1078 iotlb_init_entry(&e, da, pa, omap_pgsz);
1077
1078 iotlb_init_entry(&e, da, pa, flags);
1079 1079
1080 ret = omap_iopgtable_store_entry(oiommu, &e); 1080 ret = omap_iopgtable_store_entry(oiommu, &e);
1081 if (ret) 1081 if (ret)
@@ -1248,12 +1248,6 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1248 return ret; 1248 return ret;
1249} 1249}
1250 1250
1251static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
1252 unsigned long cap)
1253{
1254 return 0;
1255}
1256
1257static int omap_iommu_add_device(struct device *dev) 1251static int omap_iommu_add_device(struct device *dev)
1258{ 1252{
1259 struct omap_iommu_arch_data *arch_data; 1253 struct omap_iommu_arch_data *arch_data;
@@ -1305,7 +1299,6 @@ static struct iommu_ops omap_iommu_ops = {
1305 .map = omap_iommu_map, 1299 .map = omap_iommu_map,
1306 .unmap = omap_iommu_unmap, 1300 .unmap = omap_iommu_unmap,
1307 .iova_to_phys = omap_iommu_iova_to_phys, 1301 .iova_to_phys = omap_iommu_iova_to_phys,
1308 .domain_has_cap = omap_iommu_domain_has_cap,
1309 .add_device = omap_iommu_add_device, 1302 .add_device = omap_iommu_add_device,
1310 .remove_device = omap_iommu_remove_device, 1303 .remove_device = omap_iommu_remove_device,
1311 .pgsize_bitmap = OMAP_IOMMU_PGSIZES, 1304 .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h
index b6f9a51746ca..f891683e3f05 100644
--- a/drivers/iommu/omap-iopgtable.h
+++ b/drivers/iommu/omap-iopgtable.h
@@ -93,6 +93,3 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask)
93/* to find an entry in the second-level page table. */ 93/* to find an entry in the second-level page table. */
94#define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1)) 94#define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1))
95#define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da)) 95#define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da))
96
97#define to_iommu(dev) \
98 (platform_get_drvdata(to_platform_device(dev)))
diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c
index e3bc2e19b6dd..bd97adecb1fd 100644
--- a/drivers/iommu/shmobile-ipmmu.c
+++ b/drivers/iommu/shmobile-ipmmu.c
@@ -94,11 +94,6 @@ static int ipmmu_probe(struct platform_device *pdev)
94 struct resource *res; 94 struct resource *res;
95 struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data; 95 struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data;
96 96
97 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
98 if (!res) {
99 dev_err(&pdev->dev, "cannot get platform resources\n");
100 return -ENOENT;
101 }
102 ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL); 97 ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL);
103 if (!ipmmu) { 98 if (!ipmmu) {
104 dev_err(&pdev->dev, "cannot allocate device data\n"); 99 dev_err(&pdev->dev, "cannot allocate device data\n");
@@ -106,19 +101,18 @@ static int ipmmu_probe(struct platform_device *pdev)
106 } 101 }
107 spin_lock_init(&ipmmu->flush_lock); 102 spin_lock_init(&ipmmu->flush_lock);
108 ipmmu->dev = &pdev->dev; 103 ipmmu->dev = &pdev->dev;
109 ipmmu->ipmmu_base = devm_ioremap_nocache(&pdev->dev, res->start, 104
110 resource_size(res)); 105 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
111 if (!ipmmu->ipmmu_base) { 106 ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res);
112 dev_err(&pdev->dev, "ioremap_nocache failed\n"); 107 if (IS_ERR(ipmmu->ipmmu_base))
113 return -ENOMEM; 108 return PTR_ERR(ipmmu->ipmmu_base);
114 } 109
115 ipmmu->dev_names = pdata->dev_names; 110 ipmmu->dev_names = pdata->dev_names;
116 ipmmu->num_dev_names = pdata->num_dev_names; 111 ipmmu->num_dev_names = pdata->num_dev_names;
117 platform_set_drvdata(pdev, ipmmu); 112 platform_set_drvdata(pdev, ipmmu);
118 ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */ 113 ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */
119 ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */ 114 ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */
120 ipmmu_iommu_init(ipmmu); 115 return ipmmu_iommu_init(ipmmu);
121 return 0;
122} 116}
123 117
124static struct platform_driver ipmmu_driver = { 118static struct platform_driver ipmmu_driver = {
diff --git a/include/linux/platform_data/ipmmu-vmsa.h b/include/linux/platform_data/ipmmu-vmsa.h
new file mode 100644
index 000000000000..5275b3ac6d37
--- /dev/null
+++ b/include/linux/platform_data/ipmmu-vmsa.h
@@ -0,0 +1,24 @@
1/*
2 * IPMMU VMSA Platform Data
3 *
4 * Copyright (C) 2014 Renesas Electronics Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 */
10
11#ifndef __IPMMU_VMSA_H__
12#define __IPMMU_VMSA_H__
13
14struct ipmmu_vmsa_master {
15 const char *name;
16 unsigned int utlb;
17};
18
19struct ipmmu_vmsa_platform_data {
20 const struct ipmmu_vmsa_master *masters;
21 unsigned int num_masters;
22};
23
24#endif /* __IPMMU_VMSA_H__ */
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-10 23:02:52 -0400