aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
authorHiroshi DOYU <hdoyu@nvidia.com>2011-11-17 00:31:31 -0500
committerJoerg Roedel <joerg.roedel@amd.com>2012-01-26 07:59:02 -0500
commit7a31f6f48bb6e37b3195fcecd69d30740f6b0036 (patch)
tree8f917ff520e78eaa7a0e4ebadfaacdfa7bccf54b /drivers/iommu
parentd53e54b4d43b2559c5a424188894fd63d145b5d7 (diff)
ARM: IOMMU: Tegra30: Add iommu_ops for SMMU driver
Tegra 30 IOMMU H/W, SMMU (System Memory Management Unit). This patch implements struct iommu_ops for SMMU for the upper IOMMU API. This H/W module supports multiple virtual address spaces(domain x4), and manages 2 level H/W translation pagetable. Signed-off-by: Hiroshi DOYU <hdoyu@nvidia.com> Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig10
-rw-r--r--drivers/iommu/Makefile1
-rw-r--r--drivers/iommu/tegra-smmu.c1034
3 files changed, 1045 insertions, 0 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 76c86da2b41..3bd9fff5c58 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -152,4 +152,14 @@ config TEGRA_IOMMU_GART
152 space through the GART (Graphics Address Relocation Table) 152 space through the GART (Graphics Address Relocation Table)
153 hardware included on Tegra SoCs. 153 hardware included on Tegra SoCs.
154 154
155config TEGRA_IOMMU_SMMU
156 bool "Tegra SMMU IOMMU Support"
157 depends on ARCH_TEGRA_3x_SOC
158 select IOMMU_API
159 help
160 Enables support for remapping discontiguous physical memory
161 shared with the operating system into contiguous I/O virtual
162 space through the SMMU (System Memory Management Unit)
163 hardware included on Tegra SoCs.
164
155endif # IOMMU_SUPPORT 165endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 3238a31d260..7ad7a3bc124 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
9obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o 9obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
10obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o 10obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
11obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o 11obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
12obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
new file mode 100644
index 00000000000..eb93c821f59
--- /dev/null
+++ b/drivers/iommu/tegra-smmu.c
@@ -0,0 +1,1034 @@
1/*
2 * IOMMU API for SMMU in Tegra30
3 *
4 * Copyright (c) 2011-2012, NVIDIA CORPORATION. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 */
19
20#define pr_fmt(fmt) "%s(): " fmt, __func__
21
22#include <linux/module.h>
23#include <linux/platform_device.h>
24#include <linux/spinlock.h>
25#include <linux/slab.h>
26#include <linux/vmalloc.h>
27#include <linux/mm.h>
28#include <linux/pagemap.h>
29#include <linux/device.h>
30#include <linux/sched.h>
31#include <linux/iommu.h>
32#include <linux/io.h>
33
34#include <asm/page.h>
35#include <asm/cacheflush.h>
36
37#include <mach/iomap.h>
38#include <mach/smmu.h>
39
40/* bitmap of the page sizes currently supported */
41#define SMMU_IOMMU_PGSIZES (SZ_4K)
42
43#define SMMU_CONFIG 0x10
44#define SMMU_CONFIG_DISABLE 0
45#define SMMU_CONFIG_ENABLE 1
46
47#define SMMU_TLB_CONFIG 0x14
48#define SMMU_TLB_CONFIG_STATS__MASK (1 << 31)
49#define SMMU_TLB_CONFIG_STATS__ENABLE (1 << 31)
50#define SMMU_TLB_CONFIG_HIT_UNDER_MISS__ENABLE (1 << 29)
51#define SMMU_TLB_CONFIG_ACTIVE_LINES__VALUE 0x10
52#define SMMU_TLB_CONFIG_RESET_VAL 0x20000010
53
54#define SMMU_PTC_CONFIG 0x18
55#define SMMU_PTC_CONFIG_STATS__MASK (1 << 31)
56#define SMMU_PTC_CONFIG_STATS__ENABLE (1 << 31)
57#define SMMU_PTC_CONFIG_CACHE__ENABLE (1 << 29)
58#define SMMU_PTC_CONFIG_INDEX_MAP__PATTERN 0x3f
59#define SMMU_PTC_CONFIG_RESET_VAL 0x2000003f
60
61#define SMMU_PTB_ASID 0x1c
62#define SMMU_PTB_ASID_CURRENT_SHIFT 0
63
64#define SMMU_PTB_DATA 0x20
65#define SMMU_PTB_DATA_RESET_VAL 0
66#define SMMU_PTB_DATA_ASID_NONSECURE_SHIFT 29
67#define SMMU_PTB_DATA_ASID_WRITABLE_SHIFT 30
68#define SMMU_PTB_DATA_ASID_READABLE_SHIFT 31
69
70#define SMMU_TLB_FLUSH 0x30
71#define SMMU_TLB_FLUSH_VA_MATCH_ALL 0
72#define SMMU_TLB_FLUSH_VA_MATCH_SECTION 2
73#define SMMU_TLB_FLUSH_VA_MATCH_GROUP 3
74#define SMMU_TLB_FLUSH_ASID_SHIFT 29
75#define SMMU_TLB_FLUSH_ASID_MATCH_DISABLE 0
76#define SMMU_TLB_FLUSH_ASID_MATCH_ENABLE 1
77#define SMMU_TLB_FLUSH_ASID_MATCH_SHIFT 31
78
79#define SMMU_PTC_FLUSH 0x34
80#define SMMU_PTC_FLUSH_TYPE_ALL 0
81#define SMMU_PTC_FLUSH_TYPE_ADR 1
82#define SMMU_PTC_FLUSH_ADR_SHIFT 4
83
84#define SMMU_ASID_SECURITY 0x38
85
86#define SMMU_STATS_TLB_HIT_COUNT 0x1f0
87#define SMMU_STATS_TLB_MISS_COUNT 0x1f4
88#define SMMU_STATS_PTC_HIT_COUNT 0x1f8
89#define SMMU_STATS_PTC_MISS_COUNT 0x1fc
90
91#define SMMU_TRANSLATION_ENABLE_0 0x228
92#define SMMU_TRANSLATION_ENABLE_1 0x22c
93#define SMMU_TRANSLATION_ENABLE_2 0x230
94
95#define SMMU_AFI_ASID 0x238 /* PCIE */
96#define SMMU_AVPC_ASID 0x23c /* AVP */
97#define SMMU_DC_ASID 0x240 /* Display controller */
98#define SMMU_DCB_ASID 0x244 /* Display controller B */
99#define SMMU_EPP_ASID 0x248 /* Encoder pre-processor */
100#define SMMU_G2_ASID 0x24c /* 2D engine */
101#define SMMU_HC_ASID 0x250 /* Host1x */
102#define SMMU_HDA_ASID 0x254 /* High-def audio */
103#define SMMU_ISP_ASID 0x258 /* Image signal processor */
104#define SMMU_MPE_ASID 0x264 /* MPEG encoder */
105#define SMMU_NV_ASID 0x268 /* (3D) */
106#define SMMU_NV2_ASID 0x26c /* (3D) */
107#define SMMU_PPCS_ASID 0x270 /* AHB */
108#define SMMU_SATA_ASID 0x278 /* SATA */
109#define SMMU_VDE_ASID 0x27c /* Video decoder */
110#define SMMU_VI_ASID 0x280 /* Video input */
111
112#define SMMU_PDE_NEXT_SHIFT 28
113
114/* AHB Arbiter Registers */
115#define AHB_XBAR_CTRL 0xe0
116#define AHB_XBAR_CTRL_SMMU_INIT_DONE_DONE 1
117#define AHB_XBAR_CTRL_SMMU_INIT_DONE_SHIFT 17
118
119#define SMMU_NUM_ASIDS 4
120#define SMMU_TLB_FLUSH_VA_SECTION__MASK 0xffc00000
121#define SMMU_TLB_FLUSH_VA_SECTION__SHIFT 12 /* right shift */
122#define SMMU_TLB_FLUSH_VA_GROUP__MASK 0xffffc000
123#define SMMU_TLB_FLUSH_VA_GROUP__SHIFT 12 /* right shift */
124#define SMMU_TLB_FLUSH_VA(iova, which) \
125 ((((iova) & SMMU_TLB_FLUSH_VA_##which##__MASK) >> \
126 SMMU_TLB_FLUSH_VA_##which##__SHIFT) | \
127 SMMU_TLB_FLUSH_VA_MATCH_##which)
128#define SMMU_PTB_ASID_CUR(n) \
129 ((n) << SMMU_PTB_ASID_CURRENT_SHIFT)
130#define SMMU_TLB_FLUSH_ASID_MATCH_disable \
131 (SMMU_TLB_FLUSH_ASID_MATCH_DISABLE << \
132 SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
133#define SMMU_TLB_FLUSH_ASID_MATCH__ENABLE \
134 (SMMU_TLB_FLUSH_ASID_MATCH_ENABLE << \
135 SMMU_TLB_FLUSH_ASID_MATCH_SHIFT)
136
137#define SMMU_PAGE_SHIFT 12
138#define SMMU_PAGE_SIZE (1 << SMMU_PAGE_SHIFT)
139
140#define SMMU_PDIR_COUNT 1024
141#define SMMU_PDIR_SIZE (sizeof(unsigned long) * SMMU_PDIR_COUNT)
142#define SMMU_PTBL_COUNT 1024
143#define SMMU_PTBL_SIZE (sizeof(unsigned long) * SMMU_PTBL_COUNT)
144#define SMMU_PDIR_SHIFT 12
145#define SMMU_PDE_SHIFT 12
146#define SMMU_PTE_SHIFT 12
147#define SMMU_PFN_MASK 0x000fffff
148
149#define SMMU_ADDR_TO_PFN(addr) ((addr) >> 12)
150#define SMMU_ADDR_TO_PDN(addr) ((addr) >> 22)
151#define SMMU_PDN_TO_ADDR(addr) ((pdn) << 22)
152
153#define _READABLE (1 << SMMU_PTB_DATA_ASID_READABLE_SHIFT)
154#define _WRITABLE (1 << SMMU_PTB_DATA_ASID_WRITABLE_SHIFT)
155#define _NONSECURE (1 << SMMU_PTB_DATA_ASID_NONSECURE_SHIFT)
156#define _PDE_NEXT (1 << SMMU_PDE_NEXT_SHIFT)
157#define _MASK_ATTR (_READABLE | _WRITABLE | _NONSECURE)
158
159#define _PDIR_ATTR (_READABLE | _WRITABLE | _NONSECURE)
160
161#define _PDE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
162#define _PDE_ATTR_N (_PDE_ATTR | _PDE_NEXT)
163#define _PDE_VACANT(pdn) (((pdn) << 10) | _PDE_ATTR)
164
165#define _PTE_ATTR (_READABLE | _WRITABLE | _NONSECURE)
166#define _PTE_VACANT(addr) (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
167
168#define SMMU_MK_PDIR(page, attr) \
169 ((page_to_phys(page) >> SMMU_PDIR_SHIFT) | (attr))
170#define SMMU_MK_PDE(page, attr) \
171 (unsigned long)((page_to_phys(page) >> SMMU_PDE_SHIFT) | (attr))
172#define SMMU_EX_PTBL_PAGE(pde) \
173 pfn_to_page((unsigned long)(pde) & SMMU_PFN_MASK)
174#define SMMU_PFN_TO_PTE(pfn, attr) (unsigned long)((pfn) | (attr))
175
176#define SMMU_ASID_ENABLE(asid) ((asid) | (1 << 31))
177#define SMMU_ASID_DISABLE 0
178#define SMMU_ASID_ASID(n) ((n) & ~SMMU_ASID_ENABLE(0))
179
180#define smmu_client_enable_hwgrp(c, m) smmu_client_set_hwgrp(c, m, 1)
181#define smmu_client_disable_hwgrp(c) smmu_client_set_hwgrp(c, 0, 0)
182#define __smmu_client_enable_hwgrp(c, m) __smmu_client_set_hwgrp(c, m, 1)
183#define __smmu_client_disable_hwgrp(c) __smmu_client_set_hwgrp(c, 0, 0)
184
185#define HWGRP_INIT(client) [HWGRP_##client] = SMMU_##client##_ASID
186
187static const u32 smmu_hwgrp_asid_reg[] = {
188 HWGRP_INIT(AFI),
189 HWGRP_INIT(AVPC),
190 HWGRP_INIT(DC),
191 HWGRP_INIT(DCB),
192 HWGRP_INIT(EPP),
193 HWGRP_INIT(G2),
194 HWGRP_INIT(HC),
195 HWGRP_INIT(HDA),
196 HWGRP_INIT(ISP),
197 HWGRP_INIT(MPE),
198 HWGRP_INIT(NV),
199 HWGRP_INIT(NV2),
200 HWGRP_INIT(PPCS),
201 HWGRP_INIT(SATA),
202 HWGRP_INIT(VDE),
203 HWGRP_INIT(VI),
204};
205#define HWGRP_ASID_REG(x) (smmu_hwgrp_asid_reg[x])
206
207/*
208 * Per client for address space
209 */
210struct smmu_client {
211 struct device *dev;
212 struct list_head list;
213 struct smmu_as *as;
214 u32 hwgrp;
215};
216
217/*
218 * Per address space
219 */
220struct smmu_as {
221 struct smmu_device *smmu; /* back pointer to container */
222 unsigned int asid;
223 spinlock_t lock; /* for pagetable */
224 struct page *pdir_page;
225 unsigned long pdir_attr;
226 unsigned long pde_attr;
227 unsigned long pte_attr;
228 unsigned int *pte_count;
229
230 struct list_head client;
231 spinlock_t client_lock; /* for client list */
232};
233
234/*
235 * Per SMMU device - IOMMU device
236 */
237struct smmu_device {
238 void __iomem *regs, *regs_ahbarb;
239 unsigned long iovmm_base; /* remappable base address */
240 unsigned long page_count; /* total remappable size */
241 spinlock_t lock;
242 char *name;
243 struct device *dev;
244 int num_as;
245 struct smmu_as *as; /* Run-time allocated array */
246 struct page *avp_vector_page; /* dummy page shared by all AS's */
247
248 /*
249 * Register image savers for suspend/resume
250 */
251 unsigned long translation_enable_0;
252 unsigned long translation_enable_1;
253 unsigned long translation_enable_2;
254 unsigned long asid_security;
255};
256
257static struct smmu_device *smmu_handle; /* unique for a system */
258
259/*
260 * SMMU/AHB register accessors
261 */
262static inline u32 smmu_read(struct smmu_device *smmu, size_t offs)
263{
264 return readl(smmu->regs + offs);
265}
266static inline void smmu_write(struct smmu_device *smmu, u32 val, size_t offs)
267{
268 writel(val, smmu->regs + offs);
269}
270
271static inline u32 ahb_read(struct smmu_device *smmu, size_t offs)
272{
273 return readl(smmu->regs_ahbarb + offs);
274}
275static inline void ahb_write(struct smmu_device *smmu, u32 val, size_t offs)
276{
277 writel(val, smmu->regs_ahbarb + offs);
278}
279
280#define VA_PAGE_TO_PA(va, page) \
281 (page_to_phys(page) + ((unsigned long)(va) & ~PAGE_MASK))
282
283#define FLUSH_CPU_DCACHE(va, page, size) \
284 do { \
285 unsigned long _pa_ = VA_PAGE_TO_PA(va, page); \
286 __cpuc_flush_dcache_area((void *)(va), (size_t)(size)); \
287 outer_flush_range(_pa_, _pa_+(size_t)(size)); \
288 } while (0)
289
290/*
291 * Any interaction between any block on PPSB and a block on APB or AHB
292 * must have these read-back barriers to ensure the APB/AHB bus
293 * transaction is complete before initiating activity on the PPSB
294 * block.
295 */
296#define FLUSH_SMMU_REGS(smmu) smmu_read(smmu, SMMU_CONFIG)
297
298#define smmu_client_hwgrp(c) (u32)((c)->dev->platform_data)
299
300static int __smmu_client_set_hwgrp(struct smmu_client *c,
301 unsigned long map, int on)
302{
303 int i;
304 struct smmu_as *as = c->as;
305 u32 val, offs, mask = SMMU_ASID_ENABLE(as->asid);
306 struct smmu_device *smmu = as->smmu;
307
308 WARN_ON(!on && map);
309 if (on && !map)
310 return -EINVAL;
311 if (!on)
312 map = smmu_client_hwgrp(c);
313
314 for_each_set_bit(i, &map, HWGRP_COUNT) {
315 offs = HWGRP_ASID_REG(i);
316 val = smmu_read(smmu, offs);
317 if (on) {
318 if (WARN_ON(val & mask))
319 goto err_hw_busy;
320 val |= mask;
321 } else {
322 WARN_ON((val & mask) == mask);
323 val &= ~mask;
324 }
325 smmu_write(smmu, val, offs);
326 }
327 FLUSH_SMMU_REGS(smmu);
328 c->hwgrp = map;
329 return 0;
330
331err_hw_busy:
332 for_each_set_bit(i, &map, HWGRP_COUNT) {
333 offs = HWGRP_ASID_REG(i);
334 val = smmu_read(smmu, offs);
335 val &= ~mask;
336 smmu_write(smmu, val, offs);
337 }
338 return -EBUSY;
339}
340
341static int smmu_client_set_hwgrp(struct smmu_client *c, u32 map, int on)
342{
343 u32 val;
344 unsigned long flags;
345 struct smmu_as *as = c->as;
346 struct smmu_device *smmu = as->smmu;
347
348 spin_lock_irqsave(&smmu->lock, flags);
349 val = __smmu_client_set_hwgrp(c, map, on);
350 spin_unlock_irqrestore(&smmu->lock, flags);
351 return val;
352}
353
354/*
355 * Flush all TLB entries and all PTC entries
356 * Caller must lock smmu
357 */
358static void smmu_flush_regs(struct smmu_device *smmu, int enable)
359{
360 u32 val;
361
362 smmu_write(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
363 FLUSH_SMMU_REGS(smmu);
364 val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
365 SMMU_TLB_FLUSH_ASID_MATCH_disable;
366 smmu_write(smmu, val, SMMU_TLB_FLUSH);
367
368 if (enable)
369 smmu_write(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
370 FLUSH_SMMU_REGS(smmu);
371}
372
373static void smmu_setup_regs(struct smmu_device *smmu)
374{
375 int i;
376 u32 val;
377
378 for (i = 0; i < smmu->num_as; i++) {
379 struct smmu_as *as = &smmu->as[i];
380 struct smmu_client *c;
381
382 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
383 val = as->pdir_page ?
384 SMMU_MK_PDIR(as->pdir_page, as->pdir_attr) :
385 SMMU_PTB_DATA_RESET_VAL;
386 smmu_write(smmu, val, SMMU_PTB_DATA);
387
388 list_for_each_entry(c, &as->client, list)
389 __smmu_client_set_hwgrp(c, c->hwgrp, 1);
390 }
391
392 smmu_write(smmu, smmu->translation_enable_0, SMMU_TRANSLATION_ENABLE_0);
393 smmu_write(smmu, smmu->translation_enable_1, SMMU_TRANSLATION_ENABLE_1);
394 smmu_write(smmu, smmu->translation_enable_2, SMMU_TRANSLATION_ENABLE_2);
395 smmu_write(smmu, smmu->asid_security, SMMU_ASID_SECURITY);
396 smmu_write(smmu, SMMU_TLB_CONFIG_RESET_VAL, SMMU_TLB_CONFIG);
397 smmu_write(smmu, SMMU_PTC_CONFIG_RESET_VAL, SMMU_PTC_CONFIG);
398
399 smmu_flush_regs(smmu, 1);
400
401 val = ahb_read(smmu, AHB_XBAR_CTRL);
402 val |= AHB_XBAR_CTRL_SMMU_INIT_DONE_DONE <<
403 AHB_XBAR_CTRL_SMMU_INIT_DONE_SHIFT;
404 ahb_write(smmu, val, AHB_XBAR_CTRL);
405}
406
407static void flush_ptc_and_tlb(struct smmu_device *smmu,
408 struct smmu_as *as, dma_addr_t iova,
409 unsigned long *pte, struct page *page, int is_pde)
410{
411 u32 val;
412 unsigned long tlb_flush_va = is_pde
413 ? SMMU_TLB_FLUSH_VA(iova, SECTION)
414 : SMMU_TLB_FLUSH_VA(iova, GROUP);
415
416 val = SMMU_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pte, page);
417 smmu_write(smmu, val, SMMU_PTC_FLUSH);
418 FLUSH_SMMU_REGS(smmu);
419 val = tlb_flush_va |
420 SMMU_TLB_FLUSH_ASID_MATCH__ENABLE |
421 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT);
422 smmu_write(smmu, val, SMMU_TLB_FLUSH);
423 FLUSH_SMMU_REGS(smmu);
424}
425
426static void free_ptbl(struct smmu_as *as, dma_addr_t iova)
427{
428 unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
429 unsigned long *pdir = (unsigned long *)page_address(as->pdir_page);
430
431 if (pdir[pdn] != _PDE_VACANT(pdn)) {
432 dev_dbg(as->smmu->dev, "pdn: %lx\n", pdn);
433
434 ClearPageReserved(SMMU_EX_PTBL_PAGE(pdir[pdn]));
435 __free_page(SMMU_EX_PTBL_PAGE(pdir[pdn]));
436 pdir[pdn] = _PDE_VACANT(pdn);
437 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
438 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
439 as->pdir_page, 1);
440 }
441}
442
443static void free_pdir(struct smmu_as *as)
444{
445 unsigned addr;
446 int count;
447 struct device *dev = as->smmu->dev;
448
449 if (!as->pdir_page)
450 return;
451
452 addr = as->smmu->iovmm_base;
453 count = as->smmu->page_count;
454 while (count-- > 0) {
455 free_ptbl(as, addr);
456 addr += SMMU_PAGE_SIZE * SMMU_PTBL_COUNT;
457 }
458 ClearPageReserved(as->pdir_page);
459 __free_page(as->pdir_page);
460 as->pdir_page = NULL;
461 devm_kfree(dev, as->pte_count);
462 as->pte_count = NULL;
463}
464
465/*
466 * Maps PTBL for given iova and returns the PTE address
467 * Caller must unmap the mapped PTBL returned in *ptbl_page_p
468 */
469static unsigned long *locate_pte(struct smmu_as *as,
470 dma_addr_t iova, bool allocate,
471 struct page **ptbl_page_p,
472 unsigned int **count)
473{
474 unsigned long ptn = SMMU_ADDR_TO_PFN(iova);
475 unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
476 unsigned long *pdir = page_address(as->pdir_page);
477 unsigned long *ptbl;
478
479 if (pdir[pdn] != _PDE_VACANT(pdn)) {
480 /* Mapped entry table already exists */
481 *ptbl_page_p = SMMU_EX_PTBL_PAGE(pdir[pdn]);
482 ptbl = page_address(*ptbl_page_p);
483 } else if (!allocate) {
484 return NULL;
485 } else {
486 int pn;
487 unsigned long addr = SMMU_PDN_TO_ADDR(pdn);
488
489 /* Vacant - allocate a new page table */
490 dev_dbg(as->smmu->dev, "New PTBL pdn: %lx\n", pdn);
491
492 *ptbl_page_p = alloc_page(GFP_ATOMIC);
493 if (!*ptbl_page_p) {
494 dev_err(as->smmu->dev,
495 "failed to allocate smmu_device page table\n");
496 return NULL;
497 }
498 SetPageReserved(*ptbl_page_p);
499 ptbl = (unsigned long *)page_address(*ptbl_page_p);
500 for (pn = 0; pn < SMMU_PTBL_COUNT;
501 pn++, addr += SMMU_PAGE_SIZE) {
502 ptbl[pn] = _PTE_VACANT(addr);
503 }
504 FLUSH_CPU_DCACHE(ptbl, *ptbl_page_p, SMMU_PTBL_SIZE);
505 pdir[pdn] = SMMU_MK_PDE(*ptbl_page_p,
506 as->pde_attr | _PDE_NEXT);
507 FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
508 flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
509 as->pdir_page, 1);
510 }
511 *count = &as->pte_count[pdn];
512
513 return &ptbl[ptn % SMMU_PTBL_COUNT];
514}
515
516#ifdef CONFIG_SMMU_SIG_DEBUG
517static void put_signature(struct smmu_as *as,
518 dma_addr_t iova, unsigned long pfn)
519{
520 struct page *page;
521 unsigned long *vaddr;
522
523 page = pfn_to_page(pfn);
524 vaddr = page_address(page);
525 if (!vaddr)
526 return;
527
528 vaddr[0] = iova;
529 vaddr[1] = pfn << PAGE_SHIFT;
530 FLUSH_CPU_DCACHE(vaddr, page, sizeof(vaddr[0]) * 2);
531}
532#else
533static inline void put_signature(struct smmu_as *as,
534 unsigned long addr, unsigned long pfn)
535{
536}
537#endif
538
539/*
540 * Caller must lock/unlock as
541 */
542static int alloc_pdir(struct smmu_as *as)
543{
544 unsigned long *pdir;
545 int pdn;
546 u32 val;
547 struct smmu_device *smmu = as->smmu;
548
549 if (as->pdir_page)
550 return 0;
551
552 as->pte_count = devm_kzalloc(smmu->dev,
553 sizeof(as->pte_count[0]) * SMMU_PDIR_COUNT, GFP_KERNEL);
554 if (!as->pte_count) {
555 dev_err(smmu->dev,
556 "failed to allocate smmu_device PTE cunters\n");
557 return -ENOMEM;
558 }
559 as->pdir_page = alloc_page(GFP_KERNEL | __GFP_DMA);
560 if (!as->pdir_page) {
561 dev_err(smmu->dev,
562 "failed to allocate smmu_device page directory\n");
563 devm_kfree(smmu->dev, as->pte_count);
564 as->pte_count = NULL;
565 return -ENOMEM;
566 }
567 SetPageReserved(as->pdir_page);
568 pdir = page_address(as->pdir_page);
569
570 for (pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
571 pdir[pdn] = _PDE_VACANT(pdn);
572 FLUSH_CPU_DCACHE(pdir, as->pdir_page, SMMU_PDIR_SIZE);
573 val = SMMU_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pdir, as->pdir_page);
574 smmu_write(smmu, val, SMMU_PTC_FLUSH);
575 FLUSH_SMMU_REGS(as->smmu);
576 val = SMMU_TLB_FLUSH_VA_MATCH_ALL |
577 SMMU_TLB_FLUSH_ASID_MATCH__ENABLE |
578 (as->asid << SMMU_TLB_FLUSH_ASID_SHIFT);
579 smmu_write(smmu, val, SMMU_TLB_FLUSH);
580 FLUSH_SMMU_REGS(as->smmu);
581
582 return 0;
583}
584
585static void __smmu_iommu_unmap(struct smmu_as *as, dma_addr_t iova)
586{
587 unsigned long *pte;
588 struct page *page;
589 unsigned int *count;
590
591 pte = locate_pte(as, iova, false, &page, &count);
592 if (WARN_ON(!pte))
593 return;
594
595 if (WARN_ON(*pte == _PTE_VACANT(iova)))
596 return;
597
598 *pte = _PTE_VACANT(iova);
599 FLUSH_CPU_DCACHE(pte, page, sizeof(*pte));
600 flush_ptc_and_tlb(as->smmu, as, iova, pte, page, 0);
601 if (!--(*count)) {
602 free_ptbl(as, iova);
603 smmu_flush_regs(as->smmu, 0);
604 }
605}
606
607static void __smmu_iommu_map_pfn(struct smmu_as *as, dma_addr_t iova,
608 unsigned long pfn)
609{
610 struct smmu_device *smmu = as->smmu;
611 unsigned long *pte;
612 unsigned int *count;
613 struct page *page;
614
615 pte = locate_pte(as, iova, true, &page, &count);
616 if (WARN_ON(!pte))
617 return;
618
619 if (*pte == _PTE_VACANT(iova))
620 (*count)++;
621 *pte = SMMU_PFN_TO_PTE(pfn, as->pte_attr);
622 if (unlikely((*pte == _PTE_VACANT(iova))))
623 (*count)--;
624 FLUSH_CPU_DCACHE(pte, page, sizeof(*pte));
625 flush_ptc_and_tlb(smmu, as, iova, pte, page, 0);
626 put_signature(as, iova, pfn);
627}
628
629static int smmu_iommu_map(struct iommu_domain *domain, unsigned long iova,
630 phys_addr_t pa, size_t bytes, int prot)
631{
632 struct smmu_as *as = domain->priv;
633 unsigned long pfn = __phys_to_pfn(pa);
634 unsigned long flags;
635
636 dev_dbg(as->smmu->dev, "[%d] %08lx:%08x\n", as->asid, iova, pa);
637
638 if (!pfn_valid(pfn))
639 return -ENOMEM;
640
641 spin_lock_irqsave(&as->lock, flags);
642 __smmu_iommu_map_pfn(as, iova, pfn);
643 spin_unlock_irqrestore(&as->lock, flags);
644 return 0;
645}
646
647static size_t smmu_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
648 size_t bytes)
649{
650 struct smmu_as *as = domain->priv;
651 unsigned long flags;
652
653 dev_dbg(as->smmu->dev, "[%d] %08lx\n", as->asid, iova);
654
655 spin_lock_irqsave(&as->lock, flags);
656 __smmu_iommu_unmap(as, iova);
657 spin_unlock_irqrestore(&as->lock, flags);
658 return SMMU_PAGE_SIZE;
659}
660
661static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain,
662 unsigned long iova)
663{
664 struct smmu_as *as = domain->priv;
665 unsigned long *pte;
666 unsigned int *count;
667 struct page *page;
668 unsigned long pfn;
669 unsigned long flags;
670
671 spin_lock_irqsave(&as->lock, flags);
672
673 pte = locate_pte(as, iova, true, &page, &count);
674 pfn = *pte & SMMU_PFN_MASK;
675 WARN_ON(!pfn_valid(pfn));
676 dev_dbg(as->smmu->dev,
677 "iova:%08lx pfn:%08lx asid:%d\n", iova, pfn, as->asid);
678
679 spin_unlock_irqrestore(&as->lock, flags);
680 return PFN_PHYS(pfn);
681}
682
683static int smmu_iommu_domain_has_cap(struct iommu_domain *domain,
684 unsigned long cap)
685{
686 return 0;
687}
688
689static int smmu_iommu_attach_dev(struct iommu_domain *domain,
690 struct device *dev)
691{
692 struct smmu_as *as = domain->priv;
693 struct smmu_device *smmu = as->smmu;
694 struct smmu_client *client, *c;
695 u32 map;
696 int err;
697
698 client = devm_kzalloc(smmu->dev, sizeof(*c), GFP_KERNEL);
699 if (!client)
700 return -ENOMEM;
701 client->dev = dev;
702 client->as = as;
703 map = (unsigned long)dev->platform_data;
704 if (!map)
705 return -EINVAL;
706
707 err = smmu_client_enable_hwgrp(client, map);
708 if (err)
709 goto err_hwgrp;
710
711 spin_lock(&as->client_lock);
712 list_for_each_entry(c, &as->client, list) {
713 if (c->dev == dev) {
714 dev_err(smmu->dev,
715 "%s is already attached\n", dev_name(c->dev));
716 err = -EINVAL;
717 goto err_client;
718 }
719 }
720 list_add(&client->list, &as->client);
721 spin_unlock(&as->client_lock);
722
723 /*
724 * Reserve "page zero" for AVP vectors using a common dummy
725 * page.
726 */
727 if (map & HWG_AVPC) {
728 struct page *page;
729
730 page = as->smmu->avp_vector_page;
731 __smmu_iommu_map_pfn(as, 0, page_to_pfn(page));
732
733 pr_info("Reserve \"page zero\" for AVP vectors using a common dummy\n");
734 }
735
736 dev_dbg(smmu->dev, "%s is attached\n", dev_name(c->dev));
737 return 0;
738
739err_client:
740 smmu_client_disable_hwgrp(client);
741 spin_unlock(&as->client_lock);
742err_hwgrp:
743 devm_kfree(smmu->dev, client);
744 return err;
745}
746
747static void smmu_iommu_detach_dev(struct iommu_domain *domain,
748 struct device *dev)
749{
750 struct smmu_as *as = domain->priv;
751 struct smmu_device *smmu = as->smmu;
752 struct smmu_client *c;
753
754 spin_lock(&as->client_lock);
755
756 list_for_each_entry(c, &as->client, list) {
757 if (c->dev == dev) {
758 smmu_client_disable_hwgrp(c);
759 list_del(&c->list);
760 devm_kfree(smmu->dev, c);
761 c->as = NULL;
762 dev_dbg(smmu->dev,
763 "%s is detached\n", dev_name(c->dev));
764 goto out;
765 }
766 }
767 dev_err(smmu->dev, "Couldn't find %s\n", dev_name(c->dev));
768out:
769 spin_unlock(&as->client_lock);
770}
771
772static int smmu_iommu_domain_init(struct iommu_domain *domain)
773{
774 int i;
775 unsigned long flags;
776 struct smmu_as *as;
777 struct smmu_device *smmu = smmu_handle;
778
779 /* Look for a free AS with lock held */
780 for (i = 0; i < smmu->num_as; i++) {
781 struct smmu_as *tmp = &smmu->as[i];
782
783 spin_lock_irqsave(&tmp->lock, flags);
784 if (!tmp->pdir_page) {
785 as = tmp;
786 goto found;
787 }
788 spin_unlock_irqrestore(&tmp->lock, flags);
789 }
790 dev_err(smmu->dev, "no free AS\n");
791 return -ENODEV;
792
793found:
794 if (alloc_pdir(as) < 0)
795 goto err_alloc_pdir;
796
797 spin_lock(&smmu->lock);
798
799 /* Update PDIR register */
800 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
801 smmu_write(smmu,
802 SMMU_MK_PDIR(as->pdir_page, as->pdir_attr), SMMU_PTB_DATA);
803 FLUSH_SMMU_REGS(smmu);
804
805 spin_unlock(&smmu->lock);
806
807 spin_unlock_irqrestore(&as->lock, flags);
808 domain->priv = as;
809
810 dev_dbg(smmu->dev, "smmu_as@%p\n", as);
811 return 0;
812
813err_alloc_pdir:
814 spin_unlock_irqrestore(&as->lock, flags);
815 return -ENODEV;
816}
817
818static void smmu_iommu_domain_destroy(struct iommu_domain *domain)
819{
820 struct smmu_as *as = domain->priv;
821 struct smmu_device *smmu = as->smmu;
822 unsigned long flags;
823
824 spin_lock_irqsave(&as->lock, flags);
825
826 if (as->pdir_page) {
827 spin_lock(&smmu->lock);
828 smmu_write(smmu, SMMU_PTB_ASID_CUR(as->asid), SMMU_PTB_ASID);
829 smmu_write(smmu, SMMU_PTB_DATA_RESET_VAL, SMMU_PTB_DATA);
830 FLUSH_SMMU_REGS(smmu);
831 spin_unlock(&smmu->lock);
832
833 free_pdir(as);
834 }
835
836 if (!list_empty(&as->client)) {
837 struct smmu_client *c;
838
839 list_for_each_entry(c, &as->client, list)
840 smmu_iommu_detach_dev(domain, c->dev);
841 }
842
843 spin_unlock_irqrestore(&as->lock, flags);
844
845 domain->priv = NULL;
846 dev_dbg(smmu->dev, "smmu_as@%p\n", as);
847}
848
849static struct iommu_ops smmu_iommu_ops = {
850 .domain_init = smmu_iommu_domain_init,
851 .domain_destroy = smmu_iommu_domain_destroy,
852 .attach_dev = smmu_iommu_attach_dev,
853 .detach_dev = smmu_iommu_detach_dev,
854 .map = smmu_iommu_map,
855 .unmap = smmu_iommu_unmap,
856 .iova_to_phys = smmu_iommu_iova_to_phys,
857 .domain_has_cap = smmu_iommu_domain_has_cap,
858 .pgsize_bitmap = SMMU_IOMMU_PGSIZES,
859};
860
861static int tegra_smmu_suspend(struct device *dev)
862{
863 struct smmu_device *smmu = dev_get_drvdata(dev);
864
865 smmu->translation_enable_0 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_0);
866 smmu->translation_enable_1 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_1);
867 smmu->translation_enable_2 = smmu_read(smmu, SMMU_TRANSLATION_ENABLE_2);
868 smmu->asid_security = smmu_read(smmu, SMMU_ASID_SECURITY);
869 return 0;
870}
871
872static int tegra_smmu_resume(struct device *dev)
873{
874 struct smmu_device *smmu = dev_get_drvdata(dev);
875 unsigned long flags;
876
877 spin_lock_irqsave(&smmu->lock, flags);
878 smmu_setup_regs(smmu);
879 spin_unlock_irqrestore(&smmu->lock, flags);
880 return 0;
881}
882
883static int tegra_smmu_probe(struct platform_device *pdev)
884{
885 struct smmu_device *smmu;
886 struct resource *regs, *regs2, *window;
887 struct device *dev = &pdev->dev;
888 int i, err = 0;
889
890 if (smmu_handle)
891 return -EIO;
892
893 BUILD_BUG_ON(PAGE_SHIFT != SMMU_PAGE_SHIFT);
894
895 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
896 regs2 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
897 window = platform_get_resource(pdev, IORESOURCE_MEM, 2);
898 if (!regs || !regs2 || !window) {
899 dev_err(dev, "No SMMU resources\n");
900 return -ENODEV;
901 }
902
903 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
904 if (!smmu) {
905 dev_err(dev, "failed to allocate smmu_device\n");
906 return -ENOMEM;
907 }
908
909 smmu->dev = dev;
910 smmu->num_as = SMMU_NUM_ASIDS;
911 smmu->iovmm_base = (unsigned long)window->start;
912 smmu->page_count = resource_size(window) >> SMMU_PAGE_SHIFT;
913 smmu->regs = devm_ioremap(dev, regs->start, resource_size(regs));
914 smmu->regs_ahbarb = devm_ioremap(dev, regs2->start,
915 resource_size(regs2));
916 if (!smmu->regs || !smmu->regs_ahbarb) {
917 dev_err(dev, "failed to remap SMMU registers\n");
918 err = -ENXIO;
919 goto fail;
920 }
921
922 smmu->translation_enable_0 = ~0;
923 smmu->translation_enable_1 = ~0;
924 smmu->translation_enable_2 = ~0;
925 smmu->asid_security = 0;
926
927 smmu->as = devm_kzalloc(dev,
928 sizeof(smmu->as[0]) * smmu->num_as, GFP_KERNEL);
929 if (!smmu->as) {
930 dev_err(dev, "failed to allocate smmu_as\n");
931 err = -ENOMEM;
932 goto fail;
933 }
934
935 for (i = 0; i < smmu->num_as; i++) {
936 struct smmu_as *as = &smmu->as[i];
937
938 as->smmu = smmu;
939 as->asid = i;
940 as->pdir_attr = _PDIR_ATTR;
941 as->pde_attr = _PDE_ATTR;
942 as->pte_attr = _PTE_ATTR;
943
944 spin_lock_init(&as->lock);
945 INIT_LIST_HEAD(&as->client);
946 }
947 spin_lock_init(&smmu->lock);
948 smmu_setup_regs(smmu);
949 platform_set_drvdata(pdev, smmu);
950
951 smmu->avp_vector_page = alloc_page(GFP_KERNEL);
952 if (!smmu->avp_vector_page)
953 goto fail;
954
955 smmu_handle = smmu;
956 return 0;
957
958fail:
959 if (smmu->avp_vector_page)
960 __free_page(smmu->avp_vector_page);
961 if (smmu->regs)
962 devm_iounmap(dev, smmu->regs);
963 if (smmu->regs_ahbarb)
964 devm_iounmap(dev, smmu->regs_ahbarb);
965 if (smmu && smmu->as) {
966 for (i = 0; i < smmu->num_as; i++) {
967 if (smmu->as[i].pdir_page) {
968 ClearPageReserved(smmu->as[i].pdir_page);
969 __free_page(smmu->as[i].pdir_page);
970 }
971 }
972 devm_kfree(dev, smmu->as);
973 }
974 devm_kfree(dev, smmu);
975 return err;
976}
977
978static int tegra_smmu_remove(struct platform_device *pdev)
979{
980 struct smmu_device *smmu = platform_get_drvdata(pdev);
981 struct device *dev = smmu->dev;
982
983 smmu_write(smmu, SMMU_CONFIG_DISABLE, SMMU_CONFIG);
984 platform_set_drvdata(pdev, NULL);
985 if (smmu->as) {
986 int i;
987
988 for (i = 0; i < smmu->num_as; i++)
989 free_pdir(&smmu->as[i]);
990 devm_kfree(dev, smmu->as);
991 }
992 if (smmu->avp_vector_page)
993 __free_page(smmu->avp_vector_page);
994 if (smmu->regs)
995 devm_iounmap(dev, smmu->regs);
996 if (smmu->regs_ahbarb)
997 devm_iounmap(dev, smmu->regs_ahbarb);
998 devm_kfree(dev, smmu);
999 smmu_handle = NULL;
1000 return 0;
1001}
1002
1003const struct dev_pm_ops tegra_smmu_pm_ops = {
1004 .suspend = tegra_smmu_suspend,
1005 .resume = tegra_smmu_resume,
1006};
1007
1008static struct platform_driver tegra_smmu_driver = {
1009 .probe = tegra_smmu_probe,
1010 .remove = tegra_smmu_remove,
1011 .driver = {
1012 .owner = THIS_MODULE,
1013 .name = "tegra-smmu",
1014 .pm = &tegra_smmu_pm_ops,
1015 },
1016};
1017
1018static int __devinit tegra_smmu_init(void)
1019{
1020 bus_set_iommu(&platform_bus_type, &smmu_iommu_ops);
1021 return platform_driver_register(&tegra_smmu_driver);
1022}
1023
1024static void __exit tegra_smmu_exit(void)
1025{
1026 platform_driver_unregister(&tegra_smmu_driver);
1027}
1028
1029subsys_initcall(tegra_smmu_init);
1030module_exit(tegra_smmu_exit);
1031
1032MODULE_DESCRIPTION("IOMMU API for SMMU in Tegra30");
1033MODULE_AUTHOR("Hiroshi DOYU <hdoyu@nvidia.com>");
1034MODULE_LICENSE("GPL v2");
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-18 14:14:36 -0400