aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/plat-omap/Kconfig14
-rw-r--r--arch/arm/plat-omap/Makefile2
-rw-r--r--arch/arm/plat-omap/include/plat/iopgtable.h (renamed from arch/arm/plat-omap/iopgtable.h)0
-rw-r--r--arch/arm/plat-omap/iommu-debug.c418
-rw-r--r--arch/arm/plat-omap/iommu.c1326
-rw-r--r--arch/arm/plat-omap/iovmm.c923
6 files changed, 0 insertions, 2683 deletions
diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig
index e1e954d7486d..fa62037f1df6 100644
--- a/arch/arm/plat-omap/Kconfig
+++ b/arch/arm/plat-omap/Kconfig
@@ -132,20 +132,6 @@ config OMAP_MBOX_KFIFO_SIZE
132 This can also be changed at runtime (via the mbox_kfifo_size 132 This can also be changed at runtime (via the mbox_kfifo_size
133 module parameter). 133 module parameter).
134 134
135#can't be tristate; iommu api doesn't support un-registration
136config OMAP_IOMMU
137 bool
138 select IOMMU_API
139
140config OMAP_IOMMU_DEBUG
141 tristate "Export OMAP IOMMU internals in DebugFS"
142 depends on OMAP_IOMMU && DEBUG_FS
143 help
144 Select this to see extensive information about
145 the internal state of OMAP IOMMU in debugfs.
146
147 Say N unless you know you need this.
148
149config OMAP_IOMMU_IVA2 135config OMAP_IOMMU_IVA2
150 bool 136 bool
151 137
diff --git a/arch/arm/plat-omap/Makefile b/arch/arm/plat-omap/Makefile
index f0233e6abcdf..985262242f25 100644
--- a/arch/arm/plat-omap/Makefile
+++ b/arch/arm/plat-omap/Makefile
@@ -18,8 +18,6 @@ obj-$(CONFIG_ARCH_OMAP3) += omap_device.o
18obj-$(CONFIG_ARCH_OMAP4) += omap_device.o 18obj-$(CONFIG_ARCH_OMAP4) += omap_device.o
19 19
20obj-$(CONFIG_OMAP_MCBSP) += mcbsp.o 20obj-$(CONFIG_OMAP_MCBSP) += mcbsp.o
21obj-$(CONFIG_OMAP_IOMMU) += iommu.o iovmm.o
22obj-$(CONFIG_OMAP_IOMMU_DEBUG) += iommu-debug.o
23 21
24obj-$(CONFIG_CPU_FREQ) += cpu-omap.o 22obj-$(CONFIG_CPU_FREQ) += cpu-omap.o
25obj-$(CONFIG_OMAP_DM_TIMER) += dmtimer.o 23obj-$(CONFIG_OMAP_DM_TIMER) += dmtimer.o
diff --git a/arch/arm/plat-omap/iopgtable.h b/arch/arm/plat-omap/include/plat/iopgtable.h
index 33c7aa986f53..33c7aa986f53 100644
--- a/arch/arm/plat-omap/iopgtable.h
+++ b/arch/arm/plat-omap/include/plat/iopgtable.h
diff --git a/arch/arm/plat-omap/iommu-debug.c b/arch/arm/plat-omap/iommu-debug.c
deleted file mode 100644
index f07cf2f08e09..000000000000
--- a/arch/arm/plat-omap/iommu-debug.c
+++ /dev/null
@@ -1,418 +0,0 @@
1/*
2 * omap iommu: debugfs interface
3 *
4 * Copyright (C) 2008-2009 Nokia Corporation
5 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/err.h>
14#include <linux/clk.h>
15#include <linux/io.h>
16#include <linux/slab.h>
17#include <linux/uaccess.h>
18#include <linux/platform_device.h>
19#include <linux/debugfs.h>
20
21#include <plat/iommu.h>
22#include <plat/iovmm.h>
23
24#include "iopgtable.h"
25
26#define MAXCOLUMN 100 /* for short messages */
27
28static DEFINE_MUTEX(iommu_debug_lock);
29
30static struct dentry *iommu_debug_root;
31
32static ssize_t debug_read_ver(struct file *file, char __user *userbuf,
33 size_t count, loff_t *ppos)
34{
35 u32 ver = iommu_arch_version();
36 char buf[MAXCOLUMN], *p = buf;
37
38 p += sprintf(p, "H/W version: %d.%d\n", (ver >> 4) & 0xf , ver & 0xf);
39
40 return simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
41}
42
43static ssize_t debug_read_regs(struct file *file, char __user *userbuf,
44 size_t count, loff_t *ppos)
45{
46 struct iommu *obj = file->private_data;
47 char *p, *buf;
48 ssize_t bytes;
49
50 buf = kmalloc(count, GFP_KERNEL);
51 if (!buf)
52 return -ENOMEM;
53 p = buf;
54
55 mutex_lock(&iommu_debug_lock);
56
57 bytes = iommu_dump_ctx(obj, p, count);
58 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, bytes);
59
60 mutex_unlock(&iommu_debug_lock);
61 kfree(buf);
62
63 return bytes;
64}
65
66static ssize_t debug_read_tlb(struct file *file, char __user *userbuf,
67 size_t count, loff_t *ppos)
68{
69 struct iommu *obj = file->private_data;
70 char *p, *buf;
71 ssize_t bytes, rest;
72
73 buf = kmalloc(count, GFP_KERNEL);
74 if (!buf)
75 return -ENOMEM;
76 p = buf;
77
78 mutex_lock(&iommu_debug_lock);
79
80 p += sprintf(p, "%8s %8s\n", "cam:", "ram:");
81 p += sprintf(p, "-----------------------------------------\n");
82 rest = count - (p - buf);
83 p += dump_tlb_entries(obj, p, rest);
84
85 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
86
87 mutex_unlock(&iommu_debug_lock);
88 kfree(buf);
89
90 return bytes;
91}
92
93static ssize_t debug_write_pagetable(struct file *file,
94 const char __user *userbuf, size_t count, loff_t *ppos)
95{
96 struct iotlb_entry e;
97 struct cr_regs cr;
98 int err;
99 struct iommu *obj = file->private_data;
100 char buf[MAXCOLUMN], *p = buf;
101
102 count = min(count, sizeof(buf));
103
104 mutex_lock(&iommu_debug_lock);
105 if (copy_from_user(p, userbuf, count)) {
106 mutex_unlock(&iommu_debug_lock);
107 return -EFAULT;
108 }
109
110 sscanf(p, "%x %x", &cr.cam, &cr.ram);
111 if (!cr.cam || !cr.ram) {
112 mutex_unlock(&iommu_debug_lock);
113 return -EINVAL;
114 }
115
116 iotlb_cr_to_e(&cr, &e);
117 err = iopgtable_store_entry(obj, &e);
118 if (err)
119 dev_err(obj->dev, "%s: fail to store cr\n", __func__);
120
121 mutex_unlock(&iommu_debug_lock);
122 return count;
123}
124
125#define dump_ioptable_entry_one(lv, da, val) \
126 ({ \
127 int __err = 0; \
128 ssize_t bytes; \
129 const int maxcol = 22; \
130 const char *str = "%d: %08x %08x\n"; \
131 bytes = snprintf(p, maxcol, str, lv, da, val); \
132 p += bytes; \
133 len -= bytes; \
134 if (len < maxcol) \
135 __err = -ENOMEM; \
136 __err; \
137 })
138
139static ssize_t dump_ioptable(struct iommu *obj, char *buf, ssize_t len)
140{
141 int i;
142 u32 *iopgd;
143 char *p = buf;
144
145 spin_lock(&obj->page_table_lock);
146
147 iopgd = iopgd_offset(obj, 0);
148 for (i = 0; i < PTRS_PER_IOPGD; i++, iopgd++) {
149 int j, err;
150 u32 *iopte;
151 u32 da;
152
153 if (!*iopgd)
154 continue;
155
156 if (!(*iopgd & IOPGD_TABLE)) {
157 da = i << IOPGD_SHIFT;
158
159 err = dump_ioptable_entry_one(1, da, *iopgd);
160 if (err)
161 goto out;
162 continue;
163 }
164
165 iopte = iopte_offset(iopgd, 0);
166
167 for (j = 0; j < PTRS_PER_IOPTE; j++, iopte++) {
168 if (!*iopte)
169 continue;
170
171 da = (i << IOPGD_SHIFT) + (j << IOPTE_SHIFT);
172 err = dump_ioptable_entry_one(2, da, *iopgd);
173 if (err)
174 goto out;
175 }
176 }
177out:
178 spin_unlock(&obj->page_table_lock);
179
180 return p - buf;
181}
182
183static ssize_t debug_read_pagetable(struct file *file, char __user *userbuf,
184 size_t count, loff_t *ppos)
185{
186 struct iommu *obj = file->private_data;
187 char *p, *buf;
188 size_t bytes;
189
190 buf = (char *)__get_free_page(GFP_KERNEL);
191 if (!buf)
192 return -ENOMEM;
193 p = buf;
194
195 p += sprintf(p, "L: %8s %8s\n", "da:", "pa:");
196 p += sprintf(p, "-----------------------------------------\n");
197
198 mutex_lock(&iommu_debug_lock);
199
200 bytes = PAGE_SIZE - (p - buf);
201 p += dump_ioptable(obj, p, bytes);
202
203 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
204
205 mutex_unlock(&iommu_debug_lock);
206 free_page((unsigned long)buf);
207
208 return bytes;
209}
210
211static ssize_t debug_read_mmap(struct file *file, char __user *userbuf,
212 size_t count, loff_t *ppos)
213{
214 struct iommu *obj = file->private_data;
215 char *p, *buf;
216 struct iovm_struct *tmp;
217 int uninitialized_var(i);
218 ssize_t bytes;
219
220 buf = (char *)__get_free_page(GFP_KERNEL);
221 if (!buf)
222 return -ENOMEM;
223 p = buf;
224
225 p += sprintf(p, "%-3s %-8s %-8s %6s %8s\n",
226 "No", "start", "end", "size", "flags");
227 p += sprintf(p, "-------------------------------------------------\n");
228
229 mutex_lock(&iommu_debug_lock);
230
231 list_for_each_entry(tmp, &obj->mmap, list) {
232 size_t len;
233 const char *str = "%3d %08x-%08x %6x %8x\n";
234 const int maxcol = 39;
235
236 len = tmp->da_end - tmp->da_start;
237 p += snprintf(p, maxcol, str,
238 i, tmp->da_start, tmp->da_end, len, tmp->flags);
239
240 if (PAGE_SIZE - (p - buf) < maxcol)
241 break;
242 i++;
243 }
244
245 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
246
247 mutex_unlock(&iommu_debug_lock);
248 free_page((unsigned long)buf);
249
250 return bytes;
251}
252
253static ssize_t debug_read_mem(struct file *file, char __user *userbuf,
254 size_t count, loff_t *ppos)
255{
256 struct iommu *obj = file->private_data;
257 char *p, *buf;
258 struct iovm_struct *area;
259 ssize_t bytes;
260
261 count = min_t(ssize_t, count, PAGE_SIZE);
262
263 buf = (char *)__get_free_page(GFP_KERNEL);
264 if (!buf)
265 return -ENOMEM;
266 p = buf;
267
268 mutex_lock(&iommu_debug_lock);
269
270 area = find_iovm_area(obj, (u32)ppos);
271 if (IS_ERR(area)) {
272 bytes = -EINVAL;
273 goto err_out;
274 }
275 memcpy(p, area->va, count);
276 p += count;
277
278 bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf);
279err_out:
280 mutex_unlock(&iommu_debug_lock);
281 free_page((unsigned long)buf);
282
283 return bytes;
284}
285
286static ssize_t debug_write_mem(struct file *file, const char __user *userbuf,
287 size_t count, loff_t *ppos)
288{
289 struct iommu *obj = file->private_data;
290 struct iovm_struct *area;
291 char *p, *buf;
292
293 count = min_t(size_t, count, PAGE_SIZE);
294
295 buf = (char *)__get_free_page(GFP_KERNEL);
296 if (!buf)
297 return -ENOMEM;
298 p = buf;
299
300 mutex_lock(&iommu_debug_lock);
301
302 if (copy_from_user(p, userbuf, count)) {
303 count = -EFAULT;
304 goto err_out;
305 }
306
307 area = find_iovm_area(obj, (u32)ppos);
308 if (IS_ERR(area)) {
309 count = -EINVAL;
310 goto err_out;
311 }
312 memcpy(area->va, p, count);
313err_out:
314 mutex_unlock(&iommu_debug_lock);
315 free_page((unsigned long)buf);
316
317 return count;
318}
319
320static int debug_open_generic(struct inode *inode, struct file *file)
321{
322 file->private_data = inode->i_private;
323 return 0;
324}
325
326#define DEBUG_FOPS(name) \
327 static const struct file_operations debug_##name##_fops = { \
328 .open = debug_open_generic, \
329 .read = debug_read_##name, \
330 .write = debug_write_##name, \
331 .llseek = generic_file_llseek, \
332 };
333
334#define DEBUG_FOPS_RO(name) \
335 static const struct file_operations debug_##name##_fops = { \
336 .open = debug_open_generic, \
337 .read = debug_read_##name, \
338 .llseek = generic_file_llseek, \
339 };
340
341DEBUG_FOPS_RO(ver);
342DEBUG_FOPS_RO(regs);
343DEBUG_FOPS_RO(tlb);
344DEBUG_FOPS(pagetable);
345DEBUG_FOPS_RO(mmap);
346DEBUG_FOPS(mem);
347
348#define __DEBUG_ADD_FILE(attr, mode) \
349 { \
350 struct dentry *dent; \
351 dent = debugfs_create_file(#attr, mode, parent, \
352 obj, &debug_##attr##_fops); \
353 if (!dent) \
354 return -ENOMEM; \
355 }
356
357#define DEBUG_ADD_FILE(name) __DEBUG_ADD_FILE(name, 600)
358#define DEBUG_ADD_FILE_RO(name) __DEBUG_ADD_FILE(name, 400)
359
360static int iommu_debug_register(struct device *dev, void *data)
361{
362 struct platform_device *pdev = to_platform_device(dev);
363 struct iommu *obj = platform_get_drvdata(pdev);
364 struct dentry *d, *parent;
365
366 if (!obj || !obj->dev)
367 return -EINVAL;
368
369 d = debugfs_create_dir(obj->name, iommu_debug_root);
370 if (!d)
371 return -ENOMEM;
372 parent = d;
373
374 d = debugfs_create_u8("nr_tlb_entries", 400, parent,
375 (u8 *)&obj->nr_tlb_entries);
376 if (!d)
377 return -ENOMEM;
378
379 DEBUG_ADD_FILE_RO(ver);
380 DEBUG_ADD_FILE_RO(regs);
381 DEBUG_ADD_FILE_RO(tlb);
382 DEBUG_ADD_FILE(pagetable);
383 DEBUG_ADD_FILE_RO(mmap);
384 DEBUG_ADD_FILE(mem);
385
386 return 0;
387}
388
389static int __init iommu_debug_init(void)
390{
391 struct dentry *d;
392 int err;
393
394 d = debugfs_create_dir("iommu", NULL);
395 if (!d)
396 return -ENOMEM;
397 iommu_debug_root = d;
398
399 err = foreach_iommu_device(d, iommu_debug_register);
400 if (err)
401 goto err_out;
402 return 0;
403
404err_out:
405 debugfs_remove_recursive(iommu_debug_root);
406 return err;
407}
408module_init(iommu_debug_init)
409
410static void __exit iommu_debugfs_exit(void)
411{
412 debugfs_remove_recursive(iommu_debug_root);
413}
414module_exit(iommu_debugfs_exit)
415
416MODULE_DESCRIPTION("omap iommu: debugfs interface");
417MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
418MODULE_LICENSE("GPL v2");
diff --git a/arch/arm/plat-omap/iommu.c b/arch/arm/plat-omap/iommu.c
deleted file mode 100644
index 51aa008d8223..000000000000
--- a/arch/arm/plat-omap/iommu.c
+++ /dev/null
@@ -1,1326 +0,0 @@
1/*
2 * omap iommu: tlb and pagetable primitives
3 *
4 * Copyright (C) 2008-2010 Nokia Corporation
5 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/err.h>
15#include <linux/module.h>
16#include <linux/slab.h>
17#include <linux/interrupt.h>
18#include <linux/ioport.h>
19#include <linux/clk.h>
20#include <linux/platform_device.h>
21#include <linux/iommu.h>
22#include <linux/mutex.h>
23#include <linux/spinlock.h>
24
25#include <asm/cacheflush.h>
26
27#include <plat/iommu.h>
28
29#include "iopgtable.h"
30
31#define for_each_iotlb_cr(obj, n, __i, cr) \
32 for (__i = 0; \
33 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
34 __i++)
35
36/**
37 * struct omap_iommu_domain - omap iommu domain
38 * @pgtable: the page table
39 * @iommu_dev: an omap iommu device attached to this domain. only a single
40 * iommu device can be attached for now.
41 * @lock: domain lock, should be taken when attaching/detaching
42 */
43struct omap_iommu_domain {
44 u32 *pgtable;
45 struct iommu *iommu_dev;
46 spinlock_t lock;
47};
48
49/* accommodate the difference between omap1 and omap2/3 */
50static const struct iommu_functions *arch_iommu;
51
52static struct platform_driver omap_iommu_driver;
53static struct kmem_cache *iopte_cachep;
54
55/**
56 * install_iommu_arch - Install archtecure specific iommu functions
57 * @ops: a pointer to architecture specific iommu functions
58 *
59 * There are several kind of iommu algorithm(tlb, pagetable) among
60 * omap series. This interface installs such an iommu algorighm.
61 **/
62int install_iommu_arch(const struct iommu_functions *ops)
63{
64 if (arch_iommu)
65 return -EBUSY;
66
67 arch_iommu = ops;
68 return 0;
69}
70EXPORT_SYMBOL_GPL(install_iommu_arch);
71
72/**
73 * uninstall_iommu_arch - Uninstall archtecure specific iommu functions
74 * @ops: a pointer to architecture specific iommu functions
75 *
76 * This interface uninstalls the iommu algorighm installed previously.
77 **/
78void uninstall_iommu_arch(const struct iommu_functions *ops)
79{
80 if (arch_iommu != ops)
81 pr_err("%s: not your arch\n", __func__);
82
83 arch_iommu = NULL;
84}
85EXPORT_SYMBOL_GPL(uninstall_iommu_arch);
86
87/**
88 * iommu_save_ctx - Save registers for pm off-mode support
89 * @obj: target iommu
90 **/
91void iommu_save_ctx(struct iommu *obj)
92{
93 arch_iommu->save_ctx(obj);
94}
95EXPORT_SYMBOL_GPL(iommu_save_ctx);
96
97/**
98 * iommu_restore_ctx - Restore registers for pm off-mode support
99 * @obj: target iommu
100 **/
101void iommu_restore_ctx(struct iommu *obj)
102{
103 arch_iommu->restore_ctx(obj);
104}
105EXPORT_SYMBOL_GPL(iommu_restore_ctx);
106
107/**
108 * iommu_arch_version - Return running iommu arch version
109 **/
110u32 iommu_arch_version(void)
111{
112 return arch_iommu->version;
113}
114EXPORT_SYMBOL_GPL(iommu_arch_version);
115
116static int iommu_enable(struct iommu *obj)
117{
118 int err;
119
120 if (!obj)
121 return -EINVAL;
122
123 if (!arch_iommu)
124 return -ENODEV;
125
126 clk_enable(obj->clk);
127
128 err = arch_iommu->enable(obj);
129
130 clk_disable(obj->clk);
131 return err;
132}
133
134static void iommu_disable(struct iommu *obj)
135{
136 if (!obj)
137 return;
138
139 clk_enable(obj->clk);
140
141 arch_iommu->disable(obj);
142
143 clk_disable(obj->clk);
144}
145
146/*
147 * TLB operations
148 */
149void iotlb_cr_to_e(struct cr_regs *cr, struct iotlb_entry *e)
150{
151 BUG_ON(!cr || !e);
152
153 arch_iommu->cr_to_e(cr, e);
154}
155EXPORT_SYMBOL_GPL(iotlb_cr_to_e);
156
157static inline int iotlb_cr_valid(struct cr_regs *cr)
158{
159 if (!cr)
160 return -EINVAL;
161
162 return arch_iommu->cr_valid(cr);
163}
164
165static inline struct cr_regs *iotlb_alloc_cr(struct iommu *obj,
166 struct iotlb_entry *e)
167{
168 if (!e)
169 return NULL;
170
171 return arch_iommu->alloc_cr(obj, e);
172}
173
174u32 iotlb_cr_to_virt(struct cr_regs *cr)
175{
176 return arch_iommu->cr_to_virt(cr);
177}
178EXPORT_SYMBOL_GPL(iotlb_cr_to_virt);
179
180static u32 get_iopte_attr(struct iotlb_entry *e)
181{
182 return arch_iommu->get_pte_attr(e);
183}
184
185static u32 iommu_report_fault(struct iommu *obj, u32 *da)
186{
187 return arch_iommu->fault_isr(obj, da);
188}
189
190static void iotlb_lock_get(struct iommu *obj, struct iotlb_lock *l)
191{
192 u32 val;
193
194 val = iommu_read_reg(obj, MMU_LOCK);
195
196 l->base = MMU_LOCK_BASE(val);
197 l->vict = MMU_LOCK_VICT(val);
198
199}
200
201static void iotlb_lock_set(struct iommu *obj, struct iotlb_lock *l)
202{
203 u32 val;
204
205 val = (l->base << MMU_LOCK_BASE_SHIFT);
206 val |= (l->vict << MMU_LOCK_VICT_SHIFT);
207
208 iommu_write_reg(obj, val, MMU_LOCK);
209}
210
211static void iotlb_read_cr(struct iommu *obj, struct cr_regs *cr)
212{
213 arch_iommu->tlb_read_cr(obj, cr);
214}
215
216static void iotlb_load_cr(struct iommu *obj, struct cr_regs *cr)
217{
218 arch_iommu->tlb_load_cr(obj, cr);
219
220 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
221 iommu_write_reg(obj, 1, MMU_LD_TLB);
222}
223
224/**
225 * iotlb_dump_cr - Dump an iommu tlb entry into buf
226 * @obj: target iommu
227 * @cr: contents of cam and ram register
228 * @buf: output buffer
229 **/
230static inline ssize_t iotlb_dump_cr(struct iommu *obj, struct cr_regs *cr,
231 char *buf)
232{
233 BUG_ON(!cr || !buf);
234
235 return arch_iommu->dump_cr(obj, cr, buf);
236}
237
238/* only used in iotlb iteration for-loop */
239static struct cr_regs __iotlb_read_cr(struct iommu *obj, int n)
240{
241 struct cr_regs cr;
242 struct iotlb_lock l;
243
244 iotlb_lock_get(obj, &l);
245 l.vict = n;
246 iotlb_lock_set(obj, &l);
247 iotlb_read_cr(obj, &cr);
248
249 return cr;
250}
251
252/**
253 * load_iotlb_entry - Set an iommu tlb entry
254 * @obj: target iommu
255 * @e: an iommu tlb entry info
256 **/
257int load_iotlb_entry(struct iommu *obj, struct iotlb_entry *e)
258{
259 int err = 0;
260 struct iotlb_lock l;
261 struct cr_regs *cr;
262
263 if (!obj || !obj->nr_tlb_entries || !e)
264 return -EINVAL;
265
266 clk_enable(obj->clk);
267
268 iotlb_lock_get(obj, &l);
269 if (l.base == obj->nr_tlb_entries) {
270 dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
271 err = -EBUSY;
272 goto out;
273 }
274 if (!e->prsvd) {
275 int i;
276 struct cr_regs tmp;
277
278 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
279 if (!iotlb_cr_valid(&tmp))
280 break;
281
282 if (i == obj->nr_tlb_entries) {
283 dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
284 err = -EBUSY;
285 goto out;
286 }
287
288 iotlb_lock_get(obj, &l);
289 } else {
290 l.vict = l.base;
291 iotlb_lock_set(obj, &l);
292 }
293
294 cr = iotlb_alloc_cr(obj, e);
295 if (IS_ERR(cr)) {
296 clk_disable(obj->clk);
297 return PTR_ERR(cr);
298 }
299
300 iotlb_load_cr(obj, cr);
301 kfree(cr);
302
303 if (e->prsvd)
304 l.base++;
305 /* increment victim for next tlb load */
306 if (++l.vict == obj->nr_tlb_entries)
307 l.vict = l.base;
308 iotlb_lock_set(obj, &l);
309out:
310 clk_disable(obj->clk);
311 return err;
312}
313EXPORT_SYMBOL_GPL(load_iotlb_entry);
314
315/**
316 * flush_iotlb_page - Clear an iommu tlb entry
317 * @obj: target iommu
318 * @da: iommu device virtual address
319 *
320 * Clear an iommu tlb entry which includes 'da' address.
321 **/
322void flush_iotlb_page(struct iommu *obj, u32 da)
323{
324 int i;
325 struct cr_regs cr;
326
327 clk_enable(obj->clk);
328
329 for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
330 u32 start;
331 size_t bytes;
332
333 if (!iotlb_cr_valid(&cr))
334 continue;
335
336 start = iotlb_cr_to_virt(&cr);
337 bytes = iopgsz_to_bytes(cr.cam & 3);
338
339 if ((start <= da) && (da < start + bytes)) {
340 dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",
341 __func__, start, da, bytes);
342 iotlb_load_cr(obj, &cr);
343 iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
344 }
345 }
346 clk_disable(obj->clk);
347
348 if (i == obj->nr_tlb_entries)
349 dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
350}
351EXPORT_SYMBOL_GPL(flush_iotlb_page);
352
353/**
354 * flush_iotlb_range - Clear an iommu tlb entries
355 * @obj: target iommu
356 * @start: iommu device virtual address(start)
357 * @end: iommu device virtual address(end)
358 *
359 * Clear an iommu tlb entry which includes 'da' address.
360 **/
361void flush_iotlb_range(struct iommu *obj, u32 start, u32 end)
362{
363 u32 da = start;
364
365 while (da < end) {
366 flush_iotlb_page(obj, da);
367 /* FIXME: Optimize for multiple page size */
368 da += IOPTE_SIZE;
369 }
370}
371EXPORT_SYMBOL_GPL(flush_iotlb_range);
372
373/**
374 * flush_iotlb_all - Clear all iommu tlb entries
375 * @obj: target iommu
376 **/
377void flush_iotlb_all(struct iommu *obj)
378{
379 struct iotlb_lock l;
380
381 clk_enable(obj->clk);
382
383 l.base = 0;
384 l.vict = 0;
385 iotlb_lock_set(obj, &l);
386
387 iommu_write_reg(obj, 1, MMU_GFLUSH);
388
389 clk_disable(obj->clk);
390}
391EXPORT_SYMBOL_GPL(flush_iotlb_all);
392
393/**
394 * iommu_set_twl - enable/disable table walking logic
395 * @obj: target iommu
396 * @on: enable/disable
397 *
398 * Function used to enable/disable TWL. If one wants to work
399 * exclusively with locked TLB entries and receive notifications
400 * for TLB miss then call this function to disable TWL.
401 */
402void iommu_set_twl(struct iommu *obj, bool on)
403{
404 clk_enable(obj->clk);
405 arch_iommu->set_twl(obj, on);
406 clk_disable(obj->clk);
407}
408EXPORT_SYMBOL_GPL(iommu_set_twl);
409
410#if defined(CONFIG_OMAP_IOMMU_DEBUG_MODULE)
411
412ssize_t iommu_dump_ctx(struct iommu *obj, char *buf, ssize_t bytes)
413{
414 if (!obj || !buf)
415 return -EINVAL;
416
417 clk_enable(obj->clk);
418
419 bytes = arch_iommu->dump_ctx(obj, buf, bytes);
420
421 clk_disable(obj->clk);
422
423 return bytes;
424}
425EXPORT_SYMBOL_GPL(iommu_dump_ctx);
426
427static int __dump_tlb_entries(struct iommu *obj, struct cr_regs *crs, int num)
428{
429 int i;
430 struct iotlb_lock saved;
431 struct cr_regs tmp;
432 struct cr_regs *p = crs;
433
434 clk_enable(obj->clk);
435 iotlb_lock_get(obj, &saved);
436
437 for_each_iotlb_cr(obj, num, i, tmp) {
438 if (!iotlb_cr_valid(&tmp))
439 continue;
440 *p++ = tmp;
441 }
442
443 iotlb_lock_set(obj, &saved);
444 clk_disable(obj->clk);
445
446 return p - crs;
447}
448
449/**
450 * dump_tlb_entries - dump cr arrays to given buffer
451 * @obj: target iommu
452 * @buf: output buffer
453 **/
454size_t dump_tlb_entries(struct iommu *obj, char *buf, ssize_t bytes)
455{
456 int i, num;
457 struct cr_regs *cr;
458 char *p = buf;
459
460 num = bytes / sizeof(*cr);
461 num = min(obj->nr_tlb_entries, num);
462
463 cr = kcalloc(num, sizeof(*cr), GFP_KERNEL);
464 if (!cr)
465 return 0;
466
467 num = __dump_tlb_entries(obj, cr, num);
468 for (i = 0; i < num; i++)
469 p += iotlb_dump_cr(obj, cr + i, p);
470 kfree(cr);
471
472 return p - buf;
473}
474EXPORT_SYMBOL_GPL(dump_tlb_entries);
475
476int foreach_iommu_device(void *data, int (*fn)(struct device *, void *))
477{
478 return driver_for_each_device(&omap_iommu_driver.driver,
479 NULL, data, fn);
480}
481EXPORT_SYMBOL_GPL(foreach_iommu_device);
482
483#endif /* CONFIG_OMAP_IOMMU_DEBUG_MODULE */
484
485/*
486 * H/W pagetable operations
487 */
488static void flush_iopgd_range(u32 *first, u32 *last)
489{
490 /* FIXME: L2 cache should be taken care of if it exists */
491 do {
492 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
493 : : "r" (first));
494 first += L1_CACHE_BYTES / sizeof(*first);
495 } while (first <= last);
496}
497
498static void flush_iopte_range(u32 *first, u32 *last)
499{
500 /* FIXME: L2 cache should be taken care of if it exists */
501 do {
502 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
503 : : "r" (first));
504 first += L1_CACHE_BYTES / sizeof(*first);
505 } while (first <= last);
506}
507
508static void iopte_free(u32 *iopte)
509{
510 /* Note: freed iopte's must be clean ready for re-use */
511 kmem_cache_free(iopte_cachep, iopte);
512}
513
514static u32 *iopte_alloc(struct iommu *obj, u32 *iopgd, u32 da)
515{
516 u32 *iopte;
517
518 /* a table has already existed */
519 if (*iopgd)
520 goto pte_ready;
521
522 /*
523 * do the allocation outside the page table lock
524 */
525 spin_unlock(&obj->page_table_lock);
526 iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
527 spin_lock(&obj->page_table_lock);
528
529 if (!*iopgd) {
530 if (!iopte)
531 return ERR_PTR(-ENOMEM);
532
533 *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
534 flush_iopgd_range(iopgd, iopgd);
535
536 dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
537 } else {
538 /* We raced, free the reduniovant table */
539 iopte_free(iopte);
540 }
541
542pte_ready:
543 iopte = iopte_offset(iopgd, da);
544
545 dev_vdbg(obj->dev,
546 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
547 __func__, da, iopgd, *iopgd, iopte, *iopte);
548
549 return iopte;
550}
551
552static int iopgd_alloc_section(struct iommu *obj, u32 da, u32 pa, u32 prot)
553{
554 u32 *iopgd = iopgd_offset(obj, da);
555
556 if ((da | pa) & ~IOSECTION_MASK) {
557 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
558 __func__, da, pa, IOSECTION_SIZE);
559 return -EINVAL;
560 }
561
562 *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
563 flush_iopgd_range(iopgd, iopgd);
564 return 0;
565}
566
567static int iopgd_alloc_super(struct iommu *obj, u32 da, u32 pa, u32 prot)
568{
569 u32 *iopgd = iopgd_offset(obj, da);
570 int i;
571
572 if ((da | pa) & ~IOSUPER_MASK) {
573 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
574 __func__, da, pa, IOSUPER_SIZE);
575 return -EINVAL;
576 }
577
578 for (i = 0; i < 16; i++)
579 *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
580 flush_iopgd_range(iopgd, iopgd + 15);
581 return 0;
582}
583
584static int iopte_alloc_page(struct iommu *obj, u32 da, u32 pa, u32 prot)
585{
586 u32 *iopgd = iopgd_offset(obj, da);
587 u32 *iopte = iopte_alloc(obj, iopgd, da);
588
589 if (IS_ERR(iopte))
590 return PTR_ERR(iopte);
591
592 *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
593 flush_iopte_range(iopte, iopte);
594
595 dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
596 __func__, da, pa, iopte, *iopte);
597
598 return 0;
599}
600
601static int iopte_alloc_large(struct iommu *obj, u32 da, u32 pa, u32 prot)
602{
603 u32 *iopgd = iopgd_offset(obj, da);
604 u32 *iopte = iopte_alloc(obj, iopgd, da);
605 int i;
606
607 if ((da | pa) & ~IOLARGE_MASK) {
608 dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
609 __func__, da, pa, IOLARGE_SIZE);
610 return -EINVAL;
611 }
612
613 if (IS_ERR(iopte))
614 return PTR_ERR(iopte);
615
616 for (i = 0; i < 16; i++)
617 *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
618 flush_iopte_range(iopte, iopte + 15);
619 return 0;
620}
621
622static int iopgtable_store_entry_core(struct iommu *obj, struct iotlb_entry *e)
623{
624 int (*fn)(struct iommu *, u32, u32, u32);
625 u32 prot;
626 int err;
627
628 if (!obj || !e)
629 return -EINVAL;
630
631 switch (e->pgsz) {
632 case MMU_CAM_PGSZ_16M:
633 fn = iopgd_alloc_super;
634 break;
635 case MMU_CAM_PGSZ_1M:
636 fn = iopgd_alloc_section;
637 break;
638 case MMU_CAM_PGSZ_64K:
639 fn = iopte_alloc_large;
640 break;
641 case MMU_CAM_PGSZ_4K:
642 fn = iopte_alloc_page;
643 break;
644 default:
645 fn = NULL;
646 BUG();
647 break;
648 }
649
650 prot = get_iopte_attr(e);
651
652 spin_lock(&obj->page_table_lock);
653 err = fn(obj, e->da, e->pa, prot);
654 spin_unlock(&obj->page_table_lock);
655
656 return err;
657}
658
659/**
660 * iopgtable_store_entry - Make an iommu pte entry
661 * @obj: target iommu
662 * @e: an iommu tlb entry info
663 **/
664int iopgtable_store_entry(struct iommu *obj, struct iotlb_entry *e)
665{
666 int err;
667
668 flush_iotlb_page(obj, e->da);
669 err = iopgtable_store_entry_core(obj, e);
670#ifdef PREFETCH_IOTLB
671 if (!err)
672 load_iotlb_entry(obj, e);
673#endif
674 return err;
675}
676EXPORT_SYMBOL_GPL(iopgtable_store_entry);
677
678/**
679 * iopgtable_lookup_entry - Lookup an iommu pte entry
680 * @obj: target iommu
681 * @da: iommu device virtual address
682 * @ppgd: iommu pgd entry pointer to be returned
683 * @ppte: iommu pte entry pointer to be returned
684 **/
685void iopgtable_lookup_entry(struct iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
686{
687 u32 *iopgd, *iopte = NULL;
688
689 iopgd = iopgd_offset(obj, da);
690 if (!*iopgd)
691 goto out;
692
693 if (iopgd_is_table(*iopgd))
694 iopte = iopte_offset(iopgd, da);
695out:
696 *ppgd = iopgd;
697 *ppte = iopte;
698}
699EXPORT_SYMBOL_GPL(iopgtable_lookup_entry);
700
701static size_t iopgtable_clear_entry_core(struct iommu *obj, u32 da)
702{
703 size_t bytes;
704 u32 *iopgd = iopgd_offset(obj, da);
705 int nent = 1;
706
707 if (!*iopgd)
708 return 0;
709
710 if (iopgd_is_table(*iopgd)) {
711 int i;
712 u32 *iopte = iopte_offset(iopgd, da);
713
714 bytes = IOPTE_SIZE;
715 if (*iopte & IOPTE_LARGE) {
716 nent *= 16;
717 /* rewind to the 1st entry */
718 iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
719 }
720 bytes *= nent;
721 memset(iopte, 0, nent * sizeof(*iopte));
722 flush_iopte_range(iopte, iopte + (nent - 1) * sizeof(*iopte));
723
724 /*
725 * do table walk to check if this table is necessary or not
726 */
727 iopte = iopte_offset(iopgd, 0);
728 for (i = 0; i < PTRS_PER_IOPTE; i++)
729 if (iopte[i])
730 goto out;
731
732 iopte_free(iopte);
733 nent = 1; /* for the next L1 entry */
734 } else {
735 bytes = IOPGD_SIZE;
736 if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
737 nent *= 16;
738 /* rewind to the 1st entry */
739 iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
740 }
741 bytes *= nent;
742 }
743 memset(iopgd, 0, nent * sizeof(*iopgd));
744 flush_iopgd_range(iopgd, iopgd + (nent - 1) * sizeof(*iopgd));
745out:
746 return bytes;
747}
748
749/**
750 * iopgtable_clear_entry - Remove an iommu pte entry
751 * @obj: target iommu
752 * @da: iommu device virtual address
753 **/
754size_t iopgtable_clear_entry(struct iommu *obj, u32 da)
755{
756 size_t bytes;
757
758 spin_lock(&obj->page_table_lock);
759
760 bytes = iopgtable_clear_entry_core(obj, da);
761 flush_iotlb_page(obj, da);
762
763 spin_unlock(&obj->page_table_lock);
764
765 return bytes;
766}
767EXPORT_SYMBOL_GPL(iopgtable_clear_entry);
768
769static void iopgtable_clear_entry_all(struct iommu *obj)
770{
771 int i;
772
773 spin_lock(&obj->page_table_lock);
774
775 for (i = 0; i < PTRS_PER_IOPGD; i++) {
776 u32 da;
777 u32 *iopgd;
778
779 da = i << IOPGD_SHIFT;
780 iopgd = iopgd_offset(obj, da);
781
782 if (!*iopgd)
783 continue;
784
785 if (iopgd_is_table(*iopgd))
786 iopte_free(iopte_offset(iopgd, 0));
787
788 *iopgd = 0;
789 flush_iopgd_range(iopgd, iopgd);
790 }
791
792 flush_iotlb_all(obj);
793
794 spin_unlock(&obj->page_table_lock);
795}
796
797/*
798 * Device IOMMU generic operations
799 */
800static irqreturn_t iommu_fault_handler(int irq, void *data)
801{
802 u32 da, errs;
803 u32 *iopgd, *iopte;
804 struct iommu *obj = data;
805
806 if (!obj->refcount)
807 return IRQ_NONE;
808
809 clk_enable(obj->clk);
810 errs = iommu_report_fault(obj, &da);
811 clk_disable(obj->clk);
812 if (errs == 0)
813 return IRQ_HANDLED;
814
815 /* Fault callback or TLB/PTE Dynamic loading */
816 if (obj->isr && !obj->isr(obj, da, errs, obj->isr_priv))
817 return IRQ_HANDLED;
818
819 iommu_disable(obj);
820
821 iopgd = iopgd_offset(obj, da);
822
823 if (!iopgd_is_table(*iopgd)) {
824 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p "
825 "*pgd:px%08x\n", obj->name, errs, da, iopgd, *iopgd);
826 return IRQ_NONE;
827 }
828
829 iopte = iopte_offset(iopgd, da);
830
831 dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x "
832 "pte:0x%p *pte:0x%08x\n", obj->name, errs, da, iopgd, *iopgd,
833 iopte, *iopte);
834
835 return IRQ_NONE;
836}
837
838static int device_match_by_alias(struct device *dev, void *data)
839{
840 struct iommu *obj = to_iommu(dev);
841 const char *name = data;
842
843 pr_debug("%s: %s %s\n", __func__, obj->name, name);
844
845 return strcmp(obj->name, name) == 0;
846}
847
848/**
849 * iommu_set_da_range - Set a valid device address range
850 * @obj: target iommu
851 * @start Start of valid range
852 * @end End of valid range
853 **/
854int iommu_set_da_range(struct iommu *obj, u32 start, u32 end)
855{
856
857 if (!obj)
858 return -EFAULT;
859
860 if (end < start || !PAGE_ALIGN(start | end))
861 return -EINVAL;
862
863 obj->da_start = start;
864 obj->da_end = end;
865
866 return 0;
867}
868EXPORT_SYMBOL_GPL(iommu_set_da_range);
869
870/**
871 * omap_find_iommu_device() - find an omap iommu device by name
872 * @name: name of the iommu device
873 *
874 * The generic iommu API requires the caller to provide the device
875 * he wishes to attach to a certain iommu domain.
876 *
877 * Drivers generally should not bother with this as it should just
878 * be taken care of by the DMA-API using dev_archdata.
879 *
880 * This function is provided as an interim solution until the latter
881 * materializes, and omap3isp is fully migrated to the DMA-API.
882 */
883struct device *omap_find_iommu_device(const char *name)
884{
885 return driver_find_device(&omap_iommu_driver.driver, NULL,
886 (void *)name,
887 device_match_by_alias);
888}
889EXPORT_SYMBOL_GPL(omap_find_iommu_device);
890
891/**
892 * omap_iommu_attach() - attach iommu device to an iommu domain
893 * @dev: target omap iommu device
894 * @iopgd: page table
895 **/
896static struct iommu *omap_iommu_attach(struct device *dev, u32 *iopgd)
897{
898 int err = -ENOMEM;
899 struct iommu *obj = to_iommu(dev);
900
901 spin_lock(&obj->iommu_lock);
902
903 /* an iommu device can only be attached once */
904 if (++obj->refcount > 1) {
905 dev_err(dev, "%s: already attached!\n", obj->name);
906 err = -EBUSY;
907 goto err_enable;
908 }
909
910 obj->iopgd = iopgd;
911 err = iommu_enable(obj);
912 if (err)
913 goto err_enable;
914 flush_iotlb_all(obj);
915
916 if (!try_module_get(obj->owner))
917 goto err_module;
918
919 spin_unlock(&obj->iommu_lock);
920
921 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
922 return obj;
923
924err_module:
925 if (obj->refcount == 1)
926 iommu_disable(obj);
927err_enable:
928 obj->refcount--;
929 spin_unlock(&obj->iommu_lock);
930 return ERR_PTR(err);
931}
932
933/**
934 * omap_iommu_detach - release iommu device
935 * @obj: target iommu
936 **/
937static void omap_iommu_detach(struct iommu *obj)
938{
939 if (!obj || IS_ERR(obj))
940 return;
941
942 spin_lock(&obj->iommu_lock);
943
944 if (--obj->refcount == 0)
945 iommu_disable(obj);
946
947 module_put(obj->owner);
948
949 obj->iopgd = NULL;
950
951 spin_unlock(&obj->iommu_lock);
952
953 dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
954}
955
956int iommu_set_isr(const char *name,
957 int (*isr)(struct iommu *obj, u32 da, u32 iommu_errs,
958 void *priv),
959 void *isr_priv)
960{
961 struct device *dev;
962 struct iommu *obj;
963
964 dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name,
965 device_match_by_alias);
966 if (!dev)
967 return -ENODEV;
968
969 obj = to_iommu(dev);
970 mutex_lock(&obj->iommu_lock);
971 if (obj->refcount != 0) {
972 mutex_unlock(&obj->iommu_lock);
973 return -EBUSY;
974 }
975 obj->isr = isr;
976 obj->isr_priv = isr_priv;
977 mutex_unlock(&obj->iommu_lock);
978
979 return 0;
980}
981EXPORT_SYMBOL_GPL(iommu_set_isr);
982
983/*
984 * OMAP Device MMU(IOMMU) detection
985 */
986static int __devinit omap_iommu_probe(struct platform_device *pdev)
987{
988 int err = -ENODEV;
989 int irq;
990 struct iommu *obj;
991 struct resource *res;
992 struct iommu_platform_data *pdata = pdev->dev.platform_data;
993
994 if (pdev->num_resources != 2)
995 return -EINVAL;
996
997 obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
998 if (!obj)
999 return -ENOMEM;
1000
1001 obj->clk = clk_get(&pdev->dev, pdata->clk_name);
1002 if (IS_ERR(obj->clk))
1003 goto err_clk;
1004
1005 obj->nr_tlb_entries = pdata->nr_tlb_entries;
1006 obj->name = pdata->name;
1007 obj->dev = &pdev->dev;
1008 obj->ctx = (void *)obj + sizeof(*obj);
1009 obj->da_start = pdata->da_start;
1010 obj->da_end = pdata->da_end;
1011
1012 spin_lock_init(&obj->iommu_lock);
1013 mutex_init(&obj->mmap_lock);
1014 spin_lock_init(&obj->page_table_lock);
1015 INIT_LIST_HEAD(&obj->mmap);
1016
1017 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1018 if (!res) {
1019 err = -ENODEV;
1020 goto err_mem;
1021 }
1022
1023 res = request_mem_region(res->start, resource_size(res),
1024 dev_name(&pdev->dev));
1025 if (!res) {
1026 err = -EIO;
1027 goto err_mem;
1028 }
1029
1030 obj->regbase = ioremap(res->start, resource_size(res));
1031 if (!obj->regbase) {
1032 err = -ENOMEM;
1033 goto err_ioremap;
1034 }
1035
1036 irq = platform_get_irq(pdev, 0);
1037 if (irq < 0) {
1038 err = -ENODEV;
1039 goto err_irq;
1040 }
1041 err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
1042 dev_name(&pdev->dev), obj);
1043 if (err < 0)
1044 goto err_irq;
1045 platform_set_drvdata(pdev, obj);
1046
1047 dev_info(&pdev->dev, "%s registered\n", obj->name);
1048 return 0;
1049
1050err_irq:
1051 iounmap(obj->regbase);
1052err_ioremap:
1053 release_mem_region(res->start, resource_size(res));
1054err_mem:
1055 clk_put(obj->clk);
1056err_clk:
1057 kfree(obj);
1058 return err;
1059}
1060
1061static int __devexit omap_iommu_remove(struct platform_device *pdev)
1062{
1063 int irq;
1064 struct resource *res;
1065 struct iommu *obj = platform_get_drvdata(pdev);
1066
1067 platform_set_drvdata(pdev, NULL);
1068
1069 iopgtable_clear_entry_all(obj);
1070
1071 irq = platform_get_irq(pdev, 0);
1072 free_irq(irq, obj);
1073 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1074 release_mem_region(res->start, resource_size(res));
1075 iounmap(obj->regbase);
1076
1077 clk_put(obj->clk);
1078 dev_info(&pdev->dev, "%s removed\n", obj->name);
1079 kfree(obj);
1080 return 0;
1081}
1082
1083static struct platform_driver omap_iommu_driver = {
1084 .probe = omap_iommu_probe,
1085 .remove = __devexit_p(omap_iommu_remove),
1086 .driver = {
1087 .name = "omap-iommu",
1088 },
1089};
1090
1091static void iopte_cachep_ctor(void *iopte)
1092{
1093 clean_dcache_area(iopte, IOPTE_TABLE_SIZE);
1094}
1095
1096static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
1097 phys_addr_t pa, int order, int prot)
1098{
1099 struct omap_iommu_domain *omap_domain = domain->priv;
1100 struct iommu *oiommu = omap_domain->iommu_dev;
1101 struct device *dev = oiommu->dev;
1102 size_t bytes = PAGE_SIZE << order;
1103 struct iotlb_entry e;
1104 int omap_pgsz;
1105 u32 ret, flags;
1106
1107 /* we only support mapping a single iommu page for now */
1108 omap_pgsz = bytes_to_iopgsz(bytes);
1109 if (omap_pgsz < 0) {
1110 dev_err(dev, "invalid size to map: %d\n", bytes);
1111 return -EINVAL;
1112 }
1113
1114 dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes);
1115
1116 flags = omap_pgsz | prot;
1117
1118 iotlb_init_entry(&e, da, pa, flags);
1119
1120 ret = iopgtable_store_entry(oiommu, &e);
1121 if (ret) {
1122 dev_err(dev, "iopgtable_store_entry failed: %d\n", ret);
1123 return ret;
1124 }
1125
1126 return 0;
1127}
1128
1129static int omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
1130 int order)
1131{
1132 struct omap_iommu_domain *omap_domain = domain->priv;
1133 struct iommu *oiommu = omap_domain->iommu_dev;
1134 struct device *dev = oiommu->dev;
1135 size_t bytes = PAGE_SIZE << order;
1136 size_t ret;
1137
1138 dev_dbg(dev, "unmapping da 0x%lx size 0x%x\n", da, bytes);
1139
1140 ret = iopgtable_clear_entry(oiommu, da);
1141 if (ret != bytes) {
1142 dev_err(dev, "entry @ 0x%lx was %d; not %d\n", da, ret, bytes);
1143 return -EINVAL;
1144 }
1145
1146 return 0;
1147}
1148
1149static int
1150omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
1151{
1152 struct omap_iommu_domain *omap_domain = domain->priv;
1153 struct iommu *oiommu;
1154 int ret = 0;
1155
1156 spin_lock(&omap_domain->lock);
1157
1158 /* only a single device is supported per domain for now */
1159 if (omap_domain->iommu_dev) {
1160 dev_err(dev, "iommu domain is already attached\n");
1161 ret = -EBUSY;
1162 goto out;
1163 }
1164
1165 /* get a handle to and enable the omap iommu */
1166 oiommu = omap_iommu_attach(dev, omap_domain->pgtable);
1167 if (IS_ERR(oiommu)) {
1168 ret = PTR_ERR(oiommu);
1169 dev_err(dev, "can't get omap iommu: %d\n", ret);
1170 goto out;
1171 }
1172
1173 omap_domain->iommu_dev = oiommu;
1174
1175out:
1176 spin_unlock(&omap_domain->lock);
1177 return ret;
1178}
1179
1180static void omap_iommu_detach_dev(struct iommu_domain *domain,
1181 struct device *dev)
1182{
1183 struct omap_iommu_domain *omap_domain = domain->priv;
1184 struct iommu *oiommu = to_iommu(dev);
1185
1186 spin_lock(&omap_domain->lock);
1187
1188 /* only a single device is supported per domain for now */
1189 if (omap_domain->iommu_dev != oiommu) {
1190 dev_err(dev, "invalid iommu device\n");
1191 goto out;
1192 }
1193
1194 iopgtable_clear_entry_all(oiommu);
1195
1196 omap_iommu_detach(oiommu);
1197
1198 omap_domain->iommu_dev = NULL;
1199
1200out:
1201 spin_unlock(&omap_domain->lock);
1202}
1203
1204static int omap_iommu_domain_init(struct iommu_domain *domain)
1205{
1206 struct omap_iommu_domain *omap_domain;
1207
1208 omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
1209 if (!omap_domain) {
1210 pr_err("kzalloc failed\n");
1211 goto out;
1212 }
1213
1214 omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL);
1215 if (!omap_domain->pgtable) {
1216 pr_err("kzalloc failed\n");
1217 goto fail_nomem;
1218 }
1219
1220 /*
1221 * should never fail, but please keep this around to ensure
1222 * we keep the hardware happy
1223 */
1224 BUG_ON(!IS_ALIGNED((long)omap_domain->pgtable, IOPGD_TABLE_SIZE));
1225
1226 clean_dcache_area(omap_domain->pgtable, IOPGD_TABLE_SIZE);
1227 spin_lock_init(&omap_domain->lock);
1228
1229 domain->priv = omap_domain;
1230
1231 return 0;
1232
1233fail_nomem:
1234 kfree(omap_domain);
1235out:
1236 return -ENOMEM;
1237}
1238
1239/* assume device was already detached */
1240static void omap_iommu_domain_destroy(struct iommu_domain *domain)
1241{
1242 struct omap_iommu_domain *omap_domain = domain->priv;
1243
1244 domain->priv = NULL;
1245
1246 kfree(omap_domain->pgtable);
1247 kfree(omap_domain);
1248}
1249
1250static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
1251 unsigned long da)
1252{
1253 struct omap_iommu_domain *omap_domain = domain->priv;
1254 struct iommu *oiommu = omap_domain->iommu_dev;
1255 struct device *dev = oiommu->dev;
1256 u32 *pgd, *pte;
1257 phys_addr_t ret = 0;
1258
1259 iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
1260
1261 if (pte) {
1262 if (iopte_is_small(*pte))
1263 ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
1264 else if (iopte_is_large(*pte))
1265 ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
1266 else
1267 dev_err(dev, "bogus pte 0x%x", *pte);
1268 } else {
1269 if (iopgd_is_section(*pgd))
1270 ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
1271 else if (iopgd_is_super(*pgd))
1272 ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
1273 else
1274 dev_err(dev, "bogus pgd 0x%x", *pgd);
1275 }
1276
1277 return ret;
1278}
1279
1280static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
1281 unsigned long cap)
1282{
1283 return 0;
1284}
1285
1286static struct iommu_ops omap_iommu_ops = {
1287 .domain_init = omap_iommu_domain_init,
1288 .domain_destroy = omap_iommu_domain_destroy,
1289 .attach_dev = omap_iommu_attach_dev,
1290 .detach_dev = omap_iommu_detach_dev,
1291 .map = omap_iommu_map,
1292 .unmap = omap_iommu_unmap,
1293 .iova_to_phys = omap_iommu_iova_to_phys,
1294 .domain_has_cap = omap_iommu_domain_has_cap,
1295};
1296
1297static int __init omap_iommu_init(void)
1298{
1299 struct kmem_cache *p;
1300 const unsigned long flags = SLAB_HWCACHE_ALIGN;
1301 size_t align = 1 << 10; /* L2 pagetable alignement */
1302
1303 p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
1304 iopte_cachep_ctor);
1305 if (!p)
1306 return -ENOMEM;
1307 iopte_cachep = p;
1308
1309 register_iommu(&omap_iommu_ops);
1310
1311 return platform_driver_register(&omap_iommu_driver);
1312}
1313module_init(omap_iommu_init);
1314
1315static void __exit omap_iommu_exit(void)
1316{
1317 kmem_cache_destroy(iopte_cachep);
1318
1319 platform_driver_unregister(&omap_iommu_driver);
1320}
1321module_exit(omap_iommu_exit);
1322
1323MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1324MODULE_ALIAS("platform:omap-iommu");
1325MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1326MODULE_LICENSE("GPL v2");
diff --git a/arch/arm/plat-omap/iovmm.c b/arch/arm/plat-omap/iovmm.c
deleted file mode 100644
index aa2c47893b02..000000000000
--- a/arch/arm/plat-omap/iovmm.c
+++ /dev/null
@@ -1,923 +0,0 @@
1/*
2 * omap iommu: simple virtual address space management
3 *
4 * Copyright (C) 2008-2009 Nokia Corporation
5 *
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/err.h>
14#include <linux/slab.h>
15#include <linux/vmalloc.h>
16#include <linux/device.h>
17#include <linux/scatterlist.h>
18#include <linux/iommu.h>
19
20#include <asm/cacheflush.h>
21#include <asm/mach/map.h>
22
23#include <plat/iommu.h>
24#include <plat/iovmm.h>
25
26#include "iopgtable.h"
27
28/*
29 * A device driver needs to create address mappings between:
30 *
31 * - iommu/device address
32 * - physical address
33 * - mpu virtual address
34 *
35 * There are 4 possible patterns for them:
36 *
37 * |iova/ mapping iommu_ page
38 * | da pa va (d)-(p)-(v) function type
39 * ---------------------------------------------------------------------------
40 * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s
41 * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s
42 * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s
43 * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n*
44 *
45 *
46 * 'iova': device iommu virtual address
47 * 'da': alias of 'iova'
48 * 'pa': physical address
49 * 'va': mpu virtual address
50 *
51 * 'c': contiguous memory area
52 * 'd': discontiguous memory area
53 * 'a': anonymous memory allocation
54 * '()': optional feature
55 *
56 * 'n': a normal page(4KB) size is used.
57 * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used.
58 *
59 * '*': not yet, but feasible.
60 */
61
62static struct kmem_cache *iovm_area_cachep;
63
64/* return total bytes of sg buffers */
65static size_t sgtable_len(const struct sg_table *sgt)
66{
67 unsigned int i, total = 0;
68 struct scatterlist *sg;
69
70 if (!sgt)
71 return 0;
72
73 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
74 size_t bytes;
75
76 bytes = sg->length;
77
78 if (!iopgsz_ok(bytes)) {
79 pr_err("%s: sg[%d] not iommu pagesize(%x)\n",
80 __func__, i, bytes);
81 return 0;
82 }
83
84 total += bytes;
85 }
86
87 return total;
88}
89#define sgtable_ok(x) (!!sgtable_len(x))
90
91static unsigned max_alignment(u32 addr)
92{
93 int i;
94 unsigned pagesize[] = { SZ_16M, SZ_1M, SZ_64K, SZ_4K, };
95 for (i = 0; i < ARRAY_SIZE(pagesize) && addr & (pagesize[i] - 1); i++)
96 ;
97 return (i < ARRAY_SIZE(pagesize)) ? pagesize[i] : 0;
98}
99
100/*
101 * calculate the optimal number sg elements from total bytes based on
102 * iommu superpages
103 */
104static unsigned sgtable_nents(size_t bytes, u32 da, u32 pa)
105{
106 unsigned nr_entries = 0, ent_sz;
107
108 if (!IS_ALIGNED(bytes, PAGE_SIZE)) {
109 pr_err("%s: wrong size %08x\n", __func__, bytes);
110 return 0;
111 }
112
113 while (bytes) {
114 ent_sz = max_alignment(da | pa);
115 ent_sz = min_t(unsigned, ent_sz, iopgsz_max(bytes));
116 nr_entries++;
117 da += ent_sz;
118 pa += ent_sz;
119 bytes -= ent_sz;
120 }
121
122 return nr_entries;
123}
124
125/* allocate and initialize sg_table header(a kind of 'superblock') */
126static struct sg_table *sgtable_alloc(const size_t bytes, u32 flags,
127 u32 da, u32 pa)
128{
129 unsigned int nr_entries;
130 int err;
131 struct sg_table *sgt;
132
133 if (!bytes)
134 return ERR_PTR(-EINVAL);
135
136 if (!IS_ALIGNED(bytes, PAGE_SIZE))
137 return ERR_PTR(-EINVAL);
138
139 if (flags & IOVMF_LINEAR) {
140 nr_entries = sgtable_nents(bytes, da, pa);
141 if (!nr_entries)
142 return ERR_PTR(-EINVAL);
143 } else
144 nr_entries = bytes / PAGE_SIZE;
145
146 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
147 if (!sgt)
148 return ERR_PTR(-ENOMEM);
149
150 err = sg_alloc_table(sgt, nr_entries, GFP_KERNEL);
151 if (err) {
152 kfree(sgt);
153 return ERR_PTR(err);
154 }
155
156 pr_debug("%s: sgt:%p(%d entries)\n", __func__, sgt, nr_entries);
157
158 return sgt;
159}
160
161/* free sg_table header(a kind of superblock) */
162static void sgtable_free(struct sg_table *sgt)
163{
164 if (!sgt)
165 return;
166
167 sg_free_table(sgt);
168 kfree(sgt);
169
170 pr_debug("%s: sgt:%p\n", __func__, sgt);
171}
172
173/* map 'sglist' to a contiguous mpu virtual area and return 'va' */
174static void *vmap_sg(const struct sg_table *sgt)
175{
176 u32 va;
177 size_t total;
178 unsigned int i;
179 struct scatterlist *sg;
180 struct vm_struct *new;
181 const struct mem_type *mtype;
182
183 mtype = get_mem_type(MT_DEVICE);
184 if (!mtype)
185 return ERR_PTR(-EINVAL);
186
187 total = sgtable_len(sgt);
188 if (!total)
189 return ERR_PTR(-EINVAL);
190
191 new = __get_vm_area(total, VM_IOREMAP, VMALLOC_START, VMALLOC_END);
192 if (!new)
193 return ERR_PTR(-ENOMEM);
194 va = (u32)new->addr;
195
196 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
197 size_t bytes;
198 u32 pa;
199 int err;
200
201 pa = sg_phys(sg);
202 bytes = sg->length;
203
204 BUG_ON(bytes != PAGE_SIZE);
205
206 err = ioremap_page(va, pa, mtype);
207 if (err)
208 goto err_out;
209
210 va += bytes;
211 }
212
213 flush_cache_vmap((unsigned long)new->addr,
214 (unsigned long)(new->addr + total));
215 return new->addr;
216
217err_out:
218 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
219 vunmap(new->addr);
220 return ERR_PTR(-EAGAIN);
221}
222
223static inline void vunmap_sg(const void *va)
224{
225 vunmap(va);
226}
227
228static struct iovm_struct *__find_iovm_area(struct iommu *obj, const u32 da)
229{
230 struct iovm_struct *tmp;
231
232 list_for_each_entry(tmp, &obj->mmap, list) {
233 if ((da >= tmp->da_start) && (da < tmp->da_end)) {
234 size_t len;
235
236 len = tmp->da_end - tmp->da_start;
237
238 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n",
239 __func__, tmp->da_start, da, tmp->da_end, len,
240 tmp->flags);
241
242 return tmp;
243 }
244 }
245
246 return NULL;
247}
248
249/**
250 * find_iovm_area - find iovma which includes @da
251 * @da: iommu device virtual address
252 *
253 * Find the existing iovma starting at @da
254 */
255struct iovm_struct *find_iovm_area(struct iommu *obj, u32 da)
256{
257 struct iovm_struct *area;
258
259 mutex_lock(&obj->mmap_lock);
260 area = __find_iovm_area(obj, da);
261 mutex_unlock(&obj->mmap_lock);
262
263 return area;
264}
265EXPORT_SYMBOL_GPL(find_iovm_area);
266
267/*
268 * This finds the hole(area) which fits the requested address and len
269 * in iovmas mmap, and returns the new allocated iovma.
270 */
271static struct iovm_struct *alloc_iovm_area(struct iommu *obj, u32 da,
272 size_t bytes, u32 flags)
273{
274 struct iovm_struct *new, *tmp;
275 u32 start, prev_end, alignment;
276
277 if (!obj || !bytes)
278 return ERR_PTR(-EINVAL);
279
280 start = da;
281 alignment = PAGE_SIZE;
282
283 if (~flags & IOVMF_DA_FIXED) {
284 /* Don't map address 0 */
285 start = obj->da_start ? obj->da_start : alignment;
286
287 if (flags & IOVMF_LINEAR)
288 alignment = iopgsz_max(bytes);
289 start = roundup(start, alignment);
290 } else if (start < obj->da_start || start > obj->da_end ||
291 obj->da_end - start < bytes) {
292 return ERR_PTR(-EINVAL);
293 }
294
295 tmp = NULL;
296 if (list_empty(&obj->mmap))
297 goto found;
298
299 prev_end = 0;
300 list_for_each_entry(tmp, &obj->mmap, list) {
301
302 if (prev_end > start)
303 break;
304
305 if (tmp->da_start > start && (tmp->da_start - start) >= bytes)
306 goto found;
307
308 if (tmp->da_end >= start && ~flags & IOVMF_DA_FIXED)
309 start = roundup(tmp->da_end + 1, alignment);
310
311 prev_end = tmp->da_end;
312 }
313
314 if ((start >= prev_end) && (obj->da_end - start >= bytes))
315 goto found;
316
317 dev_dbg(obj->dev, "%s: no space to fit %08x(%x) flags: %08x\n",
318 __func__, da, bytes, flags);
319
320 return ERR_PTR(-EINVAL);
321
322found:
323 new = kmem_cache_zalloc(iovm_area_cachep, GFP_KERNEL);
324 if (!new)
325 return ERR_PTR(-ENOMEM);
326
327 new->iommu = obj;
328 new->da_start = start;
329 new->da_end = start + bytes;
330 new->flags = flags;
331
332 /*
333 * keep ascending order of iovmas
334 */
335 if (tmp)
336 list_add_tail(&new->list, &tmp->list);
337 else
338 list_add(&new->list, &obj->mmap);
339
340 dev_dbg(obj->dev, "%s: found %08x-%08x-%08x(%x) %08x\n",
341 __func__, new->da_start, start, new->da_end, bytes, flags);
342
343 return new;
344}
345
346static void free_iovm_area(struct iommu *obj, struct iovm_struct *area)
347{
348 size_t bytes;
349
350 BUG_ON(!obj || !area);
351
352 bytes = area->da_end - area->da_start;
353
354 dev_dbg(obj->dev, "%s: %08x-%08x(%x) %08x\n",
355 __func__, area->da_start, area->da_end, bytes, area->flags);
356
357 list_del(&area->list);
358 kmem_cache_free(iovm_area_cachep, area);
359}
360
361/**
362 * da_to_va - convert (d) to (v)
363 * @obj: objective iommu
364 * @da: iommu device virtual address
365 * @va: mpu virtual address
366 *
367 * Returns mpu virtual addr which corresponds to a given device virtual addr
368 */
369void *da_to_va(struct iommu *obj, u32 da)
370{
371 void *va = NULL;
372 struct iovm_struct *area;
373
374 mutex_lock(&obj->mmap_lock);
375
376 area = __find_iovm_area(obj, da);
377 if (!area) {
378 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
379 goto out;
380 }
381 va = area->va;
382out:
383 mutex_unlock(&obj->mmap_lock);
384
385 return va;
386}
387EXPORT_SYMBOL_GPL(da_to_va);
388
389static void sgtable_fill_vmalloc(struct sg_table *sgt, void *_va)
390{
391 unsigned int i;
392 struct scatterlist *sg;
393 void *va = _va;
394 void *va_end;
395
396 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
397 struct page *pg;
398 const size_t bytes = PAGE_SIZE;
399
400 /*
401 * iommu 'superpage' isn't supported with 'iommu_vmalloc()'
402 */
403 pg = vmalloc_to_page(va);
404 BUG_ON(!pg);
405 sg_set_page(sg, pg, bytes, 0);
406
407 va += bytes;
408 }
409
410 va_end = _va + PAGE_SIZE * i;
411}
412
413static inline void sgtable_drain_vmalloc(struct sg_table *sgt)
414{
415 /*
416 * Actually this is not necessary at all, just exists for
417 * consistency of the code readability.
418 */
419 BUG_ON(!sgt);
420}
421
422static void sgtable_fill_kmalloc(struct sg_table *sgt, u32 pa, u32 da,
423 size_t len)
424{
425 unsigned int i;
426 struct scatterlist *sg;
427
428 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
429 unsigned bytes;
430
431 bytes = max_alignment(da | pa);
432 bytes = min_t(unsigned, bytes, iopgsz_max(len));
433
434 BUG_ON(!iopgsz_ok(bytes));
435
436 sg_set_buf(sg, phys_to_virt(pa), bytes);
437 /*
438 * 'pa' is cotinuous(linear).
439 */
440 pa += bytes;
441 da += bytes;
442 len -= bytes;
443 }
444 BUG_ON(len);
445}
446
447static inline void sgtable_drain_kmalloc(struct sg_table *sgt)
448{
449 /*
450 * Actually this is not necessary at all, just exists for
451 * consistency of the code readability
452 */
453 BUG_ON(!sgt);
454}
455
456/* create 'da' <-> 'pa' mapping from 'sgt' */
457static int map_iovm_area(struct iommu_domain *domain, struct iovm_struct *new,
458 const struct sg_table *sgt, u32 flags)
459{
460 int err;
461 unsigned int i, j;
462 struct scatterlist *sg;
463 u32 da = new->da_start;
464 int order;
465
466 if (!domain || !sgt)
467 return -EINVAL;
468
469 BUG_ON(!sgtable_ok(sgt));
470
471 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
472 u32 pa;
473 size_t bytes;
474
475 pa = sg_phys(sg);
476 bytes = sg->length;
477
478 flags &= ~IOVMF_PGSZ_MASK;
479
480 if (bytes_to_iopgsz(bytes) < 0)
481 goto err_out;
482
483 order = get_order(bytes);
484
485 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__,
486 i, da, pa, bytes);
487
488 err = iommu_map(domain, da, pa, order, flags);
489 if (err)
490 goto err_out;
491
492 da += bytes;
493 }
494 return 0;
495
496err_out:
497 da = new->da_start;
498
499 for_each_sg(sgt->sgl, sg, i, j) {
500 size_t bytes;
501
502 bytes = sg->length;
503 order = get_order(bytes);
504
505 /* ignore failures.. we're already handling one */
506 iommu_unmap(domain, da, order);
507
508 da += bytes;
509 }
510 return err;
511}
512
513/* release 'da' <-> 'pa' mapping */
514static void unmap_iovm_area(struct iommu_domain *domain, struct iommu *obj,
515 struct iovm_struct *area)
516{
517 u32 start;
518 size_t total = area->da_end - area->da_start;
519 const struct sg_table *sgt = area->sgt;
520 struct scatterlist *sg;
521 int i, err;
522
523 BUG_ON(!sgtable_ok(sgt));
524 BUG_ON((!total) || !IS_ALIGNED(total, PAGE_SIZE));
525
526 start = area->da_start;
527 for_each_sg(sgt->sgl, sg, sgt->nents, i) {
528 size_t bytes;
529 int order;
530
531 bytes = sg->length;
532 order = get_order(bytes);
533
534 err = iommu_unmap(domain, start, order);
535 if (err)
536 break;
537
538 dev_dbg(obj->dev, "%s: unmap %08x(%x) %08x\n",
539 __func__, start, bytes, area->flags);
540
541 BUG_ON(!IS_ALIGNED(bytes, PAGE_SIZE));
542
543 total -= bytes;
544 start += bytes;
545 }
546 BUG_ON(total);
547}
548
549/* template function for all unmapping */
550static struct sg_table *unmap_vm_area(struct iommu_domain *domain,
551 struct iommu *obj, const u32 da,
552 void (*fn)(const void *), u32 flags)
553{
554 struct sg_table *sgt = NULL;
555 struct iovm_struct *area;
556
557 if (!IS_ALIGNED(da, PAGE_SIZE)) {
558 dev_err(obj->dev, "%s: alignment err(%08x)\n", __func__, da);
559 return NULL;
560 }
561
562 mutex_lock(&obj->mmap_lock);
563
564 area = __find_iovm_area(obj, da);
565 if (!area) {
566 dev_dbg(obj->dev, "%s: no da area(%08x)\n", __func__, da);
567 goto out;
568 }
569
570 if ((area->flags & flags) != flags) {
571 dev_err(obj->dev, "%s: wrong flags(%08x)\n", __func__,
572 area->flags);
573 goto out;
574 }
575 sgt = (struct sg_table *)area->sgt;
576
577 unmap_iovm_area(domain, obj, area);
578
579 fn(area->va);
580
581 dev_dbg(obj->dev, "%s: %08x-%08x-%08x(%x) %08x\n", __func__,
582 area->da_start, da, area->da_end,
583 area->da_end - area->da_start, area->flags);
584
585 free_iovm_area(obj, area);
586out:
587 mutex_unlock(&obj->mmap_lock);
588
589 return sgt;
590}
591
592static u32 map_iommu_region(struct iommu_domain *domain, struct iommu *obj,
593 u32 da, const struct sg_table *sgt, void *va,
594 size_t bytes, u32 flags)
595{
596 int err = -ENOMEM;
597 struct iovm_struct *new;
598
599 mutex_lock(&obj->mmap_lock);
600
601 new = alloc_iovm_area(obj, da, bytes, flags);
602 if (IS_ERR(new)) {
603 err = PTR_ERR(new);
604 goto err_alloc_iovma;
605 }
606 new->va = va;
607 new->sgt = sgt;
608
609 if (map_iovm_area(domain, new, sgt, new->flags))
610 goto err_map;
611
612 mutex_unlock(&obj->mmap_lock);
613
614 dev_dbg(obj->dev, "%s: da:%08x(%x) flags:%08x va:%p\n",
615 __func__, new->da_start, bytes, new->flags, va);
616
617 return new->da_start;
618
619err_map:
620 free_iovm_area(obj, new);
621err_alloc_iovma:
622 mutex_unlock(&obj->mmap_lock);
623 return err;
624}
625
626static inline u32 __iommu_vmap(struct iommu_domain *domain, struct iommu *obj,
627 u32 da, const struct sg_table *sgt,
628 void *va, size_t bytes, u32 flags)
629{
630 return map_iommu_region(domain, obj, da, sgt, va, bytes, flags);
631}
632
633/**
634 * iommu_vmap - (d)-(p)-(v) address mapper
635 * @obj: objective iommu
636 * @sgt: address of scatter gather table
637 * @flags: iovma and page property
638 *
639 * Creates 1-n-1 mapping with given @sgt and returns @da.
640 * All @sgt element must be io page size aligned.
641 */
642u32 iommu_vmap(struct iommu_domain *domain, struct iommu *obj, u32 da,
643 const struct sg_table *sgt, u32 flags)
644{
645 size_t bytes;
646 void *va = NULL;
647
648 if (!obj || !obj->dev || !sgt)
649 return -EINVAL;
650
651 bytes = sgtable_len(sgt);
652 if (!bytes)
653 return -EINVAL;
654 bytes = PAGE_ALIGN(bytes);
655
656 if (flags & IOVMF_MMIO) {
657 va = vmap_sg(sgt);
658 if (IS_ERR(va))
659 return PTR_ERR(va);
660 }
661
662 flags |= IOVMF_DISCONT;
663 flags |= IOVMF_MMIO;
664
665 da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags);
666 if (IS_ERR_VALUE(da))
667 vunmap_sg(va);
668
669 return da;
670}
671EXPORT_SYMBOL_GPL(iommu_vmap);
672
673/**
674 * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()'
675 * @obj: objective iommu
676 * @da: iommu device virtual address
677 *
678 * Free the iommu virtually contiguous memory area starting at
679 * @da, which was returned by 'iommu_vmap()'.
680 */
681struct sg_table *
682iommu_vunmap(struct iommu_domain *domain, struct iommu *obj, u32 da)
683{
684 struct sg_table *sgt;
685 /*
686 * 'sgt' is allocated before 'iommu_vmalloc()' is called.
687 * Just returns 'sgt' to the caller to free
688 */
689 sgt = unmap_vm_area(domain, obj, da, vunmap_sg,
690 IOVMF_DISCONT | IOVMF_MMIO);
691 if (!sgt)
692 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
693 return sgt;
694}
695EXPORT_SYMBOL_GPL(iommu_vunmap);
696
697/**
698 * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
699 * @obj: objective iommu
700 * @da: contiguous iommu virtual memory
701 * @bytes: allocation size
702 * @flags: iovma and page property
703 *
704 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
705 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set.
706 */
707u32 iommu_vmalloc(struct iommu_domain *domain, struct iommu *obj, u32 da,
708 size_t bytes, u32 flags)
709{
710 void *va;
711 struct sg_table *sgt;
712
713 if (!obj || !obj->dev || !bytes)
714 return -EINVAL;
715
716 bytes = PAGE_ALIGN(bytes);
717
718 va = vmalloc(bytes);
719 if (!va)
720 return -ENOMEM;
721
722 flags |= IOVMF_DISCONT;
723 flags |= IOVMF_ALLOC;
724
725 sgt = sgtable_alloc(bytes, flags, da, 0);
726 if (IS_ERR(sgt)) {
727 da = PTR_ERR(sgt);
728 goto err_sgt_alloc;
729 }
730 sgtable_fill_vmalloc(sgt, va);
731
732 da = __iommu_vmap(domain, obj, da, sgt, va, bytes, flags);
733 if (IS_ERR_VALUE(da))
734 goto err_iommu_vmap;
735
736 return da;
737
738err_iommu_vmap:
739 sgtable_drain_vmalloc(sgt);
740 sgtable_free(sgt);
741err_sgt_alloc:
742 vfree(va);
743 return da;
744}
745EXPORT_SYMBOL_GPL(iommu_vmalloc);
746
747/**
748 * iommu_vfree - release memory allocated by 'iommu_vmalloc()'
749 * @obj: objective iommu
750 * @da: iommu device virtual address
751 *
752 * Frees the iommu virtually continuous memory area starting at
753 * @da, as obtained from 'iommu_vmalloc()'.
754 */
755void iommu_vfree(struct iommu_domain *domain, struct iommu *obj, const u32 da)
756{
757 struct sg_table *sgt;
758
759 sgt = unmap_vm_area(domain, obj, da, vfree,
760 IOVMF_DISCONT | IOVMF_ALLOC);
761 if (!sgt)
762 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
763 sgtable_free(sgt);
764}
765EXPORT_SYMBOL_GPL(iommu_vfree);
766
767static u32 __iommu_kmap(struct iommu_domain *domain, struct iommu *obj,
768 u32 da, u32 pa, void *va, size_t bytes, u32 flags)
769{
770 struct sg_table *sgt;
771
772 sgt = sgtable_alloc(bytes, flags, da, pa);
773 if (IS_ERR(sgt))
774 return PTR_ERR(sgt);
775
776 sgtable_fill_kmalloc(sgt, pa, da, bytes);
777
778 da = map_iommu_region(domain, obj, da, sgt, va, bytes, flags);
779 if (IS_ERR_VALUE(da)) {
780 sgtable_drain_kmalloc(sgt);
781 sgtable_free(sgt);
782 }
783
784 return da;
785}
786
787/**
788 * iommu_kmap - (d)-(p)-(v) address mapper
789 * @obj: objective iommu
790 * @da: contiguous iommu virtual memory
791 * @pa: contiguous physical memory
792 * @flags: iovma and page property
793 *
794 * Creates 1-1-1 mapping and returns @da again, which can be
795 * adjusted if 'IOVMF_DA_FIXED' is not set.
796 */
797u32 iommu_kmap(struct iommu_domain *domain, struct iommu *obj, u32 da, u32 pa,
798 size_t bytes, u32 flags)
799{
800 void *va;
801
802 if (!obj || !obj->dev || !bytes)
803 return -EINVAL;
804
805 bytes = PAGE_ALIGN(bytes);
806
807 va = ioremap(pa, bytes);
808 if (!va)
809 return -ENOMEM;
810
811 flags |= IOVMF_LINEAR;
812 flags |= IOVMF_MMIO;
813
814 da = __iommu_kmap(domain, obj, da, pa, va, bytes, flags);
815 if (IS_ERR_VALUE(da))
816 iounmap(va);
817
818 return da;
819}
820EXPORT_SYMBOL_GPL(iommu_kmap);
821
822/**
823 * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()'
824 * @obj: objective iommu
825 * @da: iommu device virtual address
826 *
827 * Frees the iommu virtually contiguous memory area starting at
828 * @da, which was passed to and was returned by'iommu_kmap()'.
829 */
830void iommu_kunmap(struct iommu_domain *domain, struct iommu *obj, u32 da)
831{
832 struct sg_table *sgt;
833 typedef void (*func_t)(const void *);
834
835 sgt = unmap_vm_area(domain, obj, da, (func_t)iounmap,
836 IOVMF_LINEAR | IOVMF_MMIO);
837 if (!sgt)
838 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
839 sgtable_free(sgt);
840}
841EXPORT_SYMBOL_GPL(iommu_kunmap);
842
843/**
844 * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper
845 * @obj: objective iommu
846 * @da: contiguous iommu virtual memory
847 * @bytes: bytes for allocation
848 * @flags: iovma and page property
849 *
850 * Allocate @bytes linearly and creates 1-1-1 mapping and returns
851 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set.
852 */
853u32 iommu_kmalloc(struct iommu_domain *domain, struct iommu *obj, u32 da,
854 size_t bytes, u32 flags)
855{
856 void *va;
857 u32 pa;
858
859 if (!obj || !obj->dev || !bytes)
860 return -EINVAL;
861
862 bytes = PAGE_ALIGN(bytes);
863
864 va = kmalloc(bytes, GFP_KERNEL | GFP_DMA);
865 if (!va)
866 return -ENOMEM;
867 pa = virt_to_phys(va);
868
869 flags |= IOVMF_LINEAR;
870 flags |= IOVMF_ALLOC;
871
872 da = __iommu_kmap(domain, obj, da, pa, va, bytes, flags);
873 if (IS_ERR_VALUE(da))
874 kfree(va);
875
876 return da;
877}
878EXPORT_SYMBOL_GPL(iommu_kmalloc);
879
880/**
881 * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()'
882 * @obj: objective iommu
883 * @da: iommu device virtual address
884 *
885 * Frees the iommu virtually contiguous memory area starting at
886 * @da, which was passed to and was returned by'iommu_kmalloc()'.
887 */
888void iommu_kfree(struct iommu_domain *domain, struct iommu *obj, u32 da)
889{
890 struct sg_table *sgt;
891
892 sgt = unmap_vm_area(domain, obj, da, kfree, IOVMF_LINEAR | IOVMF_ALLOC);
893 if (!sgt)
894 dev_dbg(obj->dev, "%s: No sgt\n", __func__);
895 sgtable_free(sgt);
896}
897EXPORT_SYMBOL_GPL(iommu_kfree);
898
899
900static int __init iovmm_init(void)
901{
902 const unsigned long flags = SLAB_HWCACHE_ALIGN;
903 struct kmem_cache *p;
904
905 p = kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct), 0,
906 flags, NULL);
907 if (!p)
908 return -ENOMEM;
909 iovm_area_cachep = p;
910
911 return 0;
912}
913module_init(iovmm_init);
914
915static void __exit iovmm_exit(void)
916{
917 kmem_cache_destroy(iovm_area_cachep);
918}
919module_exit(iovmm_exit);
920
921MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
922MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
923MODULE_LICENSE("GPL v2");
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-23 21:39:35 -0500