diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2015-04-17 16:19:26 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2015-04-17 16:19:26 -0400 |
commit | e2fdae7e7c5a690b10b2d2891ec819e554dc033d (patch) | |
tree | 7b48f55eba9519aeee554a58be9d1ff8a8adffd6 /arch/sparc | |
parent | 6b6e177d632ee251c7c78d8f266a851ab9704879 (diff) | |
parent | cb97201cb060d13da0b87fd1bf68208c7389c5b1 (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc
Pull sparc updates from David Miller:
"The PowerPC folks have a really nice scalable IOMMU pool allocator
that we wanted to make use of for sparc. So here we have a series
that abstracts out their code into a common layer that anyone can make
use of.
Sparc is converted, and the PowerPC folks have reviewed and ACK'd this
series and plan to convert PowerPC over as well"
* git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc:
iommu-common: Fix PARISC compile-time warnings
sparc: Make LDC use common iommu poll management functions
sparc: Make sparc64 use scalable lib/iommu-common.c functions
sparc: Break up monolithic iommu table/lock into finer graularity pools and lock
Diffstat (limited to 'arch/sparc')
-rw-r--r-- | arch/sparc/include/asm/iommu_64.h | 7 | ||||
-rw-r--r-- | arch/sparc/kernel/iommu.c | 188 | ||||
-rw-r--r-- | arch/sparc/kernel/iommu_common.h | 8 | ||||
-rw-r--r-- | arch/sparc/kernel/ldc.c | 185 | ||||
-rw-r--r-- | arch/sparc/kernel/pci_sun4v.c | 193 |
5 files changed, 257 insertions, 324 deletions
diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h index 2b9321ab064d..e3cd4493d81d 100644 --- a/arch/sparc/include/asm/iommu_64.h +++ b/arch/sparc/include/asm/iommu_64.h | |||
@@ -16,6 +16,7 @@ | |||
16 | #define IOPTE_WRITE 0x0000000000000002UL | 16 | #define IOPTE_WRITE 0x0000000000000002UL |
17 | 17 | ||
18 | #define IOMMU_NUM_CTXS 4096 | 18 | #define IOMMU_NUM_CTXS 4096 |
19 | #include <linux/iommu-common.h> | ||
19 | 20 | ||
20 | struct iommu_arena { | 21 | struct iommu_arena { |
21 | unsigned long *map; | 22 | unsigned long *map; |
@@ -24,11 +25,10 @@ struct iommu_arena { | |||
24 | }; | 25 | }; |
25 | 26 | ||
26 | struct iommu { | 27 | struct iommu { |
28 | struct iommu_table tbl; | ||
27 | spinlock_t lock; | 29 | spinlock_t lock; |
28 | struct iommu_arena arena; | 30 | u32 dma_addr_mask; |
29 | void (*flush_all)(struct iommu *); | ||
30 | iopte_t *page_table; | 31 | iopte_t *page_table; |
31 | u32 page_table_map_base; | ||
32 | unsigned long iommu_control; | 32 | unsigned long iommu_control; |
33 | unsigned long iommu_tsbbase; | 33 | unsigned long iommu_tsbbase; |
34 | unsigned long iommu_flush; | 34 | unsigned long iommu_flush; |
@@ -40,7 +40,6 @@ struct iommu { | |||
40 | unsigned long dummy_page_pa; | 40 | unsigned long dummy_page_pa; |
41 | unsigned long ctx_lowest_free; | 41 | unsigned long ctx_lowest_free; |
42 | DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); | 42 | DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); |
43 | u32 dma_addr_mask; | ||
44 | }; | 43 | }; |
45 | 44 | ||
46 | struct strbuf { | 45 | struct strbuf { |
diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index bfa4d0c2df42..9b16b341b6ae 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c | |||
@@ -13,11 +13,15 @@ | |||
13 | #include <linux/errno.h> | 13 | #include <linux/errno.h> |
14 | #include <linux/iommu-helper.h> | 14 | #include <linux/iommu-helper.h> |
15 | #include <linux/bitmap.h> | 15 | #include <linux/bitmap.h> |
16 | #include <linux/hash.h> | ||
17 | #include <linux/iommu-common.h> | ||
16 | 18 | ||
17 | #ifdef CONFIG_PCI | 19 | #ifdef CONFIG_PCI |
18 | #include <linux/pci.h> | 20 | #include <linux/pci.h> |
19 | #endif | 21 | #endif |
20 | 22 | ||
23 | static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); | ||
24 | |||
21 | #include <asm/iommu.h> | 25 | #include <asm/iommu.h> |
22 | 26 | ||
23 | #include "iommu_common.h" | 27 | #include "iommu_common.h" |
@@ -45,8 +49,9 @@ | |||
45 | "i" (ASI_PHYS_BYPASS_EC_E)) | 49 | "i" (ASI_PHYS_BYPASS_EC_E)) |
46 | 50 | ||
47 | /* Must be invoked under the IOMMU lock. */ | 51 | /* Must be invoked under the IOMMU lock. */ |
48 | static void iommu_flushall(struct iommu *iommu) | 52 | static void iommu_flushall(struct iommu_table *iommu_table) |
49 | { | 53 | { |
54 | struct iommu *iommu = container_of(iommu_table, struct iommu, tbl); | ||
50 | if (iommu->iommu_flushinv) { | 55 | if (iommu->iommu_flushinv) { |
51 | iommu_write(iommu->iommu_flushinv, ~(u64)0); | 56 | iommu_write(iommu->iommu_flushinv, ~(u64)0); |
52 | } else { | 57 | } else { |
@@ -87,94 +92,23 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) | |||
87 | iopte_val(*iopte) = val; | 92 | iopte_val(*iopte) = val; |
88 | } | 93 | } |
89 | 94 | ||
90 | /* Based almost entirely upon the ppc64 iommu allocator. If you use the 'handle' | 95 | static struct iommu_tbl_ops iommu_sparc_ops = { |
91 | * facility it must all be done in one pass while under the iommu lock. | 96 | .reset = iommu_flushall |
92 | * | 97 | }; |
93 | * On sun4u platforms, we only flush the IOMMU once every time we've passed | ||
94 | * over the entire page table doing allocations. Therefore we only ever advance | ||
95 | * the hint and cannot backtrack it. | ||
96 | */ | ||
97 | unsigned long iommu_range_alloc(struct device *dev, | ||
98 | struct iommu *iommu, | ||
99 | unsigned long npages, | ||
100 | unsigned long *handle) | ||
101 | { | ||
102 | unsigned long n, end, start, limit, boundary_size; | ||
103 | struct iommu_arena *arena = &iommu->arena; | ||
104 | int pass = 0; | ||
105 | |||
106 | /* This allocator was derived from x86_64's bit string search */ | ||
107 | |||
108 | /* Sanity check */ | ||
109 | if (unlikely(npages == 0)) { | ||
110 | if (printk_ratelimit()) | ||
111 | WARN_ON(1); | ||
112 | return DMA_ERROR_CODE; | ||
113 | } | ||
114 | |||
115 | if (handle && *handle) | ||
116 | start = *handle; | ||
117 | else | ||
118 | start = arena->hint; | ||
119 | |||
120 | limit = arena->limit; | ||
121 | |||
122 | /* The case below can happen if we have a small segment appended | ||
123 | * to a large, or when the previous alloc was at the very end of | ||
124 | * the available space. If so, go back to the beginning and flush. | ||
125 | */ | ||
126 | if (start >= limit) { | ||
127 | start = 0; | ||
128 | if (iommu->flush_all) | ||
129 | iommu->flush_all(iommu); | ||
130 | } | ||
131 | |||
132 | again: | ||
133 | |||
134 | if (dev) | ||
135 | boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, | ||
136 | 1 << IO_PAGE_SHIFT); | ||
137 | else | ||
138 | boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT); | ||
139 | |||
140 | n = iommu_area_alloc(arena->map, limit, start, npages, | ||
141 | iommu->page_table_map_base >> IO_PAGE_SHIFT, | ||
142 | boundary_size >> IO_PAGE_SHIFT, 0); | ||
143 | if (n == -1) { | ||
144 | if (likely(pass < 1)) { | ||
145 | /* First failure, rescan from the beginning. */ | ||
146 | start = 0; | ||
147 | if (iommu->flush_all) | ||
148 | iommu->flush_all(iommu); | ||
149 | pass++; | ||
150 | goto again; | ||
151 | } else { | ||
152 | /* Second failure, give up */ | ||
153 | return DMA_ERROR_CODE; | ||
154 | } | ||
155 | } | ||
156 | |||
157 | end = n + npages; | ||
158 | |||
159 | arena->hint = end; | ||
160 | |||
161 | /* Update handle for SG allocations */ | ||
162 | if (handle) | ||
163 | *handle = end; | ||
164 | |||
165 | return n; | ||
166 | } | ||
167 | 98 | ||
168 | void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages) | 99 | static void setup_iommu_pool_hash(void) |
169 | { | 100 | { |
170 | struct iommu_arena *arena = &iommu->arena; | 101 | unsigned int i; |
171 | unsigned long entry; | 102 | static bool do_once; |
172 | |||
173 | entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; | ||
174 | 103 | ||
175 | bitmap_clear(arena->map, entry, npages); | 104 | if (do_once) |
105 | return; | ||
106 | do_once = true; | ||
107 | for_each_possible_cpu(i) | ||
108 | per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); | ||
176 | } | 109 | } |
177 | 110 | ||
111 | |||
178 | int iommu_table_init(struct iommu *iommu, int tsbsize, | 112 | int iommu_table_init(struct iommu *iommu, int tsbsize, |
179 | u32 dma_offset, u32 dma_addr_mask, | 113 | u32 dma_offset, u32 dma_addr_mask, |
180 | int numa_node) | 114 | int numa_node) |
@@ -187,22 +121,22 @@ int iommu_table_init(struct iommu *iommu, int tsbsize, | |||
187 | /* Setup initial software IOMMU state. */ | 121 | /* Setup initial software IOMMU state. */ |
188 | spin_lock_init(&iommu->lock); | 122 | spin_lock_init(&iommu->lock); |
189 | iommu->ctx_lowest_free = 1; | 123 | iommu->ctx_lowest_free = 1; |
190 | iommu->page_table_map_base = dma_offset; | 124 | iommu->tbl.page_table_map_base = dma_offset; |
191 | iommu->dma_addr_mask = dma_addr_mask; | 125 | iommu->dma_addr_mask = dma_addr_mask; |
192 | 126 | ||
193 | /* Allocate and initialize the free area map. */ | 127 | /* Allocate and initialize the free area map. */ |
194 | sz = num_tsb_entries / 8; | 128 | sz = num_tsb_entries / 8; |
195 | sz = (sz + 7UL) & ~7UL; | 129 | sz = (sz + 7UL) & ~7UL; |
196 | iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node); | 130 | iommu->tbl.map = kmalloc_node(sz, GFP_KERNEL, numa_node); |
197 | if (!iommu->arena.map) { | 131 | if (!iommu->tbl.map) |
198 | printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n"); | ||
199 | return -ENOMEM; | 132 | return -ENOMEM; |
200 | } | 133 | memset(iommu->tbl.map, 0, sz); |
201 | memset(iommu->arena.map, 0, sz); | ||
202 | iommu->arena.limit = num_tsb_entries; | ||
203 | |||
204 | if (tlb_type != hypervisor) | 134 | if (tlb_type != hypervisor) |
205 | iommu->flush_all = iommu_flushall; | 135 | iommu_sparc_ops.reset = NULL; /* not needed on on sun4v */ |
136 | |||
137 | setup_iommu_pool_hash(); | ||
138 | iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, | ||
139 | &iommu_sparc_ops, false, 1); | ||
206 | 140 | ||
207 | /* Allocate and initialize the dummy page which we | 141 | /* Allocate and initialize the dummy page which we |
208 | * set inactive IO PTEs to point to. | 142 | * set inactive IO PTEs to point to. |
@@ -235,18 +169,20 @@ out_free_dummy_page: | |||
235 | iommu->dummy_page = 0UL; | 169 | iommu->dummy_page = 0UL; |
236 | 170 | ||
237 | out_free_map: | 171 | out_free_map: |
238 | kfree(iommu->arena.map); | 172 | kfree(iommu->tbl.map); |
239 | iommu->arena.map = NULL; | 173 | iommu->tbl.map = NULL; |
240 | 174 | ||
241 | return -ENOMEM; | 175 | return -ENOMEM; |
242 | } | 176 | } |
243 | 177 | ||
244 | static inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu, | 178 | static inline iopte_t *alloc_npages(struct device *dev, |
179 | struct iommu *iommu, | ||
245 | unsigned long npages) | 180 | unsigned long npages) |
246 | { | 181 | { |
247 | unsigned long entry; | 182 | unsigned long entry; |
248 | 183 | ||
249 | entry = iommu_range_alloc(dev, iommu, npages, NULL); | 184 | entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, |
185 | __this_cpu_read(iommu_pool_hash)); | ||
250 | if (unlikely(entry == DMA_ERROR_CODE)) | 186 | if (unlikely(entry == DMA_ERROR_CODE)) |
251 | return NULL; | 187 | return NULL; |
252 | 188 | ||
@@ -284,7 +220,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, | |||
284 | dma_addr_t *dma_addrp, gfp_t gfp, | 220 | dma_addr_t *dma_addrp, gfp_t gfp, |
285 | struct dma_attrs *attrs) | 221 | struct dma_attrs *attrs) |
286 | { | 222 | { |
287 | unsigned long flags, order, first_page; | 223 | unsigned long order, first_page; |
288 | struct iommu *iommu; | 224 | struct iommu *iommu; |
289 | struct page *page; | 225 | struct page *page; |
290 | int npages, nid; | 226 | int npages, nid; |
@@ -306,16 +242,14 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, | |||
306 | 242 | ||
307 | iommu = dev->archdata.iommu; | 243 | iommu = dev->archdata.iommu; |
308 | 244 | ||
309 | spin_lock_irqsave(&iommu->lock, flags); | ||
310 | iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); | 245 | iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); |
311 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
312 | 246 | ||
313 | if (unlikely(iopte == NULL)) { | 247 | if (unlikely(iopte == NULL)) { |
314 | free_pages(first_page, order); | 248 | free_pages(first_page, order); |
315 | return NULL; | 249 | return NULL; |
316 | } | 250 | } |
317 | 251 | ||
318 | *dma_addrp = (iommu->page_table_map_base + | 252 | *dma_addrp = (iommu->tbl.page_table_map_base + |
319 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); | 253 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
320 | ret = (void *) first_page; | 254 | ret = (void *) first_page; |
321 | npages = size >> IO_PAGE_SHIFT; | 255 | npages = size >> IO_PAGE_SHIFT; |
@@ -336,16 +270,12 @@ static void dma_4u_free_coherent(struct device *dev, size_t size, | |||
336 | struct dma_attrs *attrs) | 270 | struct dma_attrs *attrs) |
337 | { | 271 | { |
338 | struct iommu *iommu; | 272 | struct iommu *iommu; |
339 | unsigned long flags, order, npages; | 273 | unsigned long order, npages; |
340 | 274 | ||
341 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | 275 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
342 | iommu = dev->archdata.iommu; | 276 | iommu = dev->archdata.iommu; |
343 | 277 | ||
344 | spin_lock_irqsave(&iommu->lock, flags); | 278 | iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL); |
345 | |||
346 | iommu_range_free(iommu, dvma, npages); | ||
347 | |||
348 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
349 | 279 | ||
350 | order = get_order(size); | 280 | order = get_order(size); |
351 | if (order < 10) | 281 | if (order < 10) |
@@ -375,8 +305,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, | |||
375 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | 305 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
376 | npages >>= IO_PAGE_SHIFT; | 306 | npages >>= IO_PAGE_SHIFT; |
377 | 307 | ||
378 | spin_lock_irqsave(&iommu->lock, flags); | ||
379 | base = alloc_npages(dev, iommu, npages); | 308 | base = alloc_npages(dev, iommu, npages); |
309 | spin_lock_irqsave(&iommu->lock, flags); | ||
380 | ctx = 0; | 310 | ctx = 0; |
381 | if (iommu->iommu_ctxflush) | 311 | if (iommu->iommu_ctxflush) |
382 | ctx = iommu_alloc_ctx(iommu); | 312 | ctx = iommu_alloc_ctx(iommu); |
@@ -385,7 +315,7 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, | |||
385 | if (unlikely(!base)) | 315 | if (unlikely(!base)) |
386 | goto bad; | 316 | goto bad; |
387 | 317 | ||
388 | bus_addr = (iommu->page_table_map_base + | 318 | bus_addr = (iommu->tbl.page_table_map_base + |
389 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); | 319 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
390 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | 320 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
391 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | 321 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
@@ -496,7 +426,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, | |||
496 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); | 426 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
497 | npages >>= IO_PAGE_SHIFT; | 427 | npages >>= IO_PAGE_SHIFT; |
498 | base = iommu->page_table + | 428 | base = iommu->page_table + |
499 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 429 | ((bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT); |
500 | bus_addr &= IO_PAGE_MASK; | 430 | bus_addr &= IO_PAGE_MASK; |
501 | 431 | ||
502 | spin_lock_irqsave(&iommu->lock, flags); | 432 | spin_lock_irqsave(&iommu->lock, flags); |
@@ -515,11 +445,11 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, | |||
515 | for (i = 0; i < npages; i++) | 445 | for (i = 0; i < npages; i++) |
516 | iopte_make_dummy(iommu, base + i); | 446 | iopte_make_dummy(iommu, base + i); |
517 | 447 | ||
518 | iommu_range_free(iommu, bus_addr, npages); | ||
519 | |||
520 | iommu_free_ctx(iommu, ctx); | 448 | iommu_free_ctx(iommu, ctx); |
521 | |||
522 | spin_unlock_irqrestore(&iommu->lock, flags); | 449 | spin_unlock_irqrestore(&iommu->lock, flags); |
450 | |||
451 | iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, | ||
452 | false, NULL); | ||
523 | } | 453 | } |
524 | 454 | ||
525 | static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, | 455 | static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, |
@@ -567,7 +497,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, | |||
567 | max_seg_size = dma_get_max_seg_size(dev); | 497 | max_seg_size = dma_get_max_seg_size(dev); |
568 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, | 498 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
569 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; | 499 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; |
570 | base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; | 500 | base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT; |
571 | for_each_sg(sglist, s, nelems, i) { | 501 | for_each_sg(sglist, s, nelems, i) { |
572 | unsigned long paddr, npages, entry, out_entry = 0, slen; | 502 | unsigned long paddr, npages, entry, out_entry = 0, slen; |
573 | iopte_t *base; | 503 | iopte_t *base; |
@@ -581,7 +511,8 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, | |||
581 | /* Allocate iommu entries for that segment */ | 511 | /* Allocate iommu entries for that segment */ |
582 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); | 512 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); |
583 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); | 513 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); |
584 | entry = iommu_range_alloc(dev, iommu, npages, &handle); | 514 | entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle, |
515 | __this_cpu_read(iommu_pool_hash)); | ||
585 | 516 | ||
586 | /* Handle failure */ | 517 | /* Handle failure */ |
587 | if (unlikely(entry == DMA_ERROR_CODE)) { | 518 | if (unlikely(entry == DMA_ERROR_CODE)) { |
@@ -594,7 +525,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, | |||
594 | base = iommu->page_table + entry; | 525 | base = iommu->page_table + entry; |
595 | 526 | ||
596 | /* Convert entry to a dma_addr_t */ | 527 | /* Convert entry to a dma_addr_t */ |
597 | dma_addr = iommu->page_table_map_base + | 528 | dma_addr = iommu->tbl.page_table_map_base + |
598 | (entry << IO_PAGE_SHIFT); | 529 | (entry << IO_PAGE_SHIFT); |
599 | dma_addr |= (s->offset & ~IO_PAGE_MASK); | 530 | dma_addr |= (s->offset & ~IO_PAGE_MASK); |
600 | 531 | ||
@@ -654,15 +585,17 @@ iommu_map_failed: | |||
654 | vaddr = s->dma_address & IO_PAGE_MASK; | 585 | vaddr = s->dma_address & IO_PAGE_MASK; |
655 | npages = iommu_num_pages(s->dma_address, s->dma_length, | 586 | npages = iommu_num_pages(s->dma_address, s->dma_length, |
656 | IO_PAGE_SIZE); | 587 | IO_PAGE_SIZE); |
657 | iommu_range_free(iommu, vaddr, npages); | ||
658 | 588 | ||
659 | entry = (vaddr - iommu->page_table_map_base) | 589 | entry = (vaddr - iommu->tbl.page_table_map_base) |
660 | >> IO_PAGE_SHIFT; | 590 | >> IO_PAGE_SHIFT; |
661 | base = iommu->page_table + entry; | 591 | base = iommu->page_table + entry; |
662 | 592 | ||
663 | for (j = 0; j < npages; j++) | 593 | for (j = 0; j < npages; j++) |
664 | iopte_make_dummy(iommu, base + j); | 594 | iopte_make_dummy(iommu, base + j); |
665 | 595 | ||
596 | iommu_tbl_range_free(&iommu->tbl, vaddr, npages, | ||
597 | false, NULL); | ||
598 | |||
666 | s->dma_address = DMA_ERROR_CODE; | 599 | s->dma_address = DMA_ERROR_CODE; |
667 | s->dma_length = 0; | 600 | s->dma_length = 0; |
668 | } | 601 | } |
@@ -677,17 +610,19 @@ iommu_map_failed: | |||
677 | /* If contexts are being used, they are the same in all of the mappings | 610 | /* If contexts are being used, they are the same in all of the mappings |
678 | * we make for a particular SG. | 611 | * we make for a particular SG. |
679 | */ | 612 | */ |
680 | static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) | 613 | static unsigned long fetch_sg_ctx(struct iommu *iommu, |
614 | struct scatterlist *sg) | ||
681 | { | 615 | { |
682 | unsigned long ctx = 0; | 616 | unsigned long ctx = 0; |
683 | 617 | ||
684 | if (iommu->iommu_ctxflush) { | 618 | if (iommu->iommu_ctxflush) { |
685 | iopte_t *base; | 619 | iopte_t *base; |
686 | u32 bus_addr; | 620 | u32 bus_addr; |
621 | struct iommu_table *tbl = &iommu->tbl; | ||
687 | 622 | ||
688 | bus_addr = sg->dma_address & IO_PAGE_MASK; | 623 | bus_addr = sg->dma_address & IO_PAGE_MASK; |
689 | base = iommu->page_table + | 624 | base = iommu->page_table + |
690 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 625 | ((bus_addr - tbl->page_table_map_base) >> IO_PAGE_SHIFT); |
691 | 626 | ||
692 | ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; | 627 | ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; |
693 | } | 628 | } |
@@ -723,9 +658,8 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
723 | if (!len) | 658 | if (!len) |
724 | break; | 659 | break; |
725 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); | 660 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); |
726 | iommu_range_free(iommu, dma_handle, npages); | ||
727 | 661 | ||
728 | entry = ((dma_handle - iommu->page_table_map_base) | 662 | entry = ((dma_handle - iommu->tbl.page_table_map_base) |
729 | >> IO_PAGE_SHIFT); | 663 | >> IO_PAGE_SHIFT); |
730 | base = iommu->page_table + entry; | 664 | base = iommu->page_table + entry; |
731 | 665 | ||
@@ -737,6 +671,8 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
737 | for (i = 0; i < npages; i++) | 671 | for (i = 0; i < npages; i++) |
738 | iopte_make_dummy(iommu, base + i); | 672 | iopte_make_dummy(iommu, base + i); |
739 | 673 | ||
674 | iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, false, | ||
675 | NULL); | ||
740 | sg = sg_next(sg); | 676 | sg = sg_next(sg); |
741 | } | 677 | } |
742 | 678 | ||
@@ -770,9 +706,10 @@ static void dma_4u_sync_single_for_cpu(struct device *dev, | |||
770 | if (iommu->iommu_ctxflush && | 706 | if (iommu->iommu_ctxflush && |
771 | strbuf->strbuf_ctxflush) { | 707 | strbuf->strbuf_ctxflush) { |
772 | iopte_t *iopte; | 708 | iopte_t *iopte; |
709 | struct iommu_table *tbl = &iommu->tbl; | ||
773 | 710 | ||
774 | iopte = iommu->page_table + | 711 | iopte = iommu->page_table + |
775 | ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT); | 712 | ((bus_addr - tbl->page_table_map_base)>>IO_PAGE_SHIFT); |
776 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; | 713 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; |
777 | } | 714 | } |
778 | 715 | ||
@@ -805,9 +742,10 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, | |||
805 | if (iommu->iommu_ctxflush && | 742 | if (iommu->iommu_ctxflush && |
806 | strbuf->strbuf_ctxflush) { | 743 | strbuf->strbuf_ctxflush) { |
807 | iopte_t *iopte; | 744 | iopte_t *iopte; |
745 | struct iommu_table *tbl = &iommu->tbl; | ||
808 | 746 | ||
809 | iopte = iommu->page_table + | 747 | iopte = iommu->page_table + ((sglist[0].dma_address - |
810 | ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 748 | tbl->page_table_map_base) >> IO_PAGE_SHIFT); |
811 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; | 749 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; |
812 | } | 750 | } |
813 | 751 | ||
diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h index 1ec0de4156e7..f4be0d724fc6 100644 --- a/arch/sparc/kernel/iommu_common.h +++ b/arch/sparc/kernel/iommu_common.h | |||
@@ -48,12 +48,4 @@ static inline int is_span_boundary(unsigned long entry, | |||
48 | return iommu_is_span_boundary(entry, nr, shift, boundary_size); | 48 | return iommu_is_span_boundary(entry, nr, shift, boundary_size); |
49 | } | 49 | } |
50 | 50 | ||
51 | unsigned long iommu_range_alloc(struct device *dev, | ||
52 | struct iommu *iommu, | ||
53 | unsigned long npages, | ||
54 | unsigned long *handle); | ||
55 | void iommu_range_free(struct iommu *iommu, | ||
56 | dma_addr_t dma_addr, | ||
57 | unsigned long npages); | ||
58 | |||
59 | #endif /* _IOMMU_COMMON_H */ | 51 | #endif /* _IOMMU_COMMON_H */ |
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c index 274a9f59d95c..d485697c37c0 100644 --- a/arch/sparc/kernel/ldc.c +++ b/arch/sparc/kernel/ldc.c | |||
@@ -15,6 +15,8 @@ | |||
15 | #include <linux/list.h> | 15 | #include <linux/list.h> |
16 | #include <linux/init.h> | 16 | #include <linux/init.h> |
17 | #include <linux/bitmap.h> | 17 | #include <linux/bitmap.h> |
18 | #include <linux/hash.h> | ||
19 | #include <linux/iommu-common.h> | ||
18 | 20 | ||
19 | #include <asm/hypervisor.h> | 21 | #include <asm/hypervisor.h> |
20 | #include <asm/iommu.h> | 22 | #include <asm/iommu.h> |
@@ -27,6 +29,11 @@ | |||
27 | #define DRV_MODULE_VERSION "1.1" | 29 | #define DRV_MODULE_VERSION "1.1" |
28 | #define DRV_MODULE_RELDATE "July 22, 2008" | 30 | #define DRV_MODULE_RELDATE "July 22, 2008" |
29 | 31 | ||
32 | #define COOKIE_PGSZ_CODE 0xf000000000000000ULL | ||
33 | #define COOKIE_PGSZ_CODE_SHIFT 60ULL | ||
34 | |||
35 | static DEFINE_PER_CPU(unsigned int, ldc_pool_hash); | ||
36 | |||
30 | static char version[] = | 37 | static char version[] = |
31 | DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; | 38 | DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; |
32 | #define LDC_PACKET_SIZE 64 | 39 | #define LDC_PACKET_SIZE 64 |
@@ -98,10 +105,10 @@ static const struct ldc_mode_ops stream_ops; | |||
98 | int ldom_domaining_enabled; | 105 | int ldom_domaining_enabled; |
99 | 106 | ||
100 | struct ldc_iommu { | 107 | struct ldc_iommu { |
101 | /* Protects arena alloc/free. */ | 108 | /* Protects ldc_unmap. */ |
102 | spinlock_t lock; | 109 | spinlock_t lock; |
103 | struct iommu_arena arena; | ||
104 | struct ldc_mtable_entry *page_table; | 110 | struct ldc_mtable_entry *page_table; |
111 | struct iommu_table iommu_table; | ||
105 | }; | 112 | }; |
106 | 113 | ||
107 | struct ldc_channel { | 114 | struct ldc_channel { |
@@ -998,31 +1005,85 @@ static void free_queue(unsigned long num_entries, struct ldc_packet *q) | |||
998 | free_pages((unsigned long)q, order); | 1005 | free_pages((unsigned long)q, order); |
999 | } | 1006 | } |
1000 | 1007 | ||
1008 | static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) | ||
1009 | { | ||
1010 | u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; | ||
1011 | /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ | ||
1012 | |||
1013 | cookie &= ~COOKIE_PGSZ_CODE; | ||
1014 | |||
1015 | return (cookie >> (13ULL + (szcode * 3ULL))); | ||
1016 | } | ||
1017 | |||
1018 | struct ldc_demap_arg { | ||
1019 | struct ldc_iommu *ldc_iommu; | ||
1020 | u64 cookie; | ||
1021 | unsigned long id; | ||
1022 | }; | ||
1023 | |||
1024 | static void ldc_demap(void *arg, unsigned long entry, unsigned long npages) | ||
1025 | { | ||
1026 | struct ldc_demap_arg *ldc_demap_arg = arg; | ||
1027 | struct ldc_iommu *iommu = ldc_demap_arg->ldc_iommu; | ||
1028 | unsigned long id = ldc_demap_arg->id; | ||
1029 | u64 cookie = ldc_demap_arg->cookie; | ||
1030 | struct ldc_mtable_entry *base; | ||
1031 | unsigned long i, shift; | ||
1032 | |||
1033 | shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; | ||
1034 | base = iommu->page_table + entry; | ||
1035 | for (i = 0; i < npages; i++) { | ||
1036 | if (base->cookie) | ||
1037 | sun4v_ldc_revoke(id, cookie + (i << shift), | ||
1038 | base->cookie); | ||
1039 | base->mte = 0; | ||
1040 | } | ||
1041 | } | ||
1042 | |||
1001 | /* XXX Make this configurable... XXX */ | 1043 | /* XXX Make this configurable... XXX */ |
1002 | #define LDC_IOTABLE_SIZE (8 * 1024) | 1044 | #define LDC_IOTABLE_SIZE (8 * 1024) |
1003 | 1045 | ||
1004 | static int ldc_iommu_init(struct ldc_channel *lp) | 1046 | struct iommu_tbl_ops ldc_iommu_ops = { |
1047 | .cookie_to_index = ldc_cookie_to_index, | ||
1048 | .demap = ldc_demap, | ||
1049 | }; | ||
1050 | |||
1051 | static void setup_ldc_pool_hash(void) | ||
1052 | { | ||
1053 | unsigned int i; | ||
1054 | static bool do_once; | ||
1055 | |||
1056 | if (do_once) | ||
1057 | return; | ||
1058 | do_once = true; | ||
1059 | for_each_possible_cpu(i) | ||
1060 | per_cpu(ldc_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); | ||
1061 | } | ||
1062 | |||
1063 | |||
1064 | static int ldc_iommu_init(const char *name, struct ldc_channel *lp) | ||
1005 | { | 1065 | { |
1006 | unsigned long sz, num_tsb_entries, tsbsize, order; | 1066 | unsigned long sz, num_tsb_entries, tsbsize, order; |
1007 | struct ldc_iommu *iommu = &lp->iommu; | 1067 | struct ldc_iommu *ldc_iommu = &lp->iommu; |
1068 | struct iommu_table *iommu = &ldc_iommu->iommu_table; | ||
1008 | struct ldc_mtable_entry *table; | 1069 | struct ldc_mtable_entry *table; |
1009 | unsigned long hv_err; | 1070 | unsigned long hv_err; |
1010 | int err; | 1071 | int err; |
1011 | 1072 | ||
1012 | num_tsb_entries = LDC_IOTABLE_SIZE; | 1073 | num_tsb_entries = LDC_IOTABLE_SIZE; |
1013 | tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); | 1074 | tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); |
1014 | 1075 | setup_ldc_pool_hash(); | |
1015 | spin_lock_init(&iommu->lock); | 1076 | spin_lock_init(&ldc_iommu->lock); |
1016 | 1077 | ||
1017 | sz = num_tsb_entries / 8; | 1078 | sz = num_tsb_entries / 8; |
1018 | sz = (sz + 7UL) & ~7UL; | 1079 | sz = (sz + 7UL) & ~7UL; |
1019 | iommu->arena.map = kzalloc(sz, GFP_KERNEL); | 1080 | iommu->map = kzalloc(sz, GFP_KERNEL); |
1020 | if (!iommu->arena.map) { | 1081 | if (!iommu->map) { |
1021 | printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); | 1082 | printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); |
1022 | return -ENOMEM; | 1083 | return -ENOMEM; |
1023 | } | 1084 | } |
1024 | 1085 | iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, | |
1025 | iommu->arena.limit = num_tsb_entries; | 1086 | &ldc_iommu_ops, false, 1); |
1026 | 1087 | ||
1027 | order = get_order(tsbsize); | 1088 | order = get_order(tsbsize); |
1028 | 1089 | ||
@@ -1037,7 +1098,7 @@ static int ldc_iommu_init(struct ldc_channel *lp) | |||
1037 | 1098 | ||
1038 | memset(table, 0, PAGE_SIZE << order); | 1099 | memset(table, 0, PAGE_SIZE << order); |
1039 | 1100 | ||
1040 | iommu->page_table = table; | 1101 | ldc_iommu->page_table = table; |
1041 | 1102 | ||
1042 | hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), | 1103 | hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), |
1043 | num_tsb_entries); | 1104 | num_tsb_entries); |
@@ -1049,31 +1110,32 @@ static int ldc_iommu_init(struct ldc_channel *lp) | |||
1049 | 1110 | ||
1050 | out_free_table: | 1111 | out_free_table: |
1051 | free_pages((unsigned long) table, order); | 1112 | free_pages((unsigned long) table, order); |
1052 | iommu->page_table = NULL; | 1113 | ldc_iommu->page_table = NULL; |
1053 | 1114 | ||
1054 | out_free_map: | 1115 | out_free_map: |
1055 | kfree(iommu->arena.map); | 1116 | kfree(iommu->map); |
1056 | iommu->arena.map = NULL; | 1117 | iommu->map = NULL; |
1057 | 1118 | ||
1058 | return err; | 1119 | return err; |
1059 | } | 1120 | } |
1060 | 1121 | ||
1061 | static void ldc_iommu_release(struct ldc_channel *lp) | 1122 | static void ldc_iommu_release(struct ldc_channel *lp) |
1062 | { | 1123 | { |
1063 | struct ldc_iommu *iommu = &lp->iommu; | 1124 | struct ldc_iommu *ldc_iommu = &lp->iommu; |
1125 | struct iommu_table *iommu = &ldc_iommu->iommu_table; | ||
1064 | unsigned long num_tsb_entries, tsbsize, order; | 1126 | unsigned long num_tsb_entries, tsbsize, order; |
1065 | 1127 | ||
1066 | (void) sun4v_ldc_set_map_table(lp->id, 0, 0); | 1128 | (void) sun4v_ldc_set_map_table(lp->id, 0, 0); |
1067 | 1129 | ||
1068 | num_tsb_entries = iommu->arena.limit; | 1130 | num_tsb_entries = iommu->poolsize * iommu->nr_pools; |
1069 | tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); | 1131 | tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); |
1070 | order = get_order(tsbsize); | 1132 | order = get_order(tsbsize); |
1071 | 1133 | ||
1072 | free_pages((unsigned long) iommu->page_table, order); | 1134 | free_pages((unsigned long) ldc_iommu->page_table, order); |
1073 | iommu->page_table = NULL; | 1135 | ldc_iommu->page_table = NULL; |
1074 | 1136 | ||
1075 | kfree(iommu->arena.map); | 1137 | kfree(iommu->map); |
1076 | iommu->arena.map = NULL; | 1138 | iommu->map = NULL; |
1077 | } | 1139 | } |
1078 | 1140 | ||
1079 | struct ldc_channel *ldc_alloc(unsigned long id, | 1141 | struct ldc_channel *ldc_alloc(unsigned long id, |
@@ -1140,7 +1202,7 @@ struct ldc_channel *ldc_alloc(unsigned long id, | |||
1140 | 1202 | ||
1141 | lp->id = id; | 1203 | lp->id = id; |
1142 | 1204 | ||
1143 | err = ldc_iommu_init(lp); | 1205 | err = ldc_iommu_init(name, lp); |
1144 | if (err) | 1206 | if (err) |
1145 | goto out_free_ldc; | 1207 | goto out_free_ldc; |
1146 | 1208 | ||
@@ -1885,40 +1947,6 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) | |||
1885 | } | 1947 | } |
1886 | EXPORT_SYMBOL(ldc_read); | 1948 | EXPORT_SYMBOL(ldc_read); |
1887 | 1949 | ||
1888 | static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages) | ||
1889 | { | ||
1890 | struct iommu_arena *arena = &iommu->arena; | ||
1891 | unsigned long n, start, end, limit; | ||
1892 | int pass; | ||
1893 | |||
1894 | limit = arena->limit; | ||
1895 | start = arena->hint; | ||
1896 | pass = 0; | ||
1897 | |||
1898 | again: | ||
1899 | n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0); | ||
1900 | end = n + npages; | ||
1901 | if (unlikely(end >= limit)) { | ||
1902 | if (likely(pass < 1)) { | ||
1903 | limit = start; | ||
1904 | start = 0; | ||
1905 | pass++; | ||
1906 | goto again; | ||
1907 | } else { | ||
1908 | /* Scanned the whole thing, give up. */ | ||
1909 | return -1; | ||
1910 | } | ||
1911 | } | ||
1912 | bitmap_set(arena->map, n, npages); | ||
1913 | |||
1914 | arena->hint = end; | ||
1915 | |||
1916 | return n; | ||
1917 | } | ||
1918 | |||
1919 | #define COOKIE_PGSZ_CODE 0xf000000000000000ULL | ||
1920 | #define COOKIE_PGSZ_CODE_SHIFT 60ULL | ||
1921 | |||
1922 | static u64 pagesize_code(void) | 1950 | static u64 pagesize_code(void) |
1923 | { | 1951 | { |
1924 | switch (PAGE_SIZE) { | 1952 | switch (PAGE_SIZE) { |
@@ -1945,23 +1973,14 @@ static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) | |||
1945 | page_offset); | 1973 | page_offset); |
1946 | } | 1974 | } |
1947 | 1975 | ||
1948 | static u64 cookie_to_index(u64 cookie, unsigned long *shift) | ||
1949 | { | ||
1950 | u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; | ||
1951 | |||
1952 | cookie &= ~COOKIE_PGSZ_CODE; | ||
1953 | |||
1954 | *shift = szcode * 3; | ||
1955 | |||
1956 | return (cookie >> (13ULL + (szcode * 3ULL))); | ||
1957 | } | ||
1958 | 1976 | ||
1959 | static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, | 1977 | static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, |
1960 | unsigned long npages) | 1978 | unsigned long npages) |
1961 | { | 1979 | { |
1962 | long entry; | 1980 | long entry; |
1963 | 1981 | ||
1964 | entry = arena_alloc(iommu, npages); | 1982 | entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_table, npages, |
1983 | NULL, __this_cpu_read(ldc_pool_hash)); | ||
1965 | if (unlikely(entry < 0)) | 1984 | if (unlikely(entry < 0)) |
1966 | return NULL; | 1985 | return NULL; |
1967 | 1986 | ||
@@ -2090,7 +2109,7 @@ int ldc_map_sg(struct ldc_channel *lp, | |||
2090 | struct ldc_trans_cookie *cookies, int ncookies, | 2109 | struct ldc_trans_cookie *cookies, int ncookies, |
2091 | unsigned int map_perm) | 2110 | unsigned int map_perm) |
2092 | { | 2111 | { |
2093 | unsigned long i, npages, flags; | 2112 | unsigned long i, npages; |
2094 | struct ldc_mtable_entry *base; | 2113 | struct ldc_mtable_entry *base; |
2095 | struct cookie_state state; | 2114 | struct cookie_state state; |
2096 | struct ldc_iommu *iommu; | 2115 | struct ldc_iommu *iommu; |
@@ -2109,9 +2128,7 @@ int ldc_map_sg(struct ldc_channel *lp, | |||
2109 | 2128 | ||
2110 | iommu = &lp->iommu; | 2129 | iommu = &lp->iommu; |
2111 | 2130 | ||
2112 | spin_lock_irqsave(&iommu->lock, flags); | ||
2113 | base = alloc_npages(iommu, npages); | 2131 | base = alloc_npages(iommu, npages); |
2114 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
2115 | 2132 | ||
2116 | if (!base) | 2133 | if (!base) |
2117 | return -ENOMEM; | 2134 | return -ENOMEM; |
@@ -2136,7 +2153,7 @@ int ldc_map_single(struct ldc_channel *lp, | |||
2136 | struct ldc_trans_cookie *cookies, int ncookies, | 2153 | struct ldc_trans_cookie *cookies, int ncookies, |
2137 | unsigned int map_perm) | 2154 | unsigned int map_perm) |
2138 | { | 2155 | { |
2139 | unsigned long npages, pa, flags; | 2156 | unsigned long npages, pa; |
2140 | struct ldc_mtable_entry *base; | 2157 | struct ldc_mtable_entry *base; |
2141 | struct cookie_state state; | 2158 | struct cookie_state state; |
2142 | struct ldc_iommu *iommu; | 2159 | struct ldc_iommu *iommu; |
@@ -2152,9 +2169,7 @@ int ldc_map_single(struct ldc_channel *lp, | |||
2152 | 2169 | ||
2153 | iommu = &lp->iommu; | 2170 | iommu = &lp->iommu; |
2154 | 2171 | ||
2155 | spin_lock_irqsave(&iommu->lock, flags); | ||
2156 | base = alloc_npages(iommu, npages); | 2172 | base = alloc_npages(iommu, npages); |
2157 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
2158 | 2173 | ||
2159 | if (!base) | 2174 | if (!base) |
2160 | return -ENOMEM; | 2175 | return -ENOMEM; |
@@ -2172,35 +2187,29 @@ int ldc_map_single(struct ldc_channel *lp, | |||
2172 | } | 2187 | } |
2173 | EXPORT_SYMBOL(ldc_map_single); | 2188 | EXPORT_SYMBOL(ldc_map_single); |
2174 | 2189 | ||
2190 | |||
2175 | static void free_npages(unsigned long id, struct ldc_iommu *iommu, | 2191 | static void free_npages(unsigned long id, struct ldc_iommu *iommu, |
2176 | u64 cookie, u64 size) | 2192 | u64 cookie, u64 size) |
2177 | { | 2193 | { |
2178 | struct iommu_arena *arena = &iommu->arena; | 2194 | unsigned long npages; |
2179 | unsigned long i, shift, index, npages; | 2195 | struct ldc_demap_arg demap_arg; |
2180 | struct ldc_mtable_entry *base; | ||
2181 | 2196 | ||
2182 | npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; | 2197 | demap_arg.ldc_iommu = iommu; |
2183 | index = cookie_to_index(cookie, &shift); | 2198 | demap_arg.cookie = cookie; |
2184 | base = iommu->page_table + index; | 2199 | demap_arg.id = id; |
2185 | 2200 | ||
2186 | BUG_ON(index > arena->limit || | 2201 | npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; |
2187 | (index + npages) > arena->limit); | 2202 | iommu_tbl_range_free(&iommu->iommu_table, cookie, npages, true, |
2203 | &demap_arg); | ||
2188 | 2204 | ||
2189 | for (i = 0; i < npages; i++) { | ||
2190 | if (base->cookie) | ||
2191 | sun4v_ldc_revoke(id, cookie + (i << shift), | ||
2192 | base->cookie); | ||
2193 | base->mte = 0; | ||
2194 | __clear_bit(index + i, arena->map); | ||
2195 | } | ||
2196 | } | 2205 | } |
2197 | 2206 | ||
2198 | void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, | 2207 | void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, |
2199 | int ncookies) | 2208 | int ncookies) |
2200 | { | 2209 | { |
2201 | struct ldc_iommu *iommu = &lp->iommu; | 2210 | struct ldc_iommu *iommu = &lp->iommu; |
2202 | unsigned long flags; | ||
2203 | int i; | 2211 | int i; |
2212 | unsigned long flags; | ||
2204 | 2213 | ||
2205 | spin_lock_irqsave(&iommu->lock, flags); | 2214 | spin_lock_irqsave(&iommu->lock, flags); |
2206 | for (i = 0; i < ncookies; i++) { | 2215 | for (i = 0; i < ncookies; i++) { |
diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 47ddbd496a1e..9b76b9d639e1 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c | |||
@@ -15,6 +15,8 @@ | |||
15 | #include <linux/export.h> | 15 | #include <linux/export.h> |
16 | #include <linux/log2.h> | 16 | #include <linux/log2.h> |
17 | #include <linux/of_device.h> | 17 | #include <linux/of_device.h> |
18 | #include <linux/hash.h> | ||
19 | #include <linux/iommu-common.h> | ||
18 | 20 | ||
19 | #include <asm/iommu.h> | 21 | #include <asm/iommu.h> |
20 | #include <asm/irq.h> | 22 | #include <asm/irq.h> |
@@ -28,6 +30,7 @@ | |||
28 | 30 | ||
29 | #define DRIVER_NAME "pci_sun4v" | 31 | #define DRIVER_NAME "pci_sun4v" |
30 | #define PFX DRIVER_NAME ": " | 32 | #define PFX DRIVER_NAME ": " |
33 | static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); | ||
31 | 34 | ||
32 | static unsigned long vpci_major = 1; | 35 | static unsigned long vpci_major = 1; |
33 | static unsigned long vpci_minor = 1; | 36 | static unsigned long vpci_minor = 1; |
@@ -155,14 +158,13 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, | |||
155 | 158 | ||
156 | iommu = dev->archdata.iommu; | 159 | iommu = dev->archdata.iommu; |
157 | 160 | ||
158 | spin_lock_irqsave(&iommu->lock, flags); | 161 | entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, |
159 | entry = iommu_range_alloc(dev, iommu, npages, NULL); | 162 | __this_cpu_read(iommu_pool_hash)); |
160 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
161 | 163 | ||
162 | if (unlikely(entry == DMA_ERROR_CODE)) | 164 | if (unlikely(entry == DMA_ERROR_CODE)) |
163 | goto range_alloc_fail; | 165 | goto range_alloc_fail; |
164 | 166 | ||
165 | *dma_addrp = (iommu->page_table_map_base + | 167 | *dma_addrp = (iommu->tbl.page_table_map_base + |
166 | (entry << IO_PAGE_SHIFT)); | 168 | (entry << IO_PAGE_SHIFT)); |
167 | ret = (void *) first_page; | 169 | ret = (void *) first_page; |
168 | first_page = __pa(first_page); | 170 | first_page = __pa(first_page); |
@@ -188,45 +190,46 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, | |||
188 | return ret; | 190 | return ret; |
189 | 191 | ||
190 | iommu_map_fail: | 192 | iommu_map_fail: |
191 | /* Interrupts are disabled. */ | 193 | iommu_tbl_range_free(&iommu->tbl, *dma_addrp, npages, false, NULL); |
192 | spin_lock(&iommu->lock); | ||
193 | iommu_range_free(iommu, *dma_addrp, npages); | ||
194 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
195 | 194 | ||
196 | range_alloc_fail: | 195 | range_alloc_fail: |
197 | free_pages(first_page, order); | 196 | free_pages(first_page, order); |
198 | return NULL; | 197 | return NULL; |
199 | } | 198 | } |
200 | 199 | ||
200 | static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry, | ||
201 | unsigned long npages) | ||
202 | { | ||
203 | u32 devhandle = *(u32 *)demap_arg; | ||
204 | unsigned long num, flags; | ||
205 | |||
206 | local_irq_save(flags); | ||
207 | do { | ||
208 | num = pci_sun4v_iommu_demap(devhandle, | ||
209 | HV_PCI_TSBID(0, entry), | ||
210 | npages); | ||
211 | |||
212 | entry += num; | ||
213 | npages -= num; | ||
214 | } while (npages != 0); | ||
215 | local_irq_restore(flags); | ||
216 | } | ||
217 | |||
201 | static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, | 218 | static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, |
202 | dma_addr_t dvma, struct dma_attrs *attrs) | 219 | dma_addr_t dvma, struct dma_attrs *attrs) |
203 | { | 220 | { |
204 | struct pci_pbm_info *pbm; | 221 | struct pci_pbm_info *pbm; |
205 | struct iommu *iommu; | 222 | struct iommu *iommu; |
206 | unsigned long flags, order, npages, entry; | 223 | unsigned long order, npages, entry; |
207 | u32 devhandle; | 224 | u32 devhandle; |
208 | 225 | ||
209 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | 226 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
210 | iommu = dev->archdata.iommu; | 227 | iommu = dev->archdata.iommu; |
211 | pbm = dev->archdata.host_controller; | 228 | pbm = dev->archdata.host_controller; |
212 | devhandle = pbm->devhandle; | 229 | devhandle = pbm->devhandle; |
213 | entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 230 | entry = ((dvma - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT); |
214 | 231 | dma_4v_iommu_demap(&devhandle, entry, npages); | |
215 | spin_lock_irqsave(&iommu->lock, flags); | 232 | iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL); |
216 | |||
217 | iommu_range_free(iommu, dvma, npages); | ||
218 | |||
219 | do { | ||
220 | unsigned long num; | ||
221 | |||
222 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | ||
223 | npages); | ||
224 | entry += num; | ||
225 | npages -= num; | ||
226 | } while (npages != 0); | ||
227 | |||
228 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
229 | |||
230 | order = get_order(size); | 233 | order = get_order(size); |
231 | if (order < 10) | 234 | if (order < 10) |
232 | free_pages((unsigned long)cpu, order); | 235 | free_pages((unsigned long)cpu, order); |
@@ -253,14 +256,13 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, | |||
253 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | 256 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
254 | npages >>= IO_PAGE_SHIFT; | 257 | npages >>= IO_PAGE_SHIFT; |
255 | 258 | ||
256 | spin_lock_irqsave(&iommu->lock, flags); | 259 | entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, |
257 | entry = iommu_range_alloc(dev, iommu, npages, NULL); | 260 | __this_cpu_read(iommu_pool_hash)); |
258 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
259 | 261 | ||
260 | if (unlikely(entry == DMA_ERROR_CODE)) | 262 | if (unlikely(entry == DMA_ERROR_CODE)) |
261 | goto bad; | 263 | goto bad; |
262 | 264 | ||
263 | bus_addr = (iommu->page_table_map_base + | 265 | bus_addr = (iommu->tbl.page_table_map_base + |
264 | (entry << IO_PAGE_SHIFT)); | 266 | (entry << IO_PAGE_SHIFT)); |
265 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | 267 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
266 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | 268 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
@@ -290,11 +292,7 @@ bad: | |||
290 | return DMA_ERROR_CODE; | 292 | return DMA_ERROR_CODE; |
291 | 293 | ||
292 | iommu_map_fail: | 294 | iommu_map_fail: |
293 | /* Interrupts are disabled. */ | 295 | iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL); |
294 | spin_lock(&iommu->lock); | ||
295 | iommu_range_free(iommu, bus_addr, npages); | ||
296 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
297 | |||
298 | return DMA_ERROR_CODE; | 296 | return DMA_ERROR_CODE; |
299 | } | 297 | } |
300 | 298 | ||
@@ -304,7 +302,7 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, | |||
304 | { | 302 | { |
305 | struct pci_pbm_info *pbm; | 303 | struct pci_pbm_info *pbm; |
306 | struct iommu *iommu; | 304 | struct iommu *iommu; |
307 | unsigned long flags, npages; | 305 | unsigned long npages; |
308 | long entry; | 306 | long entry; |
309 | u32 devhandle; | 307 | u32 devhandle; |
310 | 308 | ||
@@ -321,22 +319,9 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, | |||
321 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); | 319 | npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
322 | npages >>= IO_PAGE_SHIFT; | 320 | npages >>= IO_PAGE_SHIFT; |
323 | bus_addr &= IO_PAGE_MASK; | 321 | bus_addr &= IO_PAGE_MASK; |
324 | 322 | entry = (bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT; | |
325 | spin_lock_irqsave(&iommu->lock, flags); | 323 | dma_4v_iommu_demap(&devhandle, entry, npages); |
326 | 324 | iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL); | |
327 | iommu_range_free(iommu, bus_addr, npages); | ||
328 | |||
329 | entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; | ||
330 | do { | ||
331 | unsigned long num; | ||
332 | |||
333 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | ||
334 | npages); | ||
335 | entry += num; | ||
336 | npages -= num; | ||
337 | } while (npages != 0); | ||
338 | |||
339 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
340 | } | 325 | } |
341 | 326 | ||
342 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | 327 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, |
@@ -371,14 +356,14 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | |||
371 | /* Init first segment length for backout at failure */ | 356 | /* Init first segment length for backout at failure */ |
372 | outs->dma_length = 0; | 357 | outs->dma_length = 0; |
373 | 358 | ||
374 | spin_lock_irqsave(&iommu->lock, flags); | 359 | local_irq_save(flags); |
375 | 360 | ||
376 | iommu_batch_start(dev, prot, ~0UL); | 361 | iommu_batch_start(dev, prot, ~0UL); |
377 | 362 | ||
378 | max_seg_size = dma_get_max_seg_size(dev); | 363 | max_seg_size = dma_get_max_seg_size(dev); |
379 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, | 364 | seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
380 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; | 365 | IO_PAGE_SIZE) >> IO_PAGE_SHIFT; |
381 | base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; | 366 | base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT; |
382 | for_each_sg(sglist, s, nelems, i) { | 367 | for_each_sg(sglist, s, nelems, i) { |
383 | unsigned long paddr, npages, entry, out_entry = 0, slen; | 368 | unsigned long paddr, npages, entry, out_entry = 0, slen; |
384 | 369 | ||
@@ -391,7 +376,8 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | |||
391 | /* Allocate iommu entries for that segment */ | 376 | /* Allocate iommu entries for that segment */ |
392 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); | 377 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); |
393 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); | 378 | npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); |
394 | entry = iommu_range_alloc(dev, iommu, npages, &handle); | 379 | entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle, |
380 | __this_cpu_read(iommu_pool_hash)); | ||
395 | 381 | ||
396 | /* Handle failure */ | 382 | /* Handle failure */ |
397 | if (unlikely(entry == DMA_ERROR_CODE)) { | 383 | if (unlikely(entry == DMA_ERROR_CODE)) { |
@@ -404,7 +390,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | |||
404 | iommu_batch_new_entry(entry); | 390 | iommu_batch_new_entry(entry); |
405 | 391 | ||
406 | /* Convert entry to a dma_addr_t */ | 392 | /* Convert entry to a dma_addr_t */ |
407 | dma_addr = iommu->page_table_map_base + | 393 | dma_addr = iommu->tbl.page_table_map_base + |
408 | (entry << IO_PAGE_SHIFT); | 394 | (entry << IO_PAGE_SHIFT); |
409 | dma_addr |= (s->offset & ~IO_PAGE_MASK); | 395 | dma_addr |= (s->offset & ~IO_PAGE_MASK); |
410 | 396 | ||
@@ -451,7 +437,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | |||
451 | if (unlikely(err < 0L)) | 437 | if (unlikely(err < 0L)) |
452 | goto iommu_map_failed; | 438 | goto iommu_map_failed; |
453 | 439 | ||
454 | spin_unlock_irqrestore(&iommu->lock, flags); | 440 | local_irq_restore(flags); |
455 | 441 | ||
456 | if (outcount < incount) { | 442 | if (outcount < incount) { |
457 | outs = sg_next(outs); | 443 | outs = sg_next(outs); |
@@ -469,7 +455,8 @@ iommu_map_failed: | |||
469 | vaddr = s->dma_address & IO_PAGE_MASK; | 455 | vaddr = s->dma_address & IO_PAGE_MASK; |
470 | npages = iommu_num_pages(s->dma_address, s->dma_length, | 456 | npages = iommu_num_pages(s->dma_address, s->dma_length, |
471 | IO_PAGE_SIZE); | 457 | IO_PAGE_SIZE); |
472 | iommu_range_free(iommu, vaddr, npages); | 458 | iommu_tbl_range_free(&iommu->tbl, vaddr, npages, |
459 | false, NULL); | ||
473 | /* XXX demap? XXX */ | 460 | /* XXX demap? XXX */ |
474 | s->dma_address = DMA_ERROR_CODE; | 461 | s->dma_address = DMA_ERROR_CODE; |
475 | s->dma_length = 0; | 462 | s->dma_length = 0; |
@@ -477,7 +464,7 @@ iommu_map_failed: | |||
477 | if (s == outs) | 464 | if (s == outs) |
478 | break; | 465 | break; |
479 | } | 466 | } |
480 | spin_unlock_irqrestore(&iommu->lock, flags); | 467 | local_irq_restore(flags); |
481 | 468 | ||
482 | return 0; | 469 | return 0; |
483 | } | 470 | } |
@@ -489,7 +476,7 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
489 | struct pci_pbm_info *pbm; | 476 | struct pci_pbm_info *pbm; |
490 | struct scatterlist *sg; | 477 | struct scatterlist *sg; |
491 | struct iommu *iommu; | 478 | struct iommu *iommu; |
492 | unsigned long flags; | 479 | unsigned long flags, entry; |
493 | u32 devhandle; | 480 | u32 devhandle; |
494 | 481 | ||
495 | BUG_ON(direction == DMA_NONE); | 482 | BUG_ON(direction == DMA_NONE); |
@@ -498,33 +485,27 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
498 | pbm = dev->archdata.host_controller; | 485 | pbm = dev->archdata.host_controller; |
499 | devhandle = pbm->devhandle; | 486 | devhandle = pbm->devhandle; |
500 | 487 | ||
501 | spin_lock_irqsave(&iommu->lock, flags); | 488 | local_irq_save(flags); |
502 | 489 | ||
503 | sg = sglist; | 490 | sg = sglist; |
504 | while (nelems--) { | 491 | while (nelems--) { |
505 | dma_addr_t dma_handle = sg->dma_address; | 492 | dma_addr_t dma_handle = sg->dma_address; |
506 | unsigned int len = sg->dma_length; | 493 | unsigned int len = sg->dma_length; |
507 | unsigned long npages, entry; | 494 | unsigned long npages; |
495 | struct iommu_table *tbl = &iommu->tbl; | ||
496 | unsigned long shift = IO_PAGE_SHIFT; | ||
508 | 497 | ||
509 | if (!len) | 498 | if (!len) |
510 | break; | 499 | break; |
511 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); | 500 | npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); |
512 | iommu_range_free(iommu, dma_handle, npages); | 501 | entry = ((dma_handle - tbl->page_table_map_base) >> shift); |
513 | 502 | dma_4v_iommu_demap(&devhandle, entry, npages); | |
514 | entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 503 | iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, |
515 | while (npages) { | 504 | false, NULL); |
516 | unsigned long num; | ||
517 | |||
518 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | ||
519 | npages); | ||
520 | entry += num; | ||
521 | npages -= num; | ||
522 | } | ||
523 | |||
524 | sg = sg_next(sg); | 505 | sg = sg_next(sg); |
525 | } | 506 | } |
526 | 507 | ||
527 | spin_unlock_irqrestore(&iommu->lock, flags); | 508 | local_irq_restore(flags); |
528 | } | 509 | } |
529 | 510 | ||
530 | static struct dma_map_ops sun4v_dma_ops = { | 511 | static struct dma_map_ops sun4v_dma_ops = { |
@@ -536,6 +517,8 @@ static struct dma_map_ops sun4v_dma_ops = { | |||
536 | .unmap_sg = dma_4v_unmap_sg, | 517 | .unmap_sg = dma_4v_unmap_sg, |
537 | }; | 518 | }; |
538 | 519 | ||
520 | static struct iommu_tbl_ops dma_4v_iommu_ops; | ||
521 | |||
539 | static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) | 522 | static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) |
540 | { | 523 | { |
541 | struct property *prop; | 524 | struct property *prop; |
@@ -550,30 +533,33 @@ static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) | |||
550 | } | 533 | } |
551 | 534 | ||
552 | static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, | 535 | static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, |
553 | struct iommu *iommu) | 536 | struct iommu_table *iommu) |
554 | { | 537 | { |
555 | struct iommu_arena *arena = &iommu->arena; | 538 | struct iommu_pool *pool; |
556 | unsigned long i, cnt = 0; | 539 | unsigned long i, pool_nr, cnt = 0; |
557 | u32 devhandle; | 540 | u32 devhandle; |
558 | 541 | ||
559 | devhandle = pbm->devhandle; | 542 | devhandle = pbm->devhandle; |
560 | for (i = 0; i < arena->limit; i++) { | 543 | for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { |
561 | unsigned long ret, io_attrs, ra; | 544 | pool = &(iommu->arena_pool[pool_nr]); |
562 | 545 | for (i = pool->start; i <= pool->end; i++) { | |
563 | ret = pci_sun4v_iommu_getmap(devhandle, | 546 | unsigned long ret, io_attrs, ra; |
564 | HV_PCI_TSBID(0, i), | 547 | |
565 | &io_attrs, &ra); | 548 | ret = pci_sun4v_iommu_getmap(devhandle, |
566 | if (ret == HV_EOK) { | 549 | HV_PCI_TSBID(0, i), |
567 | if (page_in_phys_avail(ra)) { | 550 | &io_attrs, &ra); |
568 | pci_sun4v_iommu_demap(devhandle, | 551 | if (ret == HV_EOK) { |
569 | HV_PCI_TSBID(0, i), 1); | 552 | if (page_in_phys_avail(ra)) { |
570 | } else { | 553 | pci_sun4v_iommu_demap(devhandle, |
571 | cnt++; | 554 | HV_PCI_TSBID(0, |
572 | __set_bit(i, arena->map); | 555 | i), 1); |
556 | } else { | ||
557 | cnt++; | ||
558 | __set_bit(i, iommu->map); | ||
559 | } | ||
573 | } | 560 | } |
574 | } | 561 | } |
575 | } | 562 | } |
576 | |||
577 | return cnt; | 563 | return cnt; |
578 | } | 564 | } |
579 | 565 | ||
@@ -601,22 +587,22 @@ static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) | |||
601 | dma_offset = vdma[0]; | 587 | dma_offset = vdma[0]; |
602 | 588 | ||
603 | /* Setup initial software IOMMU state. */ | 589 | /* Setup initial software IOMMU state. */ |
604 | spin_lock_init(&iommu->lock); | ||
605 | iommu->ctx_lowest_free = 1; | 590 | iommu->ctx_lowest_free = 1; |
606 | iommu->page_table_map_base = dma_offset; | 591 | iommu->tbl.page_table_map_base = dma_offset; |
607 | iommu->dma_addr_mask = dma_mask; | 592 | iommu->dma_addr_mask = dma_mask; |
608 | 593 | ||
609 | /* Allocate and initialize the free area map. */ | 594 | /* Allocate and initialize the free area map. */ |
610 | sz = (num_tsb_entries + 7) / 8; | 595 | sz = (num_tsb_entries + 7) / 8; |
611 | sz = (sz + 7UL) & ~7UL; | 596 | sz = (sz + 7UL) & ~7UL; |
612 | iommu->arena.map = kzalloc(sz, GFP_KERNEL); | 597 | iommu->tbl.map = kzalloc(sz, GFP_KERNEL); |
613 | if (!iommu->arena.map) { | 598 | if (!iommu->tbl.map) { |
614 | printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); | 599 | printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); |
615 | return -ENOMEM; | 600 | return -ENOMEM; |
616 | } | 601 | } |
617 | iommu->arena.limit = num_tsb_entries; | 602 | iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, |
618 | 603 | &dma_4v_iommu_ops, false /* no large_pool */, | |
619 | sz = probe_existing_entries(pbm, iommu); | 604 | 0 /* default npools */); |
605 | sz = probe_existing_entries(pbm, &iommu->tbl); | ||
620 | if (sz) | 606 | if (sz) |
621 | printk("%s: Imported %lu TSB entries from OBP\n", | 607 | printk("%s: Imported %lu TSB entries from OBP\n", |
622 | pbm->name, sz); | 608 | pbm->name, sz); |
@@ -1015,8 +1001,17 @@ static struct platform_driver pci_sun4v_driver = { | |||
1015 | .probe = pci_sun4v_probe, | 1001 | .probe = pci_sun4v_probe, |
1016 | }; | 1002 | }; |
1017 | 1003 | ||
1004 | static void setup_iommu_pool_hash(void) | ||
1005 | { | ||
1006 | unsigned int i; | ||
1007 | |||
1008 | for_each_possible_cpu(i) | ||
1009 | per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); | ||
1010 | } | ||
1011 | |||
1018 | static int __init pci_sun4v_init(void) | 1012 | static int __init pci_sun4v_init(void) |
1019 | { | 1013 | { |
1014 | setup_iommu_pool_hash(); | ||
1020 | return platform_driver_register(&pci_sun4v_driver); | 1015 | return platform_driver_register(&pci_sun4v_driver); |
1021 | } | 1016 | } |
1022 | 1017 | ||