aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/sparc/include/asm/iommu_64.h7
-rw-r--r--arch/sparc/kernel/iommu.c188
-rw-r--r--arch/sparc/kernel/iommu_common.h8
-rw-r--r--arch/sparc/kernel/ldc.c185
-rw-r--r--arch/sparc/kernel/pci_sun4v.c193
-rw-r--r--include/linux/iommu-common.h55
-rw-r--r--lib/Makefile2
-rw-r--r--lib/iommu-common.c224
8 files changed, 325 insertions, 537 deletions
diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h
index e3cd4493d81d..2b9321ab064d 100644
--- a/arch/sparc/include/asm/iommu_64.h
+++ b/arch/sparc/include/asm/iommu_64.h
@@ -16,7 +16,6 @@
16#define IOPTE_WRITE 0x0000000000000002UL 16#define IOPTE_WRITE 0x0000000000000002UL
17 17
18#define IOMMU_NUM_CTXS 4096 18#define IOMMU_NUM_CTXS 4096
19#include <linux/iommu-common.h>
20 19
21struct iommu_arena { 20struct iommu_arena {
22 unsigned long *map; 21 unsigned long *map;
@@ -25,10 +24,11 @@ struct iommu_arena {
25}; 24};
26 25
27struct iommu { 26struct iommu {
28 struct iommu_table tbl;
29 spinlock_t lock; 27 spinlock_t lock;
30 u32 dma_addr_mask; 28 struct iommu_arena arena;
29 void (*flush_all)(struct iommu *);
31 iopte_t *page_table; 30 iopte_t *page_table;
31 u32 page_table_map_base;
32 unsigned long iommu_control; 32 unsigned long iommu_control;
33 unsigned long iommu_tsbbase; 33 unsigned long iommu_tsbbase;
34 unsigned long iommu_flush; 34 unsigned long iommu_flush;
@@ -40,6 +40,7 @@ struct iommu {
40 unsigned long dummy_page_pa; 40 unsigned long dummy_page_pa;
41 unsigned long ctx_lowest_free; 41 unsigned long ctx_lowest_free;
42 DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); 42 DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS);
43 u32 dma_addr_mask;
43}; 44};
44 45
45struct strbuf { 46struct strbuf {
diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c
index 9b16b341b6ae..bfa4d0c2df42 100644
--- a/arch/sparc/kernel/iommu.c
+++ b/arch/sparc/kernel/iommu.c
@@ -13,15 +13,11 @@
13#include <linux/errno.h> 13#include <linux/errno.h>
14#include <linux/iommu-helper.h> 14#include <linux/iommu-helper.h>
15#include <linux/bitmap.h> 15#include <linux/bitmap.h>
16#include <linux/hash.h>
17#include <linux/iommu-common.h>
18 16
19#ifdef CONFIG_PCI 17#ifdef CONFIG_PCI
20#include <linux/pci.h> 18#include <linux/pci.h>
21#endif 19#endif
22 20
23static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
24
25#include <asm/iommu.h> 21#include <asm/iommu.h>
26 22
27#include "iommu_common.h" 23#include "iommu_common.h"
@@ -49,9 +45,8 @@ static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
49 "i" (ASI_PHYS_BYPASS_EC_E)) 45 "i" (ASI_PHYS_BYPASS_EC_E))
50 46
51/* Must be invoked under the IOMMU lock. */ 47/* Must be invoked under the IOMMU lock. */
52static void iommu_flushall(struct iommu_table *iommu_table) 48static void iommu_flushall(struct iommu *iommu)
53{ 49{
54 struct iommu *iommu = container_of(iommu_table, struct iommu, tbl);
55 if (iommu->iommu_flushinv) { 50 if (iommu->iommu_flushinv) {
56 iommu_write(iommu->iommu_flushinv, ~(u64)0); 51 iommu_write(iommu->iommu_flushinv, ~(u64)0);
57 } else { 52 } else {
@@ -92,22 +87,93 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte)
92 iopte_val(*iopte) = val; 87 iopte_val(*iopte) = val;
93} 88}
94 89
95static struct iommu_tbl_ops iommu_sparc_ops = { 90/* Based almost entirely upon the ppc64 iommu allocator. If you use the 'handle'
96 .reset = iommu_flushall 91 * facility it must all be done in one pass while under the iommu lock.
97}; 92 *
98 93 * On sun4u platforms, we only flush the IOMMU once every time we've passed
99static void setup_iommu_pool_hash(void) 94 * over the entire page table doing allocations. Therefore we only ever advance
95 * the hint and cannot backtrack it.
96 */
97unsigned long iommu_range_alloc(struct device *dev,
98 struct iommu *iommu,
99 unsigned long npages,
100 unsigned long *handle)
100{ 101{
101 unsigned int i; 102 unsigned long n, end, start, limit, boundary_size;
102 static bool do_once; 103 struct iommu_arena *arena = &iommu->arena;
104 int pass = 0;
103 105
104 if (do_once) 106 /* This allocator was derived from x86_64's bit string search */
105 return; 107
106 do_once = true; 108 /* Sanity check */
107 for_each_possible_cpu(i) 109 if (unlikely(npages == 0)) {
108 per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); 110 if (printk_ratelimit())
111 WARN_ON(1);
112 return DMA_ERROR_CODE;
113 }
114
115 if (handle && *handle)
116 start = *handle;
117 else
118 start = arena->hint;
119
120 limit = arena->limit;
121
122 /* The case below can happen if we have a small segment appended
123 * to a large, or when the previous alloc was at the very end of
124 * the available space. If so, go back to the beginning and flush.
125 */
126 if (start >= limit) {
127 start = 0;
128 if (iommu->flush_all)
129 iommu->flush_all(iommu);
130 }
131
132 again:
133
134 if (dev)
135 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
136 1 << IO_PAGE_SHIFT);
137 else
138 boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT);
139
140 n = iommu_area_alloc(arena->map, limit, start, npages,
141 iommu->page_table_map_base >> IO_PAGE_SHIFT,
142 boundary_size >> IO_PAGE_SHIFT, 0);
143 if (n == -1) {
144 if (likely(pass < 1)) {
145 /* First failure, rescan from the beginning. */
146 start = 0;
147 if (iommu->flush_all)
148 iommu->flush_all(iommu);
149 pass++;
150 goto again;
151 } else {
152 /* Second failure, give up */
153 return DMA_ERROR_CODE;
154 }
155 }
156
157 end = n + npages;
158
159 arena->hint = end;
160
161 /* Update handle for SG allocations */
162 if (handle)
163 *handle = end;
164
165 return n;
109} 166}
110 167
168void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages)
169{
170 struct iommu_arena *arena = &iommu->arena;
171 unsigned long entry;
172
173 entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
174
175 bitmap_clear(arena->map, entry, npages);
176}
111 177
112int iommu_table_init(struct iommu *iommu, int tsbsize, 178int iommu_table_init(struct iommu *iommu, int tsbsize,
113 u32 dma_offset, u32 dma_addr_mask, 179 u32 dma_offset, u32 dma_addr_mask,
@@ -121,22 +187,22 @@ int iommu_table_init(struct iommu *iommu, int tsbsize,
121 /* Setup initial software IOMMU state. */ 187 /* Setup initial software IOMMU state. */
122 spin_lock_init(&iommu->lock); 188 spin_lock_init(&iommu->lock);
123 iommu->ctx_lowest_free = 1; 189 iommu->ctx_lowest_free = 1;
124 iommu->tbl.page_table_map_base = dma_offset; 190 iommu->page_table_map_base = dma_offset;
125 iommu->dma_addr_mask = dma_addr_mask; 191 iommu->dma_addr_mask = dma_addr_mask;
126 192
127 /* Allocate and initialize the free area map. */ 193 /* Allocate and initialize the free area map. */
128 sz = num_tsb_entries / 8; 194 sz = num_tsb_entries / 8;
129 sz = (sz + 7UL) & ~7UL; 195 sz = (sz + 7UL) & ~7UL;
130 iommu->tbl.map = kmalloc_node(sz, GFP_KERNEL, numa_node); 196 iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node);
131 if (!iommu->tbl.map) 197 if (!iommu->arena.map) {
198 printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n");
132 return -ENOMEM; 199 return -ENOMEM;
133 memset(iommu->tbl.map, 0, sz); 200 }
134 if (tlb_type != hypervisor) 201 memset(iommu->arena.map, 0, sz);
135 iommu_sparc_ops.reset = NULL; /* not needed on on sun4v */ 202 iommu->arena.limit = num_tsb_entries;
136 203
137 setup_iommu_pool_hash(); 204 if (tlb_type != hypervisor)
138 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, 205 iommu->flush_all = iommu_flushall;
139 &iommu_sparc_ops, false, 1);
140 206
141 /* Allocate and initialize the dummy page which we 207 /* Allocate and initialize the dummy page which we
142 * set inactive IO PTEs to point to. 208 * set inactive IO PTEs to point to.
@@ -169,20 +235,18 @@ out_free_dummy_page:
169 iommu->dummy_page = 0UL; 235 iommu->dummy_page = 0UL;
170 236
171out_free_map: 237out_free_map:
172 kfree(iommu->tbl.map); 238 kfree(iommu->arena.map);
173 iommu->tbl.map = NULL; 239 iommu->arena.map = NULL;
174 240
175 return -ENOMEM; 241 return -ENOMEM;
176} 242}
177 243
178static inline iopte_t *alloc_npages(struct device *dev, 244static inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu,
179 struct iommu *iommu,
180 unsigned long npages) 245 unsigned long npages)
181{ 246{
182 unsigned long entry; 247 unsigned long entry;
183 248
184 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, 249 entry = iommu_range_alloc(dev, iommu, npages, NULL);
185 __this_cpu_read(iommu_pool_hash));
186 if (unlikely(entry == DMA_ERROR_CODE)) 250 if (unlikely(entry == DMA_ERROR_CODE))
187 return NULL; 251 return NULL;
188 252
@@ -220,7 +284,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size,
220 dma_addr_t *dma_addrp, gfp_t gfp, 284 dma_addr_t *dma_addrp, gfp_t gfp,
221 struct dma_attrs *attrs) 285 struct dma_attrs *attrs)
222{ 286{
223 unsigned long order, first_page; 287 unsigned long flags, order, first_page;
224 struct iommu *iommu; 288 struct iommu *iommu;
225 struct page *page; 289 struct page *page;
226 int npages, nid; 290 int npages, nid;
@@ -242,14 +306,16 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size,
242 306
243 iommu = dev->archdata.iommu; 307 iommu = dev->archdata.iommu;
244 308
309 spin_lock_irqsave(&iommu->lock, flags);
245 iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); 310 iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);
311 spin_unlock_irqrestore(&iommu->lock, flags);
246 312
247 if (unlikely(iopte == NULL)) { 313 if (unlikely(iopte == NULL)) {
248 free_pages(first_page, order); 314 free_pages(first_page, order);
249 return NULL; 315 return NULL;
250 } 316 }
251 317
252 *dma_addrp = (iommu->tbl.page_table_map_base + 318 *dma_addrp = (iommu->page_table_map_base +
253 ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); 319 ((iopte - iommu->page_table) << IO_PAGE_SHIFT));
254 ret = (void *) first_page; 320 ret = (void *) first_page;
255 npages = size >> IO_PAGE_SHIFT; 321 npages = size >> IO_PAGE_SHIFT;
@@ -270,12 +336,16 @@ static void dma_4u_free_coherent(struct device *dev, size_t size,
270 struct dma_attrs *attrs) 336 struct dma_attrs *attrs)
271{ 337{
272 struct iommu *iommu; 338 struct iommu *iommu;
273 unsigned long order, npages; 339 unsigned long flags, order, npages;
274 340
275 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 341 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
276 iommu = dev->archdata.iommu; 342 iommu = dev->archdata.iommu;
277 343
278 iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL); 344 spin_lock_irqsave(&iommu->lock, flags);
345
346 iommu_range_free(iommu, dvma, npages);
347
348 spin_unlock_irqrestore(&iommu->lock, flags);
279 349
280 order = get_order(size); 350 order = get_order(size);
281 if (order < 10) 351 if (order < 10)
@@ -305,8 +375,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page,
305 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 375 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
306 npages >>= IO_PAGE_SHIFT; 376 npages >>= IO_PAGE_SHIFT;
307 377
308 base = alloc_npages(dev, iommu, npages);
309 spin_lock_irqsave(&iommu->lock, flags); 378 spin_lock_irqsave(&iommu->lock, flags);
379 base = alloc_npages(dev, iommu, npages);
310 ctx = 0; 380 ctx = 0;
311 if (iommu->iommu_ctxflush) 381 if (iommu->iommu_ctxflush)
312 ctx = iommu_alloc_ctx(iommu); 382 ctx = iommu_alloc_ctx(iommu);
@@ -315,7 +385,7 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page,
315 if (unlikely(!base)) 385 if (unlikely(!base))
316 goto bad; 386 goto bad;
317 387
318 bus_addr = (iommu->tbl.page_table_map_base + 388 bus_addr = (iommu->page_table_map_base +
319 ((base - iommu->page_table) << IO_PAGE_SHIFT)); 389 ((base - iommu->page_table) << IO_PAGE_SHIFT));
320 ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 390 ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
321 base_paddr = __pa(oaddr & IO_PAGE_MASK); 391 base_paddr = __pa(oaddr & IO_PAGE_MASK);
@@ -426,7 +496,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
426 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 496 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
427 npages >>= IO_PAGE_SHIFT; 497 npages >>= IO_PAGE_SHIFT;
428 base = iommu->page_table + 498 base = iommu->page_table +
429 ((bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT); 499 ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
430 bus_addr &= IO_PAGE_MASK; 500 bus_addr &= IO_PAGE_MASK;
431 501
432 spin_lock_irqsave(&iommu->lock, flags); 502 spin_lock_irqsave(&iommu->lock, flags);
@@ -445,11 +515,11 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
445 for (i = 0; i < npages; i++) 515 for (i = 0; i < npages; i++)
446 iopte_make_dummy(iommu, base + i); 516 iopte_make_dummy(iommu, base + i);
447 517
518 iommu_range_free(iommu, bus_addr, npages);
519
448 iommu_free_ctx(iommu, ctx); 520 iommu_free_ctx(iommu, ctx);
449 spin_unlock_irqrestore(&iommu->lock, flags);
450 521
451 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, 522 spin_unlock_irqrestore(&iommu->lock, flags);
452 false, NULL);
453} 523}
454 524
455static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 525static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -497,7 +567,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
497 max_seg_size = dma_get_max_seg_size(dev); 567 max_seg_size = dma_get_max_seg_size(dev);
498 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 568 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
499 IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 569 IO_PAGE_SIZE) >> IO_PAGE_SHIFT;
500 base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT; 570 base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT;
501 for_each_sg(sglist, s, nelems, i) { 571 for_each_sg(sglist, s, nelems, i) {
502 unsigned long paddr, npages, entry, out_entry = 0, slen; 572 unsigned long paddr, npages, entry, out_entry = 0, slen;
503 iopte_t *base; 573 iopte_t *base;
@@ -511,8 +581,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
511 /* Allocate iommu entries for that segment */ 581 /* Allocate iommu entries for that segment */
512 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 582 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
513 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 583 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
514 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle, 584 entry = iommu_range_alloc(dev, iommu, npages, &handle);
515 __this_cpu_read(iommu_pool_hash));
516 585
517 /* Handle failure */ 586 /* Handle failure */
518 if (unlikely(entry == DMA_ERROR_CODE)) { 587 if (unlikely(entry == DMA_ERROR_CODE)) {
@@ -525,7 +594,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
525 base = iommu->page_table + entry; 594 base = iommu->page_table + entry;
526 595
527 /* Convert entry to a dma_addr_t */ 596 /* Convert entry to a dma_addr_t */
528 dma_addr = iommu->tbl.page_table_map_base + 597 dma_addr = iommu->page_table_map_base +
529 (entry << IO_PAGE_SHIFT); 598 (entry << IO_PAGE_SHIFT);
530 dma_addr |= (s->offset & ~IO_PAGE_MASK); 599 dma_addr |= (s->offset & ~IO_PAGE_MASK);
531 600
@@ -585,17 +654,15 @@ iommu_map_failed:
585 vaddr = s->dma_address & IO_PAGE_MASK; 654 vaddr = s->dma_address & IO_PAGE_MASK;
586 npages = iommu_num_pages(s->dma_address, s->dma_length, 655 npages = iommu_num_pages(s->dma_address, s->dma_length,
587 IO_PAGE_SIZE); 656 IO_PAGE_SIZE);
657 iommu_range_free(iommu, vaddr, npages);
588 658
589 entry = (vaddr - iommu->tbl.page_table_map_base) 659 entry = (vaddr - iommu->page_table_map_base)
590 >> IO_PAGE_SHIFT; 660 >> IO_PAGE_SHIFT;
591 base = iommu->page_table + entry; 661 base = iommu->page_table + entry;
592 662
593 for (j = 0; j < npages; j++) 663 for (j = 0; j < npages; j++)
594 iopte_make_dummy(iommu, base + j); 664 iopte_make_dummy(iommu, base + j);
595 665
596 iommu_tbl_range_free(&iommu->tbl, vaddr, npages,
597 false, NULL);
598
599 s->dma_address = DMA_ERROR_CODE; 666 s->dma_address = DMA_ERROR_CODE;
600 s->dma_length = 0; 667 s->dma_length = 0;
601 } 668 }
@@ -610,19 +677,17 @@ iommu_map_failed:
610/* If contexts are being used, they are the same in all of the mappings 677/* If contexts are being used, they are the same in all of the mappings
611 * we make for a particular SG. 678 * we make for a particular SG.
612 */ 679 */
613static unsigned long fetch_sg_ctx(struct iommu *iommu, 680static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg)
614 struct scatterlist *sg)
615{ 681{
616 unsigned long ctx = 0; 682 unsigned long ctx = 0;
617 683
618 if (iommu->iommu_ctxflush) { 684 if (iommu->iommu_ctxflush) {
619 iopte_t *base; 685 iopte_t *base;
620 u32 bus_addr; 686 u32 bus_addr;
621 struct iommu_table *tbl = &iommu->tbl;
622 687
623 bus_addr = sg->dma_address & IO_PAGE_MASK; 688 bus_addr = sg->dma_address & IO_PAGE_MASK;
624 base = iommu->page_table + 689 base = iommu->page_table +
625 ((bus_addr - tbl->page_table_map_base) >> IO_PAGE_SHIFT); 690 ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
626 691
627 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; 692 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
628 } 693 }
@@ -658,8 +723,9 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
658 if (!len) 723 if (!len)
659 break; 724 break;
660 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 725 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
726 iommu_range_free(iommu, dma_handle, npages);
661 727
662 entry = ((dma_handle - iommu->tbl.page_table_map_base) 728 entry = ((dma_handle - iommu->page_table_map_base)
663 >> IO_PAGE_SHIFT); 729 >> IO_PAGE_SHIFT);
664 base = iommu->page_table + entry; 730 base = iommu->page_table + entry;
665 731
@@ -671,8 +737,6 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
671 for (i = 0; i < npages; i++) 737 for (i = 0; i < npages; i++)
672 iopte_make_dummy(iommu, base + i); 738 iopte_make_dummy(iommu, base + i);
673 739
674 iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, false,
675 NULL);
676 sg = sg_next(sg); 740 sg = sg_next(sg);
677 } 741 }
678 742
@@ -706,10 +770,9 @@ static void dma_4u_sync_single_for_cpu(struct device *dev,
706 if (iommu->iommu_ctxflush && 770 if (iommu->iommu_ctxflush &&
707 strbuf->strbuf_ctxflush) { 771 strbuf->strbuf_ctxflush) {
708 iopte_t *iopte; 772 iopte_t *iopte;
709 struct iommu_table *tbl = &iommu->tbl;
710 773
711 iopte = iommu->page_table + 774 iopte = iommu->page_table +
712 ((bus_addr - tbl->page_table_map_base)>>IO_PAGE_SHIFT); 775 ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT);
713 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 776 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
714 } 777 }
715 778
@@ -742,10 +805,9 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev,
742 if (iommu->iommu_ctxflush && 805 if (iommu->iommu_ctxflush &&
743 strbuf->strbuf_ctxflush) { 806 strbuf->strbuf_ctxflush) {
744 iopte_t *iopte; 807 iopte_t *iopte;
745 struct iommu_table *tbl = &iommu->tbl;
746 808
747 iopte = iommu->page_table + ((sglist[0].dma_address - 809 iopte = iommu->page_table +
748 tbl->page_table_map_base) >> IO_PAGE_SHIFT); 810 ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
749 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 811 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
750 } 812 }
751 813
diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h
index f4be0d724fc6..1ec0de4156e7 100644
--- a/arch/sparc/kernel/iommu_common.h
+++ b/arch/sparc/kernel/iommu_common.h
@@ -48,4 +48,12 @@ static inline int is_span_boundary(unsigned long entry,
48 return iommu_is_span_boundary(entry, nr, shift, boundary_size); 48 return iommu_is_span_boundary(entry, nr, shift, boundary_size);
49} 49}
50 50
51unsigned long iommu_range_alloc(struct device *dev,
52 struct iommu *iommu,
53 unsigned long npages,
54 unsigned long *handle);
55void iommu_range_free(struct iommu *iommu,
56 dma_addr_t dma_addr,
57 unsigned long npages);
58
51#endif /* _IOMMU_COMMON_H */ 59#endif /* _IOMMU_COMMON_H */
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
index d485697c37c0..274a9f59d95c 100644
--- a/arch/sparc/kernel/ldc.c
+++ b/arch/sparc/kernel/ldc.c
@@ -15,8 +15,6 @@
15#include <linux/list.h> 15#include <linux/list.h>
16#include <linux/init.h> 16#include <linux/init.h>
17#include <linux/bitmap.h> 17#include <linux/bitmap.h>
18#include <linux/hash.h>
19#include <linux/iommu-common.h>
20 18
21#include <asm/hypervisor.h> 19#include <asm/hypervisor.h>
22#include <asm/iommu.h> 20#include <asm/iommu.h>
@@ -29,11 +27,6 @@
29#define DRV_MODULE_VERSION "1.1" 27#define DRV_MODULE_VERSION "1.1"
30#define DRV_MODULE_RELDATE "July 22, 2008" 28#define DRV_MODULE_RELDATE "July 22, 2008"
31 29
32#define COOKIE_PGSZ_CODE 0xf000000000000000ULL
33#define COOKIE_PGSZ_CODE_SHIFT 60ULL
34
35static DEFINE_PER_CPU(unsigned int, ldc_pool_hash);
36
37static char version[] = 30static char version[] =
38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 31 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
39#define LDC_PACKET_SIZE 64 32#define LDC_PACKET_SIZE 64
@@ -105,10 +98,10 @@ static const struct ldc_mode_ops stream_ops;
105int ldom_domaining_enabled; 98int ldom_domaining_enabled;
106 99
107struct ldc_iommu { 100struct ldc_iommu {
108 /* Protects ldc_unmap. */ 101 /* Protects arena alloc/free. */
109 spinlock_t lock; 102 spinlock_t lock;
103 struct iommu_arena arena;
110 struct ldc_mtable_entry *page_table; 104 struct ldc_mtable_entry *page_table;
111 struct iommu_table iommu_table;
112}; 105};
113 106
114struct ldc_channel { 107struct ldc_channel {
@@ -1005,85 +998,31 @@ static void free_queue(unsigned long num_entries, struct ldc_packet *q)
1005 free_pages((unsigned long)q, order); 998 free_pages((unsigned long)q, order);
1006} 999}
1007 1000
1008static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
1009{
1010 u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1011 /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
1012
1013 cookie &= ~COOKIE_PGSZ_CODE;
1014
1015 return (cookie >> (13ULL + (szcode * 3ULL)));
1016}
1017
1018struct ldc_demap_arg {
1019 struct ldc_iommu *ldc_iommu;
1020 u64 cookie;
1021 unsigned long id;
1022};
1023
1024static void ldc_demap(void *arg, unsigned long entry, unsigned long npages)
1025{
1026 struct ldc_demap_arg *ldc_demap_arg = arg;
1027 struct ldc_iommu *iommu = ldc_demap_arg->ldc_iommu;
1028 unsigned long id = ldc_demap_arg->id;
1029 u64 cookie = ldc_demap_arg->cookie;
1030 struct ldc_mtable_entry *base;
1031 unsigned long i, shift;
1032
1033 shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
1034 base = iommu->page_table + entry;
1035 for (i = 0; i < npages; i++) {
1036 if (base->cookie)
1037 sun4v_ldc_revoke(id, cookie + (i << shift),
1038 base->cookie);
1039 base->mte = 0;
1040 }
1041}
1042
1043/* XXX Make this configurable... XXX */ 1001/* XXX Make this configurable... XXX */
1044#define LDC_IOTABLE_SIZE (8 * 1024) 1002#define LDC_IOTABLE_SIZE (8 * 1024)
1045 1003
1046struct iommu_tbl_ops ldc_iommu_ops = { 1004static int ldc_iommu_init(struct ldc_channel *lp)
1047 .cookie_to_index = ldc_cookie_to_index,
1048 .demap = ldc_demap,
1049};
1050
1051static void setup_ldc_pool_hash(void)
1052{
1053 unsigned int i;
1054 static bool do_once;
1055
1056 if (do_once)
1057 return;
1058 do_once = true;
1059 for_each_possible_cpu(i)
1060 per_cpu(ldc_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
1061}
1062
1063
1064static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1065{ 1005{
1066 unsigned long sz, num_tsb_entries, tsbsize, order; 1006 unsigned long sz, num_tsb_entries, tsbsize, order;
1067 struct ldc_iommu *ldc_iommu = &lp->iommu; 1007 struct ldc_iommu *iommu = &lp->iommu;
1068 struct iommu_table *iommu = &ldc_iommu->iommu_table;
1069 struct ldc_mtable_entry *table; 1008 struct ldc_mtable_entry *table;
1070 unsigned long hv_err; 1009 unsigned long hv_err;
1071 int err; 1010 int err;
1072 1011
1073 num_tsb_entries = LDC_IOTABLE_SIZE; 1012 num_tsb_entries = LDC_IOTABLE_SIZE;
1074 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1013 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1075 setup_ldc_pool_hash(); 1014
1076 spin_lock_init(&ldc_iommu->lock); 1015 spin_lock_init(&iommu->lock);
1077 1016
1078 sz = num_tsb_entries / 8; 1017 sz = num_tsb_entries / 8;
1079 sz = (sz + 7UL) & ~7UL; 1018 sz = (sz + 7UL) & ~7UL;
1080 iommu->map = kzalloc(sz, GFP_KERNEL); 1019 iommu->arena.map = kzalloc(sz, GFP_KERNEL);
1081 if (!iommu->map) { 1020 if (!iommu->arena.map) {
1082 printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1021 printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1083 return -ENOMEM; 1022 return -ENOMEM;
1084 } 1023 }
1085 iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, 1024
1086 &ldc_iommu_ops, false, 1); 1025 iommu->arena.limit = num_tsb_entries;
1087 1026
1088 order = get_order(tsbsize); 1027 order = get_order(tsbsize);
1089 1028
@@ -1098,7 +1037,7 @@ static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1098 1037
1099 memset(table, 0, PAGE_SIZE << order); 1038 memset(table, 0, PAGE_SIZE << order);
1100 1039
1101 ldc_iommu->page_table = table; 1040 iommu->page_table = table;
1102 1041
1103 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1042 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1104 num_tsb_entries); 1043 num_tsb_entries);
@@ -1110,32 +1049,31 @@ static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1110 1049
1111out_free_table: 1050out_free_table:
1112 free_pages((unsigned long) table, order); 1051 free_pages((unsigned long) table, order);
1113 ldc_iommu->page_table = NULL; 1052 iommu->page_table = NULL;
1114 1053
1115out_free_map: 1054out_free_map:
1116 kfree(iommu->map); 1055 kfree(iommu->arena.map);
1117 iommu->map = NULL; 1056 iommu->arena.map = NULL;
1118 1057
1119 return err; 1058 return err;
1120} 1059}
1121 1060
1122static void ldc_iommu_release(struct ldc_channel *lp) 1061static void ldc_iommu_release(struct ldc_channel *lp)
1123{ 1062{
1124 struct ldc_iommu *ldc_iommu = &lp->iommu; 1063 struct ldc_iommu *iommu = &lp->iommu;
1125 struct iommu_table *iommu = &ldc_iommu->iommu_table;
1126 unsigned long num_tsb_entries, tsbsize, order; 1064 unsigned long num_tsb_entries, tsbsize, order;
1127 1065
1128 (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1066 (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1129 1067
1130 num_tsb_entries = iommu->poolsize * iommu->nr_pools; 1068 num_tsb_entries = iommu->arena.limit;
1131 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1069 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1132 order = get_order(tsbsize); 1070 order = get_order(tsbsize);
1133 1071
1134 free_pages((unsigned long) ldc_iommu->page_table, order); 1072 free_pages((unsigned long) iommu->page_table, order);
1135 ldc_iommu->page_table = NULL; 1073 iommu->page_table = NULL;
1136 1074
1137 kfree(iommu->map); 1075 kfree(iommu->arena.map);
1138 iommu->map = NULL; 1076 iommu->arena.map = NULL;
1139} 1077}
1140 1078
1141struct ldc_channel *ldc_alloc(unsigned long id, 1079struct ldc_channel *ldc_alloc(unsigned long id,
@@ -1202,7 +1140,7 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1202 1140
1203 lp->id = id; 1141 lp->id = id;
1204 1142
1205 err = ldc_iommu_init(name, lp); 1143 err = ldc_iommu_init(lp);
1206 if (err) 1144 if (err)
1207 goto out_free_ldc; 1145 goto out_free_ldc;
1208 1146
@@ -1947,6 +1885,40 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1947} 1885}
1948EXPORT_SYMBOL(ldc_read); 1886EXPORT_SYMBOL(ldc_read);
1949 1887
1888static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages)
1889{
1890 struct iommu_arena *arena = &iommu->arena;
1891 unsigned long n, start, end, limit;
1892 int pass;
1893
1894 limit = arena->limit;
1895 start = arena->hint;
1896 pass = 0;
1897
1898again:
1899 n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0);
1900 end = n + npages;
1901 if (unlikely(end >= limit)) {
1902 if (likely(pass < 1)) {
1903 limit = start;
1904 start = 0;
1905 pass++;
1906 goto again;
1907 } else {
1908 /* Scanned the whole thing, give up. */
1909 return -1;
1910 }
1911 }
1912 bitmap_set(arena->map, n, npages);
1913
1914 arena->hint = end;
1915
1916 return n;
1917}
1918
1919#define COOKIE_PGSZ_CODE 0xf000000000000000ULL
1920#define COOKIE_PGSZ_CODE_SHIFT 60ULL
1921
1950static u64 pagesize_code(void) 1922static u64 pagesize_code(void)
1951{ 1923{
1952 switch (PAGE_SIZE) { 1924 switch (PAGE_SIZE) {
@@ -1973,14 +1945,23 @@ static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
1973 page_offset); 1945 page_offset);
1974} 1946}
1975 1947
1948static u64 cookie_to_index(u64 cookie, unsigned long *shift)
1949{
1950 u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1951
1952 cookie &= ~COOKIE_PGSZ_CODE;
1953
1954 *shift = szcode * 3;
1955
1956 return (cookie >> (13ULL + (szcode * 3ULL)));
1957}
1976 1958
1977static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1959static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
1978 unsigned long npages) 1960 unsigned long npages)
1979{ 1961{
1980 long entry; 1962 long entry;
1981 1963
1982 entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_table, npages, 1964 entry = arena_alloc(iommu, npages);
1983 NULL, __this_cpu_read(ldc_pool_hash));
1984 if (unlikely(entry < 0)) 1965 if (unlikely(entry < 0))
1985 return NULL; 1966 return NULL;
1986 1967
@@ -2109,7 +2090,7 @@ int ldc_map_sg(struct ldc_channel *lp,
2109 struct ldc_trans_cookie *cookies, int ncookies, 2090 struct ldc_trans_cookie *cookies, int ncookies,
2110 unsigned int map_perm) 2091 unsigned int map_perm)
2111{ 2092{
2112 unsigned long i, npages; 2093 unsigned long i, npages, flags;
2113 struct ldc_mtable_entry *base; 2094 struct ldc_mtable_entry *base;
2114 struct cookie_state state; 2095 struct cookie_state state;
2115 struct ldc_iommu *iommu; 2096 struct ldc_iommu *iommu;
@@ -2128,7 +2109,9 @@ int ldc_map_sg(struct ldc_channel *lp,
2128 2109
2129 iommu = &lp->iommu; 2110 iommu = &lp->iommu;
2130 2111
2112 spin_lock_irqsave(&iommu->lock, flags);
2131 base = alloc_npages(iommu, npages); 2113 base = alloc_npages(iommu, npages);
2114 spin_unlock_irqrestore(&iommu->lock, flags);
2132 2115
2133 if (!base) 2116 if (!base)
2134 return -ENOMEM; 2117 return -ENOMEM;
@@ -2153,7 +2136,7 @@ int ldc_map_single(struct ldc_channel *lp,
2153 struct ldc_trans_cookie *cookies, int ncookies, 2136 struct ldc_trans_cookie *cookies, int ncookies,
2154 unsigned int map_perm) 2137 unsigned int map_perm)
2155{ 2138{
2156 unsigned long npages, pa; 2139 unsigned long npages, pa, flags;
2157 struct ldc_mtable_entry *base; 2140 struct ldc_mtable_entry *base;
2158 struct cookie_state state; 2141 struct cookie_state state;
2159 struct ldc_iommu *iommu; 2142 struct ldc_iommu *iommu;
@@ -2169,7 +2152,9 @@ int ldc_map_single(struct ldc_channel *lp,
2169 2152
2170 iommu = &lp->iommu; 2153 iommu = &lp->iommu;
2171 2154
2155 spin_lock_irqsave(&iommu->lock, flags);
2172 base = alloc_npages(iommu, npages); 2156 base = alloc_npages(iommu, npages);
2157 spin_unlock_irqrestore(&iommu->lock, flags);
2173 2158
2174 if (!base) 2159 if (!base)
2175 return -ENOMEM; 2160 return -ENOMEM;
@@ -2187,29 +2172,35 @@ int ldc_map_single(struct ldc_channel *lp,
2187} 2172}
2188EXPORT_SYMBOL(ldc_map_single); 2173EXPORT_SYMBOL(ldc_map_single);
2189 2174
2190
2191static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2175static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2192 u64 cookie, u64 size) 2176 u64 cookie, u64 size)
2193{ 2177{
2194 unsigned long npages; 2178 struct iommu_arena *arena = &iommu->arena;
2195 struct ldc_demap_arg demap_arg; 2179 unsigned long i, shift, index, npages;
2196 2180 struct ldc_mtable_entry *base;
2197 demap_arg.ldc_iommu = iommu;
2198 demap_arg.cookie = cookie;
2199 demap_arg.id = id;
2200 2181
2201 npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2182 npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2202 iommu_tbl_range_free(&iommu->iommu_table, cookie, npages, true, 2183 index = cookie_to_index(cookie, &shift);
2203 &demap_arg); 2184 base = iommu->page_table + index;
2185
2186 BUG_ON(index > arena->limit ||
2187 (index + npages) > arena->limit);
2204 2188
2189 for (i = 0; i < npages; i++) {
2190 if (base->cookie)
2191 sun4v_ldc_revoke(id, cookie + (i << shift),
2192 base->cookie);
2193 base->mte = 0;
2194 __clear_bit(index + i, arena->map);
2195 }
2205} 2196}
2206 2197
2207void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2198void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2208 int ncookies) 2199 int ncookies)
2209{ 2200{
2210 struct ldc_iommu *iommu = &lp->iommu; 2201 struct ldc_iommu *iommu = &lp->iommu;
2211 int i;
2212 unsigned long flags; 2202 unsigned long flags;
2203 int i;
2213 2204
2214 spin_lock_irqsave(&iommu->lock, flags); 2205 spin_lock_irqsave(&iommu->lock, flags);
2215 for (i = 0; i < ncookies; i++) { 2206 for (i = 0; i < ncookies; i++) {
diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c
index 9b76b9d639e1..47ddbd496a1e 100644
--- a/arch/sparc/kernel/pci_sun4v.c
+++ b/arch/sparc/kernel/pci_sun4v.c
@@ -15,8 +15,6 @@
15#include <linux/export.h> 15#include <linux/export.h>
16#include <linux/log2.h> 16#include <linux/log2.h>
17#include <linux/of_device.h> 17#include <linux/of_device.h>
18#include <linux/hash.h>
19#include <linux/iommu-common.h>
20 18
21#include <asm/iommu.h> 19#include <asm/iommu.h>
22#include <asm/irq.h> 20#include <asm/irq.h>
@@ -30,7 +28,6 @@
30 28
31#define DRIVER_NAME "pci_sun4v" 29#define DRIVER_NAME "pci_sun4v"
32#define PFX DRIVER_NAME ": " 30#define PFX DRIVER_NAME ": "
33static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
34 31
35static unsigned long vpci_major = 1; 32static unsigned long vpci_major = 1;
36static unsigned long vpci_minor = 1; 33static unsigned long vpci_minor = 1;
@@ -158,13 +155,14 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
158 155
159 iommu = dev->archdata.iommu; 156 iommu = dev->archdata.iommu;
160 157
161 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, 158 spin_lock_irqsave(&iommu->lock, flags);
162 __this_cpu_read(iommu_pool_hash)); 159 entry = iommu_range_alloc(dev, iommu, npages, NULL);
160 spin_unlock_irqrestore(&iommu->lock, flags);
163 161
164 if (unlikely(entry == DMA_ERROR_CODE)) 162 if (unlikely(entry == DMA_ERROR_CODE))
165 goto range_alloc_fail; 163 goto range_alloc_fail;
166 164
167 *dma_addrp = (iommu->tbl.page_table_map_base + 165 *dma_addrp = (iommu->page_table_map_base +
168 (entry << IO_PAGE_SHIFT)); 166 (entry << IO_PAGE_SHIFT));
169 ret = (void *) first_page; 167 ret = (void *) first_page;
170 first_page = __pa(first_page); 168 first_page = __pa(first_page);
@@ -190,46 +188,45 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
190 return ret; 188 return ret;
191 189
192iommu_map_fail: 190iommu_map_fail:
193 iommu_tbl_range_free(&iommu->tbl, *dma_addrp, npages, false, NULL); 191 /* Interrupts are disabled. */
192 spin_lock(&iommu->lock);
193 iommu_range_free(iommu, *dma_addrp, npages);
194 spin_unlock_irqrestore(&iommu->lock, flags);
194 195
195range_alloc_fail: 196range_alloc_fail:
196 free_pages(first_page, order); 197 free_pages(first_page, order);
197 return NULL; 198 return NULL;
198} 199}
199 200
200static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry,
201 unsigned long npages)
202{
203 u32 devhandle = *(u32 *)demap_arg;
204 unsigned long num, flags;
205
206 local_irq_save(flags);
207 do {
208 num = pci_sun4v_iommu_demap(devhandle,
209 HV_PCI_TSBID(0, entry),
210 npages);
211
212 entry += num;
213 npages -= num;
214 } while (npages != 0);
215 local_irq_restore(flags);
216}
217
218static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, 201static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu,
219 dma_addr_t dvma, struct dma_attrs *attrs) 202 dma_addr_t dvma, struct dma_attrs *attrs)
220{ 203{
221 struct pci_pbm_info *pbm; 204 struct pci_pbm_info *pbm;
222 struct iommu *iommu; 205 struct iommu *iommu;
223 unsigned long order, npages, entry; 206 unsigned long flags, order, npages, entry;
224 u32 devhandle; 207 u32 devhandle;
225 208
226 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 209 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
227 iommu = dev->archdata.iommu; 210 iommu = dev->archdata.iommu;
228 pbm = dev->archdata.host_controller; 211 pbm = dev->archdata.host_controller;
229 devhandle = pbm->devhandle; 212 devhandle = pbm->devhandle;
230 entry = ((dvma - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT); 213 entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
231 dma_4v_iommu_demap(&devhandle, entry, npages); 214
232 iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL); 215 spin_lock_irqsave(&iommu->lock, flags);
216
217 iommu_range_free(iommu, dvma, npages);
218
219 do {
220 unsigned long num;
221
222 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
223 npages);
224 entry += num;
225 npages -= num;
226 } while (npages != 0);
227
228 spin_unlock_irqrestore(&iommu->lock, flags);
229
233 order = get_order(size); 230 order = get_order(size);
234 if (order < 10) 231 if (order < 10)
235 free_pages((unsigned long)cpu, order); 232 free_pages((unsigned long)cpu, order);
@@ -256,13 +253,14 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
256 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 253 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
257 npages >>= IO_PAGE_SHIFT; 254 npages >>= IO_PAGE_SHIFT;
258 255
259 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, 256 spin_lock_irqsave(&iommu->lock, flags);
260 __this_cpu_read(iommu_pool_hash)); 257 entry = iommu_range_alloc(dev, iommu, npages, NULL);
258 spin_unlock_irqrestore(&iommu->lock, flags);
261 259
262 if (unlikely(entry == DMA_ERROR_CODE)) 260 if (unlikely(entry == DMA_ERROR_CODE))
263 goto bad; 261 goto bad;
264 262
265 bus_addr = (iommu->tbl.page_table_map_base + 263 bus_addr = (iommu->page_table_map_base +
266 (entry << IO_PAGE_SHIFT)); 264 (entry << IO_PAGE_SHIFT));
267 ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 265 ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
268 base_paddr = __pa(oaddr & IO_PAGE_MASK); 266 base_paddr = __pa(oaddr & IO_PAGE_MASK);
@@ -292,7 +290,11 @@ bad:
292 return DMA_ERROR_CODE; 290 return DMA_ERROR_CODE;
293 291
294iommu_map_fail: 292iommu_map_fail:
295 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL); 293 /* Interrupts are disabled. */
294 spin_lock(&iommu->lock);
295 iommu_range_free(iommu, bus_addr, npages);
296 spin_unlock_irqrestore(&iommu->lock, flags);
297
296 return DMA_ERROR_CODE; 298 return DMA_ERROR_CODE;
297} 299}
298 300
@@ -302,7 +304,7 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
302{ 304{
303 struct pci_pbm_info *pbm; 305 struct pci_pbm_info *pbm;
304 struct iommu *iommu; 306 struct iommu *iommu;
305 unsigned long npages; 307 unsigned long flags, npages;
306 long entry; 308 long entry;
307 u32 devhandle; 309 u32 devhandle;
308 310
@@ -319,9 +321,22 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
319 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 321 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
320 npages >>= IO_PAGE_SHIFT; 322 npages >>= IO_PAGE_SHIFT;
321 bus_addr &= IO_PAGE_MASK; 323 bus_addr &= IO_PAGE_MASK;
322 entry = (bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT; 324
323 dma_4v_iommu_demap(&devhandle, entry, npages); 325 spin_lock_irqsave(&iommu->lock, flags);
324 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL); 326
327 iommu_range_free(iommu, bus_addr, npages);
328
329 entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
330 do {
331 unsigned long num;
332
333 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
334 npages);
335 entry += num;
336 npages -= num;
337 } while (npages != 0);
338
339 spin_unlock_irqrestore(&iommu->lock, flags);
325} 340}
326 341
327static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, 342static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -356,14 +371,14 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
356 /* Init first segment length for backout at failure */ 371 /* Init first segment length for backout at failure */
357 outs->dma_length = 0; 372 outs->dma_length = 0;
358 373
359 local_irq_save(flags); 374 spin_lock_irqsave(&iommu->lock, flags);
360 375
361 iommu_batch_start(dev, prot, ~0UL); 376 iommu_batch_start(dev, prot, ~0UL);
362 377
363 max_seg_size = dma_get_max_seg_size(dev); 378 max_seg_size = dma_get_max_seg_size(dev);
364 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 379 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
365 IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 380 IO_PAGE_SIZE) >> IO_PAGE_SHIFT;
366 base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT; 381 base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT;
367 for_each_sg(sglist, s, nelems, i) { 382 for_each_sg(sglist, s, nelems, i) {
368 unsigned long paddr, npages, entry, out_entry = 0, slen; 383 unsigned long paddr, npages, entry, out_entry = 0, slen;
369 384
@@ -376,8 +391,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
376 /* Allocate iommu entries for that segment */ 391 /* Allocate iommu entries for that segment */
377 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 392 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
378 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 393 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
379 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle, 394 entry = iommu_range_alloc(dev, iommu, npages, &handle);
380 __this_cpu_read(iommu_pool_hash));
381 395
382 /* Handle failure */ 396 /* Handle failure */
383 if (unlikely(entry == DMA_ERROR_CODE)) { 397 if (unlikely(entry == DMA_ERROR_CODE)) {
@@ -390,7 +404,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
390 iommu_batch_new_entry(entry); 404 iommu_batch_new_entry(entry);
391 405
392 /* Convert entry to a dma_addr_t */ 406 /* Convert entry to a dma_addr_t */
393 dma_addr = iommu->tbl.page_table_map_base + 407 dma_addr = iommu->page_table_map_base +
394 (entry << IO_PAGE_SHIFT); 408 (entry << IO_PAGE_SHIFT);
395 dma_addr |= (s->offset & ~IO_PAGE_MASK); 409 dma_addr |= (s->offset & ~IO_PAGE_MASK);
396 410
@@ -437,7 +451,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
437 if (unlikely(err < 0L)) 451 if (unlikely(err < 0L))
438 goto iommu_map_failed; 452 goto iommu_map_failed;
439 453
440 local_irq_restore(flags); 454 spin_unlock_irqrestore(&iommu->lock, flags);
441 455
442 if (outcount < incount) { 456 if (outcount < incount) {
443 outs = sg_next(outs); 457 outs = sg_next(outs);
@@ -455,8 +469,7 @@ iommu_map_failed:
455 vaddr = s->dma_address & IO_PAGE_MASK; 469 vaddr = s->dma_address & IO_PAGE_MASK;
456 npages = iommu_num_pages(s->dma_address, s->dma_length, 470 npages = iommu_num_pages(s->dma_address, s->dma_length,
457 IO_PAGE_SIZE); 471 IO_PAGE_SIZE);
458 iommu_tbl_range_free(&iommu->tbl, vaddr, npages, 472 iommu_range_free(iommu, vaddr, npages);
459 false, NULL);
460 /* XXX demap? XXX */ 473 /* XXX demap? XXX */
461 s->dma_address = DMA_ERROR_CODE; 474 s->dma_address = DMA_ERROR_CODE;
462 s->dma_length = 0; 475 s->dma_length = 0;
@@ -464,7 +477,7 @@ iommu_map_failed:
464 if (s == outs) 477 if (s == outs)
465 break; 478 break;
466 } 479 }
467 local_irq_restore(flags); 480 spin_unlock_irqrestore(&iommu->lock, flags);
468 481
469 return 0; 482 return 0;
470} 483}
@@ -476,7 +489,7 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
476 struct pci_pbm_info *pbm; 489 struct pci_pbm_info *pbm;
477 struct scatterlist *sg; 490 struct scatterlist *sg;
478 struct iommu *iommu; 491 struct iommu *iommu;
479 unsigned long flags, entry; 492 unsigned long flags;
480 u32 devhandle; 493 u32 devhandle;
481 494
482 BUG_ON(direction == DMA_NONE); 495 BUG_ON(direction == DMA_NONE);
@@ -485,27 +498,33 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
485 pbm = dev->archdata.host_controller; 498 pbm = dev->archdata.host_controller;
486 devhandle = pbm->devhandle; 499 devhandle = pbm->devhandle;
487 500
488 local_irq_save(flags); 501 spin_lock_irqsave(&iommu->lock, flags);
489 502
490 sg = sglist; 503 sg = sglist;
491 while (nelems--) { 504 while (nelems--) {
492 dma_addr_t dma_handle = sg->dma_address; 505 dma_addr_t dma_handle = sg->dma_address;
493 unsigned int len = sg->dma_length; 506 unsigned int len = sg->dma_length;
494 unsigned long npages; 507 unsigned long npages, entry;
495 struct iommu_table *tbl = &iommu->tbl;
496 unsigned long shift = IO_PAGE_SHIFT;
497 508
498 if (!len) 509 if (!len)
499 break; 510 break;
500 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 511 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
501 entry = ((dma_handle - tbl->page_table_map_base) >> shift); 512 iommu_range_free(iommu, dma_handle, npages);
502 dma_4v_iommu_demap(&devhandle, entry, npages); 513
503 iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, 514 entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT);
504 false, NULL); 515 while (npages) {
516 unsigned long num;
517
518 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
519 npages);
520 entry += num;
521 npages -= num;
522 }
523
505 sg = sg_next(sg); 524 sg = sg_next(sg);
506 } 525 }
507 526
508 local_irq_restore(flags); 527 spin_unlock_irqrestore(&iommu->lock, flags);
509} 528}
510 529
511static struct dma_map_ops sun4v_dma_ops = { 530static struct dma_map_ops sun4v_dma_ops = {
@@ -517,8 +536,6 @@ static struct dma_map_ops sun4v_dma_ops = {
517 .unmap_sg = dma_4v_unmap_sg, 536 .unmap_sg = dma_4v_unmap_sg,
518}; 537};
519 538
520static struct iommu_tbl_ops dma_4v_iommu_ops;
521
522static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) 539static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent)
523{ 540{
524 struct property *prop; 541 struct property *prop;
@@ -533,33 +550,30 @@ static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent)
533} 550}
534 551
535static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, 552static unsigned long probe_existing_entries(struct pci_pbm_info *pbm,
536 struct iommu_table *iommu) 553 struct iommu *iommu)
537{ 554{
538 struct iommu_pool *pool; 555 struct iommu_arena *arena = &iommu->arena;
539 unsigned long i, pool_nr, cnt = 0; 556 unsigned long i, cnt = 0;
540 u32 devhandle; 557 u32 devhandle;
541 558
542 devhandle = pbm->devhandle; 559 devhandle = pbm->devhandle;
543 for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { 560 for (i = 0; i < arena->limit; i++) {
544 pool = &(iommu->arena_pool[pool_nr]); 561 unsigned long ret, io_attrs, ra;
545 for (i = pool->start; i <= pool->end; i++) { 562
546 unsigned long ret, io_attrs, ra; 563 ret = pci_sun4v_iommu_getmap(devhandle,
547 564 HV_PCI_TSBID(0, i),
548 ret = pci_sun4v_iommu_getmap(devhandle, 565 &io_attrs, &ra);
549 HV_PCI_TSBID(0, i), 566 if (ret == HV_EOK) {
550 &io_attrs, &ra); 567 if (page_in_phys_avail(ra)) {
551 if (ret == HV_EOK) { 568 pci_sun4v_iommu_demap(devhandle,
552 if (page_in_phys_avail(ra)) { 569 HV_PCI_TSBID(0, i), 1);
553 pci_sun4v_iommu_demap(devhandle, 570 } else {
554 HV_PCI_TSBID(0, 571 cnt++;
555 i), 1); 572 __set_bit(i, arena->map);
556 } else {
557 cnt++;
558 __set_bit(i, iommu->map);
559 }
560 } 573 }
561 } 574 }
562 } 575 }
576
563 return cnt; 577 return cnt;
564} 578}
565 579
@@ -587,22 +601,22 @@ static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm)
587 dma_offset = vdma[0]; 601 dma_offset = vdma[0];
588 602
589 /* Setup initial software IOMMU state. */ 603 /* Setup initial software IOMMU state. */
604 spin_lock_init(&iommu->lock);
590 iommu->ctx_lowest_free = 1; 605 iommu->ctx_lowest_free = 1;
591 iommu->tbl.page_table_map_base = dma_offset; 606 iommu->page_table_map_base = dma_offset;
592 iommu->dma_addr_mask = dma_mask; 607 iommu->dma_addr_mask = dma_mask;
593 608
594 /* Allocate and initialize the free area map. */ 609 /* Allocate and initialize the free area map. */
595 sz = (num_tsb_entries + 7) / 8; 610 sz = (num_tsb_entries + 7) / 8;
596 sz = (sz + 7UL) & ~7UL; 611 sz = (sz + 7UL) & ~7UL;
597 iommu->tbl.map = kzalloc(sz, GFP_KERNEL); 612 iommu->arena.map = kzalloc(sz, GFP_KERNEL);
598 if (!iommu->tbl.map) { 613 if (!iommu->arena.map) {
599 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); 614 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n");
600 return -ENOMEM; 615 return -ENOMEM;
601 } 616 }
602 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, 617 iommu->arena.limit = num_tsb_entries;
603 &dma_4v_iommu_ops, false /* no large_pool */, 618
604 0 /* default npools */); 619 sz = probe_existing_entries(pbm, iommu);
605 sz = probe_existing_entries(pbm, &iommu->tbl);
606 if (sz) 620 if (sz)
607 printk("%s: Imported %lu TSB entries from OBP\n", 621 printk("%s: Imported %lu TSB entries from OBP\n",
608 pbm->name, sz); 622 pbm->name, sz);
@@ -1001,17 +1015,8 @@ static struct platform_driver pci_sun4v_driver = {
1001 .probe = pci_sun4v_probe, 1015 .probe = pci_sun4v_probe,
1002}; 1016};
1003 1017
1004static void setup_iommu_pool_hash(void)
1005{
1006 unsigned int i;
1007
1008 for_each_possible_cpu(i)
1009 per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
1010}
1011
1012static int __init pci_sun4v_init(void) 1018static int __init pci_sun4v_init(void)
1013{ 1019{
1014 setup_iommu_pool_hash();
1015 return platform_driver_register(&pci_sun4v_driver); 1020 return platform_driver_register(&pci_sun4v_driver);
1016} 1021}
1017 1022
diff --git a/include/linux/iommu-common.h b/include/linux/iommu-common.h
deleted file mode 100644
index 6be5c863f329..000000000000
--- a/include/linux/iommu-common.h
+++ /dev/null
@@ -1,55 +0,0 @@
1#ifndef _LINUX_IOMMU_COMMON_H
2#define _LINUX_IOMMU_COMMON_H
3
4#include <linux/spinlock_types.h>
5#include <linux/device.h>
6#include <asm/page.h>
7
8#define IOMMU_POOL_HASHBITS 4
9#define IOMMU_NR_POOLS (1 << IOMMU_POOL_HASHBITS)
10
11struct iommu_pool {
12 unsigned long start;
13 unsigned long end;
14 unsigned long hint;
15 spinlock_t lock;
16};
17
18struct iommu_table;
19
20struct iommu_tbl_ops {
21 unsigned long (*cookie_to_index)(u64, void *);
22 void (*demap)(void *, unsigned long, unsigned long);
23 void (*reset)(struct iommu_table *);
24};
25
26struct iommu_table {
27 unsigned long page_table_map_base;
28 unsigned long page_table_shift;
29 unsigned long nr_pools;
30 const struct iommu_tbl_ops *iommu_tbl_ops;
31 unsigned long poolsize;
32 struct iommu_pool arena_pool[IOMMU_NR_POOLS];
33 u32 flags;
34#define IOMMU_HAS_LARGE_POOL 0x00000001
35 struct iommu_pool large_pool;
36 unsigned long *map;
37};
38
39extern void iommu_tbl_pool_init(struct iommu_table *iommu,
40 unsigned long num_entries,
41 u32 page_table_shift,
42 const struct iommu_tbl_ops *iommu_tbl_ops,
43 bool large_pool, u32 npools);
44
45extern unsigned long iommu_tbl_range_alloc(struct device *dev,
46 struct iommu_table *iommu,
47 unsigned long npages,
48 unsigned long *handle,
49 unsigned int pool_hash);
50
51extern void iommu_tbl_range_free(struct iommu_table *iommu,
52 u64 dma_addr, unsigned long npages,
53 bool do_demap, void *demap_arg);
54
55#endif
diff --git a/lib/Makefile b/lib/Makefile
index 6c37933336a0..da6116b21555 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -106,7 +106,7 @@ obj-$(CONFIG_AUDIT_GENERIC) += audit.o
106obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o 106obj-$(CONFIG_AUDIT_COMPAT_GENERIC) += compat_audit.o
107 107
108obj-$(CONFIG_SWIOTLB) += swiotlb.o 108obj-$(CONFIG_SWIOTLB) += swiotlb.o
109obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o iommu-common.o 109obj-$(CONFIG_IOMMU_HELPER) += iommu-helper.o
110obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o 110obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o
111obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o 111obj-$(CONFIG_NOTIFIER_ERROR_INJECTION) += notifier-error-inject.o
112obj-$(CONFIG_CPU_NOTIFIER_ERROR_INJECT) += cpu-notifier-error-inject.o 112obj-$(CONFIG_CPU_NOTIFIER_ERROR_INJECT) += cpu-notifier-error-inject.o
diff --git a/lib/iommu-common.c b/lib/iommu-common.c
deleted file mode 100644
index fac4f35250c9..000000000000
--- a/lib/iommu-common.c
+++ /dev/null
@@ -1,224 +0,0 @@
1/*
2 * IOMMU mmap management and range allocation functions.
3 * Based almost entirely upon the powerpc iommu allocator.
4 */
5
6#include <linux/export.h>
7#include <linux/bitmap.h>
8#include <linux/bug.h>
9#include <linux/iommu-helper.h>
10#include <linux/iommu-common.h>
11#include <linux/dma-mapping.h>
12
13#ifndef DMA_ERROR_CODE
14#define DMA_ERROR_CODE (~(dma_addr_t)0x0)
15#endif
16
17#define IOMMU_LARGE_ALLOC 15
18
19/*
20 * Initialize iommu_pool entries for the iommu_table. `num_entries'
21 * is the number of table entries. If `large_pool' is set to true,
22 * the top 1/4 of the table will be set aside for pool allocations
23 * of more than IOMMU_LARGE_ALLOC pages.
24 */
25extern void iommu_tbl_pool_init(struct iommu_table *iommu,
26 unsigned long num_entries,
27 u32 page_table_shift,
28 const struct iommu_tbl_ops *iommu_tbl_ops,
29 bool large_pool, u32 npools)
30{
31 unsigned int start, i;
32 struct iommu_pool *p = &(iommu->large_pool);
33
34 if (npools == 0)
35 iommu->nr_pools = IOMMU_NR_POOLS;
36 else
37 iommu->nr_pools = npools;
38 BUG_ON(npools > IOMMU_NR_POOLS);
39
40 iommu->page_table_shift = page_table_shift;
41 iommu->iommu_tbl_ops = iommu_tbl_ops;
42 start = 0;
43 if (large_pool)
44 iommu->flags |= IOMMU_HAS_LARGE_POOL;
45
46 if (!large_pool)
47 iommu->poolsize = num_entries/iommu->nr_pools;
48 else
49 iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools;
50 for (i = 0; i < iommu->nr_pools; i++) {
51 spin_lock_init(&(iommu->arena_pool[i].lock));
52 iommu->arena_pool[i].start = start;
53 iommu->arena_pool[i].hint = start;
54 start += iommu->poolsize; /* start for next pool */
55 iommu->arena_pool[i].end = start - 1;
56 }
57 if (!large_pool)
58 return;
59 /* initialize large_pool */
60 spin_lock_init(&(p->lock));
61 p->start = start;
62 p->hint = p->start;
63 p->end = num_entries;
64}
65EXPORT_SYMBOL(iommu_tbl_pool_init);
66
67unsigned long iommu_tbl_range_alloc(struct device *dev,
68 struct iommu_table *iommu,
69 unsigned long npages,
70 unsigned long *handle,
71 unsigned int pool_hash)
72{
73 unsigned long n, end, start, limit, boundary_size;
74 struct iommu_pool *arena;
75 int pass = 0;
76 unsigned int pool_nr;
77 unsigned int npools = iommu->nr_pools;
78 unsigned long flags;
79 bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0);
80 bool largealloc = (large_pool && npages > IOMMU_LARGE_ALLOC);
81 unsigned long shift;
82
83 /* Sanity check */
84 if (unlikely(npages == 0)) {
85 printk_ratelimited("npages == 0\n");
86 return DMA_ERROR_CODE;
87 }
88
89 if (largealloc) {
90 arena = &(iommu->large_pool);
91 spin_lock_irqsave(&arena->lock, flags);
92 pool_nr = 0; /* to keep compiler happy */
93 } else {
94 /* pick out pool_nr */
95 pool_nr = pool_hash & (npools - 1);
96 arena = &(iommu->arena_pool[pool_nr]);
97
98 /* find first available unlocked pool */
99 while (!spin_trylock_irqsave(&(arena->lock), flags)) {
100 pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
101 arena = &(iommu->arena_pool[pool_nr]);
102 }
103 }
104
105 again:
106 if (pass == 0 && handle && *handle &&
107 (*handle >= arena->start) && (*handle < arena->end))
108 start = *handle;
109 else
110 start = arena->hint;
111
112 limit = arena->end;
113
114 /* The case below can happen if we have a small segment appended
115 * to a large, or when the previous alloc was at the very end of
116 * the available space. If so, go back to the beginning and flush.
117 */
118 if (start >= limit) {
119 start = arena->start;
120 if (iommu->iommu_tbl_ops->reset != NULL)
121 iommu->iommu_tbl_ops->reset(iommu);
122 }
123
124 if (dev)
125 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
126 1 << iommu->page_table_shift);
127 else
128 boundary_size = ALIGN(1ULL << 32, 1 << iommu->page_table_shift);
129
130 shift = iommu->page_table_map_base >> iommu->page_table_shift;
131 boundary_size = boundary_size >> iommu->page_table_shift;
132 /*
133 * if the iommu has a non-trivial cookie <-> index mapping, we set
134 * things up so that iommu_is_span_boundary() merely checks if the
135 * (index + npages) < num_tsb_entries
136 */
137 if (iommu->iommu_tbl_ops->cookie_to_index != NULL) {
138 shift = 0;
139 boundary_size = iommu->poolsize * iommu->nr_pools;
140 }
141 n = iommu_area_alloc(iommu->map, limit, start, npages, shift,
142 boundary_size, 0);
143 if (n == -1) {
144 if (likely(pass == 0)) {
145 /* First failure, rescan from the beginning. */
146 arena->hint = arena->start;
147 if (iommu->iommu_tbl_ops->reset != NULL)
148 iommu->iommu_tbl_ops->reset(iommu);
149 pass++;
150 goto again;
151 } else if (!largealloc && pass <= iommu->nr_pools) {
152 spin_unlock(&(arena->lock));
153 pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
154 arena = &(iommu->arena_pool[pool_nr]);
155 while (!spin_trylock(&(arena->lock))) {
156 pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1);
157 arena = &(iommu->arena_pool[pool_nr]);
158 }
159 arena->hint = arena->start;
160 pass++;
161 goto again;
162 } else {
163 /* give up */
164 spin_unlock_irqrestore(&(arena->lock), flags);
165 return DMA_ERROR_CODE;
166 }
167 }
168
169 end = n + npages;
170
171 arena->hint = end;
172
173 /* Update handle for SG allocations */
174 if (handle)
175 *handle = end;
176 spin_unlock_irqrestore(&(arena->lock), flags);
177
178 return n;
179}
180EXPORT_SYMBOL(iommu_tbl_range_alloc);
181
182static struct iommu_pool *get_pool(struct iommu_table *tbl,
183 unsigned long entry)
184{
185 struct iommu_pool *p;
186 unsigned long largepool_start = tbl->large_pool.start;
187 bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0);
188
189 /* The large pool is the last pool at the top of the table */
190 if (large_pool && entry >= largepool_start) {
191 p = &tbl->large_pool;
192 } else {
193 unsigned int pool_nr = entry / tbl->poolsize;
194
195 BUG_ON(pool_nr >= tbl->nr_pools);
196 p = &tbl->arena_pool[pool_nr];
197 }
198 return p;
199}
200
201void iommu_tbl_range_free(struct iommu_table *iommu, u64 dma_addr,
202 unsigned long npages, bool do_demap, void *demap_arg)
203{
204 unsigned long entry;
205 struct iommu_pool *pool;
206 unsigned long flags;
207 unsigned long shift = iommu->page_table_shift;
208
209 if (iommu->iommu_tbl_ops->cookie_to_index != NULL) {
210 entry = (*iommu->iommu_tbl_ops->cookie_to_index)(dma_addr,
211 demap_arg);
212 } else {
213 entry = (dma_addr - iommu->page_table_map_base) >> shift;
214 }
215 pool = get_pool(iommu, entry);
216
217 spin_lock_irqsave(&(pool->lock), flags);
218 if (do_demap && iommu->iommu_tbl_ops->demap != NULL)
219 (*iommu->iommu_tbl_ops->demap)(demap_arg, entry, npages);
220
221 bitmap_clear(iommu->map, entry, npages);
222 spin_unlock_irqrestore(&(pool->lock), flags);
223}
224EXPORT_SYMBOL(iommu_tbl_range_free);
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-30 12:48:08 -0500