aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc')
-rw-r--r--arch/sparc/include/asm/iommu_64.h7
-rw-r--r--arch/sparc/kernel/iommu.c188
-rw-r--r--arch/sparc/kernel/iommu_common.h8
-rw-r--r--arch/sparc/kernel/ldc.c185
-rw-r--r--arch/sparc/kernel/pci_sun4v.c193
5 files changed, 257 insertions, 324 deletions
diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h
index 2b9321ab064d..e3cd4493d81d 100644
--- a/arch/sparc/include/asm/iommu_64.h
+++ b/arch/sparc/include/asm/iommu_64.h
@@ -16,6 +16,7 @@
16#define IOPTE_WRITE 0x0000000000000002UL 16#define IOPTE_WRITE 0x0000000000000002UL
17 17
18#define IOMMU_NUM_CTXS 4096 18#define IOMMU_NUM_CTXS 4096
19#include <linux/iommu-common.h>
19 20
20struct iommu_arena { 21struct iommu_arena {
21 unsigned long *map; 22 unsigned long *map;
@@ -24,11 +25,10 @@ struct iommu_arena {
24}; 25};
25 26
26struct iommu { 27struct iommu {
28 struct iommu_table tbl;
27 spinlock_t lock; 29 spinlock_t lock;
28 struct iommu_arena arena; 30 u32 dma_addr_mask;
29 void (*flush_all)(struct iommu *);
30 iopte_t *page_table; 31 iopte_t *page_table;
31 u32 page_table_map_base;
32 unsigned long iommu_control; 32 unsigned long iommu_control;
33 unsigned long iommu_tsbbase; 33 unsigned long iommu_tsbbase;
34 unsigned long iommu_flush; 34 unsigned long iommu_flush;
@@ -40,7 +40,6 @@ struct iommu {
40 unsigned long dummy_page_pa; 40 unsigned long dummy_page_pa;
41 unsigned long ctx_lowest_free; 41 unsigned long ctx_lowest_free;
42 DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); 42 DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS);
43 u32 dma_addr_mask;
44}; 43};
45 44
46struct strbuf { 45struct strbuf {
diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c
index bfa4d0c2df42..9b16b341b6ae 100644
--- a/arch/sparc/kernel/iommu.c
+++ b/arch/sparc/kernel/iommu.c
@@ -13,11 +13,15 @@
13#include <linux/errno.h> 13#include <linux/errno.h>
14#include <linux/iommu-helper.h> 14#include <linux/iommu-helper.h>
15#include <linux/bitmap.h> 15#include <linux/bitmap.h>
16#include <linux/hash.h>
17#include <linux/iommu-common.h>
16 18
17#ifdef CONFIG_PCI 19#ifdef CONFIG_PCI
18#include <linux/pci.h> 20#include <linux/pci.h>
19#endif 21#endif
20 22
23static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
24
21#include <asm/iommu.h> 25#include <asm/iommu.h>
22 26
23#include "iommu_common.h" 27#include "iommu_common.h"
@@ -45,8 +49,9 @@
45 "i" (ASI_PHYS_BYPASS_EC_E)) 49 "i" (ASI_PHYS_BYPASS_EC_E))
46 50
47/* Must be invoked under the IOMMU lock. */ 51/* Must be invoked under the IOMMU lock. */
48static void iommu_flushall(struct iommu *iommu) 52static void iommu_flushall(struct iommu_table *iommu_table)
49{ 53{
54 struct iommu *iommu = container_of(iommu_table, struct iommu, tbl);
50 if (iommu->iommu_flushinv) { 55 if (iommu->iommu_flushinv) {
51 iommu_write(iommu->iommu_flushinv, ~(u64)0); 56 iommu_write(iommu->iommu_flushinv, ~(u64)0);
52 } else { 57 } else {
@@ -87,94 +92,23 @@ static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte)
87 iopte_val(*iopte) = val; 92 iopte_val(*iopte) = val;
88} 93}
89 94
90/* Based almost entirely upon the ppc64 iommu allocator. If you use the 'handle' 95static struct iommu_tbl_ops iommu_sparc_ops = {
91 * facility it must all be done in one pass while under the iommu lock. 96 .reset = iommu_flushall
92 * 97};
93 * On sun4u platforms, we only flush the IOMMU once every time we've passed
94 * over the entire page table doing allocations. Therefore we only ever advance
95 * the hint and cannot backtrack it.
96 */
97unsigned long iommu_range_alloc(struct device *dev,
98 struct iommu *iommu,
99 unsigned long npages,
100 unsigned long *handle)
101{
102 unsigned long n, end, start, limit, boundary_size;
103 struct iommu_arena *arena = &iommu->arena;
104 int pass = 0;
105
106 /* This allocator was derived from x86_64's bit string search */
107
108 /* Sanity check */
109 if (unlikely(npages == 0)) {
110 if (printk_ratelimit())
111 WARN_ON(1);
112 return DMA_ERROR_CODE;
113 }
114
115 if (handle && *handle)
116 start = *handle;
117 else
118 start = arena->hint;
119
120 limit = arena->limit;
121
122 /* The case below can happen if we have a small segment appended
123 * to a large, or when the previous alloc was at the very end of
124 * the available space. If so, go back to the beginning and flush.
125 */
126 if (start >= limit) {
127 start = 0;
128 if (iommu->flush_all)
129 iommu->flush_all(iommu);
130 }
131
132 again:
133
134 if (dev)
135 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
136 1 << IO_PAGE_SHIFT);
137 else
138 boundary_size = ALIGN(1UL << 32, 1 << IO_PAGE_SHIFT);
139
140 n = iommu_area_alloc(arena->map, limit, start, npages,
141 iommu->page_table_map_base >> IO_PAGE_SHIFT,
142 boundary_size >> IO_PAGE_SHIFT, 0);
143 if (n == -1) {
144 if (likely(pass < 1)) {
145 /* First failure, rescan from the beginning. */
146 start = 0;
147 if (iommu->flush_all)
148 iommu->flush_all(iommu);
149 pass++;
150 goto again;
151 } else {
152 /* Second failure, give up */
153 return DMA_ERROR_CODE;
154 }
155 }
156
157 end = n + npages;
158
159 arena->hint = end;
160
161 /* Update handle for SG allocations */
162 if (handle)
163 *handle = end;
164
165 return n;
166}
167 98
168void iommu_range_free(struct iommu *iommu, dma_addr_t dma_addr, unsigned long npages) 99static void setup_iommu_pool_hash(void)
169{ 100{
170 struct iommu_arena *arena = &iommu->arena; 101 unsigned int i;
171 unsigned long entry; 102 static bool do_once;
172
173 entry = (dma_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
174 103
175 bitmap_clear(arena->map, entry, npages); 104 if (do_once)
105 return;
106 do_once = true;
107 for_each_possible_cpu(i)
108 per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
176} 109}
177 110
111
178int iommu_table_init(struct iommu *iommu, int tsbsize, 112int iommu_table_init(struct iommu *iommu, int tsbsize,
179 u32 dma_offset, u32 dma_addr_mask, 113 u32 dma_offset, u32 dma_addr_mask,
180 int numa_node) 114 int numa_node)
@@ -187,22 +121,22 @@ int iommu_table_init(struct iommu *iommu, int tsbsize,
187 /* Setup initial software IOMMU state. */ 121 /* Setup initial software IOMMU state. */
188 spin_lock_init(&iommu->lock); 122 spin_lock_init(&iommu->lock);
189 iommu->ctx_lowest_free = 1; 123 iommu->ctx_lowest_free = 1;
190 iommu->page_table_map_base = dma_offset; 124 iommu->tbl.page_table_map_base = dma_offset;
191 iommu->dma_addr_mask = dma_addr_mask; 125 iommu->dma_addr_mask = dma_addr_mask;
192 126
193 /* Allocate and initialize the free area map. */ 127 /* Allocate and initialize the free area map. */
194 sz = num_tsb_entries / 8; 128 sz = num_tsb_entries / 8;
195 sz = (sz + 7UL) & ~7UL; 129 sz = (sz + 7UL) & ~7UL;
196 iommu->arena.map = kmalloc_node(sz, GFP_KERNEL, numa_node); 130 iommu->tbl.map = kmalloc_node(sz, GFP_KERNEL, numa_node);
197 if (!iommu->arena.map) { 131 if (!iommu->tbl.map)
198 printk(KERN_ERR "IOMMU: Error, kmalloc(arena.map) failed.\n");
199 return -ENOMEM; 132 return -ENOMEM;
200 } 133 memset(iommu->tbl.map, 0, sz);
201 memset(iommu->arena.map, 0, sz);
202 iommu->arena.limit = num_tsb_entries;
203
204 if (tlb_type != hypervisor) 134 if (tlb_type != hypervisor)
205 iommu->flush_all = iommu_flushall; 135 iommu_sparc_ops.reset = NULL; /* not needed on on sun4v */
136
137 setup_iommu_pool_hash();
138 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,
139 &iommu_sparc_ops, false, 1);
206 140
207 /* Allocate and initialize the dummy page which we 141 /* Allocate and initialize the dummy page which we
208 * set inactive IO PTEs to point to. 142 * set inactive IO PTEs to point to.
@@ -235,18 +169,20 @@ out_free_dummy_page:
235 iommu->dummy_page = 0UL; 169 iommu->dummy_page = 0UL;
236 170
237out_free_map: 171out_free_map:
238 kfree(iommu->arena.map); 172 kfree(iommu->tbl.map);
239 iommu->arena.map = NULL; 173 iommu->tbl.map = NULL;
240 174
241 return -ENOMEM; 175 return -ENOMEM;
242} 176}
243 177
244static inline iopte_t *alloc_npages(struct device *dev, struct iommu *iommu, 178static inline iopte_t *alloc_npages(struct device *dev,
179 struct iommu *iommu,
245 unsigned long npages) 180 unsigned long npages)
246{ 181{
247 unsigned long entry; 182 unsigned long entry;
248 183
249 entry = iommu_range_alloc(dev, iommu, npages, NULL); 184 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
185 __this_cpu_read(iommu_pool_hash));
250 if (unlikely(entry == DMA_ERROR_CODE)) 186 if (unlikely(entry == DMA_ERROR_CODE))
251 return NULL; 187 return NULL;
252 188
@@ -284,7 +220,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size,
284 dma_addr_t *dma_addrp, gfp_t gfp, 220 dma_addr_t *dma_addrp, gfp_t gfp,
285 struct dma_attrs *attrs) 221 struct dma_attrs *attrs)
286{ 222{
287 unsigned long flags, order, first_page; 223 unsigned long order, first_page;
288 struct iommu *iommu; 224 struct iommu *iommu;
289 struct page *page; 225 struct page *page;
290 int npages, nid; 226 int npages, nid;
@@ -306,16 +242,14 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size,
306 242
307 iommu = dev->archdata.iommu; 243 iommu = dev->archdata.iommu;
308 244
309 spin_lock_irqsave(&iommu->lock, flags);
310 iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); 245 iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT);
311 spin_unlock_irqrestore(&iommu->lock, flags);
312 246
313 if (unlikely(iopte == NULL)) { 247 if (unlikely(iopte == NULL)) {
314 free_pages(first_page, order); 248 free_pages(first_page, order);
315 return NULL; 249 return NULL;
316 } 250 }
317 251
318 *dma_addrp = (iommu->page_table_map_base + 252 *dma_addrp = (iommu->tbl.page_table_map_base +
319 ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); 253 ((iopte - iommu->page_table) << IO_PAGE_SHIFT));
320 ret = (void *) first_page; 254 ret = (void *) first_page;
321 npages = size >> IO_PAGE_SHIFT; 255 npages = size >> IO_PAGE_SHIFT;
@@ -336,16 +270,12 @@ static void dma_4u_free_coherent(struct device *dev, size_t size,
336 struct dma_attrs *attrs) 270 struct dma_attrs *attrs)
337{ 271{
338 struct iommu *iommu; 272 struct iommu *iommu;
339 unsigned long flags, order, npages; 273 unsigned long order, npages;
340 274
341 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 275 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
342 iommu = dev->archdata.iommu; 276 iommu = dev->archdata.iommu;
343 277
344 spin_lock_irqsave(&iommu->lock, flags); 278 iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL);
345
346 iommu_range_free(iommu, dvma, npages);
347
348 spin_unlock_irqrestore(&iommu->lock, flags);
349 279
350 order = get_order(size); 280 order = get_order(size);
351 if (order < 10) 281 if (order < 10)
@@ -375,8 +305,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page,
375 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 305 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
376 npages >>= IO_PAGE_SHIFT; 306 npages >>= IO_PAGE_SHIFT;
377 307
378 spin_lock_irqsave(&iommu->lock, flags);
379 base = alloc_npages(dev, iommu, npages); 308 base = alloc_npages(dev, iommu, npages);
309 spin_lock_irqsave(&iommu->lock, flags);
380 ctx = 0; 310 ctx = 0;
381 if (iommu->iommu_ctxflush) 311 if (iommu->iommu_ctxflush)
382 ctx = iommu_alloc_ctx(iommu); 312 ctx = iommu_alloc_ctx(iommu);
@@ -385,7 +315,7 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page,
385 if (unlikely(!base)) 315 if (unlikely(!base))
386 goto bad; 316 goto bad;
387 317
388 bus_addr = (iommu->page_table_map_base + 318 bus_addr = (iommu->tbl.page_table_map_base +
389 ((base - iommu->page_table) << IO_PAGE_SHIFT)); 319 ((base - iommu->page_table) << IO_PAGE_SHIFT));
390 ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 320 ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
391 base_paddr = __pa(oaddr & IO_PAGE_MASK); 321 base_paddr = __pa(oaddr & IO_PAGE_MASK);
@@ -496,7 +426,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
496 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 426 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
497 npages >>= IO_PAGE_SHIFT; 427 npages >>= IO_PAGE_SHIFT;
498 base = iommu->page_table + 428 base = iommu->page_table +
499 ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 429 ((bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT);
500 bus_addr &= IO_PAGE_MASK; 430 bus_addr &= IO_PAGE_MASK;
501 431
502 spin_lock_irqsave(&iommu->lock, flags); 432 spin_lock_irqsave(&iommu->lock, flags);
@@ -515,11 +445,11 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr,
515 for (i = 0; i < npages; i++) 445 for (i = 0; i < npages; i++)
516 iopte_make_dummy(iommu, base + i); 446 iopte_make_dummy(iommu, base + i);
517 447
518 iommu_range_free(iommu, bus_addr, npages);
519
520 iommu_free_ctx(iommu, ctx); 448 iommu_free_ctx(iommu, ctx);
521
522 spin_unlock_irqrestore(&iommu->lock, flags); 449 spin_unlock_irqrestore(&iommu->lock, flags);
450
451 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages,
452 false, NULL);
523} 453}
524 454
525static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, 455static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -567,7 +497,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
567 max_seg_size = dma_get_max_seg_size(dev); 497 max_seg_size = dma_get_max_seg_size(dev);
568 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 498 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
569 IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 499 IO_PAGE_SIZE) >> IO_PAGE_SHIFT;
570 base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; 500 base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT;
571 for_each_sg(sglist, s, nelems, i) { 501 for_each_sg(sglist, s, nelems, i) {
572 unsigned long paddr, npages, entry, out_entry = 0, slen; 502 unsigned long paddr, npages, entry, out_entry = 0, slen;
573 iopte_t *base; 503 iopte_t *base;
@@ -581,7 +511,8 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
581 /* Allocate iommu entries for that segment */ 511 /* Allocate iommu entries for that segment */
582 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 512 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
583 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 513 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
584 entry = iommu_range_alloc(dev, iommu, npages, &handle); 514 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle,
515 __this_cpu_read(iommu_pool_hash));
585 516
586 /* Handle failure */ 517 /* Handle failure */
587 if (unlikely(entry == DMA_ERROR_CODE)) { 518 if (unlikely(entry == DMA_ERROR_CODE)) {
@@ -594,7 +525,7 @@ static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist,
594 base = iommu->page_table + entry; 525 base = iommu->page_table + entry;
595 526
596 /* Convert entry to a dma_addr_t */ 527 /* Convert entry to a dma_addr_t */
597 dma_addr = iommu->page_table_map_base + 528 dma_addr = iommu->tbl.page_table_map_base +
598 (entry << IO_PAGE_SHIFT); 529 (entry << IO_PAGE_SHIFT);
599 dma_addr |= (s->offset & ~IO_PAGE_MASK); 530 dma_addr |= (s->offset & ~IO_PAGE_MASK);
600 531
@@ -654,15 +585,17 @@ iommu_map_failed:
654 vaddr = s->dma_address & IO_PAGE_MASK; 585 vaddr = s->dma_address & IO_PAGE_MASK;
655 npages = iommu_num_pages(s->dma_address, s->dma_length, 586 npages = iommu_num_pages(s->dma_address, s->dma_length,
656 IO_PAGE_SIZE); 587 IO_PAGE_SIZE);
657 iommu_range_free(iommu, vaddr, npages);
658 588
659 entry = (vaddr - iommu->page_table_map_base) 589 entry = (vaddr - iommu->tbl.page_table_map_base)
660 >> IO_PAGE_SHIFT; 590 >> IO_PAGE_SHIFT;
661 base = iommu->page_table + entry; 591 base = iommu->page_table + entry;
662 592
663 for (j = 0; j < npages; j++) 593 for (j = 0; j < npages; j++)
664 iopte_make_dummy(iommu, base + j); 594 iopte_make_dummy(iommu, base + j);
665 595
596 iommu_tbl_range_free(&iommu->tbl, vaddr, npages,
597 false, NULL);
598
666 s->dma_address = DMA_ERROR_CODE; 599 s->dma_address = DMA_ERROR_CODE;
667 s->dma_length = 0; 600 s->dma_length = 0;
668 } 601 }
@@ -677,17 +610,19 @@ iommu_map_failed:
677/* If contexts are being used, they are the same in all of the mappings 610/* If contexts are being used, they are the same in all of the mappings
678 * we make for a particular SG. 611 * we make for a particular SG.
679 */ 612 */
680static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) 613static unsigned long fetch_sg_ctx(struct iommu *iommu,
614 struct scatterlist *sg)
681{ 615{
682 unsigned long ctx = 0; 616 unsigned long ctx = 0;
683 617
684 if (iommu->iommu_ctxflush) { 618 if (iommu->iommu_ctxflush) {
685 iopte_t *base; 619 iopte_t *base;
686 u32 bus_addr; 620 u32 bus_addr;
621 struct iommu_table *tbl = &iommu->tbl;
687 622
688 bus_addr = sg->dma_address & IO_PAGE_MASK; 623 bus_addr = sg->dma_address & IO_PAGE_MASK;
689 base = iommu->page_table + 624 base = iommu->page_table +
690 ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 625 ((bus_addr - tbl->page_table_map_base) >> IO_PAGE_SHIFT);
691 626
692 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; 627 ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL;
693 } 628 }
@@ -723,9 +658,8 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
723 if (!len) 658 if (!len)
724 break; 659 break;
725 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 660 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
726 iommu_range_free(iommu, dma_handle, npages);
727 661
728 entry = ((dma_handle - iommu->page_table_map_base) 662 entry = ((dma_handle - iommu->tbl.page_table_map_base)
729 >> IO_PAGE_SHIFT); 663 >> IO_PAGE_SHIFT);
730 base = iommu->page_table + entry; 664 base = iommu->page_table + entry;
731 665
@@ -737,6 +671,8 @@ static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist,
737 for (i = 0; i < npages; i++) 671 for (i = 0; i < npages; i++)
738 iopte_make_dummy(iommu, base + i); 672 iopte_make_dummy(iommu, base + i);
739 673
674 iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, false,
675 NULL);
740 sg = sg_next(sg); 676 sg = sg_next(sg);
741 } 677 }
742 678
@@ -770,9 +706,10 @@ static void dma_4u_sync_single_for_cpu(struct device *dev,
770 if (iommu->iommu_ctxflush && 706 if (iommu->iommu_ctxflush &&
771 strbuf->strbuf_ctxflush) { 707 strbuf->strbuf_ctxflush) {
772 iopte_t *iopte; 708 iopte_t *iopte;
709 struct iommu_table *tbl = &iommu->tbl;
773 710
774 iopte = iommu->page_table + 711 iopte = iommu->page_table +
775 ((bus_addr - iommu->page_table_map_base)>>IO_PAGE_SHIFT); 712 ((bus_addr - tbl->page_table_map_base)>>IO_PAGE_SHIFT);
776 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 713 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
777 } 714 }
778 715
@@ -805,9 +742,10 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev,
805 if (iommu->iommu_ctxflush && 742 if (iommu->iommu_ctxflush &&
806 strbuf->strbuf_ctxflush) { 743 strbuf->strbuf_ctxflush) {
807 iopte_t *iopte; 744 iopte_t *iopte;
745 struct iommu_table *tbl = &iommu->tbl;
808 746
809 iopte = iommu->page_table + 747 iopte = iommu->page_table + ((sglist[0].dma_address -
810 ((sglist[0].dma_address - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 748 tbl->page_table_map_base) >> IO_PAGE_SHIFT);
811 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; 749 ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL;
812 } 750 }
813 751
diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h
index 1ec0de4156e7..f4be0d724fc6 100644
--- a/arch/sparc/kernel/iommu_common.h
+++ b/arch/sparc/kernel/iommu_common.h
@@ -48,12 +48,4 @@ static inline int is_span_boundary(unsigned long entry,
48 return iommu_is_span_boundary(entry, nr, shift, boundary_size); 48 return iommu_is_span_boundary(entry, nr, shift, boundary_size);
49} 49}
50 50
51unsigned long iommu_range_alloc(struct device *dev,
52 struct iommu *iommu,
53 unsigned long npages,
54 unsigned long *handle);
55void iommu_range_free(struct iommu *iommu,
56 dma_addr_t dma_addr,
57 unsigned long npages);
58
59#endif /* _IOMMU_COMMON_H */ 51#endif /* _IOMMU_COMMON_H */
diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c
index 274a9f59d95c..d485697c37c0 100644
--- a/arch/sparc/kernel/ldc.c
+++ b/arch/sparc/kernel/ldc.c
@@ -15,6 +15,8 @@
15#include <linux/list.h> 15#include <linux/list.h>
16#include <linux/init.h> 16#include <linux/init.h>
17#include <linux/bitmap.h> 17#include <linux/bitmap.h>
18#include <linux/hash.h>
19#include <linux/iommu-common.h>
18 20
19#include <asm/hypervisor.h> 21#include <asm/hypervisor.h>
20#include <asm/iommu.h> 22#include <asm/iommu.h>
@@ -27,6 +29,11 @@
27#define DRV_MODULE_VERSION "1.1" 29#define DRV_MODULE_VERSION "1.1"
28#define DRV_MODULE_RELDATE "July 22, 2008" 30#define DRV_MODULE_RELDATE "July 22, 2008"
29 31
32#define COOKIE_PGSZ_CODE 0xf000000000000000ULL
33#define COOKIE_PGSZ_CODE_SHIFT 60ULL
34
35static DEFINE_PER_CPU(unsigned int, ldc_pool_hash);
36
30static char version[] = 37static char version[] =
31 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
32#define LDC_PACKET_SIZE 64 39#define LDC_PACKET_SIZE 64
@@ -98,10 +105,10 @@ static const struct ldc_mode_ops stream_ops;
98int ldom_domaining_enabled; 105int ldom_domaining_enabled;
99 106
100struct ldc_iommu { 107struct ldc_iommu {
101 /* Protects arena alloc/free. */ 108 /* Protects ldc_unmap. */
102 spinlock_t lock; 109 spinlock_t lock;
103 struct iommu_arena arena;
104 struct ldc_mtable_entry *page_table; 110 struct ldc_mtable_entry *page_table;
111 struct iommu_table iommu_table;
105}; 112};
106 113
107struct ldc_channel { 114struct ldc_channel {
@@ -998,31 +1005,85 @@ static void free_queue(unsigned long num_entries, struct ldc_packet *q)
998 free_pages((unsigned long)q, order); 1005 free_pages((unsigned long)q, order);
999} 1006}
1000 1007
1008static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
1009{
1010 u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1011 /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
1012
1013 cookie &= ~COOKIE_PGSZ_CODE;
1014
1015 return (cookie >> (13ULL + (szcode * 3ULL)));
1016}
1017
1018struct ldc_demap_arg {
1019 struct ldc_iommu *ldc_iommu;
1020 u64 cookie;
1021 unsigned long id;
1022};
1023
1024static void ldc_demap(void *arg, unsigned long entry, unsigned long npages)
1025{
1026 struct ldc_demap_arg *ldc_demap_arg = arg;
1027 struct ldc_iommu *iommu = ldc_demap_arg->ldc_iommu;
1028 unsigned long id = ldc_demap_arg->id;
1029 u64 cookie = ldc_demap_arg->cookie;
1030 struct ldc_mtable_entry *base;
1031 unsigned long i, shift;
1032
1033 shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
1034 base = iommu->page_table + entry;
1035 for (i = 0; i < npages; i++) {
1036 if (base->cookie)
1037 sun4v_ldc_revoke(id, cookie + (i << shift),
1038 base->cookie);
1039 base->mte = 0;
1040 }
1041}
1042
1001/* XXX Make this configurable... XXX */ 1043/* XXX Make this configurable... XXX */
1002#define LDC_IOTABLE_SIZE (8 * 1024) 1044#define LDC_IOTABLE_SIZE (8 * 1024)
1003 1045
1004static int ldc_iommu_init(struct ldc_channel *lp) 1046struct iommu_tbl_ops ldc_iommu_ops = {
1047 .cookie_to_index = ldc_cookie_to_index,
1048 .demap = ldc_demap,
1049};
1050
1051static void setup_ldc_pool_hash(void)
1052{
1053 unsigned int i;
1054 static bool do_once;
1055
1056 if (do_once)
1057 return;
1058 do_once = true;
1059 for_each_possible_cpu(i)
1060 per_cpu(ldc_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
1061}
1062
1063
1064static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1005{ 1065{
1006 unsigned long sz, num_tsb_entries, tsbsize, order; 1066 unsigned long sz, num_tsb_entries, tsbsize, order;
1007 struct ldc_iommu *iommu = &lp->iommu; 1067 struct ldc_iommu *ldc_iommu = &lp->iommu;
1068 struct iommu_table *iommu = &ldc_iommu->iommu_table;
1008 struct ldc_mtable_entry *table; 1069 struct ldc_mtable_entry *table;
1009 unsigned long hv_err; 1070 unsigned long hv_err;
1010 int err; 1071 int err;
1011 1072
1012 num_tsb_entries = LDC_IOTABLE_SIZE; 1073 num_tsb_entries = LDC_IOTABLE_SIZE;
1013 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1074 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1014 1075 setup_ldc_pool_hash();
1015 spin_lock_init(&iommu->lock); 1076 spin_lock_init(&ldc_iommu->lock);
1016 1077
1017 sz = num_tsb_entries / 8; 1078 sz = num_tsb_entries / 8;
1018 sz = (sz + 7UL) & ~7UL; 1079 sz = (sz + 7UL) & ~7UL;
1019 iommu->arena.map = kzalloc(sz, GFP_KERNEL); 1080 iommu->map = kzalloc(sz, GFP_KERNEL);
1020 if (!iommu->arena.map) { 1081 if (!iommu->map) {
1021 printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1082 printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1022 return -ENOMEM; 1083 return -ENOMEM;
1023 } 1084 }
1024 1085 iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
1025 iommu->arena.limit = num_tsb_entries; 1086 &ldc_iommu_ops, false, 1);
1026 1087
1027 order = get_order(tsbsize); 1088 order = get_order(tsbsize);
1028 1089
@@ -1037,7 +1098,7 @@ static int ldc_iommu_init(struct ldc_channel *lp)
1037 1098
1038 memset(table, 0, PAGE_SIZE << order); 1099 memset(table, 0, PAGE_SIZE << order);
1039 1100
1040 iommu->page_table = table; 1101 ldc_iommu->page_table = table;
1041 1102
1042 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1103 hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1043 num_tsb_entries); 1104 num_tsb_entries);
@@ -1049,31 +1110,32 @@ static int ldc_iommu_init(struct ldc_channel *lp)
1049 1110
1050out_free_table: 1111out_free_table:
1051 free_pages((unsigned long) table, order); 1112 free_pages((unsigned long) table, order);
1052 iommu->page_table = NULL; 1113 ldc_iommu->page_table = NULL;
1053 1114
1054out_free_map: 1115out_free_map:
1055 kfree(iommu->arena.map); 1116 kfree(iommu->map);
1056 iommu->arena.map = NULL; 1117 iommu->map = NULL;
1057 1118
1058 return err; 1119 return err;
1059} 1120}
1060 1121
1061static void ldc_iommu_release(struct ldc_channel *lp) 1122static void ldc_iommu_release(struct ldc_channel *lp)
1062{ 1123{
1063 struct ldc_iommu *iommu = &lp->iommu; 1124 struct ldc_iommu *ldc_iommu = &lp->iommu;
1125 struct iommu_table *iommu = &ldc_iommu->iommu_table;
1064 unsigned long num_tsb_entries, tsbsize, order; 1126 unsigned long num_tsb_entries, tsbsize, order;
1065 1127
1066 (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1128 (void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1067 1129
1068 num_tsb_entries = iommu->arena.limit; 1130 num_tsb_entries = iommu->poolsize * iommu->nr_pools;
1069 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1131 tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1070 order = get_order(tsbsize); 1132 order = get_order(tsbsize);
1071 1133
1072 free_pages((unsigned long) iommu->page_table, order); 1134 free_pages((unsigned long) ldc_iommu->page_table, order);
1073 iommu->page_table = NULL; 1135 ldc_iommu->page_table = NULL;
1074 1136
1075 kfree(iommu->arena.map); 1137 kfree(iommu->map);
1076 iommu->arena.map = NULL; 1138 iommu->map = NULL;
1077} 1139}
1078 1140
1079struct ldc_channel *ldc_alloc(unsigned long id, 1141struct ldc_channel *ldc_alloc(unsigned long id,
@@ -1140,7 +1202,7 @@ struct ldc_channel *ldc_alloc(unsigned long id,
1140 1202
1141 lp->id = id; 1203 lp->id = id;
1142 1204
1143 err = ldc_iommu_init(lp); 1205 err = ldc_iommu_init(name, lp);
1144 if (err) 1206 if (err)
1145 goto out_free_ldc; 1207 goto out_free_ldc;
1146 1208
@@ -1885,40 +1947,6 @@ int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1885} 1947}
1886EXPORT_SYMBOL(ldc_read); 1948EXPORT_SYMBOL(ldc_read);
1887 1949
1888static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages)
1889{
1890 struct iommu_arena *arena = &iommu->arena;
1891 unsigned long n, start, end, limit;
1892 int pass;
1893
1894 limit = arena->limit;
1895 start = arena->hint;
1896 pass = 0;
1897
1898again:
1899 n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0);
1900 end = n + npages;
1901 if (unlikely(end >= limit)) {
1902 if (likely(pass < 1)) {
1903 limit = start;
1904 start = 0;
1905 pass++;
1906 goto again;
1907 } else {
1908 /* Scanned the whole thing, give up. */
1909 return -1;
1910 }
1911 }
1912 bitmap_set(arena->map, n, npages);
1913
1914 arena->hint = end;
1915
1916 return n;
1917}
1918
1919#define COOKIE_PGSZ_CODE 0xf000000000000000ULL
1920#define COOKIE_PGSZ_CODE_SHIFT 60ULL
1921
1922static u64 pagesize_code(void) 1950static u64 pagesize_code(void)
1923{ 1951{
1924 switch (PAGE_SIZE) { 1952 switch (PAGE_SIZE) {
@@ -1945,23 +1973,14 @@ static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
1945 page_offset); 1973 page_offset);
1946} 1974}
1947 1975
1948static u64 cookie_to_index(u64 cookie, unsigned long *shift)
1949{
1950 u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1951
1952 cookie &= ~COOKIE_PGSZ_CODE;
1953
1954 *shift = szcode * 3;
1955
1956 return (cookie >> (13ULL + (szcode * 3ULL)));
1957}
1958 1976
1959static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1977static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
1960 unsigned long npages) 1978 unsigned long npages)
1961{ 1979{
1962 long entry; 1980 long entry;
1963 1981
1964 entry = arena_alloc(iommu, npages); 1982 entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_table, npages,
1983 NULL, __this_cpu_read(ldc_pool_hash));
1965 if (unlikely(entry < 0)) 1984 if (unlikely(entry < 0))
1966 return NULL; 1985 return NULL;
1967 1986
@@ -2090,7 +2109,7 @@ int ldc_map_sg(struct ldc_channel *lp,
2090 struct ldc_trans_cookie *cookies, int ncookies, 2109 struct ldc_trans_cookie *cookies, int ncookies,
2091 unsigned int map_perm) 2110 unsigned int map_perm)
2092{ 2111{
2093 unsigned long i, npages, flags; 2112 unsigned long i, npages;
2094 struct ldc_mtable_entry *base; 2113 struct ldc_mtable_entry *base;
2095 struct cookie_state state; 2114 struct cookie_state state;
2096 struct ldc_iommu *iommu; 2115 struct ldc_iommu *iommu;
@@ -2109,9 +2128,7 @@ int ldc_map_sg(struct ldc_channel *lp,
2109 2128
2110 iommu = &lp->iommu; 2129 iommu = &lp->iommu;
2111 2130
2112 spin_lock_irqsave(&iommu->lock, flags);
2113 base = alloc_npages(iommu, npages); 2131 base = alloc_npages(iommu, npages);
2114 spin_unlock_irqrestore(&iommu->lock, flags);
2115 2132
2116 if (!base) 2133 if (!base)
2117 return -ENOMEM; 2134 return -ENOMEM;
@@ -2136,7 +2153,7 @@ int ldc_map_single(struct ldc_channel *lp,
2136 struct ldc_trans_cookie *cookies, int ncookies, 2153 struct ldc_trans_cookie *cookies, int ncookies,
2137 unsigned int map_perm) 2154 unsigned int map_perm)
2138{ 2155{
2139 unsigned long npages, pa, flags; 2156 unsigned long npages, pa;
2140 struct ldc_mtable_entry *base; 2157 struct ldc_mtable_entry *base;
2141 struct cookie_state state; 2158 struct cookie_state state;
2142 struct ldc_iommu *iommu; 2159 struct ldc_iommu *iommu;
@@ -2152,9 +2169,7 @@ int ldc_map_single(struct ldc_channel *lp,
2152 2169
2153 iommu = &lp->iommu; 2170 iommu = &lp->iommu;
2154 2171
2155 spin_lock_irqsave(&iommu->lock, flags);
2156 base = alloc_npages(iommu, npages); 2172 base = alloc_npages(iommu, npages);
2157 spin_unlock_irqrestore(&iommu->lock, flags);
2158 2173
2159 if (!base) 2174 if (!base)
2160 return -ENOMEM; 2175 return -ENOMEM;
@@ -2172,35 +2187,29 @@ int ldc_map_single(struct ldc_channel *lp,
2172} 2187}
2173EXPORT_SYMBOL(ldc_map_single); 2188EXPORT_SYMBOL(ldc_map_single);
2174 2189
2190
2175static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2191static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2176 u64 cookie, u64 size) 2192 u64 cookie, u64 size)
2177{ 2193{
2178 struct iommu_arena *arena = &iommu->arena; 2194 unsigned long npages;
2179 unsigned long i, shift, index, npages; 2195 struct ldc_demap_arg demap_arg;
2180 struct ldc_mtable_entry *base;
2181 2196
2182 npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2197 demap_arg.ldc_iommu = iommu;
2183 index = cookie_to_index(cookie, &shift); 2198 demap_arg.cookie = cookie;
2184 base = iommu->page_table + index; 2199 demap_arg.id = id;
2185 2200
2186 BUG_ON(index > arena->limit || 2201 npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2187 (index + npages) > arena->limit); 2202 iommu_tbl_range_free(&iommu->iommu_table, cookie, npages, true,
2203 &demap_arg);
2188 2204
2189 for (i = 0; i < npages; i++) {
2190 if (base->cookie)
2191 sun4v_ldc_revoke(id, cookie + (i << shift),
2192 base->cookie);
2193 base->mte = 0;
2194 __clear_bit(index + i, arena->map);
2195 }
2196} 2205}
2197 2206
2198void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2207void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2199 int ncookies) 2208 int ncookies)
2200{ 2209{
2201 struct ldc_iommu *iommu = &lp->iommu; 2210 struct ldc_iommu *iommu = &lp->iommu;
2202 unsigned long flags;
2203 int i; 2211 int i;
2212 unsigned long flags;
2204 2213
2205 spin_lock_irqsave(&iommu->lock, flags); 2214 spin_lock_irqsave(&iommu->lock, flags);
2206 for (i = 0; i < ncookies; i++) { 2215 for (i = 0; i < ncookies; i++) {
diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c
index 47ddbd496a1e..9b76b9d639e1 100644
--- a/arch/sparc/kernel/pci_sun4v.c
+++ b/arch/sparc/kernel/pci_sun4v.c
@@ -15,6 +15,8 @@
15#include <linux/export.h> 15#include <linux/export.h>
16#include <linux/log2.h> 16#include <linux/log2.h>
17#include <linux/of_device.h> 17#include <linux/of_device.h>
18#include <linux/hash.h>
19#include <linux/iommu-common.h>
18 20
19#include <asm/iommu.h> 21#include <asm/iommu.h>
20#include <asm/irq.h> 22#include <asm/irq.h>
@@ -28,6 +30,7 @@
28 30
29#define DRIVER_NAME "pci_sun4v" 31#define DRIVER_NAME "pci_sun4v"
30#define PFX DRIVER_NAME ": " 32#define PFX DRIVER_NAME ": "
33static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
31 34
32static unsigned long vpci_major = 1; 35static unsigned long vpci_major = 1;
33static unsigned long vpci_minor = 1; 36static unsigned long vpci_minor = 1;
@@ -155,14 +158,13 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
155 158
156 iommu = dev->archdata.iommu; 159 iommu = dev->archdata.iommu;
157 160
158 spin_lock_irqsave(&iommu->lock, flags); 161 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
159 entry = iommu_range_alloc(dev, iommu, npages, NULL); 162 __this_cpu_read(iommu_pool_hash));
160 spin_unlock_irqrestore(&iommu->lock, flags);
161 163
162 if (unlikely(entry == DMA_ERROR_CODE)) 164 if (unlikely(entry == DMA_ERROR_CODE))
163 goto range_alloc_fail; 165 goto range_alloc_fail;
164 166
165 *dma_addrp = (iommu->page_table_map_base + 167 *dma_addrp = (iommu->tbl.page_table_map_base +
166 (entry << IO_PAGE_SHIFT)); 168 (entry << IO_PAGE_SHIFT));
167 ret = (void *) first_page; 169 ret = (void *) first_page;
168 first_page = __pa(first_page); 170 first_page = __pa(first_page);
@@ -188,45 +190,46 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
188 return ret; 190 return ret;
189 191
190iommu_map_fail: 192iommu_map_fail:
191 /* Interrupts are disabled. */ 193 iommu_tbl_range_free(&iommu->tbl, *dma_addrp, npages, false, NULL);
192 spin_lock(&iommu->lock);
193 iommu_range_free(iommu, *dma_addrp, npages);
194 spin_unlock_irqrestore(&iommu->lock, flags);
195 194
196range_alloc_fail: 195range_alloc_fail:
197 free_pages(first_page, order); 196 free_pages(first_page, order);
198 return NULL; 197 return NULL;
199} 198}
200 199
200static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry,
201 unsigned long npages)
202{
203 u32 devhandle = *(u32 *)demap_arg;
204 unsigned long num, flags;
205
206 local_irq_save(flags);
207 do {
208 num = pci_sun4v_iommu_demap(devhandle,
209 HV_PCI_TSBID(0, entry),
210 npages);
211
212 entry += num;
213 npages -= num;
214 } while (npages != 0);
215 local_irq_restore(flags);
216}
217
201static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, 218static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu,
202 dma_addr_t dvma, struct dma_attrs *attrs) 219 dma_addr_t dvma, struct dma_attrs *attrs)
203{ 220{
204 struct pci_pbm_info *pbm; 221 struct pci_pbm_info *pbm;
205 struct iommu *iommu; 222 struct iommu *iommu;
206 unsigned long flags, order, npages, entry; 223 unsigned long order, npages, entry;
207 u32 devhandle; 224 u32 devhandle;
208 225
209 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 226 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
210 iommu = dev->archdata.iommu; 227 iommu = dev->archdata.iommu;
211 pbm = dev->archdata.host_controller; 228 pbm = dev->archdata.host_controller;
212 devhandle = pbm->devhandle; 229 devhandle = pbm->devhandle;
213 entry = ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 230 entry = ((dvma - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT);
214 231 dma_4v_iommu_demap(&devhandle, entry, npages);
215 spin_lock_irqsave(&iommu->lock, flags); 232 iommu_tbl_range_free(&iommu->tbl, dvma, npages, false, NULL);
216
217 iommu_range_free(iommu, dvma, npages);
218
219 do {
220 unsigned long num;
221
222 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
223 npages);
224 entry += num;
225 npages -= num;
226 } while (npages != 0);
227
228 spin_unlock_irqrestore(&iommu->lock, flags);
229
230 order = get_order(size); 233 order = get_order(size);
231 if (order < 10) 234 if (order < 10)
232 free_pages((unsigned long)cpu, order); 235 free_pages((unsigned long)cpu, order);
@@ -253,14 +256,13 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
253 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 256 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
254 npages >>= IO_PAGE_SHIFT; 257 npages >>= IO_PAGE_SHIFT;
255 258
256 spin_lock_irqsave(&iommu->lock, flags); 259 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
257 entry = iommu_range_alloc(dev, iommu, npages, NULL); 260 __this_cpu_read(iommu_pool_hash));
258 spin_unlock_irqrestore(&iommu->lock, flags);
259 261
260 if (unlikely(entry == DMA_ERROR_CODE)) 262 if (unlikely(entry == DMA_ERROR_CODE))
261 goto bad; 263 goto bad;
262 264
263 bus_addr = (iommu->page_table_map_base + 265 bus_addr = (iommu->tbl.page_table_map_base +
264 (entry << IO_PAGE_SHIFT)); 266 (entry << IO_PAGE_SHIFT));
265 ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 267 ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
266 base_paddr = __pa(oaddr & IO_PAGE_MASK); 268 base_paddr = __pa(oaddr & IO_PAGE_MASK);
@@ -290,11 +292,7 @@ bad:
290 return DMA_ERROR_CODE; 292 return DMA_ERROR_CODE;
291 293
292iommu_map_fail: 294iommu_map_fail:
293 /* Interrupts are disabled. */ 295 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL);
294 spin_lock(&iommu->lock);
295 iommu_range_free(iommu, bus_addr, npages);
296 spin_unlock_irqrestore(&iommu->lock, flags);
297
298 return DMA_ERROR_CODE; 296 return DMA_ERROR_CODE;
299} 297}
300 298
@@ -304,7 +302,7 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
304{ 302{
305 struct pci_pbm_info *pbm; 303 struct pci_pbm_info *pbm;
306 struct iommu *iommu; 304 struct iommu *iommu;
307 unsigned long flags, npages; 305 unsigned long npages;
308 long entry; 306 long entry;
309 u32 devhandle; 307 u32 devhandle;
310 308
@@ -321,22 +319,9 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
321 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 319 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
322 npages >>= IO_PAGE_SHIFT; 320 npages >>= IO_PAGE_SHIFT;
323 bus_addr &= IO_PAGE_MASK; 321 bus_addr &= IO_PAGE_MASK;
324 322 entry = (bus_addr - iommu->tbl.page_table_map_base) >> IO_PAGE_SHIFT;
325 spin_lock_irqsave(&iommu->lock, flags); 323 dma_4v_iommu_demap(&devhandle, entry, npages);
326 324 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, false, NULL);
327 iommu_range_free(iommu, bus_addr, npages);
328
329 entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT;
330 do {
331 unsigned long num;
332
333 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
334 npages);
335 entry += num;
336 npages -= num;
337 } while (npages != 0);
338
339 spin_unlock_irqrestore(&iommu->lock, flags);
340} 325}
341 326
342static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, 327static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -371,14 +356,14 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
371 /* Init first segment length for backout at failure */ 356 /* Init first segment length for backout at failure */
372 outs->dma_length = 0; 357 outs->dma_length = 0;
373 358
374 spin_lock_irqsave(&iommu->lock, flags); 359 local_irq_save(flags);
375 360
376 iommu_batch_start(dev, prot, ~0UL); 361 iommu_batch_start(dev, prot, ~0UL);
377 362
378 max_seg_size = dma_get_max_seg_size(dev); 363 max_seg_size = dma_get_max_seg_size(dev);
379 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 364 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
380 IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 365 IO_PAGE_SIZE) >> IO_PAGE_SHIFT;
381 base_shift = iommu->page_table_map_base >> IO_PAGE_SHIFT; 366 base_shift = iommu->tbl.page_table_map_base >> IO_PAGE_SHIFT;
382 for_each_sg(sglist, s, nelems, i) { 367 for_each_sg(sglist, s, nelems, i) {
383 unsigned long paddr, npages, entry, out_entry = 0, slen; 368 unsigned long paddr, npages, entry, out_entry = 0, slen;
384 369
@@ -391,7 +376,8 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
391 /* Allocate iommu entries for that segment */ 376 /* Allocate iommu entries for that segment */
392 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 377 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
393 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 378 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
394 entry = iommu_range_alloc(dev, iommu, npages, &handle); 379 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, &handle,
380 __this_cpu_read(iommu_pool_hash));
395 381
396 /* Handle failure */ 382 /* Handle failure */
397 if (unlikely(entry == DMA_ERROR_CODE)) { 383 if (unlikely(entry == DMA_ERROR_CODE)) {
@@ -404,7 +390,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
404 iommu_batch_new_entry(entry); 390 iommu_batch_new_entry(entry);
405 391
406 /* Convert entry to a dma_addr_t */ 392 /* Convert entry to a dma_addr_t */
407 dma_addr = iommu->page_table_map_base + 393 dma_addr = iommu->tbl.page_table_map_base +
408 (entry << IO_PAGE_SHIFT); 394 (entry << IO_PAGE_SHIFT);
409 dma_addr |= (s->offset & ~IO_PAGE_MASK); 395 dma_addr |= (s->offset & ~IO_PAGE_MASK);
410 396
@@ -451,7 +437,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
451 if (unlikely(err < 0L)) 437 if (unlikely(err < 0L))
452 goto iommu_map_failed; 438 goto iommu_map_failed;
453 439
454 spin_unlock_irqrestore(&iommu->lock, flags); 440 local_irq_restore(flags);
455 441
456 if (outcount < incount) { 442 if (outcount < incount) {
457 outs = sg_next(outs); 443 outs = sg_next(outs);
@@ -469,7 +455,8 @@ iommu_map_failed:
469 vaddr = s->dma_address & IO_PAGE_MASK; 455 vaddr = s->dma_address & IO_PAGE_MASK;
470 npages = iommu_num_pages(s->dma_address, s->dma_length, 456 npages = iommu_num_pages(s->dma_address, s->dma_length,
471 IO_PAGE_SIZE); 457 IO_PAGE_SIZE);
472 iommu_range_free(iommu, vaddr, npages); 458 iommu_tbl_range_free(&iommu->tbl, vaddr, npages,
459 false, NULL);
473 /* XXX demap? XXX */ 460 /* XXX demap? XXX */
474 s->dma_address = DMA_ERROR_CODE; 461 s->dma_address = DMA_ERROR_CODE;
475 s->dma_length = 0; 462 s->dma_length = 0;
@@ -477,7 +464,7 @@ iommu_map_failed:
477 if (s == outs) 464 if (s == outs)
478 break; 465 break;
479 } 466 }
480 spin_unlock_irqrestore(&iommu->lock, flags); 467 local_irq_restore(flags);
481 468
482 return 0; 469 return 0;
483} 470}
@@ -489,7 +476,7 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
489 struct pci_pbm_info *pbm; 476 struct pci_pbm_info *pbm;
490 struct scatterlist *sg; 477 struct scatterlist *sg;
491 struct iommu *iommu; 478 struct iommu *iommu;
492 unsigned long flags; 479 unsigned long flags, entry;
493 u32 devhandle; 480 u32 devhandle;
494 481
495 BUG_ON(direction == DMA_NONE); 482 BUG_ON(direction == DMA_NONE);
@@ -498,33 +485,27 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
498 pbm = dev->archdata.host_controller; 485 pbm = dev->archdata.host_controller;
499 devhandle = pbm->devhandle; 486 devhandle = pbm->devhandle;
500 487
501 spin_lock_irqsave(&iommu->lock, flags); 488 local_irq_save(flags);
502 489
503 sg = sglist; 490 sg = sglist;
504 while (nelems--) { 491 while (nelems--) {
505 dma_addr_t dma_handle = sg->dma_address; 492 dma_addr_t dma_handle = sg->dma_address;
506 unsigned int len = sg->dma_length; 493 unsigned int len = sg->dma_length;
507 unsigned long npages, entry; 494 unsigned long npages;
495 struct iommu_table *tbl = &iommu->tbl;
496 unsigned long shift = IO_PAGE_SHIFT;
508 497
509 if (!len) 498 if (!len)
510 break; 499 break;
511 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 500 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
512 iommu_range_free(iommu, dma_handle, npages); 501 entry = ((dma_handle - tbl->page_table_map_base) >> shift);
513 502 dma_4v_iommu_demap(&devhandle, entry, npages);
514 entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); 503 iommu_tbl_range_free(&iommu->tbl, dma_handle, npages,
515 while (npages) { 504 false, NULL);
516 unsigned long num;
517
518 num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry),
519 npages);
520 entry += num;
521 npages -= num;
522 }
523
524 sg = sg_next(sg); 505 sg = sg_next(sg);
525 } 506 }
526 507
527 spin_unlock_irqrestore(&iommu->lock, flags); 508 local_irq_restore(flags);
528} 509}
529 510
530static struct dma_map_ops sun4v_dma_ops = { 511static struct dma_map_ops sun4v_dma_ops = {
@@ -536,6 +517,8 @@ static struct dma_map_ops sun4v_dma_ops = {
536 .unmap_sg = dma_4v_unmap_sg, 517 .unmap_sg = dma_4v_unmap_sg,
537}; 518};
538 519
520static struct iommu_tbl_ops dma_4v_iommu_ops;
521
539static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) 522static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent)
540{ 523{
541 struct property *prop; 524 struct property *prop;
@@ -550,30 +533,33 @@ static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent)
550} 533}
551 534
552static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, 535static unsigned long probe_existing_entries(struct pci_pbm_info *pbm,
553 struct iommu *iommu) 536 struct iommu_table *iommu)
554{ 537{
555 struct iommu_arena *arena = &iommu->arena; 538 struct iommu_pool *pool;
556 unsigned long i, cnt = 0; 539 unsigned long i, pool_nr, cnt = 0;
557 u32 devhandle; 540 u32 devhandle;
558 541
559 devhandle = pbm->devhandle; 542 devhandle = pbm->devhandle;
560 for (i = 0; i < arena->limit; i++) { 543 for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) {
561 unsigned long ret, io_attrs, ra; 544 pool = &(iommu->arena_pool[pool_nr]);
562 545 for (i = pool->start; i <= pool->end; i++) {
563 ret = pci_sun4v_iommu_getmap(devhandle, 546 unsigned long ret, io_attrs, ra;
564 HV_PCI_TSBID(0, i), 547
565 &io_attrs, &ra); 548 ret = pci_sun4v_iommu_getmap(devhandle,
566 if (ret == HV_EOK) { 549 HV_PCI_TSBID(0, i),
567 if (page_in_phys_avail(ra)) { 550 &io_attrs, &ra);
568 pci_sun4v_iommu_demap(devhandle, 551 if (ret == HV_EOK) {
569 HV_PCI_TSBID(0, i), 1); 552 if (page_in_phys_avail(ra)) {
570 } else { 553 pci_sun4v_iommu_demap(devhandle,
571 cnt++; 554 HV_PCI_TSBID(0,
572 __set_bit(i, arena->map); 555 i), 1);
556 } else {
557 cnt++;
558 __set_bit(i, iommu->map);
559 }
573 } 560 }
574 } 561 }
575 } 562 }
576
577 return cnt; 563 return cnt;
578} 564}
579 565
@@ -601,22 +587,22 @@ static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm)
601 dma_offset = vdma[0]; 587 dma_offset = vdma[0];
602 588
603 /* Setup initial software IOMMU state. */ 589 /* Setup initial software IOMMU state. */
604 spin_lock_init(&iommu->lock);
605 iommu->ctx_lowest_free = 1; 590 iommu->ctx_lowest_free = 1;
606 iommu->page_table_map_base = dma_offset; 591 iommu->tbl.page_table_map_base = dma_offset;
607 iommu->dma_addr_mask = dma_mask; 592 iommu->dma_addr_mask = dma_mask;
608 593
609 /* Allocate and initialize the free area map. */ 594 /* Allocate and initialize the free area map. */
610 sz = (num_tsb_entries + 7) / 8; 595 sz = (num_tsb_entries + 7) / 8;
611 sz = (sz + 7UL) & ~7UL; 596 sz = (sz + 7UL) & ~7UL;
612 iommu->arena.map = kzalloc(sz, GFP_KERNEL); 597 iommu->tbl.map = kzalloc(sz, GFP_KERNEL);
613 if (!iommu->arena.map) { 598 if (!iommu->tbl.map) {
614 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); 599 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n");
615 return -ENOMEM; 600 return -ENOMEM;
616 } 601 }
617 iommu->arena.limit = num_tsb_entries; 602 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT,
618 603 &dma_4v_iommu_ops, false /* no large_pool */,
619 sz = probe_existing_entries(pbm, iommu); 604 0 /* default npools */);
605 sz = probe_existing_entries(pbm, &iommu->tbl);
620 if (sz) 606 if (sz)
621 printk("%s: Imported %lu TSB entries from OBP\n", 607 printk("%s: Imported %lu TSB entries from OBP\n",
622 pbm->name, sz); 608 pbm->name, sz);
@@ -1015,8 +1001,17 @@ static struct platform_driver pci_sun4v_driver = {
1015 .probe = pci_sun4v_probe, 1001 .probe = pci_sun4v_probe,
1016}; 1002};
1017 1003
1004static void setup_iommu_pool_hash(void)
1005{
1006 unsigned int i;
1007
1008 for_each_possible_cpu(i)
1009 per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
1010}
1011
1018static int __init pci_sun4v_init(void) 1012static int __init pci_sun4v_init(void)
1019{ 1013{
1014 setup_iommu_pool_hash();
1020 return platform_driver_register(&pci_sun4v_driver); 1015 return platform_driver_register(&pci_sun4v_driver);
1021} 1016}
1022 1017
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-24 17:02:01 -0500