aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinas Vepstas <linas@austin.ibm.com>2006-10-30 00:15:59 -0500
committerPaul Mackerras <paulus@samba.org>2006-10-31 22:52:48 -0500
commit5d2efba64b231a1733c4048d1708d77e07f26426 (patch)
tree2893dd45b9c26cef6cddb5fef0c6f820c5eb534e
parentdd6c89f686bdb2a5de72fab636fc839e5a0add6d (diff)
[POWERPC] Use 4kB iommu pages even on 64kB-page systems
The 10Gigabit ethernet device drivers appear to be able to chew up all 256MB of TCE mappings on pSeries systems, as evidenced by numerous error messages: iommu_alloc failed, tbl c0000000010d5c48 vaddr c0000000d875eff0 npages 1 Some experimentation indicates that this is essentially because one 1500 byte ethernet MTU gets mapped as a 64K DMA region when the large 64K pages are enabled. Thus, it doesn't take much to exhaust all of the available DMA mappings for a high-speed card. This patch changes the iommu allocator to work with its own unique, distinct page size. Although the patch is long, its actually quite simple: it just #defines a distinct IOMMU_PAGE_SIZE and then uses this in all the places that matter. As a side effect, it also dramatically improves network performance on platforms with H-calls on iommu translation inserts/removes (since we no longer call it 16 times for a 1500 bytes packet when the iommu HW is still 4k). In the future, we might want to make the IOMMU_PAGE_SIZE a variable in the iommu_table instance, thus allowing support for different HW page sizes in the iommu itself. Signed-off-by: Linas Vepstas <linas@austin.ibm.com> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org> Acked-by: Olof Johansson <olof@lixom.net> Acked-by: Stephen Rothwell <sfr@canb.auug.org.au> Signed-off-by: Paul Mackerras <paulus@samba.org>
-rw-r--r--arch/powerpc/kernel/iommu.c77
-rw-r--r--arch/powerpc/kernel/vio.c4
-rw-r--r--arch/powerpc/platforms/iseries/iommu.c11
-rw-r--r--arch/powerpc/platforms/pseries/iommu.c35
-rw-r--r--arch/powerpc/sysdev/dart.h1
-rw-r--r--arch/powerpc/sysdev/dart_iommu.c8
-rw-r--r--include/asm-powerpc/iommu.h22
-rw-r--r--include/asm-powerpc/tce.h3
8 files changed, 80 insertions, 81 deletions
diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
index f88a2a675d90..ba6b7256084b 100644
--- a/arch/powerpc/kernel/iommu.c
+++ b/arch/powerpc/kernel/iommu.c
@@ -47,6 +47,17 @@ static int novmerge = 0;
47static int novmerge = 1; 47static int novmerge = 1;
48#endif 48#endif
49 49
50static inline unsigned long iommu_num_pages(unsigned long vaddr,
51 unsigned long slen)
52{
53 unsigned long npages;
54
55 npages = IOMMU_PAGE_ALIGN(vaddr + slen) - (vaddr & IOMMU_PAGE_MASK);
56 npages >>= IOMMU_PAGE_SHIFT;
57
58 return npages;
59}
60
50static int __init setup_iommu(char *str) 61static int __init setup_iommu(char *str)
51{ 62{
52 if (!strcmp(str, "novmerge")) 63 if (!strcmp(str, "novmerge"))
@@ -178,10 +189,10 @@ static dma_addr_t iommu_alloc(struct iommu_table *tbl, void *page,
178 } 189 }
179 190
180 entry += tbl->it_offset; /* Offset into real TCE table */ 191 entry += tbl->it_offset; /* Offset into real TCE table */
181 ret = entry << PAGE_SHIFT; /* Set the return dma address */ 192 ret = entry << IOMMU_PAGE_SHIFT; /* Set the return dma address */
182 193
183 /* Put the TCEs in the HW table */ 194 /* Put the TCEs in the HW table */
184 ppc_md.tce_build(tbl, entry, npages, (unsigned long)page & PAGE_MASK, 195 ppc_md.tce_build(tbl, entry, npages, (unsigned long)page & IOMMU_PAGE_MASK,
185 direction); 196 direction);
186 197
187 198
@@ -203,7 +214,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
203 unsigned long entry, free_entry; 214 unsigned long entry, free_entry;
204 unsigned long i; 215 unsigned long i;
205 216
206 entry = dma_addr >> PAGE_SHIFT; 217 entry = dma_addr >> IOMMU_PAGE_SHIFT;
207 free_entry = entry - tbl->it_offset; 218 free_entry = entry - tbl->it_offset;
208 219
209 if (((free_entry + npages) > tbl->it_size) || 220 if (((free_entry + npages) > tbl->it_size) ||
@@ -270,7 +281,7 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
270 /* Init first segment length for backout at failure */ 281 /* Init first segment length for backout at failure */
271 outs->dma_length = 0; 282 outs->dma_length = 0;
272 283
273 DBG("mapping %d elements:\n", nelems); 284 DBG("sg mapping %d elements:\n", nelems);
274 285
275 spin_lock_irqsave(&(tbl->it_lock), flags); 286 spin_lock_irqsave(&(tbl->it_lock), flags);
276 287
@@ -285,9 +296,8 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
285 } 296 }
286 /* Allocate iommu entries for that segment */ 297 /* Allocate iommu entries for that segment */
287 vaddr = (unsigned long)page_address(s->page) + s->offset; 298 vaddr = (unsigned long)page_address(s->page) + s->offset;
288 npages = PAGE_ALIGN(vaddr + slen) - (vaddr & PAGE_MASK); 299 npages = iommu_num_pages(vaddr, slen);
289 npages >>= PAGE_SHIFT; 300 entry = iommu_range_alloc(tbl, npages, &handle, mask >> IOMMU_PAGE_SHIFT, 0);
290 entry = iommu_range_alloc(tbl, npages, &handle, mask >> PAGE_SHIFT, 0);
291 301
292 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); 302 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);
293 303
@@ -301,14 +311,14 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
301 311
302 /* Convert entry to a dma_addr_t */ 312 /* Convert entry to a dma_addr_t */
303 entry += tbl->it_offset; 313 entry += tbl->it_offset;
304 dma_addr = entry << PAGE_SHIFT; 314 dma_addr = entry << IOMMU_PAGE_SHIFT;
305 dma_addr |= s->offset; 315 dma_addr |= (s->offset & ~IOMMU_PAGE_MASK);
306 316
307 DBG(" - %lx pages, entry: %lx, dma_addr: %lx\n", 317 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
308 npages, entry, dma_addr); 318 npages, entry, dma_addr);
309 319
310 /* Insert into HW table */ 320 /* Insert into HW table */
311 ppc_md.tce_build(tbl, entry, npages, vaddr & PAGE_MASK, direction); 321 ppc_md.tce_build(tbl, entry, npages, vaddr & IOMMU_PAGE_MASK, direction);
312 322
313 /* If we are in an open segment, try merging */ 323 /* If we are in an open segment, try merging */
314 if (segstart != s) { 324 if (segstart != s) {
@@ -323,7 +333,7 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
323 DBG(" can't merge, new segment.\n"); 333 DBG(" can't merge, new segment.\n");
324 } else { 334 } else {
325 outs->dma_length += s->length; 335 outs->dma_length += s->length;
326 DBG(" merged, new len: %lx\n", outs->dma_length); 336 DBG(" merged, new len: %ux\n", outs->dma_length);
327 } 337 }
328 } 338 }
329 339
@@ -367,9 +377,8 @@ int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
367 if (s->dma_length != 0) { 377 if (s->dma_length != 0) {
368 unsigned long vaddr, npages; 378 unsigned long vaddr, npages;
369 379
370 vaddr = s->dma_address & PAGE_MASK; 380 vaddr = s->dma_address & IOMMU_PAGE_MASK;
371 npages = (PAGE_ALIGN(s->dma_address + s->dma_length) - vaddr) 381 npages = iommu_num_pages(s->dma_address, s->dma_length);
372 >> PAGE_SHIFT;
373 __iommu_free(tbl, vaddr, npages); 382 __iommu_free(tbl, vaddr, npages);
374 s->dma_address = DMA_ERROR_CODE; 383 s->dma_address = DMA_ERROR_CODE;
375 s->dma_length = 0; 384 s->dma_length = 0;
@@ -398,8 +407,7 @@ void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist,
398 407
399 if (sglist->dma_length == 0) 408 if (sglist->dma_length == 0)
400 break; 409 break;
401 npages = (PAGE_ALIGN(dma_handle + sglist->dma_length) 410 npages = iommu_num_pages(dma_handle,sglist->dma_length);
402 - (dma_handle & PAGE_MASK)) >> PAGE_SHIFT;
403 __iommu_free(tbl, dma_handle, npages); 411 __iommu_free(tbl, dma_handle, npages);
404 sglist++; 412 sglist++;
405 } 413 }
@@ -532,12 +540,11 @@ dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr,
532 BUG_ON(direction == DMA_NONE); 540 BUG_ON(direction == DMA_NONE);
533 541
534 uaddr = (unsigned long)vaddr; 542 uaddr = (unsigned long)vaddr;
535 npages = PAGE_ALIGN(uaddr + size) - (uaddr & PAGE_MASK); 543 npages = iommu_num_pages(uaddr, size);
536 npages >>= PAGE_SHIFT;
537 544
538 if (tbl) { 545 if (tbl) {
539 dma_handle = iommu_alloc(tbl, vaddr, npages, direction, 546 dma_handle = iommu_alloc(tbl, vaddr, npages, direction,
540 mask >> PAGE_SHIFT, 0); 547 mask >> IOMMU_PAGE_SHIFT, 0);
541 if (dma_handle == DMA_ERROR_CODE) { 548 if (dma_handle == DMA_ERROR_CODE) {
542 if (printk_ratelimit()) { 549 if (printk_ratelimit()) {
543 printk(KERN_INFO "iommu_alloc failed, " 550 printk(KERN_INFO "iommu_alloc failed, "
@@ -545,7 +552,7 @@ dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr,
545 tbl, vaddr, npages); 552 tbl, vaddr, npages);
546 } 553 }
547 } else 554 } else
548 dma_handle |= (uaddr & ~PAGE_MASK); 555 dma_handle |= (uaddr & ~IOMMU_PAGE_MASK);
549 } 556 }
550 557
551 return dma_handle; 558 return dma_handle;
@@ -554,11 +561,14 @@ dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr,
554void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle, 561void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle,
555 size_t size, enum dma_data_direction direction) 562 size_t size, enum dma_data_direction direction)
556{ 563{
564 unsigned int npages;
565
557 BUG_ON(direction == DMA_NONE); 566 BUG_ON(direction == DMA_NONE);
558 567
559 if (tbl) 568 if (tbl) {
560 iommu_free(tbl, dma_handle, (PAGE_ALIGN(dma_handle + size) - 569 npages = iommu_num_pages(dma_handle, size);
561 (dma_handle & PAGE_MASK)) >> PAGE_SHIFT); 570 iommu_free(tbl, dma_handle, npages);
571 }
562} 572}
563 573
564/* Allocates a contiguous real buffer and creates mappings over it. 574/* Allocates a contiguous real buffer and creates mappings over it.
@@ -570,11 +580,11 @@ void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size,
570{ 580{
571 void *ret = NULL; 581 void *ret = NULL;
572 dma_addr_t mapping; 582 dma_addr_t mapping;
573 unsigned int npages, order; 583 unsigned int order;
584 unsigned int nio_pages, io_order;
574 struct page *page; 585 struct page *page;
575 586
576 size = PAGE_ALIGN(size); 587 size = PAGE_ALIGN(size);
577 npages = size >> PAGE_SHIFT;
578 order = get_order(size); 588 order = get_order(size);
579 589
580 /* 590 /*
@@ -598,8 +608,10 @@ void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size,
598 memset(ret, 0, size); 608 memset(ret, 0, size);
599 609
600 /* Set up tces to cover the allocated range */ 610 /* Set up tces to cover the allocated range */
601 mapping = iommu_alloc(tbl, ret, npages, DMA_BIDIRECTIONAL, 611 nio_pages = size >> IOMMU_PAGE_SHIFT;
602 mask >> PAGE_SHIFT, order); 612 io_order = get_iommu_order(size);
613 mapping = iommu_alloc(tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
614 mask >> IOMMU_PAGE_SHIFT, io_order);
603 if (mapping == DMA_ERROR_CODE) { 615 if (mapping == DMA_ERROR_CODE) {
604 free_pages((unsigned long)ret, order); 616 free_pages((unsigned long)ret, order);
605 return NULL; 617 return NULL;
@@ -611,12 +623,13 @@ void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size,
611void iommu_free_coherent(struct iommu_table *tbl, size_t size, 623void iommu_free_coherent(struct iommu_table *tbl, size_t size,
612 void *vaddr, dma_addr_t dma_handle) 624 void *vaddr, dma_addr_t dma_handle)
613{ 625{
614 unsigned int npages;
615
616 if (tbl) { 626 if (tbl) {
627 unsigned int nio_pages;
628
629 size = PAGE_ALIGN(size);
630 nio_pages = size >> IOMMU_PAGE_SHIFT;
631 iommu_free(tbl, dma_handle, nio_pages);
617 size = PAGE_ALIGN(size); 632 size = PAGE_ALIGN(size);
618 npages = size >> PAGE_SHIFT;
619 iommu_free(tbl, dma_handle, npages);
620 free_pages((unsigned long)vaddr, get_order(size)); 633 free_pages((unsigned long)vaddr, get_order(size));
621 } 634 }
622} 635}
diff --git a/arch/powerpc/kernel/vio.c b/arch/powerpc/kernel/vio.c
index cb87e71eec66..ed007878d1bf 100644
--- a/arch/powerpc/kernel/vio.c
+++ b/arch/powerpc/kernel/vio.c
@@ -92,9 +92,9 @@ static struct iommu_table *vio_build_iommu_table(struct vio_dev *dev)
92 &tbl->it_index, &offset, &size); 92 &tbl->it_index, &offset, &size);
93 93
94 /* TCE table size - measured in tce entries */ 94 /* TCE table size - measured in tce entries */
95 tbl->it_size = size >> PAGE_SHIFT; 95 tbl->it_size = size >> IOMMU_PAGE_SHIFT;
96 /* offset for VIO should always be 0 */ 96 /* offset for VIO should always be 0 */
97 tbl->it_offset = offset >> PAGE_SHIFT; 97 tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
98 tbl->it_busno = 0; 98 tbl->it_busno = 0;
99 tbl->it_type = TCE_VB; 99 tbl->it_type = TCE_VB;
100 100
diff --git a/arch/powerpc/platforms/iseries/iommu.c b/arch/powerpc/platforms/iseries/iommu.c
index f4cbbcf8773a..218817d13c5c 100644
--- a/arch/powerpc/platforms/iseries/iommu.c
+++ b/arch/powerpc/platforms/iseries/iommu.c
@@ -43,9 +43,6 @@ static void tce_build_iSeries(struct iommu_table *tbl, long index, long npages,
43 u64 rc; 43 u64 rc;
44 u64 tce, rpn; 44 u64 tce, rpn;
45 45
46 index <<= TCE_PAGE_FACTOR;
47 npages <<= TCE_PAGE_FACTOR;
48
49 while (npages--) { 46 while (npages--) {
50 rpn = virt_to_abs(uaddr) >> TCE_SHIFT; 47 rpn = virt_to_abs(uaddr) >> TCE_SHIFT;
51 tce = (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT; 48 tce = (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
@@ -75,9 +72,6 @@ static void tce_free_iSeries(struct iommu_table *tbl, long index, long npages)
75{ 72{
76 u64 rc; 73 u64 rc;
77 74
78 npages <<= TCE_PAGE_FACTOR;
79 index <<= TCE_PAGE_FACTOR;
80
81 while (npages--) { 75 while (npages--) {
82 rc = HvCallXm_setTce((u64)tbl->it_index, (u64)index, 0); 76 rc = HvCallXm_setTce((u64)tbl->it_index, (u64)index, 0);
83 if (rc) 77 if (rc)
@@ -136,10 +130,9 @@ void iommu_table_getparms_iSeries(unsigned long busno,
136 panic("PCI_DMA: parms->size is zero, parms is 0x%p", parms); 130 panic("PCI_DMA: parms->size is zero, parms is 0x%p", parms);
137 131
138 /* itc_size is in pages worth of table, it_size is in # of entries */ 132 /* itc_size is in pages worth of table, it_size is in # of entries */
139 tbl->it_size = ((parms->itc_size * TCE_PAGE_SIZE) / 133 tbl->it_size = (parms->itc_size * TCE_PAGE_SIZE) / TCE_ENTRY_SIZE;
140 TCE_ENTRY_SIZE) >> TCE_PAGE_FACTOR;
141 tbl->it_busno = parms->itc_busno; 134 tbl->it_busno = parms->itc_busno;
142 tbl->it_offset = parms->itc_offset >> TCE_PAGE_FACTOR; 135 tbl->it_offset = parms->itc_offset;
143 tbl->it_index = parms->itc_index; 136 tbl->it_index = parms->itc_index;
144 tbl->it_blocksize = 1; 137 tbl->it_blocksize = 1;
145 tbl->it_type = virtbus ? TCE_VB : TCE_PCI; 138 tbl->it_type = virtbus ? TCE_VB : TCE_PCI;
diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
index d24ba547e53f..556c279a789d 100644
--- a/arch/powerpc/platforms/pseries/iommu.c
+++ b/arch/powerpc/platforms/pseries/iommu.c
@@ -57,9 +57,6 @@ static void tce_build_pSeries(struct iommu_table *tbl, long index,
57 u64 *tcep; 57 u64 *tcep;
58 u64 rpn; 58 u64 rpn;
59 59
60 index <<= TCE_PAGE_FACTOR;
61 npages <<= TCE_PAGE_FACTOR;
62
63 proto_tce = TCE_PCI_READ; // Read allowed 60 proto_tce = TCE_PCI_READ; // Read allowed
64 61
65 if (direction != DMA_TO_DEVICE) 62 if (direction != DMA_TO_DEVICE)
@@ -82,9 +79,6 @@ static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
82{ 79{
83 u64 *tcep; 80 u64 *tcep;
84 81
85 npages <<= TCE_PAGE_FACTOR;
86 index <<= TCE_PAGE_FACTOR;
87
88 tcep = ((u64 *)tbl->it_base) + index; 82 tcep = ((u64 *)tbl->it_base) + index;
89 83
90 while (npages--) 84 while (npages--)
@@ -95,7 +89,6 @@ static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
95{ 89{
96 u64 *tcep; 90 u64 *tcep;
97 91
98 index <<= TCE_PAGE_FACTOR;
99 tcep = ((u64 *)tbl->it_base) + index; 92 tcep = ((u64 *)tbl->it_base) + index;
100 93
101 return *tcep; 94 return *tcep;
@@ -109,9 +102,6 @@ static void tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
109 u64 proto_tce, tce; 102 u64 proto_tce, tce;
110 u64 rpn; 103 u64 rpn;
111 104
112 tcenum <<= TCE_PAGE_FACTOR;
113 npages <<= TCE_PAGE_FACTOR;
114
115 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT; 105 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
116 proto_tce = TCE_PCI_READ; 106 proto_tce = TCE_PCI_READ;
117 if (direction != DMA_TO_DEVICE) 107 if (direction != DMA_TO_DEVICE)
@@ -146,7 +136,7 @@ static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
146 u64 rpn; 136 u64 rpn;
147 long l, limit; 137 long l, limit;
148 138
149 if (TCE_PAGE_FACTOR == 0 && npages == 1) 139 if (npages == 1)
150 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr, 140 return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
151 direction); 141 direction);
152 142
@@ -164,9 +154,6 @@ static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
164 __get_cpu_var(tce_page) = tcep; 154 __get_cpu_var(tce_page) = tcep;
165 } 155 }
166 156
167 tcenum <<= TCE_PAGE_FACTOR;
168 npages <<= TCE_PAGE_FACTOR;
169
170 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT; 157 rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
171 proto_tce = TCE_PCI_READ; 158 proto_tce = TCE_PCI_READ;
172 if (direction != DMA_TO_DEVICE) 159 if (direction != DMA_TO_DEVICE)
@@ -207,9 +194,6 @@ static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages
207{ 194{
208 u64 rc; 195 u64 rc;
209 196
210 tcenum <<= TCE_PAGE_FACTOR;
211 npages <<= TCE_PAGE_FACTOR;
212
213 while (npages--) { 197 while (npages--) {
214 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0); 198 rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
215 199
@@ -229,9 +213,6 @@ static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long n
229{ 213{
230 u64 rc; 214 u64 rc;
231 215
232 tcenum <<= TCE_PAGE_FACTOR;
233 npages <<= TCE_PAGE_FACTOR;
234
235 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages); 216 rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
236 217
237 if (rc && printk_ratelimit()) { 218 if (rc && printk_ratelimit()) {
@@ -248,7 +229,6 @@ static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
248 u64 rc; 229 u64 rc;
249 unsigned long tce_ret; 230 unsigned long tce_ret;
250 231
251 tcenum <<= TCE_PAGE_FACTOR;
252 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret); 232 rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);
253 233
254 if (rc && printk_ratelimit()) { 234 if (rc && printk_ratelimit()) {
@@ -289,7 +269,7 @@ static void iommu_table_setparms(struct pci_controller *phb,
289 tbl->it_busno = phb->bus->number; 269 tbl->it_busno = phb->bus->number;
290 270
291 /* Units of tce entries */ 271 /* Units of tce entries */
292 tbl->it_offset = phb->dma_window_base_cur >> PAGE_SHIFT; 272 tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
293 273
294 /* Test if we are going over 2GB of DMA space */ 274 /* Test if we are going over 2GB of DMA space */
295 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) { 275 if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
@@ -300,7 +280,7 @@ static void iommu_table_setparms(struct pci_controller *phb,
300 phb->dma_window_base_cur += phb->dma_window_size; 280 phb->dma_window_base_cur += phb->dma_window_size;
301 281
302 /* Set the tce table size - measured in entries */ 282 /* Set the tce table size - measured in entries */
303 tbl->it_size = phb->dma_window_size >> PAGE_SHIFT; 283 tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
304 284
305 tbl->it_index = 0; 285 tbl->it_index = 0;
306 tbl->it_blocksize = 16; 286 tbl->it_blocksize = 16;
@@ -325,8 +305,8 @@ static void iommu_table_setparms_lpar(struct pci_controller *phb,
325 tbl->it_base = 0; 305 tbl->it_base = 0;
326 tbl->it_blocksize = 16; 306 tbl->it_blocksize = 16;
327 tbl->it_type = TCE_PCI; 307 tbl->it_type = TCE_PCI;
328 tbl->it_offset = offset >> PAGE_SHIFT; 308 tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
329 tbl->it_size = size >> PAGE_SHIFT; 309 tbl->it_size = size >> IOMMU_PAGE_SHIFT;
330} 310}
331 311
332static void iommu_bus_setup_pSeries(struct pci_bus *bus) 312static void iommu_bus_setup_pSeries(struct pci_bus *bus)
@@ -522,8 +502,6 @@ static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
522 const void *dma_window = NULL; 502 const void *dma_window = NULL;
523 struct pci_dn *pci; 503 struct pci_dn *pci;
524 504
525 DBG("iommu_dev_setup_pSeriesLP, dev %p (%s)\n", dev, pci_name(dev));
526
527 /* dev setup for LPAR is a little tricky, since the device tree might 505 /* dev setup for LPAR is a little tricky, since the device tree might
528 * contain the dma-window properties per-device and not neccesarily 506 * contain the dma-window properties per-device and not neccesarily
529 * for the bus. So we need to search upwards in the tree until we 507 * for the bus. So we need to search upwards in the tree until we
@@ -532,6 +510,9 @@ static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
532 */ 510 */
533 dn = pci_device_to_OF_node(dev); 511 dn = pci_device_to_OF_node(dev);
534 512
513 DBG("iommu_dev_setup_pSeriesLP, dev %p (%s) %s\n",
514 dev, pci_name(dev), dn->full_name);
515
535 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table; 516 for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
536 pdn = pdn->parent) { 517 pdn = pdn->parent) {
537 dma_window = get_property(pdn, "ibm,dma-window", NULL); 518 dma_window = get_property(pdn, "ibm,dma-window", NULL);
diff --git a/arch/powerpc/sysdev/dart.h b/arch/powerpc/sysdev/dart.h
index 1c8817c4835e..ff202edb0591 100644
--- a/arch/powerpc/sysdev/dart.h
+++ b/arch/powerpc/sysdev/dart.h
@@ -72,7 +72,6 @@
72 72
73#define DART_PAGE_SHIFT 12 73#define DART_PAGE_SHIFT 12
74#define DART_PAGE_SIZE (1 << DART_PAGE_SHIFT) 74#define DART_PAGE_SIZE (1 << DART_PAGE_SHIFT)
75#define DART_PAGE_FACTOR (PAGE_SHIFT - DART_PAGE_SHIFT)
76 75
77 76
78#endif /* _POWERPC_SYSDEV_DART_H */ 77#endif /* _POWERPC_SYSDEV_DART_H */
diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c
index 03b4477dd7f0..572b7846cc77 100644
--- a/arch/powerpc/sysdev/dart_iommu.c
+++ b/arch/powerpc/sysdev/dart_iommu.c
@@ -156,9 +156,6 @@ static void dart_build(struct iommu_table *tbl, long index,
156 156
157 DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); 157 DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
158 158
159 index <<= DART_PAGE_FACTOR;
160 npages <<= DART_PAGE_FACTOR;
161
162 dp = ((unsigned int*)tbl->it_base) + index; 159 dp = ((unsigned int*)tbl->it_base) + index;
163 160
164 /* On U3, all memory is contigous, so we can move this 161 /* On U3, all memory is contigous, so we can move this
@@ -199,9 +196,6 @@ static void dart_free(struct iommu_table *tbl, long index, long npages)
199 196
200 DBG("dart: free at: %lx, %lx\n", index, npages); 197 DBG("dart: free at: %lx, %lx\n", index, npages);
201 198
202 index <<= DART_PAGE_FACTOR;
203 npages <<= DART_PAGE_FACTOR;
204
205 dp = ((unsigned int *)tbl->it_base) + index; 199 dp = ((unsigned int *)tbl->it_base) + index;
206 200
207 while (npages--) 201 while (npages--)
@@ -281,7 +275,7 @@ static void iommu_table_dart_setup(void)
281 iommu_table_dart.it_busno = 0; 275 iommu_table_dart.it_busno = 0;
282 iommu_table_dart.it_offset = 0; 276 iommu_table_dart.it_offset = 0;
283 /* it_size is in number of entries */ 277 /* it_size is in number of entries */
284 iommu_table_dart.it_size = (dart_tablesize / sizeof(u32)) >> DART_PAGE_FACTOR; 278 iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
285 279
286 /* Initialize the common IOMMU code */ 280 /* Initialize the common IOMMU code */
287 iommu_table_dart.it_base = (unsigned long)dart_vbase; 281 iommu_table_dart.it_base = (unsigned long)dart_vbase;
diff --git a/include/asm-powerpc/iommu.h b/include/asm-powerpc/iommu.h
index a5e98641a2ae..39fad685ffab 100644
--- a/include/asm-powerpc/iommu.h
+++ b/include/asm-powerpc/iommu.h
@@ -22,17 +22,35 @@
22#define _ASM_IOMMU_H 22#define _ASM_IOMMU_H
23#ifdef __KERNEL__ 23#ifdef __KERNEL__
24 24
25#include <asm/types.h> 25#include <linux/compiler.h>
26#include <linux/spinlock.h> 26#include <linux/spinlock.h>
27#include <linux/device.h> 27#include <linux/device.h>
28#include <linux/dma-mapping.h> 28#include <linux/dma-mapping.h>
29#include <asm/types.h>
30#include <asm/bitops.h>
31
32#define IOMMU_PAGE_SHIFT 12
33#define IOMMU_PAGE_SIZE (ASM_CONST(1) << IOMMU_PAGE_SHIFT)
34#define IOMMU_PAGE_MASK (~((1 << IOMMU_PAGE_SHIFT) - 1))
35#define IOMMU_PAGE_ALIGN(addr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE)
36
37#ifndef __ASSEMBLY__
38
39/* Pure 2^n version of get_order */
40static __inline__ __attribute_const__ int get_iommu_order(unsigned long size)
41{
42 return __ilog2((size - 1) >> IOMMU_PAGE_SHIFT) + 1;
43}
44
45#endif /* __ASSEMBLY__ */
46
29 47
30/* 48/*
31 * IOMAP_MAX_ORDER defines the largest contiguous block 49 * IOMAP_MAX_ORDER defines the largest contiguous block
32 * of dma space we can get. IOMAP_MAX_ORDER = 13 50 * of dma space we can get. IOMAP_MAX_ORDER = 13
33 * allows up to 2**12 pages (4096 * 4096) = 16 MB 51 * allows up to 2**12 pages (4096 * 4096) = 16 MB
34 */ 52 */
35#define IOMAP_MAX_ORDER 13 53#define IOMAP_MAX_ORDER 13
36 54
37struct iommu_table { 55struct iommu_table {
38 unsigned long it_busno; /* Bus number this table belongs to */ 56 unsigned long it_busno; /* Bus number this table belongs to */
diff --git a/include/asm-powerpc/tce.h b/include/asm-powerpc/tce.h
index c9483adbf599..f663634cccc9 100644
--- a/include/asm-powerpc/tce.h
+++ b/include/asm-powerpc/tce.h
@@ -22,6 +22,8 @@
22#define _ASM_POWERPC_TCE_H 22#define _ASM_POWERPC_TCE_H
23#ifdef __KERNEL__ 23#ifdef __KERNEL__
24 24
25#include <asm/iommu.h>
26
25/* 27/*
26 * Tces come in two formats, one for the virtual bus and a different 28 * Tces come in two formats, one for the virtual bus and a different
27 * format for PCI 29 * format for PCI
@@ -33,7 +35,6 @@
33 35
34#define TCE_SHIFT 12 36#define TCE_SHIFT 12
35#define TCE_PAGE_SIZE (1 << TCE_SHIFT) 37#define TCE_PAGE_SIZE (1 << TCE_SHIFT)
36#define TCE_PAGE_FACTOR (PAGE_SHIFT - TCE_SHIFT)
37 38
38#define TCE_ENTRY_SIZE 8 /* each TCE is 64 bits */ 39#define TCE_ENTRY_SIZE 8 /* each TCE is 64 bits */
39 40