diff options
author | David S. Miller <davem@sunset.davemloft.net> | 2005-10-14 01:15:24 -0400 |
---|---|---|
committer | David S. Miller <davem@sunset.davemloft.net> | 2005-10-14 01:15:24 -0400 |
commit | 688cb30bdc3e398d97682a6a58f825821ee838c2 (patch) | |
tree | 8fc9ca543fc3966eb961743ba4587298fce49279 | |
parent | 51e8513615ed8202b22ba9a43b0c7376ea4f6868 (diff) |
[SPARC64]: Eliminate PCI IOMMU dma mapping size limit.
The hairy fast allocator in the sparc64 PCI IOMMU code
has a hard limit of 256 pages. Certain devices can
exceed this when performing very large I/Os.
So replace with a more simple allocator, based largely
upon the arch/ppc64/kernel/iommu.c code.
Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r-- | arch/sparc64/kernel/pci_iommu.c | 365 | ||||
-rw-r--r-- | include/asm-sparc64/pbm.h | 28 |
2 files changed, 145 insertions, 248 deletions
diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c index cdee7d9fed72..a11910be1013 100644 --- a/arch/sparc64/kernel/pci_iommu.c +++ b/arch/sparc64/kernel/pci_iommu.c | |||
@@ -49,12 +49,6 @@ static void __iommu_flushall(struct pci_iommu *iommu) | |||
49 | 49 | ||
50 | /* Ensure completion of previous PIO writes. */ | 50 | /* Ensure completion of previous PIO writes. */ |
51 | (void) pci_iommu_read(iommu->write_complete_reg); | 51 | (void) pci_iommu_read(iommu->write_complete_reg); |
52 | |||
53 | /* Now update everyone's flush point. */ | ||
54 | for (entry = 0; entry < PBM_NCLUSTERS; entry++) { | ||
55 | iommu->alloc_info[entry].flush = | ||
56 | iommu->alloc_info[entry].next; | ||
57 | } | ||
58 | } | 52 | } |
59 | 53 | ||
60 | #define IOPTE_CONSISTENT(CTX) \ | 54 | #define IOPTE_CONSISTENT(CTX) \ |
@@ -80,9 +74,61 @@ static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte) | |||
80 | iopte_val(*iopte) = val; | 74 | iopte_val(*iopte) = val; |
81 | } | 75 | } |
82 | 76 | ||
77 | /* Based largely upon the ppc64 iommu allocator. */ | ||
78 | static long pci_arena_alloc(struct pci_iommu *iommu, unsigned long npages) | ||
79 | { | ||
80 | struct pci_iommu_arena *arena = &iommu->arena; | ||
81 | unsigned long n, i, start, end, limit; | ||
82 | int pass; | ||
83 | |||
84 | limit = arena->limit; | ||
85 | start = arena->hint; | ||
86 | pass = 0; | ||
87 | |||
88 | again: | ||
89 | n = find_next_zero_bit(arena->map, limit, start); | ||
90 | end = n + npages; | ||
91 | if (unlikely(end >= limit)) { | ||
92 | if (likely(pass < 1)) { | ||
93 | limit = start; | ||
94 | start = 0; | ||
95 | __iommu_flushall(iommu); | ||
96 | pass++; | ||
97 | goto again; | ||
98 | } else { | ||
99 | /* Scanned the whole thing, give up. */ | ||
100 | return -1; | ||
101 | } | ||
102 | } | ||
103 | |||
104 | for (i = n; i < end; i++) { | ||
105 | if (test_bit(i, arena->map)) { | ||
106 | start = i + 1; | ||
107 | goto again; | ||
108 | } | ||
109 | } | ||
110 | |||
111 | for (i = n; i < end; i++) | ||
112 | __set_bit(i, arena->map); | ||
113 | |||
114 | arena->hint = end; | ||
115 | |||
116 | return n; | ||
117 | } | ||
118 | |||
119 | static void pci_arena_free(struct pci_iommu_arena *arena, unsigned long base, unsigned long npages) | ||
120 | { | ||
121 | unsigned long i; | ||
122 | |||
123 | for (i = base; i < (base + npages); i++) | ||
124 | __clear_bit(i, arena->map); | ||
125 | } | ||
126 | |||
83 | void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask) | 127 | void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask) |
84 | { | 128 | { |
85 | unsigned long i, tsbbase, order; | 129 | unsigned long i, tsbbase, order, sz, num_tsb_entries; |
130 | |||
131 | num_tsb_entries = tsbsize / sizeof(iopte_t); | ||
86 | 132 | ||
87 | /* Setup initial software IOMMU state. */ | 133 | /* Setup initial software IOMMU state. */ |
88 | spin_lock_init(&iommu->lock); | 134 | spin_lock_init(&iommu->lock); |
@@ -90,27 +136,16 @@ void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, | |||
90 | iommu->page_table_map_base = dma_offset; | 136 | iommu->page_table_map_base = dma_offset; |
91 | iommu->dma_addr_mask = dma_addr_mask; | 137 | iommu->dma_addr_mask = dma_addr_mask; |
92 | 138 | ||
93 | switch (tsbsize / (8 * 1024)) { | 139 | /* Allocate and initialize the free area map. */ |
94 | case 64: | 140 | sz = num_tsb_entries / 8; |
95 | iommu->page_table_sz_bits = 16; | 141 | sz = (sz + 7UL) & ~7UL; |
96 | break; | 142 | iommu->arena.map = kmalloc(sz, GFP_KERNEL); |
97 | case 128: | 143 | if (!iommu->arena.map) { |
98 | iommu->page_table_sz_bits = 17; | 144 | prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n"); |
99 | break; | ||
100 | default: | ||
101 | prom_printf("PCI_IOMMU: Illegal TSB size %d\n", | ||
102 | tsbsize / (8 * 1024)); | ||
103 | prom_halt(); | 145 | prom_halt(); |
104 | break; | ||
105 | }; | ||
106 | |||
107 | iommu->lowest_consistent_map = | ||
108 | 1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS); | ||
109 | |||
110 | for (i = 0; i < PBM_NCLUSTERS; i++) { | ||
111 | iommu->alloc_info[i].flush = 0; | ||
112 | iommu->alloc_info[i].next = 0; | ||
113 | } | 146 | } |
147 | memset(iommu->arena.map, 0, sz); | ||
148 | iommu->arena.limit = num_tsb_entries; | ||
114 | 149 | ||
115 | /* Allocate and initialize the dummy page which we | 150 | /* Allocate and initialize the dummy page which we |
116 | * set inactive IO PTEs to point to. | 151 | * set inactive IO PTEs to point to. |
@@ -132,114 +167,24 @@ void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, | |||
132 | } | 167 | } |
133 | iommu->page_table = (iopte_t *)tsbbase; | 168 | iommu->page_table = (iopte_t *)tsbbase; |
134 | 169 | ||
135 | for (i = 0; i < tsbsize / sizeof(iopte_t); i++) | 170 | for (i = 0; i < num_tsb_entries; i++) |
136 | iopte_make_dummy(iommu, &iommu->page_table[i]); | 171 | iopte_make_dummy(iommu, &iommu->page_table[i]); |
137 | } | 172 | } |
138 | 173 | ||
139 | static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages) | 174 | static inline iopte_t *alloc_npages(struct pci_iommu *iommu, unsigned long npages) |
140 | { | 175 | { |
141 | iopte_t *iopte, *limit, *first; | 176 | long entry; |
142 | unsigned long cnum, ent, flush_point; | ||
143 | |||
144 | cnum = 0; | ||
145 | while ((1UL << cnum) < npages) | ||
146 | cnum++; | ||
147 | iopte = (iommu->page_table + | ||
148 | (cnum << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
149 | |||
150 | if (cnum == 0) | ||
151 | limit = (iommu->page_table + | ||
152 | iommu->lowest_consistent_map); | ||
153 | else | ||
154 | limit = (iopte + | ||
155 | (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
156 | 177 | ||
157 | iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); | 178 | entry = pci_arena_alloc(iommu, npages); |
158 | flush_point = iommu->alloc_info[cnum].flush; | 179 | if (unlikely(entry < 0)) |
159 | 180 | return NULL; | |
160 | first = iopte; | ||
161 | for (;;) { | ||
162 | if (IOPTE_IS_DUMMY(iommu, iopte)) { | ||
163 | if ((iopte + (1 << cnum)) >= limit) | ||
164 | ent = 0; | ||
165 | else | ||
166 | ent = ent + 1; | ||
167 | iommu->alloc_info[cnum].next = ent; | ||
168 | if (ent == flush_point) | ||
169 | __iommu_flushall(iommu); | ||
170 | break; | ||
171 | } | ||
172 | iopte += (1 << cnum); | ||
173 | ent++; | ||
174 | if (iopte >= limit) { | ||
175 | iopte = (iommu->page_table + | ||
176 | (cnum << | ||
177 | (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
178 | ent = 0; | ||
179 | } | ||
180 | if (ent == flush_point) | ||
181 | __iommu_flushall(iommu); | ||
182 | if (iopte == first) | ||
183 | goto bad; | ||
184 | } | ||
185 | |||
186 | /* I've got your streaming cluster right here buddy boy... */ | ||
187 | return iopte; | ||
188 | 181 | ||
189 | bad: | 182 | return iommu->page_table + entry; |
190 | printk(KERN_EMERG "pci_iommu: alloc_streaming_cluster of npages(%ld) failed!\n", | ||
191 | npages); | ||
192 | return NULL; | ||
193 | } | 183 | } |
194 | 184 | ||
195 | static void free_streaming_cluster(struct pci_iommu *iommu, dma_addr_t base, | 185 | static inline void free_npages(struct pci_iommu *iommu, dma_addr_t base, unsigned long npages) |
196 | unsigned long npages, unsigned long ctx) | ||
197 | { | 186 | { |
198 | unsigned long cnum, ent; | 187 | pci_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); |
199 | |||
200 | cnum = 0; | ||
201 | while ((1UL << cnum) < npages) | ||
202 | cnum++; | ||
203 | |||
204 | ent = (base << (32 - IO_PAGE_SHIFT + PBM_LOGCLUSTERS - iommu->page_table_sz_bits)) | ||
205 | >> (32 + PBM_LOGCLUSTERS + cnum - iommu->page_table_sz_bits); | ||
206 | |||
207 | /* If the global flush might not have caught this entry, | ||
208 | * adjust the flush point such that we will flush before | ||
209 | * ever trying to reuse it. | ||
210 | */ | ||
211 | #define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) | ||
212 | if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush)) | ||
213 | iommu->alloc_info[cnum].flush = ent; | ||
214 | #undef between | ||
215 | } | ||
216 | |||
217 | /* We allocate consistent mappings from the end of cluster zero. */ | ||
218 | static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long npages) | ||
219 | { | ||
220 | iopte_t *iopte; | ||
221 | |||
222 | iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)); | ||
223 | while (iopte > iommu->page_table) { | ||
224 | iopte--; | ||
225 | if (IOPTE_IS_DUMMY(iommu, iopte)) { | ||
226 | unsigned long tmp = npages; | ||
227 | |||
228 | while (--tmp) { | ||
229 | iopte--; | ||
230 | if (!IOPTE_IS_DUMMY(iommu, iopte)) | ||
231 | break; | ||
232 | } | ||
233 | if (tmp == 0) { | ||
234 | u32 entry = (iopte - iommu->page_table); | ||
235 | |||
236 | if (entry < iommu->lowest_consistent_map) | ||
237 | iommu->lowest_consistent_map = entry; | ||
238 | return iopte; | ||
239 | } | ||
240 | } | ||
241 | } | ||
242 | return NULL; | ||
243 | } | 188 | } |
244 | 189 | ||
245 | static int iommu_alloc_ctx(struct pci_iommu *iommu) | 190 | static int iommu_alloc_ctx(struct pci_iommu *iommu) |
@@ -279,7 +224,7 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
279 | struct pcidev_cookie *pcp; | 224 | struct pcidev_cookie *pcp; |
280 | struct pci_iommu *iommu; | 225 | struct pci_iommu *iommu; |
281 | iopte_t *iopte; | 226 | iopte_t *iopte; |
282 | unsigned long flags, order, first_page, ctx; | 227 | unsigned long flags, order, first_page; |
283 | void *ret; | 228 | void *ret; |
284 | int npages; | 229 | int npages; |
285 | 230 | ||
@@ -297,9 +242,10 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
297 | iommu = pcp->pbm->iommu; | 242 | iommu = pcp->pbm->iommu; |
298 | 243 | ||
299 | spin_lock_irqsave(&iommu->lock, flags); | 244 | spin_lock_irqsave(&iommu->lock, flags); |
300 | iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); | 245 | iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); |
301 | if (iopte == NULL) { | 246 | spin_unlock_irqrestore(&iommu->lock, flags); |
302 | spin_unlock_irqrestore(&iommu->lock, flags); | 247 | |
248 | if (unlikely(iopte == NULL)) { | ||
303 | free_pages(first_page, order); | 249 | free_pages(first_page, order); |
304 | return NULL; | 250 | return NULL; |
305 | } | 251 | } |
@@ -308,31 +254,15 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
308 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); | 254 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
309 | ret = (void *) first_page; | 255 | ret = (void *) first_page; |
310 | npages = size >> IO_PAGE_SHIFT; | 256 | npages = size >> IO_PAGE_SHIFT; |
311 | ctx = 0; | ||
312 | if (iommu->iommu_ctxflush) | ||
313 | ctx = iommu_alloc_ctx(iommu); | ||
314 | first_page = __pa(first_page); | 257 | first_page = __pa(first_page); |
315 | while (npages--) { | 258 | while (npages--) { |
316 | iopte_val(*iopte) = (IOPTE_CONSISTENT(ctx) | | 259 | iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) | |
317 | IOPTE_WRITE | | 260 | IOPTE_WRITE | |
318 | (first_page & IOPTE_PAGE)); | 261 | (first_page & IOPTE_PAGE)); |
319 | iopte++; | 262 | iopte++; |
320 | first_page += IO_PAGE_SIZE; | 263 | first_page += IO_PAGE_SIZE; |
321 | } | 264 | } |
322 | 265 | ||
323 | { | ||
324 | int i; | ||
325 | u32 daddr = *dma_addrp; | ||
326 | |||
327 | npages = size >> IO_PAGE_SHIFT; | ||
328 | for (i = 0; i < npages; i++) { | ||
329 | pci_iommu_write(iommu->iommu_flush, daddr); | ||
330 | daddr += IO_PAGE_SIZE; | ||
331 | } | ||
332 | } | ||
333 | |||
334 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
335 | |||
336 | return ret; | 266 | return ret; |
337 | } | 267 | } |
338 | 268 | ||
@@ -342,7 +272,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ | |||
342 | struct pcidev_cookie *pcp; | 272 | struct pcidev_cookie *pcp; |
343 | struct pci_iommu *iommu; | 273 | struct pci_iommu *iommu; |
344 | iopte_t *iopte; | 274 | iopte_t *iopte; |
345 | unsigned long flags, order, npages, i, ctx; | 275 | unsigned long flags, order, npages; |
346 | 276 | ||
347 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | 277 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
348 | pcp = pdev->sysdata; | 278 | pcp = pdev->sysdata; |
@@ -352,46 +282,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ | |||
352 | 282 | ||
353 | spin_lock_irqsave(&iommu->lock, flags); | 283 | spin_lock_irqsave(&iommu->lock, flags); |
354 | 284 | ||
355 | if ((iopte - iommu->page_table) == | 285 | free_npages(iommu, dvma, npages); |
356 | iommu->lowest_consistent_map) { | ||
357 | iopte_t *walk = iopte + npages; | ||
358 | iopte_t *limit; | ||
359 | |||
360 | limit = (iommu->page_table + | ||
361 | (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
362 | while (walk < limit) { | ||
363 | if (!IOPTE_IS_DUMMY(iommu, walk)) | ||
364 | break; | ||
365 | walk++; | ||
366 | } | ||
367 | iommu->lowest_consistent_map = | ||
368 | (walk - iommu->page_table); | ||
369 | } | ||
370 | |||
371 | /* Data for consistent mappings cannot enter the streaming | ||
372 | * buffers, so we only need to update the TSB. We flush | ||
373 | * the IOMMU here as well to prevent conflicts with the | ||
374 | * streaming mapping deferred tlb flush scheme. | ||
375 | */ | ||
376 | |||
377 | ctx = 0; | ||
378 | if (iommu->iommu_ctxflush) | ||
379 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; | ||
380 | |||
381 | for (i = 0; i < npages; i++, iopte++) | ||
382 | iopte_make_dummy(iommu, iopte); | ||
383 | |||
384 | if (iommu->iommu_ctxflush) { | ||
385 | pci_iommu_write(iommu->iommu_ctxflush, ctx); | ||
386 | } else { | ||
387 | for (i = 0; i < npages; i++) { | ||
388 | u32 daddr = dvma + (i << IO_PAGE_SHIFT); | ||
389 | |||
390 | pci_iommu_write(iommu->iommu_flush, daddr); | ||
391 | } | ||
392 | } | ||
393 | |||
394 | iommu_free_ctx(iommu, ctx); | ||
395 | 286 | ||
396 | spin_unlock_irqrestore(&iommu->lock, flags); | 287 | spin_unlock_irqrestore(&iommu->lock, flags); |
397 | 288 | ||
@@ -418,25 +309,27 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct | |||
418 | iommu = pcp->pbm->iommu; | 309 | iommu = pcp->pbm->iommu; |
419 | strbuf = &pcp->pbm->stc; | 310 | strbuf = &pcp->pbm->stc; |
420 | 311 | ||
421 | if (direction == PCI_DMA_NONE) | 312 | if (unlikely(direction == PCI_DMA_NONE)) |
422 | BUG(); | 313 | goto bad_no_ctx; |
423 | 314 | ||
424 | oaddr = (unsigned long)ptr; | 315 | oaddr = (unsigned long)ptr; |
425 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | 316 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
426 | npages >>= IO_PAGE_SHIFT; | 317 | npages >>= IO_PAGE_SHIFT; |
427 | 318 | ||
428 | spin_lock_irqsave(&iommu->lock, flags); | 319 | spin_lock_irqsave(&iommu->lock, flags); |
320 | base = alloc_npages(iommu, npages); | ||
321 | ctx = 0; | ||
322 | if (iommu->iommu_ctxflush) | ||
323 | ctx = iommu_alloc_ctx(iommu); | ||
324 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
429 | 325 | ||
430 | base = alloc_streaming_cluster(iommu, npages); | 326 | if (unlikely(!base)) |
431 | if (base == NULL) | ||
432 | goto bad; | 327 | goto bad; |
328 | |||
433 | bus_addr = (iommu->page_table_map_base + | 329 | bus_addr = (iommu->page_table_map_base + |
434 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); | 330 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
435 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | 331 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
436 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | 332 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
437 | ctx = 0; | ||
438 | if (iommu->iommu_ctxflush) | ||
439 | ctx = iommu_alloc_ctx(iommu); | ||
440 | if (strbuf->strbuf_enabled) | 333 | if (strbuf->strbuf_enabled) |
441 | iopte_protection = IOPTE_STREAMING(ctx); | 334 | iopte_protection = IOPTE_STREAMING(ctx); |
442 | else | 335 | else |
@@ -447,12 +340,13 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct | |||
447 | for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) | 340 | for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) |
448 | iopte_val(*base) = iopte_protection | base_paddr; | 341 | iopte_val(*base) = iopte_protection | base_paddr; |
449 | 342 | ||
450 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
451 | |||
452 | return ret; | 343 | return ret; |
453 | 344 | ||
454 | bad: | 345 | bad: |
455 | spin_unlock_irqrestore(&iommu->lock, flags); | 346 | iommu_free_ctx(iommu, ctx); |
347 | bad_no_ctx: | ||
348 | if (printk_ratelimit()) | ||
349 | WARN_ON(1); | ||
456 | return PCI_DMA_ERROR_CODE; | 350 | return PCI_DMA_ERROR_CODE; |
457 | } | 351 | } |
458 | 352 | ||
@@ -527,10 +421,13 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int | |||
527 | struct pci_iommu *iommu; | 421 | struct pci_iommu *iommu; |
528 | struct pci_strbuf *strbuf; | 422 | struct pci_strbuf *strbuf; |
529 | iopte_t *base; | 423 | iopte_t *base; |
530 | unsigned long flags, npages, ctx; | 424 | unsigned long flags, npages, ctx, i; |
531 | 425 | ||
532 | if (direction == PCI_DMA_NONE) | 426 | if (unlikely(direction == PCI_DMA_NONE)) { |
533 | BUG(); | 427 | if (printk_ratelimit()) |
428 | WARN_ON(1); | ||
429 | return; | ||
430 | } | ||
534 | 431 | ||
535 | pcp = pdev->sysdata; | 432 | pcp = pdev->sysdata; |
536 | iommu = pcp->pbm->iommu; | 433 | iommu = pcp->pbm->iommu; |
@@ -556,13 +453,14 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int | |||
556 | 453 | ||
557 | /* Step 1: Kick data out of streaming buffers if necessary. */ | 454 | /* Step 1: Kick data out of streaming buffers if necessary. */ |
558 | if (strbuf->strbuf_enabled) | 455 | if (strbuf->strbuf_enabled) |
559 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); | 456 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, |
457 | npages, direction); | ||
560 | 458 | ||
561 | /* Step 2: Clear out first TSB entry. */ | 459 | /* Step 2: Clear out TSB entries. */ |
562 | iopte_make_dummy(iommu, base); | 460 | for (i = 0; i < npages; i++) |
461 | iopte_make_dummy(iommu, base + i); | ||
563 | 462 | ||
564 | free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, | 463 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
565 | npages, ctx); | ||
566 | 464 | ||
567 | iommu_free_ctx(iommu, ctx); | 465 | iommu_free_ctx(iommu, ctx); |
568 | 466 | ||
@@ -667,6 +565,8 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
667 | pci_map_single(pdev, | 565 | pci_map_single(pdev, |
668 | (page_address(sglist->page) + sglist->offset), | 566 | (page_address(sglist->page) + sglist->offset), |
669 | sglist->length, direction); | 567 | sglist->length, direction); |
568 | if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE)) | ||
569 | return 0; | ||
670 | sglist->dma_length = sglist->length; | 570 | sglist->dma_length = sglist->length; |
671 | return 1; | 571 | return 1; |
672 | } | 572 | } |
@@ -675,21 +575,29 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
675 | iommu = pcp->pbm->iommu; | 575 | iommu = pcp->pbm->iommu; |
676 | strbuf = &pcp->pbm->stc; | 576 | strbuf = &pcp->pbm->stc; |
677 | 577 | ||
678 | if (direction == PCI_DMA_NONE) | 578 | if (unlikely(direction == PCI_DMA_NONE)) |
679 | BUG(); | 579 | goto bad_no_ctx; |
680 | 580 | ||
681 | /* Step 1: Prepare scatter list. */ | 581 | /* Step 1: Prepare scatter list. */ |
682 | 582 | ||
683 | npages = prepare_sg(sglist, nelems); | 583 | npages = prepare_sg(sglist, nelems); |
684 | 584 | ||
685 | /* Step 2: Allocate a cluster. */ | 585 | /* Step 2: Allocate a cluster and context, if necessary. */ |
686 | 586 | ||
687 | spin_lock_irqsave(&iommu->lock, flags); | 587 | spin_lock_irqsave(&iommu->lock, flags); |
688 | 588 | ||
689 | base = alloc_streaming_cluster(iommu, npages); | 589 | base = alloc_npages(iommu, npages); |
590 | ctx = 0; | ||
591 | if (iommu->iommu_ctxflush) | ||
592 | ctx = iommu_alloc_ctx(iommu); | ||
593 | |||
594 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
595 | |||
690 | if (base == NULL) | 596 | if (base == NULL) |
691 | goto bad; | 597 | goto bad; |
692 | dma_base = iommu->page_table_map_base + ((base - iommu->page_table) << IO_PAGE_SHIFT); | 598 | |
599 | dma_base = iommu->page_table_map_base + | ||
600 | ((base - iommu->page_table) << IO_PAGE_SHIFT); | ||
693 | 601 | ||
694 | /* Step 3: Normalize DMA addresses. */ | 602 | /* Step 3: Normalize DMA addresses. */ |
695 | used = nelems; | 603 | used = nelems; |
@@ -702,30 +610,28 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
702 | } | 610 | } |
703 | used = nelems - used; | 611 | used = nelems - used; |
704 | 612 | ||
705 | /* Step 4: Choose a context if necessary. */ | 613 | /* Step 4: Create the mappings. */ |
706 | ctx = 0; | ||
707 | if (iommu->iommu_ctxflush) | ||
708 | ctx = iommu_alloc_ctx(iommu); | ||
709 | |||
710 | /* Step 5: Create the mappings. */ | ||
711 | if (strbuf->strbuf_enabled) | 614 | if (strbuf->strbuf_enabled) |
712 | iopte_protection = IOPTE_STREAMING(ctx); | 615 | iopte_protection = IOPTE_STREAMING(ctx); |
713 | else | 616 | else |
714 | iopte_protection = IOPTE_CONSISTENT(ctx); | 617 | iopte_protection = IOPTE_CONSISTENT(ctx); |
715 | if (direction != PCI_DMA_TODEVICE) | 618 | if (direction != PCI_DMA_TODEVICE) |
716 | iopte_protection |= IOPTE_WRITE; | 619 | iopte_protection |= IOPTE_WRITE; |
717 | fill_sg (base, sglist, used, nelems, iopte_protection); | 620 | |
621 | fill_sg(base, sglist, used, nelems, iopte_protection); | ||
622 | |||
718 | #ifdef VERIFY_SG | 623 | #ifdef VERIFY_SG |
719 | verify_sglist(sglist, nelems, base, npages); | 624 | verify_sglist(sglist, nelems, base, npages); |
720 | #endif | 625 | #endif |
721 | 626 | ||
722 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
723 | |||
724 | return used; | 627 | return used; |
725 | 628 | ||
726 | bad: | 629 | bad: |
727 | spin_unlock_irqrestore(&iommu->lock, flags); | 630 | iommu_free_ctx(iommu, ctx); |
728 | return PCI_DMA_ERROR_CODE; | 631 | bad_no_ctx: |
632 | if (printk_ratelimit()) | ||
633 | WARN_ON(1); | ||
634 | return 0; | ||
729 | } | 635 | } |
730 | 636 | ||
731 | /* Unmap a set of streaming mode DMA translations. */ | 637 | /* Unmap a set of streaming mode DMA translations. */ |
@@ -738,8 +644,10 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
738 | unsigned long flags, ctx, i, npages; | 644 | unsigned long flags, ctx, i, npages; |
739 | u32 bus_addr; | 645 | u32 bus_addr; |
740 | 646 | ||
741 | if (direction == PCI_DMA_NONE) | 647 | if (unlikely(direction == PCI_DMA_NONE)) { |
742 | BUG(); | 648 | if (printk_ratelimit()) |
649 | WARN_ON(1); | ||
650 | } | ||
743 | 651 | ||
744 | pcp = pdev->sysdata; | 652 | pcp = pdev->sysdata; |
745 | iommu = pcp->pbm->iommu; | 653 | iommu = pcp->pbm->iommu; |
@@ -751,7 +659,8 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
751 | if (sglist[i].dma_length == 0) | 659 | if (sglist[i].dma_length == 0) |
752 | break; | 660 | break; |
753 | i--; | 661 | i--; |
754 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> IO_PAGE_SHIFT; | 662 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - |
663 | bus_addr) >> IO_PAGE_SHIFT; | ||
755 | 664 | ||
756 | base = iommu->page_table + | 665 | base = iommu->page_table + |
757 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 666 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
@@ -772,11 +681,11 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
772 | if (strbuf->strbuf_enabled) | 681 | if (strbuf->strbuf_enabled) |
773 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); | 682 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); |
774 | 683 | ||
775 | /* Step 2: Clear out first TSB entry. */ | 684 | /* Step 2: Clear out the TSB entries. */ |
776 | iopte_make_dummy(iommu, base); | 685 | for (i = 0; i < npages; i++) |
686 | iopte_make_dummy(iommu, base + i); | ||
777 | 687 | ||
778 | free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, | 688 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
779 | npages, ctx); | ||
780 | 689 | ||
781 | iommu_free_ctx(iommu, ctx); | 690 | iommu_free_ctx(iommu, ctx); |
782 | 691 | ||
diff --git a/include/asm-sparc64/pbm.h b/include/asm-sparc64/pbm.h index c067407de0b0..dd35a2c7798a 100644 --- a/include/asm-sparc64/pbm.h +++ b/include/asm-sparc64/pbm.h | |||
@@ -27,23 +27,27 @@ | |||
27 | * PCI bus. | 27 | * PCI bus. |
28 | */ | 28 | */ |
29 | 29 | ||
30 | #define PBM_LOGCLUSTERS 3 | ||
31 | #define PBM_NCLUSTERS (1 << PBM_LOGCLUSTERS) | ||
32 | |||
33 | struct pci_controller_info; | 30 | struct pci_controller_info; |
34 | 31 | ||
35 | /* This contains the software state necessary to drive a PCI | 32 | /* This contains the software state necessary to drive a PCI |
36 | * controller's IOMMU. | 33 | * controller's IOMMU. |
37 | */ | 34 | */ |
35 | struct pci_iommu_arena { | ||
36 | unsigned long *map; | ||
37 | unsigned int hint; | ||
38 | unsigned int limit; | ||
39 | }; | ||
40 | |||
38 | struct pci_iommu { | 41 | struct pci_iommu { |
39 | /* This protects the controller's IOMMU and all | 42 | /* This protects the controller's IOMMU and all |
40 | * streaming buffers underneath. | 43 | * streaming buffers underneath. |
41 | */ | 44 | */ |
42 | spinlock_t lock; | 45 | spinlock_t lock; |
43 | 46 | ||
47 | struct pci_iommu_arena arena; | ||
48 | |||
44 | /* IOMMU page table, a linear array of ioptes. */ | 49 | /* IOMMU page table, a linear array of ioptes. */ |
45 | iopte_t *page_table; /* The page table itself. */ | 50 | iopte_t *page_table; /* The page table itself. */ |
46 | int page_table_sz_bits; /* log2 of ow many pages does it map? */ | ||
47 | 51 | ||
48 | /* Base PCI memory space address where IOMMU mappings | 52 | /* Base PCI memory space address where IOMMU mappings |
49 | * begin. | 53 | * begin. |
@@ -62,12 +66,6 @@ struct pci_iommu { | |||
62 | */ | 66 | */ |
63 | unsigned long write_complete_reg; | 67 | unsigned long write_complete_reg; |
64 | 68 | ||
65 | /* The lowest used consistent mapping entry. Since | ||
66 | * we allocate consistent maps out of cluster 0 this | ||
67 | * is relative to the beginning of closter 0. | ||
68 | */ | ||
69 | u32 lowest_consistent_map; | ||
70 | |||
71 | /* In order to deal with some buggy third-party PCI bridges that | 69 | /* In order to deal with some buggy third-party PCI bridges that |
72 | * do wrong prefetching, we never mark valid mappings as invalid. | 70 | * do wrong prefetching, we never mark valid mappings as invalid. |
73 | * Instead we point them at this dummy page. | 71 | * Instead we point them at this dummy page. |
@@ -75,16 +73,6 @@ struct pci_iommu { | |||
75 | unsigned long dummy_page; | 73 | unsigned long dummy_page; |
76 | unsigned long dummy_page_pa; | 74 | unsigned long dummy_page_pa; |
77 | 75 | ||
78 | /* If PBM_NCLUSTERS is ever decreased to 4 or lower, | ||
79 | * or if largest supported page_table_sz * 8K goes above | ||
80 | * 2GB, you must increase the size of the type of | ||
81 | * these counters. You have been duly warned. -DaveM | ||
82 | */ | ||
83 | struct { | ||
84 | u16 next; | ||
85 | u16 flush; | ||
86 | } alloc_info[PBM_NCLUSTERS]; | ||
87 | |||
88 | /* CTX allocation. */ | 76 | /* CTX allocation. */ |
89 | unsigned long ctx_lowest_free; | 77 | unsigned long ctx_lowest_free; |
90 | unsigned long ctx_bitmap[IOMMU_NUM_CTXS / (sizeof(unsigned long) * 8)]; | 78 | unsigned long ctx_bitmap[IOMMU_NUM_CTXS / (sizeof(unsigned long) * 8)]; |