diff options
Diffstat (limited to 'arch/sparc64/kernel/pci_iommu.c')
-rw-r--r-- | arch/sparc64/kernel/pci_iommu.c | 363 |
1 files changed, 159 insertions, 204 deletions
diff --git a/arch/sparc64/kernel/pci_iommu.c b/arch/sparc64/kernel/pci_iommu.c index 425c60cfea19..a11910be1013 100644 --- a/arch/sparc64/kernel/pci_iommu.c +++ b/arch/sparc64/kernel/pci_iommu.c | |||
@@ -49,12 +49,6 @@ static void __iommu_flushall(struct pci_iommu *iommu) | |||
49 | 49 | ||
50 | /* Ensure completion of previous PIO writes. */ | 50 | /* Ensure completion of previous PIO writes. */ |
51 | (void) pci_iommu_read(iommu->write_complete_reg); | 51 | (void) pci_iommu_read(iommu->write_complete_reg); |
52 | |||
53 | /* Now update everyone's flush point. */ | ||
54 | for (entry = 0; entry < PBM_NCLUSTERS; entry++) { | ||
55 | iommu->alloc_info[entry].flush = | ||
56 | iommu->alloc_info[entry].next; | ||
57 | } | ||
58 | } | 52 | } |
59 | 53 | ||
60 | #define IOPTE_CONSISTENT(CTX) \ | 54 | #define IOPTE_CONSISTENT(CTX) \ |
@@ -80,120 +74,117 @@ static void inline iopte_make_dummy(struct pci_iommu *iommu, iopte_t *iopte) | |||
80 | iopte_val(*iopte) = val; | 74 | iopte_val(*iopte) = val; |
81 | } | 75 | } |
82 | 76 | ||
83 | void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize) | 77 | /* Based largely upon the ppc64 iommu allocator. */ |
78 | static long pci_arena_alloc(struct pci_iommu *iommu, unsigned long npages) | ||
84 | { | 79 | { |
85 | int i; | 80 | struct pci_iommu_arena *arena = &iommu->arena; |
86 | 81 | unsigned long n, i, start, end, limit; | |
87 | tsbsize /= sizeof(iopte_t); | 82 | int pass; |
88 | 83 | ||
89 | for (i = 0; i < tsbsize; i++) | 84 | limit = arena->limit; |
90 | iopte_make_dummy(iommu, &iommu->page_table[i]); | 85 | start = arena->hint; |
91 | } | 86 | pass = 0; |
92 | 87 | ||
93 | static iopte_t *alloc_streaming_cluster(struct pci_iommu *iommu, unsigned long npages) | 88 | again: |
94 | { | 89 | n = find_next_zero_bit(arena->map, limit, start); |
95 | iopte_t *iopte, *limit, *first; | 90 | end = n + npages; |
96 | unsigned long cnum, ent, flush_point; | 91 | if (unlikely(end >= limit)) { |
97 | 92 | if (likely(pass < 1)) { | |
98 | cnum = 0; | 93 | limit = start; |
99 | while ((1UL << cnum) < npages) | 94 | start = 0; |
100 | cnum++; | 95 | __iommu_flushall(iommu); |
101 | iopte = (iommu->page_table + | 96 | pass++; |
102 | (cnum << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | 97 | goto again; |
103 | 98 | } else { | |
104 | if (cnum == 0) | 99 | /* Scanned the whole thing, give up. */ |
105 | limit = (iommu->page_table + | 100 | return -1; |
106 | iommu->lowest_consistent_map); | ||
107 | else | ||
108 | limit = (iopte + | ||
109 | (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
110 | |||
111 | iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); | ||
112 | flush_point = iommu->alloc_info[cnum].flush; | ||
113 | |||
114 | first = iopte; | ||
115 | for (;;) { | ||
116 | if (IOPTE_IS_DUMMY(iommu, iopte)) { | ||
117 | if ((iopte + (1 << cnum)) >= limit) | ||
118 | ent = 0; | ||
119 | else | ||
120 | ent = ent + 1; | ||
121 | iommu->alloc_info[cnum].next = ent; | ||
122 | if (ent == flush_point) | ||
123 | __iommu_flushall(iommu); | ||
124 | break; | ||
125 | } | 101 | } |
126 | iopte += (1 << cnum); | 102 | } |
127 | ent++; | 103 | |
128 | if (iopte >= limit) { | 104 | for (i = n; i < end; i++) { |
129 | iopte = (iommu->page_table + | 105 | if (test_bit(i, arena->map)) { |
130 | (cnum << | 106 | start = i + 1; |
131 | (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | 107 | goto again; |
132 | ent = 0; | ||
133 | } | 108 | } |
134 | if (ent == flush_point) | ||
135 | __iommu_flushall(iommu); | ||
136 | if (iopte == first) | ||
137 | goto bad; | ||
138 | } | 109 | } |
139 | 110 | ||
140 | /* I've got your streaming cluster right here buddy boy... */ | 111 | for (i = n; i < end; i++) |
141 | return iopte; | 112 | __set_bit(i, arena->map); |
142 | 113 | ||
143 | bad: | 114 | arena->hint = end; |
144 | printk(KERN_EMERG "pci_iommu: alloc_streaming_cluster of npages(%ld) failed!\n", | 115 | |
145 | npages); | 116 | return n; |
146 | return NULL; | ||
147 | } | 117 | } |
148 | 118 | ||
149 | static void free_streaming_cluster(struct pci_iommu *iommu, dma_addr_t base, | 119 | static void pci_arena_free(struct pci_iommu_arena *arena, unsigned long base, unsigned long npages) |
150 | unsigned long npages, unsigned long ctx) | ||
151 | { | 120 | { |
152 | unsigned long cnum, ent; | 121 | unsigned long i; |
153 | 122 | ||
154 | cnum = 0; | 123 | for (i = base; i < (base + npages); i++) |
155 | while ((1UL << cnum) < npages) | 124 | __clear_bit(i, arena->map); |
156 | cnum++; | 125 | } |
157 | 126 | ||
158 | ent = (base << (32 - IO_PAGE_SHIFT + PBM_LOGCLUSTERS - iommu->page_table_sz_bits)) | 127 | void pci_iommu_table_init(struct pci_iommu *iommu, int tsbsize, u32 dma_offset, u32 dma_addr_mask) |
159 | >> (32 + PBM_LOGCLUSTERS + cnum - iommu->page_table_sz_bits); | 128 | { |
129 | unsigned long i, tsbbase, order, sz, num_tsb_entries; | ||
130 | |||
131 | num_tsb_entries = tsbsize / sizeof(iopte_t); | ||
132 | |||
133 | /* Setup initial software IOMMU state. */ | ||
134 | spin_lock_init(&iommu->lock); | ||
135 | iommu->ctx_lowest_free = 1; | ||
136 | iommu->page_table_map_base = dma_offset; | ||
137 | iommu->dma_addr_mask = dma_addr_mask; | ||
138 | |||
139 | /* Allocate and initialize the free area map. */ | ||
140 | sz = num_tsb_entries / 8; | ||
141 | sz = (sz + 7UL) & ~7UL; | ||
142 | iommu->arena.map = kmalloc(sz, GFP_KERNEL); | ||
143 | if (!iommu->arena.map) { | ||
144 | prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n"); | ||
145 | prom_halt(); | ||
146 | } | ||
147 | memset(iommu->arena.map, 0, sz); | ||
148 | iommu->arena.limit = num_tsb_entries; | ||
160 | 149 | ||
161 | /* If the global flush might not have caught this entry, | 150 | /* Allocate and initialize the dummy page which we |
162 | * adjust the flush point such that we will flush before | 151 | * set inactive IO PTEs to point to. |
163 | * ever trying to reuse it. | ||
164 | */ | 152 | */ |
165 | #define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) | 153 | iommu->dummy_page = __get_free_pages(GFP_KERNEL, 0); |
166 | if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush)) | 154 | if (!iommu->dummy_page) { |
167 | iommu->alloc_info[cnum].flush = ent; | 155 | prom_printf("PCI_IOMMU: Error, gfp(dummy_page) failed.\n"); |
168 | #undef between | 156 | prom_halt(); |
157 | } | ||
158 | memset((void *)iommu->dummy_page, 0, PAGE_SIZE); | ||
159 | iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); | ||
160 | |||
161 | /* Now allocate and setup the IOMMU page table itself. */ | ||
162 | order = get_order(tsbsize); | ||
163 | tsbbase = __get_free_pages(GFP_KERNEL, order); | ||
164 | if (!tsbbase) { | ||
165 | prom_printf("PCI_IOMMU: Error, gfp(tsb) failed.\n"); | ||
166 | prom_halt(); | ||
167 | } | ||
168 | iommu->page_table = (iopte_t *)tsbbase; | ||
169 | |||
170 | for (i = 0; i < num_tsb_entries; i++) | ||
171 | iopte_make_dummy(iommu, &iommu->page_table[i]); | ||
169 | } | 172 | } |
170 | 173 | ||
171 | /* We allocate consistent mappings from the end of cluster zero. */ | 174 | static inline iopte_t *alloc_npages(struct pci_iommu *iommu, unsigned long npages) |
172 | static iopte_t *alloc_consistent_cluster(struct pci_iommu *iommu, unsigned long npages) | ||
173 | { | 175 | { |
174 | iopte_t *iopte; | 176 | long entry; |
175 | 177 | ||
176 | iopte = iommu->page_table + (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS)); | 178 | entry = pci_arena_alloc(iommu, npages); |
177 | while (iopte > iommu->page_table) { | 179 | if (unlikely(entry < 0)) |
178 | iopte--; | 180 | return NULL; |
179 | if (IOPTE_IS_DUMMY(iommu, iopte)) { | ||
180 | unsigned long tmp = npages; | ||
181 | 181 | ||
182 | while (--tmp) { | 182 | return iommu->page_table + entry; |
183 | iopte--; | 183 | } |
184 | if (!IOPTE_IS_DUMMY(iommu, iopte)) | ||
185 | break; | ||
186 | } | ||
187 | if (tmp == 0) { | ||
188 | u32 entry = (iopte - iommu->page_table); | ||
189 | 184 | ||
190 | if (entry < iommu->lowest_consistent_map) | 185 | static inline void free_npages(struct pci_iommu *iommu, dma_addr_t base, unsigned long npages) |
191 | iommu->lowest_consistent_map = entry; | 186 | { |
192 | return iopte; | 187 | pci_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); |
193 | } | ||
194 | } | ||
195 | } | ||
196 | return NULL; | ||
197 | } | 188 | } |
198 | 189 | ||
199 | static int iommu_alloc_ctx(struct pci_iommu *iommu) | 190 | static int iommu_alloc_ctx(struct pci_iommu *iommu) |
@@ -233,7 +224,7 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
233 | struct pcidev_cookie *pcp; | 224 | struct pcidev_cookie *pcp; |
234 | struct pci_iommu *iommu; | 225 | struct pci_iommu *iommu; |
235 | iopte_t *iopte; | 226 | iopte_t *iopte; |
236 | unsigned long flags, order, first_page, ctx; | 227 | unsigned long flags, order, first_page; |
237 | void *ret; | 228 | void *ret; |
238 | int npages; | 229 | int npages; |
239 | 230 | ||
@@ -251,9 +242,10 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
251 | iommu = pcp->pbm->iommu; | 242 | iommu = pcp->pbm->iommu; |
252 | 243 | ||
253 | spin_lock_irqsave(&iommu->lock, flags); | 244 | spin_lock_irqsave(&iommu->lock, flags); |
254 | iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); | 245 | iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); |
255 | if (iopte == NULL) { | 246 | spin_unlock_irqrestore(&iommu->lock, flags); |
256 | spin_unlock_irqrestore(&iommu->lock, flags); | 247 | |
248 | if (unlikely(iopte == NULL)) { | ||
257 | free_pages(first_page, order); | 249 | free_pages(first_page, order); |
258 | return NULL; | 250 | return NULL; |
259 | } | 251 | } |
@@ -262,31 +254,15 @@ void *pci_alloc_consistent(struct pci_dev *pdev, size_t size, dma_addr_t *dma_ad | |||
262 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); | 254 | ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
263 | ret = (void *) first_page; | 255 | ret = (void *) first_page; |
264 | npages = size >> IO_PAGE_SHIFT; | 256 | npages = size >> IO_PAGE_SHIFT; |
265 | ctx = 0; | ||
266 | if (iommu->iommu_ctxflush) | ||
267 | ctx = iommu_alloc_ctx(iommu); | ||
268 | first_page = __pa(first_page); | 257 | first_page = __pa(first_page); |
269 | while (npages--) { | 258 | while (npages--) { |
270 | iopte_val(*iopte) = (IOPTE_CONSISTENT(ctx) | | 259 | iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) | |
271 | IOPTE_WRITE | | 260 | IOPTE_WRITE | |
272 | (first_page & IOPTE_PAGE)); | 261 | (first_page & IOPTE_PAGE)); |
273 | iopte++; | 262 | iopte++; |
274 | first_page += IO_PAGE_SIZE; | 263 | first_page += IO_PAGE_SIZE; |
275 | } | 264 | } |
276 | 265 | ||
277 | { | ||
278 | int i; | ||
279 | u32 daddr = *dma_addrp; | ||
280 | |||
281 | npages = size >> IO_PAGE_SHIFT; | ||
282 | for (i = 0; i < npages; i++) { | ||
283 | pci_iommu_write(iommu->iommu_flush, daddr); | ||
284 | daddr += IO_PAGE_SIZE; | ||
285 | } | ||
286 | } | ||
287 | |||
288 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
289 | |||
290 | return ret; | 266 | return ret; |
291 | } | 267 | } |
292 | 268 | ||
@@ -296,7 +272,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ | |||
296 | struct pcidev_cookie *pcp; | 272 | struct pcidev_cookie *pcp; |
297 | struct pci_iommu *iommu; | 273 | struct pci_iommu *iommu; |
298 | iopte_t *iopte; | 274 | iopte_t *iopte; |
299 | unsigned long flags, order, npages, i, ctx; | 275 | unsigned long flags, order, npages; |
300 | 276 | ||
301 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; | 277 | npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
302 | pcp = pdev->sysdata; | 278 | pcp = pdev->sysdata; |
@@ -306,46 +282,7 @@ void pci_free_consistent(struct pci_dev *pdev, size_t size, void *cpu, dma_addr_ | |||
306 | 282 | ||
307 | spin_lock_irqsave(&iommu->lock, flags); | 283 | spin_lock_irqsave(&iommu->lock, flags); |
308 | 284 | ||
309 | if ((iopte - iommu->page_table) == | 285 | free_npages(iommu, dvma, npages); |
310 | iommu->lowest_consistent_map) { | ||
311 | iopte_t *walk = iopte + npages; | ||
312 | iopte_t *limit; | ||
313 | |||
314 | limit = (iommu->page_table + | ||
315 | (1 << (iommu->page_table_sz_bits - PBM_LOGCLUSTERS))); | ||
316 | while (walk < limit) { | ||
317 | if (!IOPTE_IS_DUMMY(iommu, walk)) | ||
318 | break; | ||
319 | walk++; | ||
320 | } | ||
321 | iommu->lowest_consistent_map = | ||
322 | (walk - iommu->page_table); | ||
323 | } | ||
324 | |||
325 | /* Data for consistent mappings cannot enter the streaming | ||
326 | * buffers, so we only need to update the TSB. We flush | ||
327 | * the IOMMU here as well to prevent conflicts with the | ||
328 | * streaming mapping deferred tlb flush scheme. | ||
329 | */ | ||
330 | |||
331 | ctx = 0; | ||
332 | if (iommu->iommu_ctxflush) | ||
333 | ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; | ||
334 | |||
335 | for (i = 0; i < npages; i++, iopte++) | ||
336 | iopte_make_dummy(iommu, iopte); | ||
337 | |||
338 | if (iommu->iommu_ctxflush) { | ||
339 | pci_iommu_write(iommu->iommu_ctxflush, ctx); | ||
340 | } else { | ||
341 | for (i = 0; i < npages; i++) { | ||
342 | u32 daddr = dvma + (i << IO_PAGE_SHIFT); | ||
343 | |||
344 | pci_iommu_write(iommu->iommu_flush, daddr); | ||
345 | } | ||
346 | } | ||
347 | |||
348 | iommu_free_ctx(iommu, ctx); | ||
349 | 286 | ||
350 | spin_unlock_irqrestore(&iommu->lock, flags); | 287 | spin_unlock_irqrestore(&iommu->lock, flags); |
351 | 288 | ||
@@ -372,25 +309,27 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct | |||
372 | iommu = pcp->pbm->iommu; | 309 | iommu = pcp->pbm->iommu; |
373 | strbuf = &pcp->pbm->stc; | 310 | strbuf = &pcp->pbm->stc; |
374 | 311 | ||
375 | if (direction == PCI_DMA_NONE) | 312 | if (unlikely(direction == PCI_DMA_NONE)) |
376 | BUG(); | 313 | goto bad_no_ctx; |
377 | 314 | ||
378 | oaddr = (unsigned long)ptr; | 315 | oaddr = (unsigned long)ptr; |
379 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); | 316 | npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
380 | npages >>= IO_PAGE_SHIFT; | 317 | npages >>= IO_PAGE_SHIFT; |
381 | 318 | ||
382 | spin_lock_irqsave(&iommu->lock, flags); | 319 | spin_lock_irqsave(&iommu->lock, flags); |
320 | base = alloc_npages(iommu, npages); | ||
321 | ctx = 0; | ||
322 | if (iommu->iommu_ctxflush) | ||
323 | ctx = iommu_alloc_ctx(iommu); | ||
324 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
383 | 325 | ||
384 | base = alloc_streaming_cluster(iommu, npages); | 326 | if (unlikely(!base)) |
385 | if (base == NULL) | ||
386 | goto bad; | 327 | goto bad; |
328 | |||
387 | bus_addr = (iommu->page_table_map_base + | 329 | bus_addr = (iommu->page_table_map_base + |
388 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); | 330 | ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
389 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); | 331 | ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
390 | base_paddr = __pa(oaddr & IO_PAGE_MASK); | 332 | base_paddr = __pa(oaddr & IO_PAGE_MASK); |
391 | ctx = 0; | ||
392 | if (iommu->iommu_ctxflush) | ||
393 | ctx = iommu_alloc_ctx(iommu); | ||
394 | if (strbuf->strbuf_enabled) | 333 | if (strbuf->strbuf_enabled) |
395 | iopte_protection = IOPTE_STREAMING(ctx); | 334 | iopte_protection = IOPTE_STREAMING(ctx); |
396 | else | 335 | else |
@@ -401,12 +340,13 @@ dma_addr_t pci_map_single(struct pci_dev *pdev, void *ptr, size_t sz, int direct | |||
401 | for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) | 340 | for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) |
402 | iopte_val(*base) = iopte_protection | base_paddr; | 341 | iopte_val(*base) = iopte_protection | base_paddr; |
403 | 342 | ||
404 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
405 | |||
406 | return ret; | 343 | return ret; |
407 | 344 | ||
408 | bad: | 345 | bad: |
409 | spin_unlock_irqrestore(&iommu->lock, flags); | 346 | iommu_free_ctx(iommu, ctx); |
347 | bad_no_ctx: | ||
348 | if (printk_ratelimit()) | ||
349 | WARN_ON(1); | ||
410 | return PCI_DMA_ERROR_CODE; | 350 | return PCI_DMA_ERROR_CODE; |
411 | } | 351 | } |
412 | 352 | ||
@@ -481,10 +421,13 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int | |||
481 | struct pci_iommu *iommu; | 421 | struct pci_iommu *iommu; |
482 | struct pci_strbuf *strbuf; | 422 | struct pci_strbuf *strbuf; |
483 | iopte_t *base; | 423 | iopte_t *base; |
484 | unsigned long flags, npages, ctx; | 424 | unsigned long flags, npages, ctx, i; |
485 | 425 | ||
486 | if (direction == PCI_DMA_NONE) | 426 | if (unlikely(direction == PCI_DMA_NONE)) { |
487 | BUG(); | 427 | if (printk_ratelimit()) |
428 | WARN_ON(1); | ||
429 | return; | ||
430 | } | ||
488 | 431 | ||
489 | pcp = pdev->sysdata; | 432 | pcp = pdev->sysdata; |
490 | iommu = pcp->pbm->iommu; | 433 | iommu = pcp->pbm->iommu; |
@@ -510,13 +453,14 @@ void pci_unmap_single(struct pci_dev *pdev, dma_addr_t bus_addr, size_t sz, int | |||
510 | 453 | ||
511 | /* Step 1: Kick data out of streaming buffers if necessary. */ | 454 | /* Step 1: Kick data out of streaming buffers if necessary. */ |
512 | if (strbuf->strbuf_enabled) | 455 | if (strbuf->strbuf_enabled) |
513 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); | 456 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, |
457 | npages, direction); | ||
514 | 458 | ||
515 | /* Step 2: Clear out first TSB entry. */ | 459 | /* Step 2: Clear out TSB entries. */ |
516 | iopte_make_dummy(iommu, base); | 460 | for (i = 0; i < npages; i++) |
461 | iopte_make_dummy(iommu, base + i); | ||
517 | 462 | ||
518 | free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, | 463 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
519 | npages, ctx); | ||
520 | 464 | ||
521 | iommu_free_ctx(iommu, ctx); | 465 | iommu_free_ctx(iommu, ctx); |
522 | 466 | ||
@@ -621,6 +565,8 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
621 | pci_map_single(pdev, | 565 | pci_map_single(pdev, |
622 | (page_address(sglist->page) + sglist->offset), | 566 | (page_address(sglist->page) + sglist->offset), |
623 | sglist->length, direction); | 567 | sglist->length, direction); |
568 | if (unlikely(sglist->dma_address == PCI_DMA_ERROR_CODE)) | ||
569 | return 0; | ||
624 | sglist->dma_length = sglist->length; | 570 | sglist->dma_length = sglist->length; |
625 | return 1; | 571 | return 1; |
626 | } | 572 | } |
@@ -629,21 +575,29 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
629 | iommu = pcp->pbm->iommu; | 575 | iommu = pcp->pbm->iommu; |
630 | strbuf = &pcp->pbm->stc; | 576 | strbuf = &pcp->pbm->stc; |
631 | 577 | ||
632 | if (direction == PCI_DMA_NONE) | 578 | if (unlikely(direction == PCI_DMA_NONE)) |
633 | BUG(); | 579 | goto bad_no_ctx; |
634 | 580 | ||
635 | /* Step 1: Prepare scatter list. */ | 581 | /* Step 1: Prepare scatter list. */ |
636 | 582 | ||
637 | npages = prepare_sg(sglist, nelems); | 583 | npages = prepare_sg(sglist, nelems); |
638 | 584 | ||
639 | /* Step 2: Allocate a cluster. */ | 585 | /* Step 2: Allocate a cluster and context, if necessary. */ |
640 | 586 | ||
641 | spin_lock_irqsave(&iommu->lock, flags); | 587 | spin_lock_irqsave(&iommu->lock, flags); |
642 | 588 | ||
643 | base = alloc_streaming_cluster(iommu, npages); | 589 | base = alloc_npages(iommu, npages); |
590 | ctx = 0; | ||
591 | if (iommu->iommu_ctxflush) | ||
592 | ctx = iommu_alloc_ctx(iommu); | ||
593 | |||
594 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
595 | |||
644 | if (base == NULL) | 596 | if (base == NULL) |
645 | goto bad; | 597 | goto bad; |
646 | dma_base = iommu->page_table_map_base + ((base - iommu->page_table) << IO_PAGE_SHIFT); | 598 | |
599 | dma_base = iommu->page_table_map_base + | ||
600 | ((base - iommu->page_table) << IO_PAGE_SHIFT); | ||
647 | 601 | ||
648 | /* Step 3: Normalize DMA addresses. */ | 602 | /* Step 3: Normalize DMA addresses. */ |
649 | used = nelems; | 603 | used = nelems; |
@@ -656,30 +610,28 @@ int pci_map_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, int | |||
656 | } | 610 | } |
657 | used = nelems - used; | 611 | used = nelems - used; |
658 | 612 | ||
659 | /* Step 4: Choose a context if necessary. */ | 613 | /* Step 4: Create the mappings. */ |
660 | ctx = 0; | ||
661 | if (iommu->iommu_ctxflush) | ||
662 | ctx = iommu_alloc_ctx(iommu); | ||
663 | |||
664 | /* Step 5: Create the mappings. */ | ||
665 | if (strbuf->strbuf_enabled) | 614 | if (strbuf->strbuf_enabled) |
666 | iopte_protection = IOPTE_STREAMING(ctx); | 615 | iopte_protection = IOPTE_STREAMING(ctx); |
667 | else | 616 | else |
668 | iopte_protection = IOPTE_CONSISTENT(ctx); | 617 | iopte_protection = IOPTE_CONSISTENT(ctx); |
669 | if (direction != PCI_DMA_TODEVICE) | 618 | if (direction != PCI_DMA_TODEVICE) |
670 | iopte_protection |= IOPTE_WRITE; | 619 | iopte_protection |= IOPTE_WRITE; |
671 | fill_sg (base, sglist, used, nelems, iopte_protection); | 620 | |
621 | fill_sg(base, sglist, used, nelems, iopte_protection); | ||
622 | |||
672 | #ifdef VERIFY_SG | 623 | #ifdef VERIFY_SG |
673 | verify_sglist(sglist, nelems, base, npages); | 624 | verify_sglist(sglist, nelems, base, npages); |
674 | #endif | 625 | #endif |
675 | 626 | ||
676 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
677 | |||
678 | return used; | 627 | return used; |
679 | 628 | ||
680 | bad: | 629 | bad: |
681 | spin_unlock_irqrestore(&iommu->lock, flags); | 630 | iommu_free_ctx(iommu, ctx); |
682 | return PCI_DMA_ERROR_CODE; | 631 | bad_no_ctx: |
632 | if (printk_ratelimit()) | ||
633 | WARN_ON(1); | ||
634 | return 0; | ||
683 | } | 635 | } |
684 | 636 | ||
685 | /* Unmap a set of streaming mode DMA translations. */ | 637 | /* Unmap a set of streaming mode DMA translations. */ |
@@ -692,8 +644,10 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
692 | unsigned long flags, ctx, i, npages; | 644 | unsigned long flags, ctx, i, npages; |
693 | u32 bus_addr; | 645 | u32 bus_addr; |
694 | 646 | ||
695 | if (direction == PCI_DMA_NONE) | 647 | if (unlikely(direction == PCI_DMA_NONE)) { |
696 | BUG(); | 648 | if (printk_ratelimit()) |
649 | WARN_ON(1); | ||
650 | } | ||
697 | 651 | ||
698 | pcp = pdev->sysdata; | 652 | pcp = pdev->sysdata; |
699 | iommu = pcp->pbm->iommu; | 653 | iommu = pcp->pbm->iommu; |
@@ -705,7 +659,8 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
705 | if (sglist[i].dma_length == 0) | 659 | if (sglist[i].dma_length == 0) |
706 | break; | 660 | break; |
707 | i--; | 661 | i--; |
708 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - bus_addr) >> IO_PAGE_SHIFT; | 662 | npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - |
663 | bus_addr) >> IO_PAGE_SHIFT; | ||
709 | 664 | ||
710 | base = iommu->page_table + | 665 | base = iommu->page_table + |
711 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | 666 | ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); |
@@ -726,11 +681,11 @@ void pci_unmap_sg(struct pci_dev *pdev, struct scatterlist *sglist, int nelems, | |||
726 | if (strbuf->strbuf_enabled) | 681 | if (strbuf->strbuf_enabled) |
727 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); | 682 | pci_strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); |
728 | 683 | ||
729 | /* Step 2: Clear out first TSB entry. */ | 684 | /* Step 2: Clear out the TSB entries. */ |
730 | iopte_make_dummy(iommu, base); | 685 | for (i = 0; i < npages; i++) |
686 | iopte_make_dummy(iommu, base + i); | ||
731 | 687 | ||
732 | free_streaming_cluster(iommu, bus_addr - iommu->page_table_map_base, | 688 | free_npages(iommu, bus_addr - iommu->page_table_map_base, npages); |
733 | npages, ctx); | ||
734 | 689 | ||
735 | iommu_free_ctx(iommu, ctx); | 690 | iommu_free_ctx(iommu, ctx); |
736 | 691 | ||