diff options
author | David S. Miller <davem@huronp11.davemloft.net> | 2008-02-09 06:11:01 -0500 |
---|---|---|
committer | David S. Miller <davem@davemloft.net> | 2008-02-09 06:15:36 -0500 |
commit | 13fa14e185614066d96f90f09da08eebe58cbc8f (patch) | |
tree | 153e69e8ea564109b7509ad4a0ce6daacbedb860 /arch/sparc64/kernel/pci_sun4v.c | |
parent | d284142cbad66832d5072a0aebeca7bd9ca841b7 (diff) |
[SPARC64]: Add SG merging support back into IOMMU code.
Mimicks almost perfectly the powerpc IOMMU code, except that it
doesn't have the IOMMU_PAGE_SIZE != PAGE_SIZE handling, and it also
lacks the device dma mask support bits.
I'll add that later as time permits, but this gets us at least back to
where we were beforehand.
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc64/kernel/pci_sun4v.c')
-rw-r--r-- | arch/sparc64/kernel/pci_sun4v.c | 206 |
1 files changed, 132 insertions, 74 deletions
diff --git a/arch/sparc64/kernel/pci_sun4v.c b/arch/sparc64/kernel/pci_sun4v.c index c8b6199a5dc4..ddca6c6c0b49 100644 --- a/arch/sparc64/kernel/pci_sun4v.c +++ b/arch/sparc64/kernel/pci_sun4v.c | |||
@@ -89,6 +89,17 @@ static long iommu_batch_flush(struct iommu_batch *p) | |||
89 | return 0; | 89 | return 0; |
90 | } | 90 | } |
91 | 91 | ||
92 | static inline void iommu_batch_new_entry(unsigned long entry) | ||
93 | { | ||
94 | struct iommu_batch *p = &__get_cpu_var(iommu_batch); | ||
95 | |||
96 | if (p->entry + p->npages == entry) | ||
97 | return; | ||
98 | if (p->entry != ~0UL) | ||
99 | iommu_batch_flush(p); | ||
100 | p->entry = entry; | ||
101 | } | ||
102 | |||
92 | /* Interrupts must be disabled. */ | 103 | /* Interrupts must be disabled. */ |
93 | static inline long iommu_batch_add(u64 phys_page) | 104 | static inline long iommu_batch_add(u64 phys_page) |
94 | { | 105 | { |
@@ -320,88 +331,131 @@ static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr, | |||
320 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, | 331 | static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, |
321 | int nelems, enum dma_data_direction direction) | 332 | int nelems, enum dma_data_direction direction) |
322 | { | 333 | { |
323 | unsigned long flags, npages, i, prot; | 334 | struct scatterlist *s, *outs, *segstart; |
324 | u32 dma_base, orig_dma_base; | 335 | unsigned long flags, handle, prot; |
325 | struct scatterlist *sg; | 336 | dma_addr_t dma_next = 0, dma_addr; |
337 | unsigned int max_seg_size; | ||
338 | int outcount, incount, i; | ||
326 | struct iommu *iommu; | 339 | struct iommu *iommu; |
327 | long entry, err; | 340 | long err; |
328 | 341 | ||
329 | /* Fast path single entry scatterlists. */ | 342 | BUG_ON(direction == DMA_NONE); |
330 | if (nelems == 1) { | ||
331 | sglist->dma_address = | ||
332 | dma_4v_map_single(dev, sg_virt(sglist), | ||
333 | sglist->length, direction); | ||
334 | if (unlikely(sglist->dma_address == DMA_ERROR_CODE)) | ||
335 | return 0; | ||
336 | sglist->dma_length = sglist->length; | ||
337 | return 1; | ||
338 | } | ||
339 | 343 | ||
340 | iommu = dev->archdata.iommu; | 344 | iommu = dev->archdata.iommu; |
345 | if (nelems == 0 || !iommu) | ||
346 | return 0; | ||
341 | 347 | ||
342 | if (unlikely(direction == DMA_NONE)) | 348 | prot = HV_PCI_MAP_ATTR_READ; |
343 | goto bad; | 349 | if (direction != DMA_TO_DEVICE) |
344 | 350 | prot |= HV_PCI_MAP_ATTR_WRITE; | |
345 | npages = calc_npages(sglist, nelems); | ||
346 | 351 | ||
347 | spin_lock_irqsave(&iommu->lock, flags); | 352 | outs = s = segstart = &sglist[0]; |
348 | entry = iommu_range_alloc(dev, iommu, npages, NULL); | 353 | outcount = 1; |
349 | spin_unlock_irqrestore(&iommu->lock, flags); | 354 | incount = nelems; |
355 | handle = 0; | ||
350 | 356 | ||
351 | if (unlikely(entry == DMA_ERROR_CODE)) | 357 | /* Init first segment length for backout at failure */ |
352 | goto bad; | 358 | outs->dma_length = 0; |
353 | 359 | ||
354 | orig_dma_base = dma_base = iommu->page_table_map_base + | 360 | spin_lock_irqsave(&iommu->lock, flags); |
355 | (entry << IO_PAGE_SHIFT); | ||
356 | 361 | ||
357 | prot = HV_PCI_MAP_ATTR_READ; | 362 | iommu_batch_start(dev, prot, ~0UL); |
358 | if (direction != DMA_TO_DEVICE) | ||
359 | prot |= HV_PCI_MAP_ATTR_WRITE; | ||
360 | 363 | ||
361 | local_irq_save(flags); | 364 | max_seg_size = dma_get_max_seg_size(dev); |
365 | for_each_sg(sglist, s, nelems, i) { | ||
366 | unsigned long paddr, npages, entry, slen; | ||
362 | 367 | ||
363 | iommu_batch_start(dev, prot, entry); | 368 | slen = s->length; |
369 | /* Sanity check */ | ||
370 | if (slen == 0) { | ||
371 | dma_next = 0; | ||
372 | continue; | ||
373 | } | ||
374 | /* Allocate iommu entries for that segment */ | ||
375 | paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); | ||
376 | npages = iommu_num_pages(paddr, slen); | ||
377 | entry = iommu_range_alloc(dev, iommu, npages, &handle); | ||
364 | 378 | ||
365 | for_each_sg(sglist, sg, nelems, i) { | 379 | /* Handle failure */ |
366 | unsigned long paddr = SG_ENT_PHYS_ADDRESS(sg); | 380 | if (unlikely(entry == DMA_ERROR_CODE)) { |
367 | unsigned long slen = sg->length; | 381 | if (printk_ratelimit()) |
368 | unsigned long this_npages; | 382 | printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" |
383 | " npages %lx\n", iommu, paddr, npages); | ||
384 | goto iommu_map_failed; | ||
385 | } | ||
369 | 386 | ||
370 | this_npages = iommu_num_pages(paddr, slen); | 387 | iommu_batch_new_entry(entry); |
371 | 388 | ||
372 | sg->dma_address = dma_base | (paddr & ~IO_PAGE_MASK); | 389 | /* Convert entry to a dma_addr_t */ |
373 | sg->dma_length = slen; | 390 | dma_addr = iommu->page_table_map_base + |
391 | (entry << IO_PAGE_SHIFT); | ||
392 | dma_addr |= (s->offset & ~IO_PAGE_MASK); | ||
374 | 393 | ||
394 | /* Insert into HW table */ | ||
375 | paddr &= IO_PAGE_MASK; | 395 | paddr &= IO_PAGE_MASK; |
376 | while (this_npages--) { | 396 | while (npages--) { |
377 | err = iommu_batch_add(paddr); | 397 | err = iommu_batch_add(paddr); |
378 | if (unlikely(err < 0L)) { | 398 | if (unlikely(err < 0L)) |
379 | local_irq_restore(flags); | ||
380 | goto iommu_map_failed; | 399 | goto iommu_map_failed; |
400 | paddr += IO_PAGE_SIZE; | ||
401 | } | ||
402 | |||
403 | /* If we are in an open segment, try merging */ | ||
404 | if (segstart != s) { | ||
405 | /* We cannot merge if: | ||
406 | * - allocated dma_addr isn't contiguous to previous allocation | ||
407 | */ | ||
408 | if ((dma_addr != dma_next) || | ||
409 | (outs->dma_length + s->length > max_seg_size)) { | ||
410 | /* Can't merge: create a new segment */ | ||
411 | segstart = s; | ||
412 | outcount++; | ||
413 | outs = sg_next(outs); | ||
414 | } else { | ||
415 | outs->dma_length += s->length; | ||
381 | } | 416 | } |
417 | } | ||
382 | 418 | ||
383 | paddr += IO_PAGE_SIZE; | 419 | if (segstart == s) { |
384 | dma_base += IO_PAGE_SIZE; | 420 | /* This is a new segment, fill entries */ |
421 | outs->dma_address = dma_addr; | ||
422 | outs->dma_length = slen; | ||
385 | } | 423 | } |
424 | |||
425 | /* Calculate next page pointer for contiguous check */ | ||
426 | dma_next = dma_addr + slen; | ||
386 | } | 427 | } |
387 | 428 | ||
388 | err = iommu_batch_end(); | 429 | err = iommu_batch_end(); |
389 | 430 | ||
390 | local_irq_restore(flags); | ||
391 | |||
392 | if (unlikely(err < 0L)) | 431 | if (unlikely(err < 0L)) |
393 | goto iommu_map_failed; | 432 | goto iommu_map_failed; |
394 | 433 | ||
395 | return nelems; | 434 | spin_unlock_irqrestore(&iommu->lock, flags); |
396 | 435 | ||
397 | bad: | 436 | if (outcount < incount) { |
398 | if (printk_ratelimit()) | 437 | outs = sg_next(outs); |
399 | WARN_ON(1); | 438 | outs->dma_address = DMA_ERROR_CODE; |
400 | return 0; | 439 | outs->dma_length = 0; |
440 | } | ||
441 | |||
442 | return outcount; | ||
401 | 443 | ||
402 | iommu_map_failed: | 444 | iommu_map_failed: |
403 | spin_lock_irqsave(&iommu->lock, flags); | 445 | for_each_sg(sglist, s, nelems, i) { |
404 | iommu_range_free(iommu, orig_dma_base, npages); | 446 | if (s->dma_length != 0) { |
447 | unsigned long vaddr, npages; | ||
448 | |||
449 | vaddr = s->dma_address & IO_PAGE_MASK; | ||
450 | npages = iommu_num_pages(s->dma_address, s->dma_length); | ||
451 | iommu_range_free(iommu, vaddr, npages); | ||
452 | /* XXX demap? XXX */ | ||
453 | s->dma_address = DMA_ERROR_CODE; | ||
454 | s->dma_length = 0; | ||
455 | } | ||
456 | if (s == outs) | ||
457 | break; | ||
458 | } | ||
405 | spin_unlock_irqrestore(&iommu->lock, flags); | 459 | spin_unlock_irqrestore(&iommu->lock, flags); |
406 | 460 | ||
407 | return 0; | 461 | return 0; |
@@ -410,39 +464,43 @@ iommu_map_failed: | |||
410 | static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, | 464 | static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, |
411 | int nelems, enum dma_data_direction direction) | 465 | int nelems, enum dma_data_direction direction) |
412 | { | 466 | { |
413 | unsigned long flags, npages; | ||
414 | struct pci_pbm_info *pbm; | 467 | struct pci_pbm_info *pbm; |
415 | u32 devhandle, bus_addr; | 468 | struct scatterlist *sg; |
416 | struct iommu *iommu; | 469 | struct iommu *iommu; |
417 | long entry; | 470 | unsigned long flags; |
471 | u32 devhandle; | ||
418 | 472 | ||
419 | if (unlikely(direction == DMA_NONE)) { | 473 | BUG_ON(direction == DMA_NONE); |
420 | if (printk_ratelimit()) | ||
421 | WARN_ON(1); | ||
422 | } | ||
423 | 474 | ||
424 | iommu = dev->archdata.iommu; | 475 | iommu = dev->archdata.iommu; |
425 | pbm = dev->archdata.host_controller; | 476 | pbm = dev->archdata.host_controller; |
426 | devhandle = pbm->devhandle; | 477 | devhandle = pbm->devhandle; |
427 | 478 | ||
428 | bus_addr = sglist->dma_address & IO_PAGE_MASK; | ||
429 | |||
430 | npages = calc_npages(sglist, nelems); | ||
431 | |||
432 | entry = ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | ||
433 | |||
434 | spin_lock_irqsave(&iommu->lock, flags); | 479 | spin_lock_irqsave(&iommu->lock, flags); |
435 | 480 | ||
436 | iommu_range_free(iommu, bus_addr, npages); | 481 | sg = sglist; |
437 | 482 | while (nelems--) { | |
438 | do { | 483 | dma_addr_t dma_handle = sg->dma_address; |
439 | unsigned long num; | 484 | unsigned int len = sg->dma_length; |
485 | unsigned long npages, entry; | ||
486 | |||
487 | if (!len) | ||
488 | break; | ||
489 | npages = iommu_num_pages(dma_handle, len); | ||
490 | iommu_range_free(iommu, dma_handle, npages); | ||
491 | |||
492 | entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); | ||
493 | while (npages) { | ||
494 | unsigned long num; | ||
495 | |||
496 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | ||
497 | npages); | ||
498 | entry += num; | ||
499 | npages -= num; | ||
500 | } | ||
440 | 501 | ||
441 | num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), | 502 | sg = sg_next(sg); |
442 | npages); | 503 | } |
443 | entry += num; | ||
444 | npages -= num; | ||
445 | } while (npages != 0); | ||
446 | 504 | ||
447 | spin_unlock_irqrestore(&iommu->lock, flags); | 505 | spin_unlock_irqrestore(&iommu->lock, flags); |
448 | } | 506 | } |