diff options
Diffstat (limited to 'arch/x86/kernel/amd_iommu.c')
-rw-r--r-- | arch/x86/kernel/amd_iommu.c | 328 |
1 files changed, 267 insertions, 61 deletions
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 042fdc27bc92..34e4d112b1ef 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c | |||
@@ -33,6 +33,10 @@ | |||
33 | 33 | ||
34 | static DEFINE_RWLOCK(amd_iommu_devtable_lock); | 34 | static DEFINE_RWLOCK(amd_iommu_devtable_lock); |
35 | 35 | ||
36 | /* A list of preallocated protection domains */ | ||
37 | static LIST_HEAD(iommu_pd_list); | ||
38 | static DEFINE_SPINLOCK(iommu_pd_list_lock); | ||
39 | |||
36 | /* | 40 | /* |
37 | * general struct to manage commands send to an IOMMU | 41 | * general struct to manage commands send to an IOMMU |
38 | */ | 42 | */ |
@@ -51,6 +55,102 @@ static int iommu_has_npcache(struct amd_iommu *iommu) | |||
51 | 55 | ||
52 | /**************************************************************************** | 56 | /**************************************************************************** |
53 | * | 57 | * |
58 | * Interrupt handling functions | ||
59 | * | ||
60 | ****************************************************************************/ | ||
61 | |||
62 | static void iommu_print_event(void *__evt) | ||
63 | { | ||
64 | u32 *event = __evt; | ||
65 | int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK; | ||
66 | int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK; | ||
67 | int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK; | ||
68 | int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; | ||
69 | u64 address = (u64)(((u64)event[3]) << 32) | event[2]; | ||
70 | |||
71 | printk(KERN_ERR "AMD IOMMU: Event logged ["); | ||
72 | |||
73 | switch (type) { | ||
74 | case EVENT_TYPE_ILL_DEV: | ||
75 | printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " | ||
76 | "address=0x%016llx flags=0x%04x]\n", | ||
77 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
78 | address, flags); | ||
79 | break; | ||
80 | case EVENT_TYPE_IO_FAULT: | ||
81 | printk("IO_PAGE_FAULT device=%02x:%02x.%x " | ||
82 | "domain=0x%04x address=0x%016llx flags=0x%04x]\n", | ||
83 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
84 | domid, address, flags); | ||
85 | break; | ||
86 | case EVENT_TYPE_DEV_TAB_ERR: | ||
87 | printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " | ||
88 | "address=0x%016llx flags=0x%04x]\n", | ||
89 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
90 | address, flags); | ||
91 | break; | ||
92 | case EVENT_TYPE_PAGE_TAB_ERR: | ||
93 | printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " | ||
94 | "domain=0x%04x address=0x%016llx flags=0x%04x]\n", | ||
95 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
96 | domid, address, flags); | ||
97 | break; | ||
98 | case EVENT_TYPE_ILL_CMD: | ||
99 | printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); | ||
100 | break; | ||
101 | case EVENT_TYPE_CMD_HARD_ERR: | ||
102 | printk("COMMAND_HARDWARE_ERROR address=0x%016llx " | ||
103 | "flags=0x%04x]\n", address, flags); | ||
104 | break; | ||
105 | case EVENT_TYPE_IOTLB_INV_TO: | ||
106 | printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x " | ||
107 | "address=0x%016llx]\n", | ||
108 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
109 | address); | ||
110 | break; | ||
111 | case EVENT_TYPE_INV_DEV_REQ: | ||
112 | printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x " | ||
113 | "address=0x%016llx flags=0x%04x]\n", | ||
114 | PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), | ||
115 | address, flags); | ||
116 | break; | ||
117 | default: | ||
118 | printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type); | ||
119 | } | ||
120 | } | ||
121 | |||
122 | static void iommu_poll_events(struct amd_iommu *iommu) | ||
123 | { | ||
124 | u32 head, tail; | ||
125 | unsigned long flags; | ||
126 | |||
127 | spin_lock_irqsave(&iommu->lock, flags); | ||
128 | |||
129 | head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); | ||
130 | tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); | ||
131 | |||
132 | while (head != tail) { | ||
133 | iommu_print_event(iommu->evt_buf + head); | ||
134 | head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size; | ||
135 | } | ||
136 | |||
137 | writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); | ||
138 | |||
139 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
140 | } | ||
141 | |||
142 | irqreturn_t amd_iommu_int_handler(int irq, void *data) | ||
143 | { | ||
144 | struct amd_iommu *iommu; | ||
145 | |||
146 | list_for_each_entry(iommu, &amd_iommu_list, list) | ||
147 | iommu_poll_events(iommu); | ||
148 | |||
149 | return IRQ_HANDLED; | ||
150 | } | ||
151 | |||
152 | /**************************************************************************** | ||
153 | * | ||
54 | * IOMMU command queuing functions | 154 | * IOMMU command queuing functions |
55 | * | 155 | * |
56 | ****************************************************************************/ | 156 | ****************************************************************************/ |
@@ -213,6 +313,14 @@ static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid, | |||
213 | return 0; | 313 | return 0; |
214 | } | 314 | } |
215 | 315 | ||
316 | /* Flush the whole IO/TLB for a given protection domain */ | ||
317 | static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid) | ||
318 | { | ||
319 | u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; | ||
320 | |||
321 | iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1); | ||
322 | } | ||
323 | |||
216 | /**************************************************************************** | 324 | /**************************************************************************** |
217 | * | 325 | * |
218 | * The functions below are used the create the page table mappings for | 326 | * The functions below are used the create the page table mappings for |
@@ -372,11 +480,6 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, | |||
372 | * efficient allocator. | 480 | * efficient allocator. |
373 | * | 481 | * |
374 | ****************************************************************************/ | 482 | ****************************************************************************/ |
375 | static unsigned long dma_mask_to_pages(unsigned long mask) | ||
376 | { | ||
377 | return (mask >> PAGE_SHIFT) + | ||
378 | (PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT); | ||
379 | } | ||
380 | 483 | ||
381 | /* | 484 | /* |
382 | * The address allocator core function. | 485 | * The address allocator core function. |
@@ -385,25 +488,31 @@ static unsigned long dma_mask_to_pages(unsigned long mask) | |||
385 | */ | 488 | */ |
386 | static unsigned long dma_ops_alloc_addresses(struct device *dev, | 489 | static unsigned long dma_ops_alloc_addresses(struct device *dev, |
387 | struct dma_ops_domain *dom, | 490 | struct dma_ops_domain *dom, |
388 | unsigned int pages) | 491 | unsigned int pages, |
492 | unsigned long align_mask, | ||
493 | u64 dma_mask) | ||
389 | { | 494 | { |
390 | unsigned long limit = dma_mask_to_pages(*dev->dma_mask); | 495 | unsigned long limit; |
391 | unsigned long address; | 496 | unsigned long address; |
392 | unsigned long size = dom->aperture_size >> PAGE_SHIFT; | ||
393 | unsigned long boundary_size; | 497 | unsigned long boundary_size; |
394 | 498 | ||
395 | boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, | 499 | boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, |
396 | PAGE_SIZE) >> PAGE_SHIFT; | 500 | PAGE_SIZE) >> PAGE_SHIFT; |
397 | limit = limit < size ? limit : size; | 501 | limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0, |
502 | dma_mask >> PAGE_SHIFT); | ||
398 | 503 | ||
399 | if (dom->next_bit >= limit) | 504 | if (dom->next_bit >= limit) { |
400 | dom->next_bit = 0; | 505 | dom->next_bit = 0; |
506 | dom->need_flush = true; | ||
507 | } | ||
401 | 508 | ||
402 | address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages, | 509 | address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages, |
403 | 0 , boundary_size, 0); | 510 | 0 , boundary_size, align_mask); |
404 | if (address == -1) | 511 | if (address == -1) { |
405 | address = iommu_area_alloc(dom->bitmap, limit, 0, pages, | 512 | address = iommu_area_alloc(dom->bitmap, limit, 0, pages, |
406 | 0, boundary_size, 0); | 513 | 0, boundary_size, align_mask); |
514 | dom->need_flush = true; | ||
515 | } | ||
407 | 516 | ||
408 | if (likely(address != -1)) { | 517 | if (likely(address != -1)) { |
409 | dom->next_bit = address + pages; | 518 | dom->next_bit = address + pages; |
@@ -469,7 +578,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom, | |||
469 | if (start_page + pages > last_page) | 578 | if (start_page + pages > last_page) |
470 | pages = last_page - start_page; | 579 | pages = last_page - start_page; |
471 | 580 | ||
472 | set_bit_string(dom->bitmap, start_page, pages); | 581 | iommu_area_reserve(dom->bitmap, start_page, pages); |
473 | } | 582 | } |
474 | 583 | ||
475 | static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom) | 584 | static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom) |
@@ -563,6 +672,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu, | |||
563 | dma_dom->bitmap[0] = 1; | 672 | dma_dom->bitmap[0] = 1; |
564 | dma_dom->next_bit = 0; | 673 | dma_dom->next_bit = 0; |
565 | 674 | ||
675 | dma_dom->need_flush = false; | ||
676 | dma_dom->target_dev = 0xffff; | ||
677 | |||
566 | /* Intialize the exclusion range if necessary */ | 678 | /* Intialize the exclusion range if necessary */ |
567 | if (iommu->exclusion_start && | 679 | if (iommu->exclusion_start && |
568 | iommu->exclusion_start < dma_dom->aperture_size) { | 680 | iommu->exclusion_start < dma_dom->aperture_size) { |
@@ -633,12 +745,13 @@ static void set_device_domain(struct amd_iommu *iommu, | |||
633 | 745 | ||
634 | u64 pte_root = virt_to_phys(domain->pt_root); | 746 | u64 pte_root = virt_to_phys(domain->pt_root); |
635 | 747 | ||
636 | pte_root |= (domain->mode & 0x07) << 9; | 748 | pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) |
637 | pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | 2; | 749 | << DEV_ENTRY_MODE_SHIFT; |
750 | pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; | ||
638 | 751 | ||
639 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); | 752 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); |
640 | amd_iommu_dev_table[devid].data[0] = pte_root; | 753 | amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); |
641 | amd_iommu_dev_table[devid].data[1] = pte_root >> 32; | 754 | amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); |
642 | amd_iommu_dev_table[devid].data[2] = domain->id; | 755 | amd_iommu_dev_table[devid].data[2] = domain->id; |
643 | 756 | ||
644 | amd_iommu_pd_table[devid] = domain; | 757 | amd_iommu_pd_table[devid] = domain; |
@@ -656,6 +769,45 @@ static void set_device_domain(struct amd_iommu *iommu, | |||
656 | *****************************************************************************/ | 769 | *****************************************************************************/ |
657 | 770 | ||
658 | /* | 771 | /* |
772 | * This function checks if the driver got a valid device from the caller to | ||
773 | * avoid dereferencing invalid pointers. | ||
774 | */ | ||
775 | static bool check_device(struct device *dev) | ||
776 | { | ||
777 | if (!dev || !dev->dma_mask) | ||
778 | return false; | ||
779 | |||
780 | return true; | ||
781 | } | ||
782 | |||
783 | /* | ||
784 | * In this function the list of preallocated protection domains is traversed to | ||
785 | * find the domain for a specific device | ||
786 | */ | ||
787 | static struct dma_ops_domain *find_protection_domain(u16 devid) | ||
788 | { | ||
789 | struct dma_ops_domain *entry, *ret = NULL; | ||
790 | unsigned long flags; | ||
791 | |||
792 | if (list_empty(&iommu_pd_list)) | ||
793 | return NULL; | ||
794 | |||
795 | spin_lock_irqsave(&iommu_pd_list_lock, flags); | ||
796 | |||
797 | list_for_each_entry(entry, &iommu_pd_list, list) { | ||
798 | if (entry->target_dev == devid) { | ||
799 | ret = entry; | ||
800 | list_del(&ret->list); | ||
801 | break; | ||
802 | } | ||
803 | } | ||
804 | |||
805 | spin_unlock_irqrestore(&iommu_pd_list_lock, flags); | ||
806 | |||
807 | return ret; | ||
808 | } | ||
809 | |||
810 | /* | ||
659 | * In the dma_ops path we only have the struct device. This function | 811 | * In the dma_ops path we only have the struct device. This function |
660 | * finds the corresponding IOMMU, the protection domain and the | 812 | * finds the corresponding IOMMU, the protection domain and the |
661 | * requestor id for a given device. | 813 | * requestor id for a given device. |
@@ -671,27 +823,30 @@ static int get_device_resources(struct device *dev, | |||
671 | struct pci_dev *pcidev; | 823 | struct pci_dev *pcidev; |
672 | u16 _bdf; | 824 | u16 _bdf; |
673 | 825 | ||
674 | BUG_ON(!dev || dev->bus != &pci_bus_type || !dev->dma_mask); | 826 | *iommu = NULL; |
827 | *domain = NULL; | ||
828 | *bdf = 0xffff; | ||
829 | |||
830 | if (dev->bus != &pci_bus_type) | ||
831 | return 0; | ||
675 | 832 | ||
676 | pcidev = to_pci_dev(dev); | 833 | pcidev = to_pci_dev(dev); |
677 | _bdf = calc_devid(pcidev->bus->number, pcidev->devfn); | 834 | _bdf = calc_devid(pcidev->bus->number, pcidev->devfn); |
678 | 835 | ||
679 | /* device not translated by any IOMMU in the system? */ | 836 | /* device not translated by any IOMMU in the system? */ |
680 | if (_bdf > amd_iommu_last_bdf) { | 837 | if (_bdf > amd_iommu_last_bdf) |
681 | *iommu = NULL; | ||
682 | *domain = NULL; | ||
683 | *bdf = 0xffff; | ||
684 | return 0; | 838 | return 0; |
685 | } | ||
686 | 839 | ||
687 | *bdf = amd_iommu_alias_table[_bdf]; | 840 | *bdf = amd_iommu_alias_table[_bdf]; |
688 | 841 | ||
689 | *iommu = amd_iommu_rlookup_table[*bdf]; | 842 | *iommu = amd_iommu_rlookup_table[*bdf]; |
690 | if (*iommu == NULL) | 843 | if (*iommu == NULL) |
691 | return 0; | 844 | return 0; |
692 | dma_dom = (*iommu)->default_dom; | ||
693 | *domain = domain_for_device(*bdf); | 845 | *domain = domain_for_device(*bdf); |
694 | if (*domain == NULL) { | 846 | if (*domain == NULL) { |
847 | dma_dom = find_protection_domain(*bdf); | ||
848 | if (!dma_dom) | ||
849 | dma_dom = (*iommu)->default_dom; | ||
695 | *domain = &dma_dom->domain; | 850 | *domain = &dma_dom->domain; |
696 | set_device_domain(*iommu, *domain, *bdf); | 851 | set_device_domain(*iommu, *domain, *bdf); |
697 | printk(KERN_INFO "AMD IOMMU: Using protection domain %d for " | 852 | printk(KERN_INFO "AMD IOMMU: Using protection domain %d for " |
@@ -770,17 +925,24 @@ static dma_addr_t __map_single(struct device *dev, | |||
770 | struct dma_ops_domain *dma_dom, | 925 | struct dma_ops_domain *dma_dom, |
771 | phys_addr_t paddr, | 926 | phys_addr_t paddr, |
772 | size_t size, | 927 | size_t size, |
773 | int dir) | 928 | int dir, |
929 | bool align, | ||
930 | u64 dma_mask) | ||
774 | { | 931 | { |
775 | dma_addr_t offset = paddr & ~PAGE_MASK; | 932 | dma_addr_t offset = paddr & ~PAGE_MASK; |
776 | dma_addr_t address, start; | 933 | dma_addr_t address, start; |
777 | unsigned int pages; | 934 | unsigned int pages; |
935 | unsigned long align_mask = 0; | ||
778 | int i; | 936 | int i; |
779 | 937 | ||
780 | pages = iommu_num_pages(paddr, size); | 938 | pages = iommu_num_pages(paddr, size); |
781 | paddr &= PAGE_MASK; | 939 | paddr &= PAGE_MASK; |
782 | 940 | ||
783 | address = dma_ops_alloc_addresses(dev, dma_dom, pages); | 941 | if (align) |
942 | align_mask = (1UL << get_order(size)) - 1; | ||
943 | |||
944 | address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask, | ||
945 | dma_mask); | ||
784 | if (unlikely(address == bad_dma_address)) | 946 | if (unlikely(address == bad_dma_address)) |
785 | goto out; | 947 | goto out; |
786 | 948 | ||
@@ -792,6 +954,12 @@ static dma_addr_t __map_single(struct device *dev, | |||
792 | } | 954 | } |
793 | address += offset; | 955 | address += offset; |
794 | 956 | ||
957 | if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { | ||
958 | iommu_flush_tlb(iommu, dma_dom->domain.id); | ||
959 | dma_dom->need_flush = false; | ||
960 | } else if (unlikely(iommu_has_npcache(iommu))) | ||
961 | iommu_flush_pages(iommu, dma_dom->domain.id, address, size); | ||
962 | |||
795 | out: | 963 | out: |
796 | return address; | 964 | return address; |
797 | } | 965 | } |
@@ -822,6 +990,9 @@ static void __unmap_single(struct amd_iommu *iommu, | |||
822 | } | 990 | } |
823 | 991 | ||
824 | dma_ops_free_addresses(dma_dom, dma_addr, pages); | 992 | dma_ops_free_addresses(dma_dom, dma_addr, pages); |
993 | |||
994 | if (amd_iommu_unmap_flush) | ||
995 | iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size); | ||
825 | } | 996 | } |
826 | 997 | ||
827 | /* | 998 | /* |
@@ -835,6 +1006,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr, | |||
835 | struct protection_domain *domain; | 1006 | struct protection_domain *domain; |
836 | u16 devid; | 1007 | u16 devid; |
837 | dma_addr_t addr; | 1008 | dma_addr_t addr; |
1009 | u64 dma_mask; | ||
1010 | |||
1011 | if (!check_device(dev)) | ||
1012 | return bad_dma_address; | ||
1013 | |||
1014 | dma_mask = *dev->dma_mask; | ||
838 | 1015 | ||
839 | get_device_resources(dev, &iommu, &domain, &devid); | 1016 | get_device_resources(dev, &iommu, &domain, &devid); |
840 | 1017 | ||
@@ -843,14 +1020,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr, | |||
843 | return (dma_addr_t)paddr; | 1020 | return (dma_addr_t)paddr; |
844 | 1021 | ||
845 | spin_lock_irqsave(&domain->lock, flags); | 1022 | spin_lock_irqsave(&domain->lock, flags); |
846 | addr = __map_single(dev, iommu, domain->priv, paddr, size, dir); | 1023 | addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false, |
1024 | dma_mask); | ||
847 | if (addr == bad_dma_address) | 1025 | if (addr == bad_dma_address) |
848 | goto out; | 1026 | goto out; |
849 | 1027 | ||
850 | if (iommu_has_npcache(iommu)) | 1028 | if (unlikely(iommu->need_sync)) |
851 | iommu_flush_pages(iommu, domain->id, addr, size); | ||
852 | |||
853 | if (iommu->need_sync) | ||
854 | iommu_completion_wait(iommu); | 1029 | iommu_completion_wait(iommu); |
855 | 1030 | ||
856 | out: | 1031 | out: |
@@ -870,7 +1045,8 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr, | |||
870 | struct protection_domain *domain; | 1045 | struct protection_domain *domain; |
871 | u16 devid; | 1046 | u16 devid; |
872 | 1047 | ||
873 | if (!get_device_resources(dev, &iommu, &domain, &devid)) | 1048 | if (!check_device(dev) || |
1049 | !get_device_resources(dev, &iommu, &domain, &devid)) | ||
874 | /* device not handled by any AMD IOMMU */ | 1050 | /* device not handled by any AMD IOMMU */ |
875 | return; | 1051 | return; |
876 | 1052 | ||
@@ -878,9 +1054,7 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr, | |||
878 | 1054 | ||
879 | __unmap_single(iommu, domain->priv, dma_addr, size, dir); | 1055 | __unmap_single(iommu, domain->priv, dma_addr, size, dir); |
880 | 1056 | ||
881 | iommu_flush_pages(iommu, domain->id, dma_addr, size); | 1057 | if (unlikely(iommu->need_sync)) |
882 | |||
883 | if (iommu->need_sync) | ||
884 | iommu_completion_wait(iommu); | 1058 | iommu_completion_wait(iommu); |
885 | 1059 | ||
886 | spin_unlock_irqrestore(&domain->lock, flags); | 1060 | spin_unlock_irqrestore(&domain->lock, flags); |
@@ -919,6 +1093,12 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, | |||
919 | struct scatterlist *s; | 1093 | struct scatterlist *s; |
920 | phys_addr_t paddr; | 1094 | phys_addr_t paddr; |
921 | int mapped_elems = 0; | 1095 | int mapped_elems = 0; |
1096 | u64 dma_mask; | ||
1097 | |||
1098 | if (!check_device(dev)) | ||
1099 | return 0; | ||
1100 | |||
1101 | dma_mask = *dev->dma_mask; | ||
922 | 1102 | ||
923 | get_device_resources(dev, &iommu, &domain, &devid); | 1103 | get_device_resources(dev, &iommu, &domain, &devid); |
924 | 1104 | ||
@@ -931,19 +1111,17 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, | |||
931 | paddr = sg_phys(s); | 1111 | paddr = sg_phys(s); |
932 | 1112 | ||
933 | s->dma_address = __map_single(dev, iommu, domain->priv, | 1113 | s->dma_address = __map_single(dev, iommu, domain->priv, |
934 | paddr, s->length, dir); | 1114 | paddr, s->length, dir, false, |
1115 | dma_mask); | ||
935 | 1116 | ||
936 | if (s->dma_address) { | 1117 | if (s->dma_address) { |
937 | s->dma_length = s->length; | 1118 | s->dma_length = s->length; |
938 | mapped_elems++; | 1119 | mapped_elems++; |
939 | } else | 1120 | } else |
940 | goto unmap; | 1121 | goto unmap; |
941 | if (iommu_has_npcache(iommu)) | ||
942 | iommu_flush_pages(iommu, domain->id, s->dma_address, | ||
943 | s->dma_length); | ||
944 | } | 1122 | } |
945 | 1123 | ||
946 | if (iommu->need_sync) | 1124 | if (unlikely(iommu->need_sync)) |
947 | iommu_completion_wait(iommu); | 1125 | iommu_completion_wait(iommu); |
948 | 1126 | ||
949 | out: | 1127 | out: |
@@ -977,7 +1155,8 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
977 | u16 devid; | 1155 | u16 devid; |
978 | int i; | 1156 | int i; |
979 | 1157 | ||
980 | if (!get_device_resources(dev, &iommu, &domain, &devid)) | 1158 | if (!check_device(dev) || |
1159 | !get_device_resources(dev, &iommu, &domain, &devid)) | ||
981 | return; | 1160 | return; |
982 | 1161 | ||
983 | spin_lock_irqsave(&domain->lock, flags); | 1162 | spin_lock_irqsave(&domain->lock, flags); |
@@ -985,12 +1164,10 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
985 | for_each_sg(sglist, s, nelems, i) { | 1164 | for_each_sg(sglist, s, nelems, i) { |
986 | __unmap_single(iommu, domain->priv, s->dma_address, | 1165 | __unmap_single(iommu, domain->priv, s->dma_address, |
987 | s->dma_length, dir); | 1166 | s->dma_length, dir); |
988 | iommu_flush_pages(iommu, domain->id, s->dma_address, | ||
989 | s->dma_length); | ||
990 | s->dma_address = s->dma_length = 0; | 1167 | s->dma_address = s->dma_length = 0; |
991 | } | 1168 | } |
992 | 1169 | ||
993 | if (iommu->need_sync) | 1170 | if (unlikely(iommu->need_sync)) |
994 | iommu_completion_wait(iommu); | 1171 | iommu_completion_wait(iommu); |
995 | 1172 | ||
996 | spin_unlock_irqrestore(&domain->lock, flags); | 1173 | spin_unlock_irqrestore(&domain->lock, flags); |
@@ -1008,25 +1185,33 @@ static void *alloc_coherent(struct device *dev, size_t size, | |||
1008 | struct protection_domain *domain; | 1185 | struct protection_domain *domain; |
1009 | u16 devid; | 1186 | u16 devid; |
1010 | phys_addr_t paddr; | 1187 | phys_addr_t paddr; |
1188 | u64 dma_mask = dev->coherent_dma_mask; | ||
1189 | |||
1190 | if (!check_device(dev)) | ||
1191 | return NULL; | ||
1192 | |||
1193 | if (!get_device_resources(dev, &iommu, &domain, &devid)) | ||
1194 | flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); | ||
1011 | 1195 | ||
1196 | flag |= __GFP_ZERO; | ||
1012 | virt_addr = (void *)__get_free_pages(flag, get_order(size)); | 1197 | virt_addr = (void *)__get_free_pages(flag, get_order(size)); |
1013 | if (!virt_addr) | 1198 | if (!virt_addr) |
1014 | return 0; | 1199 | return 0; |
1015 | 1200 | ||
1016 | memset(virt_addr, 0, size); | ||
1017 | paddr = virt_to_phys(virt_addr); | 1201 | paddr = virt_to_phys(virt_addr); |
1018 | 1202 | ||
1019 | get_device_resources(dev, &iommu, &domain, &devid); | ||
1020 | |||
1021 | if (!iommu || !domain) { | 1203 | if (!iommu || !domain) { |
1022 | *dma_addr = (dma_addr_t)paddr; | 1204 | *dma_addr = (dma_addr_t)paddr; |
1023 | return virt_addr; | 1205 | return virt_addr; |
1024 | } | 1206 | } |
1025 | 1207 | ||
1208 | if (!dma_mask) | ||
1209 | dma_mask = *dev->dma_mask; | ||
1210 | |||
1026 | spin_lock_irqsave(&domain->lock, flags); | 1211 | spin_lock_irqsave(&domain->lock, flags); |
1027 | 1212 | ||
1028 | *dma_addr = __map_single(dev, iommu, domain->priv, paddr, | 1213 | *dma_addr = __map_single(dev, iommu, domain->priv, paddr, |
1029 | size, DMA_BIDIRECTIONAL); | 1214 | size, DMA_BIDIRECTIONAL, true, dma_mask); |
1030 | 1215 | ||
1031 | if (*dma_addr == bad_dma_address) { | 1216 | if (*dma_addr == bad_dma_address) { |
1032 | free_pages((unsigned long)virt_addr, get_order(size)); | 1217 | free_pages((unsigned long)virt_addr, get_order(size)); |
@@ -1034,10 +1219,7 @@ static void *alloc_coherent(struct device *dev, size_t size, | |||
1034 | goto out; | 1219 | goto out; |
1035 | } | 1220 | } |
1036 | 1221 | ||
1037 | if (iommu_has_npcache(iommu)) | 1222 | if (unlikely(iommu->need_sync)) |
1038 | iommu_flush_pages(iommu, domain->id, *dma_addr, size); | ||
1039 | |||
1040 | if (iommu->need_sync) | ||
1041 | iommu_completion_wait(iommu); | 1223 | iommu_completion_wait(iommu); |
1042 | 1224 | ||
1043 | out: | 1225 | out: |
@@ -1048,8 +1230,6 @@ out: | |||
1048 | 1230 | ||
1049 | /* | 1231 | /* |
1050 | * The exported free_coherent function for dma_ops. | 1232 | * The exported free_coherent function for dma_ops. |
1051 | * FIXME: fix the generic x86 DMA layer so that it actually calls that | ||
1052 | * function. | ||
1053 | */ | 1233 | */ |
1054 | static void free_coherent(struct device *dev, size_t size, | 1234 | static void free_coherent(struct device *dev, size_t size, |
1055 | void *virt_addr, dma_addr_t dma_addr) | 1235 | void *virt_addr, dma_addr_t dma_addr) |
@@ -1059,6 +1239,9 @@ static void free_coherent(struct device *dev, size_t size, | |||
1059 | struct protection_domain *domain; | 1239 | struct protection_domain *domain; |
1060 | u16 devid; | 1240 | u16 devid; |
1061 | 1241 | ||
1242 | if (!check_device(dev)) | ||
1243 | return; | ||
1244 | |||
1062 | get_device_resources(dev, &iommu, &domain, &devid); | 1245 | get_device_resources(dev, &iommu, &domain, &devid); |
1063 | 1246 | ||
1064 | if (!iommu || !domain) | 1247 | if (!iommu || !domain) |
@@ -1067,9 +1250,8 @@ static void free_coherent(struct device *dev, size_t size, | |||
1067 | spin_lock_irqsave(&domain->lock, flags); | 1250 | spin_lock_irqsave(&domain->lock, flags); |
1068 | 1251 | ||
1069 | __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); | 1252 | __unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); |
1070 | iommu_flush_pages(iommu, domain->id, dma_addr, size); | ||
1071 | 1253 | ||
1072 | if (iommu->need_sync) | 1254 | if (unlikely(iommu->need_sync)) |
1073 | iommu_completion_wait(iommu); | 1255 | iommu_completion_wait(iommu); |
1074 | 1256 | ||
1075 | spin_unlock_irqrestore(&domain->lock, flags); | 1257 | spin_unlock_irqrestore(&domain->lock, flags); |
@@ -1079,6 +1261,30 @@ free_mem: | |||
1079 | } | 1261 | } |
1080 | 1262 | ||
1081 | /* | 1263 | /* |
1264 | * This function is called by the DMA layer to find out if we can handle a | ||
1265 | * particular device. It is part of the dma_ops. | ||
1266 | */ | ||
1267 | static int amd_iommu_dma_supported(struct device *dev, u64 mask) | ||
1268 | { | ||
1269 | u16 bdf; | ||
1270 | struct pci_dev *pcidev; | ||
1271 | |||
1272 | /* No device or no PCI device */ | ||
1273 | if (!dev || dev->bus != &pci_bus_type) | ||
1274 | return 0; | ||
1275 | |||
1276 | pcidev = to_pci_dev(dev); | ||
1277 | |||
1278 | bdf = calc_devid(pcidev->bus->number, pcidev->devfn); | ||
1279 | |||
1280 | /* Out of our scope? */ | ||
1281 | if (bdf > amd_iommu_last_bdf) | ||
1282 | return 0; | ||
1283 | |||
1284 | return 1; | ||
1285 | } | ||
1286 | |||
1287 | /* | ||
1082 | * The function for pre-allocating protection domains. | 1288 | * The function for pre-allocating protection domains. |
1083 | * | 1289 | * |
1084 | * If the driver core informs the DMA layer if a driver grabs a device | 1290 | * If the driver core informs the DMA layer if a driver grabs a device |
@@ -1107,10 +1313,9 @@ void prealloc_protection_domains(void) | |||
1107 | if (!dma_dom) | 1313 | if (!dma_dom) |
1108 | continue; | 1314 | continue; |
1109 | init_unity_mappings_for_device(dma_dom, devid); | 1315 | init_unity_mappings_for_device(dma_dom, devid); |
1110 | set_device_domain(iommu, &dma_dom->domain, devid); | 1316 | dma_dom->target_dev = devid; |
1111 | printk(KERN_INFO "AMD IOMMU: Allocated domain %d for device ", | 1317 | |
1112 | dma_dom->domain.id); | 1318 | list_add_tail(&dma_dom->list, &iommu_pd_list); |
1113 | print_devid(devid, 1); | ||
1114 | } | 1319 | } |
1115 | } | 1320 | } |
1116 | 1321 | ||
@@ -1121,6 +1326,7 @@ static struct dma_mapping_ops amd_iommu_dma_ops = { | |||
1121 | .unmap_single = unmap_single, | 1326 | .unmap_single = unmap_single, |
1122 | .map_sg = map_sg, | 1327 | .map_sg = map_sg, |
1123 | .unmap_sg = unmap_sg, | 1328 | .unmap_sg = unmap_sg, |
1329 | .dma_supported = amd_iommu_dma_supported, | ||
1124 | }; | 1330 | }; |
1125 | 1331 | ||
1126 | /* | 1332 | /* |