aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu/intel-iommu.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu/intel-iommu.c')
-rw-r--r--drivers/iommu/intel-iommu.c882
1 files changed, 440 insertions, 442 deletions
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 51b6b77dc3e5..5619f264862d 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -45,7 +45,6 @@
45#include <asm/iommu.h> 45#include <asm/iommu.h>
46 46
47#include "irq_remapping.h" 47#include "irq_remapping.h"
48#include "pci.h"
49 48
50#define ROOT_SIZE VTD_PAGE_SIZE 49#define ROOT_SIZE VTD_PAGE_SIZE
51#define CONTEXT_SIZE VTD_PAGE_SIZE 50#define CONTEXT_SIZE VTD_PAGE_SIZE
@@ -304,7 +303,7 @@ static inline bool dma_pte_present(struct dma_pte *pte)
304 303
305static inline bool dma_pte_superpage(struct dma_pte *pte) 304static inline bool dma_pte_superpage(struct dma_pte *pte)
306{ 305{
307 return (pte->val & (1 << 7)); 306 return (pte->val & DMA_PTE_LARGE_PAGE);
308} 307}
309 308
310static inline int first_pte_in_page(struct dma_pte *pte) 309static inline int first_pte_in_page(struct dma_pte *pte)
@@ -321,16 +320,13 @@ static inline int first_pte_in_page(struct dma_pte *pte)
321static struct dmar_domain *si_domain; 320static struct dmar_domain *si_domain;
322static int hw_pass_through = 1; 321static int hw_pass_through = 1;
323 322
324/* devices under the same p2p bridge are owned in one domain */
325#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326
327/* domain represents a virtual machine, more than one devices 323/* domain represents a virtual machine, more than one devices
328 * across iommus may be owned in one domain, e.g. kvm guest. 324 * across iommus may be owned in one domain, e.g. kvm guest.
329 */ 325 */
330#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1) 326#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
331 327
332/* si_domain contains mulitple devices */ 328/* si_domain contains mulitple devices */
333#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2) 329#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
334 330
335/* define the limit of IOMMUs supported in each domain */ 331/* define the limit of IOMMUs supported in each domain */
336#ifdef CONFIG_X86 332#ifdef CONFIG_X86
@@ -429,6 +425,8 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
429 struct device *dev); 425 struct device *dev);
430static void iommu_detach_dependent_devices(struct intel_iommu *iommu, 426static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
431 struct device *dev); 427 struct device *dev);
428static int domain_detach_iommu(struct dmar_domain *domain,
429 struct intel_iommu *iommu);
432 430
433#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON 431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
434int dmar_disabled = 0; 432int dmar_disabled = 0;
@@ -451,7 +449,7 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
451static DEFINE_SPINLOCK(device_domain_lock); 449static DEFINE_SPINLOCK(device_domain_lock);
452static LIST_HEAD(device_domain_list); 450static LIST_HEAD(device_domain_list);
453 451
454static struct iommu_ops intel_iommu_ops; 452static const struct iommu_ops intel_iommu_ops;
455 453
456static int __init intel_iommu_setup(char *str) 454static int __init intel_iommu_setup(char *str)
457{ 455{
@@ -540,6 +538,24 @@ void free_iova_mem(struct iova *iova)
540 kmem_cache_free(iommu_iova_cache, iova); 538 kmem_cache_free(iommu_iova_cache, iova);
541} 539}
542 540
541static inline int domain_type_is_vm(struct dmar_domain *domain)
542{
543 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
544}
545
546static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
547{
548 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
549 DOMAIN_FLAG_STATIC_IDENTITY);
550}
551
552static inline int domain_pfn_supported(struct dmar_domain *domain,
553 unsigned long pfn)
554{
555 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
556
557 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
558}
543 559
544static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw) 560static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
545{ 561{
@@ -580,9 +596,7 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
580 int iommu_id; 596 int iommu_id;
581 597
582 /* si_domain and vm domain should not get here. */ 598 /* si_domain and vm domain should not get here. */
583 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE); 599 BUG_ON(domain_type_is_vm_or_si(domain));
584 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
585
586 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus); 600 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
587 if (iommu_id < 0 || iommu_id >= g_num_of_iommus) 601 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
588 return NULL; 602 return NULL;
@@ -619,50 +633,56 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain)
619 rcu_read_unlock(); 633 rcu_read_unlock();
620} 634}
621 635
622static void domain_update_iommu_snooping(struct dmar_domain *domain) 636static int domain_update_iommu_snooping(struct intel_iommu *skip)
623{ 637{
624 int i; 638 struct dmar_drhd_unit *drhd;
625 639 struct intel_iommu *iommu;
626 domain->iommu_snooping = 1; 640 int ret = 1;
627 641
628 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) { 642 rcu_read_lock();
629 if (!ecap_sc_support(g_iommus[i]->ecap)) { 643 for_each_active_iommu(iommu, drhd) {
630 domain->iommu_snooping = 0; 644 if (iommu != skip) {
631 break; 645 if (!ecap_sc_support(iommu->ecap)) {
646 ret = 0;
647 break;
648 }
632 } 649 }
633 } 650 }
651 rcu_read_unlock();
652
653 return ret;
634} 654}
635 655
636static void domain_update_iommu_superpage(struct dmar_domain *domain) 656static int domain_update_iommu_superpage(struct intel_iommu *skip)
637{ 657{
638 struct dmar_drhd_unit *drhd; 658 struct dmar_drhd_unit *drhd;
639 struct intel_iommu *iommu = NULL; 659 struct intel_iommu *iommu;
640 int mask = 0xf; 660 int mask = 0xf;
641 661
642 if (!intel_iommu_superpage) { 662 if (!intel_iommu_superpage) {
643 domain->iommu_superpage = 0; 663 return 0;
644 return;
645 } 664 }
646 665
647 /* set iommu_superpage to the smallest common denominator */ 666 /* set iommu_superpage to the smallest common denominator */
648 rcu_read_lock(); 667 rcu_read_lock();
649 for_each_active_iommu(iommu, drhd) { 668 for_each_active_iommu(iommu, drhd) {
650 mask &= cap_super_page_val(iommu->cap); 669 if (iommu != skip) {
651 if (!mask) { 670 mask &= cap_super_page_val(iommu->cap);
652 break; 671 if (!mask)
672 break;
653 } 673 }
654 } 674 }
655 rcu_read_unlock(); 675 rcu_read_unlock();
656 676
657 domain->iommu_superpage = fls(mask); 677 return fls(mask);
658} 678}
659 679
660/* Some capabilities may be different across iommus */ 680/* Some capabilities may be different across iommus */
661static void domain_update_iommu_cap(struct dmar_domain *domain) 681static void domain_update_iommu_cap(struct dmar_domain *domain)
662{ 682{
663 domain_update_iommu_coherency(domain); 683 domain_update_iommu_coherency(domain);
664 domain_update_iommu_snooping(domain); 684 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
665 domain_update_iommu_superpage(domain); 685 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
666} 686}
667 687
668static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn) 688static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
@@ -671,7 +691,7 @@ static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devf
671 struct intel_iommu *iommu; 691 struct intel_iommu *iommu;
672 struct device *tmp; 692 struct device *tmp;
673 struct pci_dev *ptmp, *pdev = NULL; 693 struct pci_dev *ptmp, *pdev = NULL;
674 u16 segment; 694 u16 segment = 0;
675 int i; 695 int i;
676 696
677 if (dev_is_pci(dev)) { 697 if (dev_is_pci(dev)) {
@@ -816,14 +836,13 @@ out:
816static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, 836static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
817 unsigned long pfn, int *target_level) 837 unsigned long pfn, int *target_level)
818{ 838{
819 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
820 struct dma_pte *parent, *pte = NULL; 839 struct dma_pte *parent, *pte = NULL;
821 int level = agaw_to_level(domain->agaw); 840 int level = agaw_to_level(domain->agaw);
822 int offset; 841 int offset;
823 842
824 BUG_ON(!domain->pgd); 843 BUG_ON(!domain->pgd);
825 844
826 if (addr_width < BITS_PER_LONG && pfn >> addr_width) 845 if (!domain_pfn_supported(domain, pfn))
827 /* Address beyond IOMMU's addressing capabilities. */ 846 /* Address beyond IOMMU's addressing capabilities. */
828 return NULL; 847 return NULL;
829 848
@@ -849,13 +868,11 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
849 868
850 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE); 869 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
851 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE; 870 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
852 if (cmpxchg64(&pte->val, 0ULL, pteval)) { 871 if (cmpxchg64(&pte->val, 0ULL, pteval))
853 /* Someone else set it while we were thinking; use theirs. */ 872 /* Someone else set it while we were thinking; use theirs. */
854 free_pgtable_page(tmp_page); 873 free_pgtable_page(tmp_page);
855 } else { 874 else
856 dma_pte_addr(pte);
857 domain_flush_cache(domain, pte, sizeof(*pte)); 875 domain_flush_cache(domain, pte, sizeof(*pte));
858 }
859 } 876 }
860 if (level == 1) 877 if (level == 1)
861 break; 878 break;
@@ -892,7 +909,7 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
892 break; 909 break;
893 } 910 }
894 911
895 if (pte->val & DMA_PTE_LARGE_PAGE) { 912 if (dma_pte_superpage(pte)) {
896 *large_page = total; 913 *large_page = total;
897 return pte; 914 return pte;
898 } 915 }
@@ -908,12 +925,11 @@ static void dma_pte_clear_range(struct dmar_domain *domain,
908 unsigned long start_pfn, 925 unsigned long start_pfn,
909 unsigned long last_pfn) 926 unsigned long last_pfn)
910{ 927{
911 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
912 unsigned int large_page = 1; 928 unsigned int large_page = 1;
913 struct dma_pte *first_pte, *pte; 929 struct dma_pte *first_pte, *pte;
914 930
915 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); 931 BUG_ON(!domain_pfn_supported(domain, start_pfn));
916 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); 932 BUG_ON(!domain_pfn_supported(domain, last_pfn));
917 BUG_ON(start_pfn > last_pfn); 933 BUG_ON(start_pfn > last_pfn);
918 934
919 /* we don't need lock here; nobody else touches the iova range */ 935 /* we don't need lock here; nobody else touches the iova range */
@@ -974,12 +990,12 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
974 unsigned long start_pfn, 990 unsigned long start_pfn,
975 unsigned long last_pfn) 991 unsigned long last_pfn)
976{ 992{
977 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT; 993 BUG_ON(!domain_pfn_supported(domain, start_pfn));
978 994 BUG_ON(!domain_pfn_supported(domain, last_pfn));
979 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
980 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
981 BUG_ON(start_pfn > last_pfn); 995 BUG_ON(start_pfn > last_pfn);
982 996
997 dma_pte_clear_range(domain, start_pfn, last_pfn);
998
983 /* We don't need lock here; nobody else touches the iova range */ 999 /* We don't need lock here; nobody else touches the iova range */
984 dma_pte_free_level(domain, agaw_to_level(domain->agaw), 1000 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
985 domain->pgd, 0, start_pfn, last_pfn); 1001 domain->pgd, 0, start_pfn, last_pfn);
@@ -1077,11 +1093,10 @@ struct page *domain_unmap(struct dmar_domain *domain,
1077 unsigned long start_pfn, 1093 unsigned long start_pfn,
1078 unsigned long last_pfn) 1094 unsigned long last_pfn)
1079{ 1095{
1080 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1081 struct page *freelist = NULL; 1096 struct page *freelist = NULL;
1082 1097
1083 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width); 1098 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1084 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width); 1099 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1085 BUG_ON(start_pfn > last_pfn); 1100 BUG_ON(start_pfn > last_pfn);
1086 1101
1087 /* we don't need lock here; nobody else touches the iova range */ 1102 /* we don't need lock here; nobody else touches the iova range */
@@ -1275,7 +1290,8 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1275 1290
1276 spin_lock_irqsave(&device_domain_lock, flags); 1291 spin_lock_irqsave(&device_domain_lock, flags);
1277 list_for_each_entry(info, &domain->devices, link) 1292 list_for_each_entry(info, &domain->devices, link)
1278 if (info->bus == bus && info->devfn == devfn) { 1293 if (info->iommu == iommu && info->bus == bus &&
1294 info->devfn == devfn) {
1279 found = 1; 1295 found = 1;
1280 break; 1296 break;
1281 } 1297 }
@@ -1384,7 +1400,7 @@ static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1384 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1385} 1401}
1386 1402
1387static int iommu_enable_translation(struct intel_iommu *iommu) 1403static void iommu_enable_translation(struct intel_iommu *iommu)
1388{ 1404{
1389 u32 sts; 1405 u32 sts;
1390 unsigned long flags; 1406 unsigned long flags;
@@ -1398,10 +1414,9 @@ static int iommu_enable_translation(struct intel_iommu *iommu)
1398 readl, (sts & DMA_GSTS_TES), sts); 1414 readl, (sts & DMA_GSTS_TES), sts);
1399 1415
1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flags); 1416 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1401 return 0;
1402} 1417}
1403 1418
1404static int iommu_disable_translation(struct intel_iommu *iommu) 1419static void iommu_disable_translation(struct intel_iommu *iommu)
1405{ 1420{
1406 u32 sts; 1421 u32 sts;
1407 unsigned long flag; 1422 unsigned long flag;
@@ -1415,7 +1430,6 @@ static int iommu_disable_translation(struct intel_iommu *iommu)
1415 readl, (!(sts & DMA_GSTS_TES)), sts); 1430 readl, (!(sts & DMA_GSTS_TES)), sts);
1416 1431
1417 raw_spin_unlock_irqrestore(&iommu->register_lock, flag); 1432 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1418 return 0;
1419} 1433}
1420 1434
1421 1435
@@ -1462,8 +1476,7 @@ static int iommu_init_domains(struct intel_iommu *iommu)
1462static void free_dmar_iommu(struct intel_iommu *iommu) 1476static void free_dmar_iommu(struct intel_iommu *iommu)
1463{ 1477{
1464 struct dmar_domain *domain; 1478 struct dmar_domain *domain;
1465 int i, count; 1479 int i;
1466 unsigned long flags;
1467 1480
1468 if ((iommu->domains) && (iommu->domain_ids)) { 1481 if ((iommu->domains) && (iommu->domain_ids)) {
1469 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) { 1482 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
@@ -1476,11 +1489,8 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
1476 1489
1477 domain = iommu->domains[i]; 1490 domain = iommu->domains[i];
1478 clear_bit(i, iommu->domain_ids); 1491 clear_bit(i, iommu->domain_ids);
1479 1492 if (domain_detach_iommu(domain, iommu) == 0 &&
1480 spin_lock_irqsave(&domain->iommu_lock, flags); 1493 !domain_type_is_vm(domain))
1481 count = --domain->iommu_count;
1482 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1483 if (count == 0)
1484 domain_exit(domain); 1494 domain_exit(domain);
1485 } 1495 }
1486 } 1496 }
@@ -1499,7 +1509,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
1499 free_context_table(iommu); 1509 free_context_table(iommu);
1500} 1510}
1501 1511
1502static struct dmar_domain *alloc_domain(bool vm) 1512static struct dmar_domain *alloc_domain(int flags)
1503{ 1513{
1504 /* domain id for virtual machine, it won't be set in context */ 1514 /* domain id for virtual machine, it won't be set in context */
1505 static atomic_t vm_domid = ATOMIC_INIT(0); 1515 static atomic_t vm_domid = ATOMIC_INIT(0);
@@ -1509,46 +1519,62 @@ static struct dmar_domain *alloc_domain(bool vm)
1509 if (!domain) 1519 if (!domain)
1510 return NULL; 1520 return NULL;
1511 1521
1522 memset(domain, 0, sizeof(*domain));
1512 domain->nid = -1; 1523 domain->nid = -1;
1513 domain->iommu_count = 0; 1524 domain->flags = flags;
1514 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1515 domain->flags = 0;
1516 spin_lock_init(&domain->iommu_lock); 1525 spin_lock_init(&domain->iommu_lock);
1517 INIT_LIST_HEAD(&domain->devices); 1526 INIT_LIST_HEAD(&domain->devices);
1518 if (vm) { 1527 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1519 domain->id = atomic_inc_return(&vm_domid); 1528 domain->id = atomic_inc_return(&vm_domid);
1520 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
1521 }
1522 1529
1523 return domain; 1530 return domain;
1524} 1531}
1525 1532
1526static int iommu_attach_domain(struct dmar_domain *domain, 1533static int __iommu_attach_domain(struct dmar_domain *domain,
1527 struct intel_iommu *iommu) 1534 struct intel_iommu *iommu)
1528{ 1535{
1529 int num; 1536 int num;
1530 unsigned long ndomains; 1537 unsigned long ndomains;
1531 unsigned long flags;
1532 1538
1533 ndomains = cap_ndoms(iommu->cap); 1539 ndomains = cap_ndoms(iommu->cap);
1534
1535 spin_lock_irqsave(&iommu->lock, flags);
1536
1537 num = find_first_zero_bit(iommu->domain_ids, ndomains); 1540 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1538 if (num >= ndomains) { 1541 if (num < ndomains) {
1539 spin_unlock_irqrestore(&iommu->lock, flags); 1542 set_bit(num, iommu->domain_ids);
1540 printk(KERN_ERR "IOMMU: no free domain ids\n"); 1543 iommu->domains[num] = domain;
1541 return -ENOMEM; 1544 } else {
1545 num = -ENOSPC;
1542 } 1546 }
1543 1547
1544 domain->id = num; 1548 return num;
1545 domain->iommu_count++; 1549}
1546 set_bit(num, iommu->domain_ids); 1550
1547 set_bit(iommu->seq_id, domain->iommu_bmp); 1551static int iommu_attach_domain(struct dmar_domain *domain,
1548 iommu->domains[num] = domain; 1552 struct intel_iommu *iommu)
1553{
1554 int num;
1555 unsigned long flags;
1556
1557 spin_lock_irqsave(&iommu->lock, flags);
1558 num = __iommu_attach_domain(domain, iommu);
1549 spin_unlock_irqrestore(&iommu->lock, flags); 1559 spin_unlock_irqrestore(&iommu->lock, flags);
1560 if (num < 0)
1561 pr_err("IOMMU: no free domain ids\n");
1550 1562
1551 return 0; 1563 return num;
1564}
1565
1566static int iommu_attach_vm_domain(struct dmar_domain *domain,
1567 struct intel_iommu *iommu)
1568{
1569 int num;
1570 unsigned long ndomains;
1571
1572 ndomains = cap_ndoms(iommu->cap);
1573 for_each_set_bit(num, iommu->domain_ids, ndomains)
1574 if (iommu->domains[num] == domain)
1575 return num;
1576
1577 return __iommu_attach_domain(domain, iommu);
1552} 1578}
1553 1579
1554static void iommu_detach_domain(struct dmar_domain *domain, 1580static void iommu_detach_domain(struct dmar_domain *domain,
@@ -1558,17 +1584,53 @@ static void iommu_detach_domain(struct dmar_domain *domain,
1558 int num, ndomains; 1584 int num, ndomains;
1559 1585
1560 spin_lock_irqsave(&iommu->lock, flags); 1586 spin_lock_irqsave(&iommu->lock, flags);
1561 ndomains = cap_ndoms(iommu->cap); 1587 if (domain_type_is_vm_or_si(domain)) {
1562 for_each_set_bit(num, iommu->domain_ids, ndomains) { 1588 ndomains = cap_ndoms(iommu->cap);
1563 if (iommu->domains[num] == domain) { 1589 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1564 clear_bit(num, iommu->domain_ids); 1590 if (iommu->domains[num] == domain) {
1565 iommu->domains[num] = NULL; 1591 clear_bit(num, iommu->domain_ids);
1566 break; 1592 iommu->domains[num] = NULL;
1593 break;
1594 }
1567 } 1595 }
1596 } else {
1597 clear_bit(domain->id, iommu->domain_ids);
1598 iommu->domains[domain->id] = NULL;
1568 } 1599 }
1569 spin_unlock_irqrestore(&iommu->lock, flags); 1600 spin_unlock_irqrestore(&iommu->lock, flags);
1570} 1601}
1571 1602
1603static void domain_attach_iommu(struct dmar_domain *domain,
1604 struct intel_iommu *iommu)
1605{
1606 unsigned long flags;
1607
1608 spin_lock_irqsave(&domain->iommu_lock, flags);
1609 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1610 domain->iommu_count++;
1611 if (domain->iommu_count == 1)
1612 domain->nid = iommu->node;
1613 domain_update_iommu_cap(domain);
1614 }
1615 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1616}
1617
1618static int domain_detach_iommu(struct dmar_domain *domain,
1619 struct intel_iommu *iommu)
1620{
1621 unsigned long flags;
1622 int count = INT_MAX;
1623
1624 spin_lock_irqsave(&domain->iommu_lock, flags);
1625 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1626 count = --domain->iommu_count;
1627 domain_update_iommu_cap(domain);
1628 }
1629 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1630
1631 return count;
1632}
1633
1572static struct iova_domain reserved_iova_list; 1634static struct iova_domain reserved_iova_list;
1573static struct lock_class_key reserved_rbtree_key; 1635static struct lock_class_key reserved_rbtree_key;
1574 1636
@@ -1706,9 +1768,7 @@ static void domain_exit(struct dmar_domain *domain)
1706 /* clear attached or cached domains */ 1768 /* clear attached or cached domains */
1707 rcu_read_lock(); 1769 rcu_read_lock();
1708 for_each_active_iommu(iommu, drhd) 1770 for_each_active_iommu(iommu, drhd)
1709 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE || 1771 iommu_detach_domain(domain, iommu);
1710 test_bit(iommu->seq_id, domain->iommu_bmp))
1711 iommu_detach_domain(domain, iommu);
1712 rcu_read_unlock(); 1772 rcu_read_unlock();
1713 1773
1714 dma_free_pagelist(freelist); 1774 dma_free_pagelist(freelist);
@@ -1723,8 +1783,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1723 struct context_entry *context; 1783 struct context_entry *context;
1724 unsigned long flags; 1784 unsigned long flags;
1725 struct dma_pte *pgd; 1785 struct dma_pte *pgd;
1726 unsigned long num;
1727 unsigned long ndomains;
1728 int id; 1786 int id;
1729 int agaw; 1787 int agaw;
1730 struct device_domain_info *info = NULL; 1788 struct device_domain_info *info = NULL;
@@ -1748,31 +1806,14 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1748 id = domain->id; 1806 id = domain->id;
1749 pgd = domain->pgd; 1807 pgd = domain->pgd;
1750 1808
1751 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE || 1809 if (domain_type_is_vm_or_si(domain)) {
1752 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) { 1810 if (domain_type_is_vm(domain)) {
1753 int found = 0; 1811 id = iommu_attach_vm_domain(domain, iommu);
1754 1812 if (id < 0) {
1755 /* find an available domain id for this device in iommu */
1756 ndomains = cap_ndoms(iommu->cap);
1757 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1758 if (iommu->domains[num] == domain) {
1759 id = num;
1760 found = 1;
1761 break;
1762 }
1763 }
1764
1765 if (found == 0) {
1766 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1767 if (num >= ndomains) {
1768 spin_unlock_irqrestore(&iommu->lock, flags); 1813 spin_unlock_irqrestore(&iommu->lock, flags);
1769 printk(KERN_ERR "IOMMU: no free domain ids\n"); 1814 pr_err("IOMMU: no free domain ids\n");
1770 return -EFAULT; 1815 return -EFAULT;
1771 } 1816 }
1772
1773 set_bit(num, iommu->domain_ids);
1774 iommu->domains[num] = domain;
1775 id = num;
1776 } 1817 }
1777 1818
1778 /* Skip top levels of page tables for 1819 /* Skip top levels of page tables for
@@ -1824,72 +1865,68 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1824 (((u16)bus) << 8) | devfn, 1865 (((u16)bus) << 8) | devfn,
1825 DMA_CCMD_MASK_NOBIT, 1866 DMA_CCMD_MASK_NOBIT,
1826 DMA_CCMD_DEVICE_INVL); 1867 DMA_CCMD_DEVICE_INVL);
1827 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH); 1868 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1828 } else { 1869 } else {
1829 iommu_flush_write_buffer(iommu); 1870 iommu_flush_write_buffer(iommu);
1830 } 1871 }
1831 iommu_enable_dev_iotlb(info); 1872 iommu_enable_dev_iotlb(info);
1832 spin_unlock_irqrestore(&iommu->lock, flags); 1873 spin_unlock_irqrestore(&iommu->lock, flags);
1833 1874
1834 spin_lock_irqsave(&domain->iommu_lock, flags); 1875 domain_attach_iommu(domain, iommu);
1835 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) { 1876
1836 domain->iommu_count++;
1837 if (domain->iommu_count == 1)
1838 domain->nid = iommu->node;
1839 domain_update_iommu_cap(domain);
1840 }
1841 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1842 return 0; 1877 return 0;
1843} 1878}
1844 1879
1880struct domain_context_mapping_data {
1881 struct dmar_domain *domain;
1882 struct intel_iommu *iommu;
1883 int translation;
1884};
1885
1886static int domain_context_mapping_cb(struct pci_dev *pdev,
1887 u16 alias, void *opaque)
1888{
1889 struct domain_context_mapping_data *data = opaque;
1890
1891 return domain_context_mapping_one(data->domain, data->iommu,
1892 PCI_BUS_NUM(alias), alias & 0xff,
1893 data->translation);
1894}
1895
1845static int 1896static int
1846domain_context_mapping(struct dmar_domain *domain, struct device *dev, 1897domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1847 int translation) 1898 int translation)
1848{ 1899{
1849 int ret;
1850 struct pci_dev *pdev, *tmp, *parent;
1851 struct intel_iommu *iommu; 1900 struct intel_iommu *iommu;
1852 u8 bus, devfn; 1901 u8 bus, devfn;
1902 struct domain_context_mapping_data data;
1853 1903
1854 iommu = device_to_iommu(dev, &bus, &devfn); 1904 iommu = device_to_iommu(dev, &bus, &devfn);
1855 if (!iommu) 1905 if (!iommu)
1856 return -ENODEV; 1906 return -ENODEV;
1857 1907
1858 ret = domain_context_mapping_one(domain, iommu, bus, devfn, 1908 if (!dev_is_pci(dev))
1859 translation); 1909 return domain_context_mapping_one(domain, iommu, bus, devfn,
1860 if (ret || !dev_is_pci(dev))
1861 return ret;
1862
1863 /* dependent device mapping */
1864 pdev = to_pci_dev(dev);
1865 tmp = pci_find_upstream_pcie_bridge(pdev);
1866 if (!tmp)
1867 return 0;
1868 /* Secondary interface's bus number and devfn 0 */
1869 parent = pdev->bus->self;
1870 while (parent != tmp) {
1871 ret = domain_context_mapping_one(domain, iommu,
1872 parent->bus->number,
1873 parent->devfn, translation);
1874 if (ret)
1875 return ret;
1876 parent = parent->bus->self;
1877 }
1878 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1879 return domain_context_mapping_one(domain, iommu,
1880 tmp->subordinate->number, 0,
1881 translation);
1882 else /* this is a legacy PCI bridge */
1883 return domain_context_mapping_one(domain, iommu,
1884 tmp->bus->number,
1885 tmp->devfn,
1886 translation); 1910 translation);
1911
1912 data.domain = domain;
1913 data.iommu = iommu;
1914 data.translation = translation;
1915
1916 return pci_for_each_dma_alias(to_pci_dev(dev),
1917 &domain_context_mapping_cb, &data);
1918}
1919
1920static int domain_context_mapped_cb(struct pci_dev *pdev,
1921 u16 alias, void *opaque)
1922{
1923 struct intel_iommu *iommu = opaque;
1924
1925 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
1887} 1926}
1888 1927
1889static int domain_context_mapped(struct device *dev) 1928static int domain_context_mapped(struct device *dev)
1890{ 1929{
1891 int ret;
1892 struct pci_dev *pdev, *tmp, *parent;
1893 struct intel_iommu *iommu; 1930 struct intel_iommu *iommu;
1894 u8 bus, devfn; 1931 u8 bus, devfn;
1895 1932
@@ -1897,30 +1934,11 @@ static int domain_context_mapped(struct device *dev)
1897 if (!iommu) 1934 if (!iommu)
1898 return -ENODEV; 1935 return -ENODEV;
1899 1936
1900 ret = device_context_mapped(iommu, bus, devfn); 1937 if (!dev_is_pci(dev))
1901 if (!ret || !dev_is_pci(dev)) 1938 return device_context_mapped(iommu, bus, devfn);
1902 return ret;
1903 1939
1904 /* dependent device mapping */ 1940 return !pci_for_each_dma_alias(to_pci_dev(dev),
1905 pdev = to_pci_dev(dev); 1941 domain_context_mapped_cb, iommu);
1906 tmp = pci_find_upstream_pcie_bridge(pdev);
1907 if (!tmp)
1908 return ret;
1909 /* Secondary interface's bus number and devfn 0 */
1910 parent = pdev->bus->self;
1911 while (parent != tmp) {
1912 ret = device_context_mapped(iommu, parent->bus->number,
1913 parent->devfn);
1914 if (!ret)
1915 return ret;
1916 parent = parent->bus->self;
1917 }
1918 if (pci_is_pcie(tmp))
1919 return device_context_mapped(iommu, tmp->subordinate->number,
1920 0);
1921 else
1922 return device_context_mapped(iommu, tmp->bus->number,
1923 tmp->devfn);
1924} 1942}
1925 1943
1926/* Returns a number of VTD pages, but aligned to MM page size */ 1944/* Returns a number of VTD pages, but aligned to MM page size */
@@ -1965,12 +1983,11 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1965{ 1983{
1966 struct dma_pte *first_pte = NULL, *pte = NULL; 1984 struct dma_pte *first_pte = NULL, *pte = NULL;
1967 phys_addr_t uninitialized_var(pteval); 1985 phys_addr_t uninitialized_var(pteval);
1968 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1969 unsigned long sg_res; 1986 unsigned long sg_res;
1970 unsigned int largepage_lvl = 0; 1987 unsigned int largepage_lvl = 0;
1971 unsigned long lvl_pages = 0; 1988 unsigned long lvl_pages = 0;
1972 1989
1973 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width); 1990 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
1974 1991
1975 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0) 1992 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1976 return -EINVAL; 1993 return -EINVAL;
@@ -2004,12 +2021,14 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2004 /* It is large page*/ 2021 /* It is large page*/
2005 if (largepage_lvl > 1) { 2022 if (largepage_lvl > 1) {
2006 pteval |= DMA_PTE_LARGE_PAGE; 2023 pteval |= DMA_PTE_LARGE_PAGE;
2007 /* Ensure that old small page tables are removed to make room 2024 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2008 for superpage, if they exist. */ 2025 /*
2009 dma_pte_clear_range(domain, iov_pfn, 2026 * Ensure that old small page tables are
2010 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1); 2027 * removed to make room for superpage,
2028 * if they exist.
2029 */
2011 dma_pte_free_pagetable(domain, iov_pfn, 2030 dma_pte_free_pagetable(domain, iov_pfn,
2012 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1); 2031 iov_pfn + lvl_pages - 1);
2013 } else { 2032 } else {
2014 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE; 2033 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2015 } 2034 }
@@ -2102,31 +2121,20 @@ static inline void unlink_domain_info(struct device_domain_info *info)
2102 2121
2103static void domain_remove_dev_info(struct dmar_domain *domain) 2122static void domain_remove_dev_info(struct dmar_domain *domain)
2104{ 2123{
2105 struct device_domain_info *info; 2124 struct device_domain_info *info, *tmp;
2106 unsigned long flags, flags2; 2125 unsigned long flags;
2107 2126
2108 spin_lock_irqsave(&device_domain_lock, flags); 2127 spin_lock_irqsave(&device_domain_lock, flags);
2109 while (!list_empty(&domain->devices)) { 2128 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2110 info = list_entry(domain->devices.next,
2111 struct device_domain_info, link);
2112 unlink_domain_info(info); 2129 unlink_domain_info(info);
2113 spin_unlock_irqrestore(&device_domain_lock, flags); 2130 spin_unlock_irqrestore(&device_domain_lock, flags);
2114 2131
2115 iommu_disable_dev_iotlb(info); 2132 iommu_disable_dev_iotlb(info);
2116 iommu_detach_dev(info->iommu, info->bus, info->devfn); 2133 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2117 2134
2118 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) { 2135 if (domain_type_is_vm(domain)) {
2119 iommu_detach_dependent_devices(info->iommu, info->dev); 2136 iommu_detach_dependent_devices(info->iommu, info->dev);
2120 /* clear this iommu in iommu_bmp, update iommu count 2137 domain_detach_iommu(domain, info->iommu);
2121 * and capabilities
2122 */
2123 spin_lock_irqsave(&domain->iommu_lock, flags2);
2124 if (test_and_clear_bit(info->iommu->seq_id,
2125 domain->iommu_bmp)) {
2126 domain->iommu_count--;
2127 domain_update_iommu_cap(domain);
2128 }
2129 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2130 } 2138 }
2131 2139
2132 free_devinfo_mem(info); 2140 free_devinfo_mem(info);
@@ -2181,8 +2189,6 @@ static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2181 info->dev = dev; 2189 info->dev = dev;
2182 info->domain = domain; 2190 info->domain = domain;
2183 info->iommu = iommu; 2191 info->iommu = iommu;
2184 if (!dev)
2185 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2186 2192
2187 spin_lock_irqsave(&device_domain_lock, flags); 2193 spin_lock_irqsave(&device_domain_lock, flags);
2188 if (dev) 2194 if (dev)
@@ -2209,79 +2215,86 @@ static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2209 return domain; 2215 return domain;
2210} 2216}
2211 2217
2218static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2219{
2220 *(u16 *)opaque = alias;
2221 return 0;
2222}
2223
2212/* domain is initialized */ 2224/* domain is initialized */
2213static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) 2225static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2214{ 2226{
2215 struct dmar_domain *domain, *free = NULL; 2227 struct dmar_domain *domain, *tmp;
2216 struct intel_iommu *iommu = NULL; 2228 struct intel_iommu *iommu;
2217 struct device_domain_info *info; 2229 struct device_domain_info *info;
2218 struct pci_dev *dev_tmp = NULL; 2230 u16 dma_alias;
2219 unsigned long flags; 2231 unsigned long flags;
2220 u8 bus, devfn, bridge_bus, bridge_devfn; 2232 u8 bus, devfn;
2221 2233
2222 domain = find_domain(dev); 2234 domain = find_domain(dev);
2223 if (domain) 2235 if (domain)
2224 return domain; 2236 return domain;
2225 2237
2238 iommu = device_to_iommu(dev, &bus, &devfn);
2239 if (!iommu)
2240 return NULL;
2241
2226 if (dev_is_pci(dev)) { 2242 if (dev_is_pci(dev)) {
2227 struct pci_dev *pdev = to_pci_dev(dev); 2243 struct pci_dev *pdev = to_pci_dev(dev);
2228 u16 segment;
2229 2244
2230 segment = pci_domain_nr(pdev->bus); 2245 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2231 dev_tmp = pci_find_upstream_pcie_bridge(pdev); 2246
2232 if (dev_tmp) { 2247 spin_lock_irqsave(&device_domain_lock, flags);
2233 if (pci_is_pcie(dev_tmp)) { 2248 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2234 bridge_bus = dev_tmp->subordinate->number; 2249 PCI_BUS_NUM(dma_alias),
2235 bridge_devfn = 0; 2250 dma_alias & 0xff);
2236 } else { 2251 if (info) {
2237 bridge_bus = dev_tmp->bus->number; 2252 iommu = info->iommu;
2238 bridge_devfn = dev_tmp->devfn; 2253 domain = info->domain;
2239 }
2240 spin_lock_irqsave(&device_domain_lock, flags);
2241 info = dmar_search_domain_by_dev_info(segment,
2242 bridge_bus,
2243 bridge_devfn);
2244 if (info) {
2245 iommu = info->iommu;
2246 domain = info->domain;
2247 }
2248 spin_unlock_irqrestore(&device_domain_lock, flags);
2249 /* pcie-pci bridge already has a domain, uses it */
2250 if (info)
2251 goto found_domain;
2252 } 2254 }
2253 } 2255 spin_unlock_irqrestore(&device_domain_lock, flags);
2254 2256
2255 iommu = device_to_iommu(dev, &bus, &devfn); 2257 /* DMA alias already has a domain, uses it */
2256 if (!iommu) 2258 if (info)
2257 goto error; 2259 goto found_domain;
2260 }
2258 2261
2259 /* Allocate and initialize new domain for the device */ 2262 /* Allocate and initialize new domain for the device */
2260 domain = alloc_domain(false); 2263 domain = alloc_domain(0);
2261 if (!domain) 2264 if (!domain)
2262 goto error; 2265 return NULL;
2263 if (iommu_attach_domain(domain, iommu)) { 2266 domain->id = iommu_attach_domain(domain, iommu);
2267 if (domain->id < 0) {
2264 free_domain_mem(domain); 2268 free_domain_mem(domain);
2265 domain = NULL; 2269 return NULL;
2266 goto error; 2270 }
2271 domain_attach_iommu(domain, iommu);
2272 if (domain_init(domain, gaw)) {
2273 domain_exit(domain);
2274 return NULL;
2267 } 2275 }
2268 free = domain;
2269 if (domain_init(domain, gaw))
2270 goto error;
2271 2276
2272 /* register pcie-to-pci device */ 2277 /* register PCI DMA alias device */
2273 if (dev_tmp) { 2278 if (dev_is_pci(dev)) {
2274 domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn, 2279 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2275 NULL, domain); 2280 dma_alias & 0xff, NULL, domain);
2281
2282 if (!tmp || tmp != domain) {
2283 domain_exit(domain);
2284 domain = tmp;
2285 }
2286
2276 if (!domain) 2287 if (!domain)
2277 goto error; 2288 return NULL;
2278 } 2289 }
2279 2290
2280found_domain: 2291found_domain:
2281 domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain); 2292 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2282error: 2293
2283 if (free != domain) 2294 if (!tmp || tmp != domain) {
2284 domain_exit(free); 2295 domain_exit(domain);
2296 domain = tmp;
2297 }
2285 2298
2286 return domain; 2299 return domain;
2287} 2300}
@@ -2405,6 +2418,7 @@ static inline void iommu_prepare_isa(void)
2405 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; " 2418 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2406 "floppy might not work\n"); 2419 "floppy might not work\n");
2407 2420
2421 pci_dev_put(pdev);
2408} 2422}
2409#else 2423#else
2410static inline void iommu_prepare_isa(void) 2424static inline void iommu_prepare_isa(void)
@@ -2420,19 +2434,25 @@ static int __init si_domain_init(int hw)
2420 struct dmar_drhd_unit *drhd; 2434 struct dmar_drhd_unit *drhd;
2421 struct intel_iommu *iommu; 2435 struct intel_iommu *iommu;
2422 int nid, ret = 0; 2436 int nid, ret = 0;
2437 bool first = true;
2423 2438
2424 si_domain = alloc_domain(false); 2439 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2425 if (!si_domain) 2440 if (!si_domain)
2426 return -EFAULT; 2441 return -EFAULT;
2427 2442
2428 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2429
2430 for_each_active_iommu(iommu, drhd) { 2443 for_each_active_iommu(iommu, drhd) {
2431 ret = iommu_attach_domain(si_domain, iommu); 2444 ret = iommu_attach_domain(si_domain, iommu);
2432 if (ret) { 2445 if (ret < 0) {
2446 domain_exit(si_domain);
2447 return -EFAULT;
2448 } else if (first) {
2449 si_domain->id = ret;
2450 first = false;
2451 } else if (si_domain->id != ret) {
2433 domain_exit(si_domain); 2452 domain_exit(si_domain);
2434 return -EFAULT; 2453 return -EFAULT;
2435 } 2454 }
2455 domain_attach_iommu(si_domain, iommu);
2436 } 2456 }
2437 2457
2438 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { 2458 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
@@ -2523,22 +2543,46 @@ static bool device_has_rmrr(struct device *dev)
2523 return false; 2543 return false;
2524} 2544}
2525 2545
2546/*
2547 * There are a couple cases where we need to restrict the functionality of
2548 * devices associated with RMRRs. The first is when evaluating a device for
2549 * identity mapping because problems exist when devices are moved in and out
2550 * of domains and their respective RMRR information is lost. This means that
2551 * a device with associated RMRRs will never be in a "passthrough" domain.
2552 * The second is use of the device through the IOMMU API. This interface
2553 * expects to have full control of the IOVA space for the device. We cannot
2554 * satisfy both the requirement that RMRR access is maintained and have an
2555 * unencumbered IOVA space. We also have no ability to quiesce the device's
2556 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2557 * We therefore prevent devices associated with an RMRR from participating in
2558 * the IOMMU API, which eliminates them from device assignment.
2559 *
2560 * In both cases we assume that PCI USB devices with RMRRs have them largely
2561 * for historical reasons and that the RMRR space is not actively used post
2562 * boot. This exclusion may change if vendors begin to abuse it.
2563 */
2564static bool device_is_rmrr_locked(struct device *dev)
2565{
2566 if (!device_has_rmrr(dev))
2567 return false;
2568
2569 if (dev_is_pci(dev)) {
2570 struct pci_dev *pdev = to_pci_dev(dev);
2571
2572 if ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
2573 return false;
2574 }
2575
2576 return true;
2577}
2578
2526static int iommu_should_identity_map(struct device *dev, int startup) 2579static int iommu_should_identity_map(struct device *dev, int startup)
2527{ 2580{
2528 2581
2529 if (dev_is_pci(dev)) { 2582 if (dev_is_pci(dev)) {
2530 struct pci_dev *pdev = to_pci_dev(dev); 2583 struct pci_dev *pdev = to_pci_dev(dev);
2531 2584
2532 /* 2585 if (device_is_rmrr_locked(dev))
2533 * We want to prevent any device associated with an RMRR from
2534 * getting placed into the SI Domain. This is done because
2535 * problems exist when devices are moved in and out of domains
2536 * and their respective RMRR info is lost. We exempt USB devices
2537 * from this process due to their usage of RMRRs that are known
2538 * to not be needed after BIOS hand-off to OS.
2539 */
2540 if (device_has_rmrr(dev) &&
2541 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2542 return 0; 2586 return 0;
2543 2587
2544 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev)) 2588 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
@@ -2850,11 +2894,7 @@ static int __init init_dmars(void)
2850 2894
2851 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); 2895 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2852 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); 2896 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2853 2897 iommu_enable_translation(iommu);
2854 ret = iommu_enable_translation(iommu);
2855 if (ret)
2856 goto free_iommu;
2857
2858 iommu_disable_protect_mem_regions(iommu); 2898 iommu_disable_protect_mem_regions(iommu);
2859 } 2899 }
2860 2900
@@ -3091,10 +3131,10 @@ static void flush_unmaps(void)
3091 /* On real hardware multiple invalidations are expensive */ 3131 /* On real hardware multiple invalidations are expensive */
3092 if (cap_caching_mode(iommu->cap)) 3132 if (cap_caching_mode(iommu->cap))
3093 iommu_flush_iotlb_psi(iommu, domain->id, 3133 iommu_flush_iotlb_psi(iommu, domain->id,
3094 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 3134 iova->pfn_lo, iova_size(iova),
3095 !deferred_flush[i].freelist[j], 0); 3135 !deferred_flush[i].freelist[j], 0);
3096 else { 3136 else {
3097 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1)); 3137 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3098 iommu_flush_dev_iotlb(deferred_flush[i].domain[j], 3138 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3099 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask); 3139 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3100 } 3140 }
@@ -3144,9 +3184,7 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *f
3144 spin_unlock_irqrestore(&async_umap_flush_lock, flags); 3184 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3145} 3185}
3146 3186
3147static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr, 3187static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3148 size_t size, enum dma_data_direction dir,
3149 struct dma_attrs *attrs)
3150{ 3188{
3151 struct dmar_domain *domain; 3189 struct dmar_domain *domain;
3152 unsigned long start_pfn, last_pfn; 3190 unsigned long start_pfn, last_pfn;
@@ -3190,6 +3228,13 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3190 } 3228 }
3191} 3229}
3192 3230
3231static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3232 size_t size, enum dma_data_direction dir,
3233 struct dma_attrs *attrs)
3234{
3235 intel_unmap(dev, dev_addr);
3236}
3237
3193static void *intel_alloc_coherent(struct device *dev, size_t size, 3238static void *intel_alloc_coherent(struct device *dev, size_t size,
3194 dma_addr_t *dma_handle, gfp_t flags, 3239 dma_addr_t *dma_handle, gfp_t flags,
3195 struct dma_attrs *attrs) 3240 struct dma_attrs *attrs)
@@ -3246,7 +3291,7 @@ static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3246 size = PAGE_ALIGN(size); 3291 size = PAGE_ALIGN(size);
3247 order = get_order(size); 3292 order = get_order(size);
3248 3293
3249 intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL); 3294 intel_unmap(dev, dma_handle);
3250 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) 3295 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3251 __free_pages(page, order); 3296 __free_pages(page, order);
3252} 3297}
@@ -3255,43 +3300,7 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3255 int nelems, enum dma_data_direction dir, 3300 int nelems, enum dma_data_direction dir,
3256 struct dma_attrs *attrs) 3301 struct dma_attrs *attrs)
3257{ 3302{
3258 struct dmar_domain *domain; 3303 intel_unmap(dev, sglist[0].dma_address);
3259 unsigned long start_pfn, last_pfn;
3260 struct iova *iova;
3261 struct intel_iommu *iommu;
3262 struct page *freelist;
3263
3264 if (iommu_no_mapping(dev))
3265 return;
3266
3267 domain = find_domain(dev);
3268 BUG_ON(!domain);
3269
3270 iommu = domain_get_iommu(domain);
3271
3272 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3273 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3274 (unsigned long long)sglist[0].dma_address))
3275 return;
3276
3277 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3278 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3279
3280 freelist = domain_unmap(domain, start_pfn, last_pfn);
3281
3282 if (intel_iommu_strict) {
3283 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3284 last_pfn - start_pfn + 1, !freelist, 0);
3285 /* free iova */
3286 __free_iova(&domain->iovad, iova);
3287 dma_free_pagelist(freelist);
3288 } else {
3289 add_unmap(domain, iova, freelist);
3290 /*
3291 * queue up the release of the unmap to save the 1/6th of the
3292 * cpu used up by the iotlb flush operation...
3293 */
3294 }
3295} 3304}
3296 3305
3297static int intel_nontranslate_map_sg(struct device *hddev, 3306static int intel_nontranslate_map_sg(struct device *hddev,
@@ -3355,13 +3364,8 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
3355 3364
3356 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot); 3365 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3357 if (unlikely(ret)) { 3366 if (unlikely(ret)) {
3358 /* clear the page */
3359 dma_pte_clear_range(domain, start_vpfn,
3360 start_vpfn + size - 1);
3361 /* free page tables */
3362 dma_pte_free_pagetable(domain, start_vpfn, 3367 dma_pte_free_pagetable(domain, start_vpfn,
3363 start_vpfn + size - 1); 3368 start_vpfn + size - 1);
3364 /* free iova */
3365 __free_iova(&domain->iovad, iova); 3369 __free_iova(&domain->iovad, iova);
3366 return 0; 3370 return 0;
3367 } 3371 }
@@ -3568,10 +3572,8 @@ static int init_iommu_hw(void)
3568 3572
3569 iommu->flush.flush_context(iommu, 0, 0, 0, 3573 iommu->flush.flush_context(iommu, 0, 0, 0,
3570 DMA_CCMD_GLOBAL_INVL); 3574 DMA_CCMD_GLOBAL_INVL);
3571 iommu->flush.flush_iotlb(iommu, 0, 0, 0, 3575 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3572 DMA_TLB_GLOBAL_FLUSH); 3576 iommu_enable_translation(iommu);
3573 if (iommu_enable_translation(iommu))
3574 return 1;
3575 iommu_disable_protect_mem_regions(iommu); 3577 iommu_disable_protect_mem_regions(iommu);
3576 } 3578 }
3577 3579
@@ -3867,15 +3869,21 @@ static int device_notifier(struct notifier_block *nb,
3867 action != BUS_NOTIFY_DEL_DEVICE) 3869 action != BUS_NOTIFY_DEL_DEVICE)
3868 return 0; 3870 return 0;
3869 3871
3872 /*
3873 * If the device is still attached to a device driver we can't
3874 * tear down the domain yet as DMA mappings may still be in use.
3875 * Wait for the BUS_NOTIFY_UNBOUND_DRIVER event to do that.
3876 */
3877 if (action == BUS_NOTIFY_DEL_DEVICE && dev->driver != NULL)
3878 return 0;
3879
3870 domain = find_domain(dev); 3880 domain = find_domain(dev);
3871 if (!domain) 3881 if (!domain)
3872 return 0; 3882 return 0;
3873 3883
3874 down_read(&dmar_global_lock); 3884 down_read(&dmar_global_lock);
3875 domain_remove_one_dev_info(domain, dev); 3885 domain_remove_one_dev_info(domain, dev);
3876 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) && 3886 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
3877 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3878 list_empty(&domain->devices))
3879 domain_exit(domain); 3887 domain_exit(domain);
3880 up_read(&dmar_global_lock); 3888 up_read(&dmar_global_lock);
3881 3889
@@ -3935,8 +3943,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
3935 rcu_read_lock(); 3943 rcu_read_lock();
3936 for_each_active_iommu(iommu, drhd) 3944 for_each_active_iommu(iommu, drhd)
3937 iommu_flush_iotlb_psi(iommu, si_domain->id, 3945 iommu_flush_iotlb_psi(iommu, si_domain->id,
3938 iova->pfn_lo, 3946 iova->pfn_lo, iova_size(iova),
3939 iova->pfn_hi - iova->pfn_lo + 1,
3940 !freelist, 0); 3947 !freelist, 0);
3941 rcu_read_unlock(); 3948 rcu_read_unlock();
3942 dma_free_pagelist(freelist); 3949 dma_free_pagelist(freelist);
@@ -3955,6 +3962,63 @@ static struct notifier_block intel_iommu_memory_nb = {
3955 .priority = 0 3962 .priority = 0
3956}; 3963};
3957 3964
3965
3966static ssize_t intel_iommu_show_version(struct device *dev,
3967 struct device_attribute *attr,
3968 char *buf)
3969{
3970 struct intel_iommu *iommu = dev_get_drvdata(dev);
3971 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3972 return sprintf(buf, "%d:%d\n",
3973 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3974}
3975static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
3976
3977static ssize_t intel_iommu_show_address(struct device *dev,
3978 struct device_attribute *attr,
3979 char *buf)
3980{
3981 struct intel_iommu *iommu = dev_get_drvdata(dev);
3982 return sprintf(buf, "%llx\n", iommu->reg_phys);
3983}
3984static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
3985
3986static ssize_t intel_iommu_show_cap(struct device *dev,
3987 struct device_attribute *attr,
3988 char *buf)
3989{
3990 struct intel_iommu *iommu = dev_get_drvdata(dev);
3991 return sprintf(buf, "%llx\n", iommu->cap);
3992}
3993static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
3994
3995static ssize_t intel_iommu_show_ecap(struct device *dev,
3996 struct device_attribute *attr,
3997 char *buf)
3998{
3999 struct intel_iommu *iommu = dev_get_drvdata(dev);
4000 return sprintf(buf, "%llx\n", iommu->ecap);
4001}
4002static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4003
4004static struct attribute *intel_iommu_attrs[] = {
4005 &dev_attr_version.attr,
4006 &dev_attr_address.attr,
4007 &dev_attr_cap.attr,
4008 &dev_attr_ecap.attr,
4009 NULL,
4010};
4011
4012static struct attribute_group intel_iommu_group = {
4013 .name = "intel-iommu",
4014 .attrs = intel_iommu_attrs,
4015};
4016
4017const struct attribute_group *intel_iommu_groups[] = {
4018 &intel_iommu_group,
4019 NULL,
4020};
4021
3958int __init intel_iommu_init(void) 4022int __init intel_iommu_init(void)
3959{ 4023{
3960 int ret = -ENODEV; 4024 int ret = -ENODEV;
@@ -4026,6 +4090,11 @@ int __init intel_iommu_init(void)
4026 4090
4027 init_iommu_pm_ops(); 4091 init_iommu_pm_ops();
4028 4092
4093 for_each_active_iommu(iommu, drhd)
4094 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4095 intel_iommu_groups,
4096 iommu->name);
4097
4029 bus_set_iommu(&pci_bus_type, &intel_iommu_ops); 4098 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4030 bus_register_notifier(&pci_bus_type, &device_nb); 4099 bus_register_notifier(&pci_bus_type, &device_nb);
4031 if (si_domain && !hw_pass_through) 4100 if (si_domain && !hw_pass_through)
@@ -4044,33 +4113,27 @@ out_free_dmar:
4044 return ret; 4113 return ret;
4045} 4114}
4046 4115
4116static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4117{
4118 struct intel_iommu *iommu = opaque;
4119
4120 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4121 return 0;
4122}
4123
4124/*
4125 * NB - intel-iommu lacks any sort of reference counting for the users of
4126 * dependent devices. If multiple endpoints have intersecting dependent
4127 * devices, unbinding the driver from any one of them will possibly leave
4128 * the others unable to operate.
4129 */
4047static void iommu_detach_dependent_devices(struct intel_iommu *iommu, 4130static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4048 struct device *dev) 4131 struct device *dev)
4049{ 4132{
4050 struct pci_dev *tmp, *parent, *pdev;
4051
4052 if (!iommu || !dev || !dev_is_pci(dev)) 4133 if (!iommu || !dev || !dev_is_pci(dev))
4053 return; 4134 return;
4054 4135
4055 pdev = to_pci_dev(dev); 4136 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4056
4057 /* dependent device detach */
4058 tmp = pci_find_upstream_pcie_bridge(pdev);
4059 /* Secondary interface's bus number and devfn 0 */
4060 if (tmp) {
4061 parent = pdev->bus->self;
4062 while (parent != tmp) {
4063 iommu_detach_dev(iommu, parent->bus->number,
4064 parent->devfn);
4065 parent = parent->bus->self;
4066 }
4067 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
4068 iommu_detach_dev(iommu,
4069 tmp->subordinate->number, 0);
4070 else /* this is a legacy PCI bridge */
4071 iommu_detach_dev(iommu, tmp->bus->number,
4072 tmp->devfn);
4073 }
4074} 4137}
4075 4138
4076static void domain_remove_one_dev_info(struct dmar_domain *domain, 4139static void domain_remove_one_dev_info(struct dmar_domain *domain,
@@ -4117,20 +4180,9 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
4117 spin_unlock_irqrestore(&device_domain_lock, flags); 4180 spin_unlock_irqrestore(&device_domain_lock, flags);
4118 4181
4119 if (found == 0) { 4182 if (found == 0) {
4120 unsigned long tmp_flags; 4183 domain_detach_iommu(domain, iommu);
4121 spin_lock_irqsave(&domain->iommu_lock, tmp_flags); 4184 if (!domain_type_is_vm_or_si(domain))
4122 clear_bit(iommu->seq_id, domain->iommu_bmp); 4185 iommu_detach_domain(domain, iommu);
4123 domain->iommu_count--;
4124 domain_update_iommu_cap(domain);
4125 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
4126
4127 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
4128 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
4129 spin_lock_irqsave(&iommu->lock, tmp_flags);
4130 clear_bit(domain->id, iommu->domain_ids);
4131 iommu->domains[domain->id] = NULL;
4132 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
4133 }
4134 } 4186 }
4135} 4187}
4136 4188
@@ -4150,7 +4202,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
4150 domain->iommu_snooping = 0; 4202 domain->iommu_snooping = 0;
4151 domain->iommu_superpage = 0; 4203 domain->iommu_superpage = 0;
4152 domain->max_addr = 0; 4204 domain->max_addr = 0;
4153 domain->nid = -1;
4154 4205
4155 /* always allocate the top pgd */ 4206 /* always allocate the top pgd */
4156 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid); 4207 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
@@ -4164,7 +4215,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain)
4164{ 4215{
4165 struct dmar_domain *dmar_domain; 4216 struct dmar_domain *dmar_domain;
4166 4217
4167 dmar_domain = alloc_domain(true); 4218 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4168 if (!dmar_domain) { 4219 if (!dmar_domain) {
4169 printk(KERN_ERR 4220 printk(KERN_ERR
4170 "intel_iommu_domain_init: dmar_domain == NULL\n"); 4221 "intel_iommu_domain_init: dmar_domain == NULL\n");
@@ -4202,14 +4253,18 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
4202 int addr_width; 4253 int addr_width;
4203 u8 bus, devfn; 4254 u8 bus, devfn;
4204 4255
4256 if (device_is_rmrr_locked(dev)) {
4257 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4258 return -EPERM;
4259 }
4260
4205 /* normally dev is not mapped */ 4261 /* normally dev is not mapped */
4206 if (unlikely(domain_context_mapped(dev))) { 4262 if (unlikely(domain_context_mapped(dev))) {
4207 struct dmar_domain *old_domain; 4263 struct dmar_domain *old_domain;
4208 4264
4209 old_domain = find_domain(dev); 4265 old_domain = find_domain(dev);
4210 if (old_domain) { 4266 if (old_domain) {
4211 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE || 4267 if (domain_type_is_vm_or_si(dmar_domain))
4212 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4213 domain_remove_one_dev_info(old_domain, dev); 4268 domain_remove_one_dev_info(old_domain, dev);
4214 else 4269 else
4215 domain_remove_dev_info(old_domain); 4270 domain_remove_dev_info(old_domain);
@@ -4373,99 +4428,42 @@ static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4373 return 0; 4428 return 0;
4374} 4429}
4375 4430
4376#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4377
4378static int intel_iommu_add_device(struct device *dev) 4431static int intel_iommu_add_device(struct device *dev)
4379{ 4432{
4380 struct pci_dev *pdev = to_pci_dev(dev); 4433 struct intel_iommu *iommu;
4381 struct pci_dev *bridge, *dma_pdev = NULL;
4382 struct iommu_group *group; 4434 struct iommu_group *group;
4383 int ret;
4384 u8 bus, devfn; 4435 u8 bus, devfn;
4385 4436
4386 if (!device_to_iommu(dev, &bus, &devfn)) 4437 iommu = device_to_iommu(dev, &bus, &devfn);
4438 if (!iommu)
4387 return -ENODEV; 4439 return -ENODEV;
4388 4440
4389 bridge = pci_find_upstream_pcie_bridge(pdev); 4441 iommu_device_link(iommu->iommu_dev, dev);
4390 if (bridge) {
4391 if (pci_is_pcie(bridge))
4392 dma_pdev = pci_get_domain_bus_and_slot(
4393 pci_domain_nr(pdev->bus),
4394 bridge->subordinate->number, 0);
4395 if (!dma_pdev)
4396 dma_pdev = pci_dev_get(bridge);
4397 } else
4398 dma_pdev = pci_dev_get(pdev);
4399
4400 /* Account for quirked devices */
4401 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4402
4403 /*
4404 * If it's a multifunction device that does not support our
4405 * required ACS flags, add to the same group as lowest numbered
4406 * function that also does not suport the required ACS flags.
4407 */
4408 if (dma_pdev->multifunction &&
4409 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4410 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4411
4412 for (i = 0; i < 8; i++) {
4413 struct pci_dev *tmp;
4414
4415 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4416 if (!tmp)
4417 continue;
4418
4419 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4420 swap_pci_ref(&dma_pdev, tmp);
4421 break;
4422 }
4423 pci_dev_put(tmp);
4424 }
4425 }
4426
4427 /*
4428 * Devices on the root bus go through the iommu. If that's not us,
4429 * find the next upstream device and test ACS up to the root bus.
4430 * Finding the next device may require skipping virtual buses.
4431 */
4432 while (!pci_is_root_bus(dma_pdev->bus)) {
4433 struct pci_bus *bus = dma_pdev->bus;
4434
4435 while (!bus->self) {
4436 if (!pci_is_root_bus(bus))
4437 bus = bus->parent;
4438 else
4439 goto root_bus;
4440 }
4441
4442 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4443 break;
4444
4445 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4446 }
4447 4442
4448root_bus: 4443 group = iommu_group_get_for_dev(dev);
4449 group = iommu_group_get(&dma_pdev->dev);
4450 pci_dev_put(dma_pdev);
4451 if (!group) {
4452 group = iommu_group_alloc();
4453 if (IS_ERR(group))
4454 return PTR_ERR(group);
4455 }
4456 4444
4457 ret = iommu_group_add_device(group, dev); 4445 if (IS_ERR(group))
4446 return PTR_ERR(group);
4458 4447
4459 iommu_group_put(group); 4448 iommu_group_put(group);
4460 return ret; 4449 return 0;
4461} 4450}
4462 4451
4463static void intel_iommu_remove_device(struct device *dev) 4452static void intel_iommu_remove_device(struct device *dev)
4464{ 4453{
4454 struct intel_iommu *iommu;
4455 u8 bus, devfn;
4456
4457 iommu = device_to_iommu(dev, &bus, &devfn);
4458 if (!iommu)
4459 return;
4460
4465 iommu_group_remove_device(dev); 4461 iommu_group_remove_device(dev);
4462
4463 iommu_device_unlink(iommu->iommu_dev, dev);
4466} 4464}
4467 4465
4468static struct iommu_ops intel_iommu_ops = { 4466static const struct iommu_ops intel_iommu_ops = {
4469 .domain_init = intel_iommu_domain_init, 4467 .domain_init = intel_iommu_domain_init,
4470 .domain_destroy = intel_iommu_domain_destroy, 4468 .domain_destroy = intel_iommu_domain_destroy,
4471 .attach_dev = intel_iommu_attach_device, 4469 .attach_dev = intel_iommu_attach_device,
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-07 16:48:03 -0500