From 74315cccd2104a953f493acca2c6b0519d6f5c6f Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 8 Jun 2011 17:29:11 -0400 Subject: iommu-api: Add missing header file If CONFIG_IOMMU_API is not defined some functions will just return -ENODEV. Add errno.h for the definition of ENODEV. Signed-off-by: Laura Abbott Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0a2ba4098996..9940319d6f9d 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -19,6 +19,8 @@ #ifndef __LINUX_IOMMU_H #define __LINUX_IOMMU_H +#include + #define IOMMU_READ (1) #define IOMMU_WRITE (2) #define IOMMU_CACHE (4) /* DMA cache coherency */ -- cgit v1.2.2 From 39c555460cbc6d35656878c89d4664fbd6c81551 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 11:52:27 +0200 Subject: x86/amd-iommu: Remove redundant device_flush_dte() calls Remove these function calls from places where the function has already been called by another function. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 7c3a95e54ec5..cc6f6da630e8 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1798,7 +1798,6 @@ static int device_change_notifier(struct notifier_block *nb, goto out; } - device_flush_dte(dev); iommu_completion_wait(iommu); out: @@ -2605,7 +2604,6 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, if (!iommu) return; - device_flush_dte(dev); iommu_completion_wait(iommu); } -- cgit v1.2.2 From 8fa5f802abf3cd374b5f07418cea72c5d9d204cc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:24:45 +0200 Subject: x86/amd-iommu: Introduce global dev_data_list This list keeps all allocated iommu_dev_data structs in a list together. This is needed for instances that have no associated device. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 56 ++++++++++++++++++++++++++++------ 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 4c9982995414..35520e31c1b4 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -310,6 +310,7 @@ struct protection_domain { */ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ + struct list_head dev_data_list; /* For global dev_data_list */ struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index cc6f6da630e8..8b8489084649 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -45,6 +45,10 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); static LIST_HEAD(iommu_pd_list); static DEFINE_SPINLOCK(iommu_pd_list_lock); +/* List of all available dev_data structures */ +static LIST_HEAD(dev_data_list); +static DEFINE_SPINLOCK(dev_data_list_lock); + /* * Domain for untranslated devices - only allocated * if iommu=pt passed on kernel cmd line. @@ -68,6 +72,35 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ +static struct iommu_dev_data *alloc_dev_data(void) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return NULL; + + atomic_set(&dev_data->bind, 0); + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_add_tail(&dev_data->dev_data_list, &dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static void free_dev_data(struct iommu_dev_data *dev_data) +{ + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_del(&dev_data->dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + kfree(dev_data); +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -139,32 +172,28 @@ static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; struct pci_dev *pdev; - u16 devid, alias; + u16 alias; if (dev->archdata.iommu) return 0; - dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + dev_data = alloc_dev_data(); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - devid = get_device_id(dev); - alias = amd_iommu_alias_table[devid]; + alias = amd_iommu_alias_table[get_device_id(dev)]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; else { - kfree(dev_data); + free_dev_data(dev_data); return -ENOTSUPP; } - atomic_set(&dev_data->bind, 0); - dev->archdata.iommu = dev_data; - return 0; } @@ -184,11 +213,16 @@ static void iommu_ignore_device(struct device *dev) static void iommu_uninit_device(struct device *dev) { - kfree(dev->archdata.iommu); + /* + * Nothing to do here - we keep dev_data around for unplugged devices + * and reuse it when the device is re-plugged - not doing so would + * introduce a ton of races. + */ } void __init amd_iommu_uninit_devices(void) { + struct iommu_dev_data *dev_data, *n; struct pci_dev *pdev = NULL; for_each_pci_dev(pdev) { @@ -198,6 +232,10 @@ void __init amd_iommu_uninit_devices(void) iommu_uninit_device(&pdev->dev); } + + /* Free all of our dev_data structures */ + list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) + free_dev_data(dev_data); } int __init amd_iommu_init_devices(void) -- cgit v1.2.2 From f62dda66b5601396c28b563cc18b481af04a91b3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:55:35 +0200 Subject: x86/amd-iommu: Store devid in dev_data This allows to use dev_data independent of struct device later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 39 +++++++++++++--------------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 35520e31c1b4..2833862311b5 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -315,6 +315,7 @@ struct iommu_dev_data { struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ + u16 devid; /* PCI Device ID */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 8b8489084649..4e8b176297b5 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -72,7 +72,7 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ -static struct iommu_dev_data *alloc_dev_data(void) +static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; unsigned long flags; @@ -81,6 +81,7 @@ static struct iommu_dev_data *alloc_dev_data(void) if (!dev_data) return NULL; + dev_data->devid = devid; atomic_set(&dev_data->bind, 0); spin_lock_irqsave(&dev_data_list_lock, flags); @@ -177,13 +178,13 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(); + dev_data = alloc_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - alias = amd_iommu_alias_table[get_device_id(dev)]; + alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; @@ -714,16 +715,16 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) */ static int device_flush_dte(struct device *dev) { + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct pci_dev *pdev; - u16 devid; int ret; - pdev = to_pci_dev(dev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + pdev = to_pci_dev(dev); + dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; - ret = iommu_flush_dte(iommu, devid); + ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; @@ -1570,11 +1571,9 @@ static void do_attach(struct device *dev, struct protection_domain *domain) struct amd_iommu *iommu; struct pci_dev *pdev; bool ats = false; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); if (amd_iommu_iotlb_sup) @@ -1583,7 +1582,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) /* Update data structures */ dev_data->domain = domain; list_add(&dev_data->list, &domain->dev_list); - set_dte_entry(devid, domain, ats); + set_dte_entry(dev_data->devid, domain, ats); /* Do reference counting */ domain->dev_iommu[iommu->index] += 1; @@ -1597,11 +1596,9 @@ static void do_detach(struct device *dev) { struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1610,7 +1607,7 @@ static void do_detach(struct device *dev) /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(devid); + clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ device_flush_dte(dev); @@ -1760,9 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; unsigned long flags; - u16 devid; - devid = get_device_id(dev); dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1897,8 +1892,7 @@ static void update_device_table(struct protection_domain *domain) list_for_each_entry(dev_data, &domain->dev_list, list) { struct pci_dev *pdev = to_pci_dev(dev_data->dev); - u16 devid = get_device_id(dev_data->dev); - set_dte_entry(devid, domain, pci_ats_enabled(pdev)); + set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); } } @@ -2652,16 +2646,13 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct iommu_dev_data *dev_data; struct amd_iommu *iommu; int ret; - u16 devid; if (!check_device(dev)) return -EINVAL; dev_data = dev->archdata.iommu; - devid = get_device_id(dev); - - iommu = amd_iommu_rlookup_table[devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; if (!iommu) return -EINVAL; -- cgit v1.2.2 From ea61cddb9dcb9da9bcabd40f1620c24abaf645c9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:56:30 +0200 Subject: x86/amd-iommu: Store ATS state in dev_data This allows the low-level functions to operate on dev_data exclusivly later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 4 ++++ arch/x86/kernel/amd_iommu.c | 44 ++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 2833862311b5..9fc045ee2fcb 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -316,6 +316,10 @@ struct iommu_dev_data { struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ + struct { + bool enabled; + int qdep; + } ats; /* ATS state */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 4e8b176297b5..9e07ef65a016 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -695,17 +695,16 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) */ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; - u16 devid; int qdep; - qdep = pci_ats_queue_depth(pdev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + dev_data = get_dev_data(dev); + qdep = dev_data->ats.qdep; + iommu = amd_iommu_rlookup_table[dev_data->devid]; - build_inv_iotlb_pages(&cmd, devid, qdep, address, size); + build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size); return iommu_queue_command(iommu, &cmd); } @@ -728,7 +727,7 @@ static int device_flush_dte(struct device *dev) if (ret) return ret; - if (pci_ats_enabled(pdev)) + if (dev_data->ats.enabled) ret = device_flush_iotlb(dev, 0, ~0UL); return ret; @@ -760,9 +759,8 @@ static void __domain_flush_pages(struct protection_domain *domain, } list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - if (!pci_ats_enabled(pdev)) + if (!dev_data->ats.enabled) continue; ret |= device_flush_iotlb(dev_data->dev, address, size); @@ -1576,8 +1574,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); - if (amd_iommu_iotlb_sup) - ats = pci_ats_enabled(pdev); + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1674,11 +1671,16 @@ static int attach_device(struct device *dev, struct protection_domain *domain) { struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; int ret; - if (amd_iommu_iotlb_sup) - pci_enable_ats(pdev, PAGE_SHIFT); + dev_data = get_dev_data(dev); + + if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) { + dev_data->ats.enabled = true; + dev_data->ats.qdep = pci_ats_queue_depth(pdev); + } write_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev, domain); @@ -1736,7 +1738,7 @@ static void __detach_device(struct device *dev) */ static void detach_device(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; /* lock device table */ @@ -1744,8 +1746,12 @@ static void detach_device(struct device *dev) __detach_device(dev); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) - pci_disable_ats(pdev); + dev_data = get_dev_data(dev); + + if (dev_data->ats.enabled) { + pci_disable_ats(to_pci_dev(dev)); + dev_data->ats.enabled = false; + } } /* @@ -1890,10 +1896,8 @@ static void update_device_table(struct protection_domain *domain) { struct iommu_dev_data *dev_data; - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); - } + list_for_each_entry(dev_data, &domain->dev_list, list) + set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled); } static void update_domain(struct protection_domain *domain) -- cgit v1.2.2 From 6c542047937ad66db1d31e290fd2a3b290424ffa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:07:31 +0200 Subject: x86/amd-iommu: Use only dev_data for dte and iotlb flushing routines This patch make the functions flushing the DTE and IOTLBs only take the dev_data structure instead of the struct device directly. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 9e07ef65a016..0a5b46504b5d 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -693,14 +693,13 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) /* * Command send function for flushing on-device TLB */ -static int device_flush_iotlb(struct device *dev, u64 address, size_t size) +static int device_flush_iotlb(struct iommu_dev_data *dev_data, + u64 address, size_t size) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; int qdep; - dev_data = get_dev_data(dev); qdep = dev_data->ats.qdep; iommu = amd_iommu_rlookup_table[dev_data->devid]; @@ -712,23 +711,19 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) /* * Command send function for invalidating a device table entry */ -static int device_flush_dte(struct device *dev) +static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; int ret; - pdev = to_pci_dev(dev); - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; if (dev_data->ats.enabled) - ret = device_flush_iotlb(dev, 0, ~0UL); + ret = device_flush_iotlb(dev_data, 0, ~0UL); return ret; } @@ -763,7 +758,7 @@ static void __domain_flush_pages(struct protection_domain *domain, if (!dev_data->ats.enabled) continue; - ret |= device_flush_iotlb(dev_data->dev, address, size); + ret |= device_flush_iotlb(dev_data, address, size); } WARN_ON(ret); @@ -815,7 +810,7 @@ static void domain_flush_devices(struct protection_domain *domain) spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(dev_data, &domain->dev_list, list) - device_flush_dte(dev_data->dev); + device_flush_dte(dev_data); spin_unlock_irqrestore(&domain->lock, flags); } @@ -1586,7 +1581,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) domain->dev_cnt += 1; /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } static void do_detach(struct device *dev) @@ -1607,7 +1602,7 @@ static void do_detach(struct device *dev) clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } /* -- cgit v1.2.2 From ec9e79ef062272a119994c4fdd5e434f3e1bf352 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:25:50 +0200 Subject: x86/amd-iommu: Use only dev_data in low-level domain attach/detach functions With this patch the low-level attach/detach functions only work on dev_data structures. This allows to remove the dev_data->dev pointer. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 - arch/x86/kernel/amd_iommu.c | 61 +++++++++++++--------------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 9fc045ee2fcb..fffbe3c9e6c2 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,6 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 0a5b46504b5d..a15a17231909 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -182,8 +182,6 @@ static int iommu_init_device(struct device *dev) if (!dev_data) return -ENOMEM; - dev_data->dev = dev; - alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) @@ -1558,18 +1556,14 @@ static void clear_dte_entry(u16 devid) amd_iommu_apply_erratum_63(devid); } -static void do_attach(struct device *dev, struct protection_domain *domain) +static void do_attach(struct iommu_dev_data *dev_data, + struct protection_domain *domain) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; - bool ats = false; - - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; - pdev = to_pci_dev(dev); + bool ats; - ats = dev_data->ats.enabled; + iommu = amd_iommu_rlookup_table[dev_data->devid]; + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1584,13 +1578,11 @@ static void do_attach(struct device *dev, struct protection_domain *domain) device_flush_dte(dev_data); } -static void do_detach(struct device *dev) +static void do_detach(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1609,13 +1601,12 @@ static void do_detach(struct device *dev) * If a device is not yet associated with a domain, this function does * assigns it visible for the hardware */ -static int __attach_device(struct device *dev, +static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *alias_data; int ret; - dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1635,16 +1626,15 @@ static int __attach_device(struct device *dev, goto out_unlock; /* Do real assignment */ - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (alias_data->domain == NULL) - do_attach(dev_data->alias, domain); + do_attach(alias_data, domain); atomic_inc(&alias_data->bind); } if (dev_data->domain == NULL) - do_attach(dev, domain); + do_attach(dev_data, domain); atomic_inc(&dev_data->bind); @@ -1678,7 +1668,7 @@ static int attach_device(struct device *dev, } write_lock_irqsave(&amd_iommu_devtable_lock, flags); - ret = __attach_device(dev, domain); + ret = __attach_device(dev_data, domain); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); /* @@ -1694,9 +1684,8 @@ static int attach_device(struct device *dev, /* * Removes a device from a protection domain (unlocked) */ -static void __detach_device(struct device *dev) +static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data = get_dev_data(dev); struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1707,14 +1696,14 @@ static void __detach_device(struct device *dev) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (atomic_dec_and_test(&alias_data->bind)) - do_detach(dev_data->alias); + do_detach(alias_data); } if (atomic_dec_and_test(&dev_data->bind)) - do_detach(dev); + do_detach(dev_data); spin_unlock_irqrestore(&domain->lock, flags); @@ -1725,7 +1714,7 @@ static void __detach_device(struct device *dev) */ if (iommu_pass_through && (dev_data->domain == NULL && domain != pt_domain)) - __attach_device(dev, pt_domain); + __attach_device(dev_data, pt_domain); } /* @@ -1736,13 +1725,13 @@ static void detach_device(struct device *dev) struct iommu_dev_data *dev_data; unsigned long flags; + dev_data = get_dev_data(dev); + /* lock device table */ write_lock_irqsave(&amd_iommu_devtable_lock, flags); - __detach_device(dev); + __detach_device(dev_data); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - dev_data = get_dev_data(dev); - if (dev_data->ats.enabled) { pci_disable_ats(to_pci_dev(dev)); dev_data->ats.enabled = false; @@ -1768,7 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) dom = dev_data->domain; if (dom == NULL && alias_data->domain != NULL) { - __attach_device(dev, alias_data->domain); + __attach_device(dev_data, alias_data->domain); dom = alias_data->domain; } @@ -2527,9 +2516,7 @@ static void cleanup_domain(struct protection_domain *domain) write_lock_irqsave(&amd_iommu_devtable_lock, flags); list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { - struct device *dev = dev_data->dev; - - __detach_device(dev); + __detach_device(dev_data); atomic_set(&dev_data->bind, 0); } -- cgit v1.2.2 From 2b02b091ab3d1685926bfc76486bf7a22d92979a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:48:39 +0200 Subject: x86/amd-iommu: Allow dev_data->alias to be NULL Let dev_data->alias be just NULL if the device has no alias. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 77 ++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index a15a17231909..2d914a49b19a 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -183,12 +183,14 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; alias = amd_iommu_alias_table[dev_data->devid]; - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { - free_dev_data(dev_data); - return -ENOTSUPP; + if (alias != dev_data->devid) { + pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); + if (pdev) + dev_data->alias = &pdev->dev; + else { + free_dev_data(dev_data); + return -ENOTSUPP; + } } dev->archdata.iommu = dev_data; @@ -1604,29 +1606,31 @@ static void do_detach(struct iommu_dev_data *dev_data) static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *alias_data; int ret; - alias_data = get_dev_data(dev_data->alias); - - if (!alias_data) - return -EINVAL; - /* lock domain */ spin_lock(&domain->lock); - /* Some sanity checks */ - ret = -EBUSY; - if (alias_data->domain != NULL && - alias_data->domain != domain) - goto out_unlock; + if (dev_data->alias != NULL) { + struct iommu_dev_data *alias_data; + + alias_data = get_dev_data(dev_data->alias); + + ret = -EINVAL; + if (!alias_data) + goto out_unlock; - if (dev_data->domain != NULL && - dev_data->domain != domain) - goto out_unlock; + /* Some sanity checks */ + ret = -EBUSY; + if (alias_data->domain != NULL && + alias_data->domain != domain) + goto out_unlock; - /* Do real assignment */ - if (dev_data->devid != alias_data->devid) { + if (dev_data->domain != NULL && + dev_data->domain != domain) + goto out_unlock; + + /* Do real assignment */ if (alias_data->domain == NULL) do_attach(alias_data, domain); @@ -1696,8 +1700,8 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - alias_data = get_dev_data(dev_data->alias); - if (dev_data->devid != alias_data->devid) { + if (dev_data->alias) { + alias_data = get_dev_data(dev_data->alias); if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1744,24 +1748,25 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; + struct protection_domain *dom = NULL; unsigned long flags; dev_data = get_dev_data(dev); - alias_data = get_dev_data(dev_data->alias); - if (!alias_data) - return NULL; - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - dom = dev_data->domain; - if (dom == NULL && - alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } + if (dev_data->domain) + return dev_data->domain; - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + if (dev_data->alias != NULL) { + alias_data = get_dev_data(dev_data->alias); + + read_lock_irqsave(&amd_iommu_devtable_lock, flags); + if (alias_data->domain != NULL) { + __attach_device(dev_data, alias_data->domain); + dom = alias_data->domain; + } + read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + } return dom; } -- cgit v1.2.2 From 3b03bb745eb9fcafc6ef80fcbd04f0f0512acdc9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 18:53:25 +0200 Subject: x86/amd-iommu: Search for existind dev_data before allocting a new one Search for existing dev_data first will allow to switch dev_data->alias to just store dev_data instead of struct device. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 2d914a49b19a..f64799878d45 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -102,6 +102,37 @@ static void free_dev_data(struct iommu_dev_data *dev_data) kfree(dev_data); } +static struct iommu_dev_data *search_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { + if (dev_data->devid == devid) + goto out_unlock; + } + + dev_data = NULL; + +out_unlock: + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static struct iommu_dev_data *find_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + + dev_data = search_dev_data(devid); + + if (dev_data == NULL) + dev_data = alloc_dev_data(devid); + + return dev_data; +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -178,7 +209,7 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(get_device_id(dev)); + dev_data = find_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; -- cgit v1.2.2 From 71f77580906b60eb43daff0efb99792e76a3d06d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 19:03:15 +0200 Subject: x86/amd-iommu: Store device alias as dev_data pointer This finally allows PCI-Device-IDs to be handled by the IOMMU driver that have no corresponding struct device present in the system. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 2 +- arch/x86/kernel/amd_iommu.c | 34 +++++++++++++++------------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index fffbe3c9e6c2..5b9c5075e81a 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,7 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *alias; /* The Alias Device */ + struct iommu_dev_data *alias_data;/* The alias dev_data */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index f64799878d45..0ed1c940d76b 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -203,7 +203,6 @@ static bool check_device(struct device *dev) static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct pci_dev *pdev; u16 alias; if (dev->archdata.iommu) @@ -215,13 +214,16 @@ static int iommu_init_device(struct device *dev) alias = amd_iommu_alias_table[dev_data->devid]; if (alias != dev_data->devid) { - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { + struct iommu_dev_data *alias_data; + + alias_data = find_dev_data(alias); + if (alias_data == NULL) { + pr_err("AMD-Vi: Warning: Unhandled device %s\n", + dev_name(dev)); free_dev_data(dev_data); return -ENOTSUPP; } + dev_data->alias_data = alias_data; } dev->archdata.iommu = dev_data; @@ -1642,14 +1644,8 @@ static int __attach_device(struct iommu_dev_data *dev_data, /* lock domain */ spin_lock(&domain->lock); - if (dev_data->alias != NULL) { - struct iommu_dev_data *alias_data; - - alias_data = get_dev_data(dev_data->alias); - - ret = -EINVAL; - if (!alias_data) - goto out_unlock; + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; /* Some sanity checks */ ret = -EBUSY; @@ -1721,7 +1717,6 @@ static int attach_device(struct device *dev, */ static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1731,8 +1726,9 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; + if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1779,7 +1775,7 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *dev_data; struct protection_domain *dom = NULL; unsigned long flags; @@ -1788,8 +1784,8 @@ static struct protection_domain *domain_for_device(struct device *dev) if (dev_data->domain) return dev_data->domain; - if (dev_data->alias != NULL) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; read_lock_irqsave(&amd_iommu_devtable_lock, flags); if (alias_data->domain != NULL) { -- cgit v1.2.2 From ab493a0f0f55d28636ac860ea682d57b84257f10 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Thu, 2 Jun 2011 02:48:05 +0300 Subject: drivers: iommu: move to a dedicated folder Create a dedicated folder for iommu drivers, and move the base iommu implementation over there. Grouping the various iommu drivers in a single location will help finding similar problems shared by different platforms, so they could be solved once, in the iommu framework, instead of solved differently (or duplicated) in each driver. Signed-off-by: Ohad Ben-Cohen Signed-off-by: Joerg Roedel --- arch/arm/mach-msm/Kconfig | 3 -- arch/ia64/Kconfig | 3 -- arch/x86/Kconfig | 5 +- drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/base/Makefile | 1 - drivers/base/iommu.c | 124 ---------------------------------------------- drivers/iommu/Kconfig | 3 ++ drivers/iommu/Makefile | 1 + drivers/iommu/iommu.c | 124 ++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 133 insertions(+), 134 deletions(-) delete mode 100644 drivers/base/iommu.c create mode 100644 drivers/iommu/Kconfig create mode 100644 drivers/iommu/Makefile create mode 100644 drivers/iommu/iommu.c diff --git a/arch/arm/mach-msm/Kconfig b/arch/arm/mach-msm/Kconfig index 1516896e8d17..efb7b7dfc20b 100644 --- a/arch/arm/mach-msm/Kconfig +++ b/arch/arm/mach-msm/Kconfig @@ -205,9 +205,6 @@ config MSM_GPIOMUX config MSM_V2_TLMM bool -config IOMMU_API - bool - config MSM_SCM bool endif diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 38280ef4a2af..9929e4e11ea0 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -681,6 +681,3 @@ source "lib/Kconfig" config IOMMU_HELPER def_bool (IA64_HP_ZX1 || IA64_HP_ZX1_SWIOTLB || IA64_GENERIC || SWIOTLB) - -config IOMMU_API - def_bool (DMAR) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index da349723d411..460d57370016 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -685,6 +685,7 @@ config AMD_IOMMU select SWIOTLB select PCI_MSI select PCI_IOV + select IOMMU_API depends on X86_64 && PCI && ACPI ---help--- With this option you can enable support for AMD IOMMU hardware in @@ -720,9 +721,6 @@ config SWIOTLB config IOMMU_HELPER def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU) -config IOMMU_API - def_bool (AMD_IOMMU || DMAR) - config MAXSMP bool "Enable Maximum number of SMP Processors and NUMA Nodes" depends on X86_64 && SMP && DEBUG_KERNEL && EXPERIMENTAL @@ -1945,6 +1943,7 @@ config PCI_CNB20LE_QUIRK config DMAR bool "Support for DMA Remapping Devices (EXPERIMENTAL)" depends on PCI_MSI && ACPI && EXPERIMENTAL + select IOMMU_API help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. diff --git a/drivers/Kconfig b/drivers/Kconfig index 3bb154d8c8cc..9d513188b47a 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -126,4 +126,6 @@ source "drivers/hwspinlock/Kconfig" source "drivers/clocksource/Kconfig" +source "drivers/iommu/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 09f3232bcdcd..2f7a71a933de 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -122,3 +122,4 @@ obj-y += ieee802154/ obj-y += clk/ obj-$(CONFIG_HWSPINLOCK) += hwspinlock/ +obj-$(CONFIG_IOMMU_API) += iommu/ diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 4c5701c15f53..5ab0d07c4578 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -13,7 +13,6 @@ obj-$(CONFIG_FW_LOADER) += firmware_class.o obj-$(CONFIG_NUMA) += node.o obj-$(CONFIG_MEMORY_HOTPLUG_SPARSE) += memory.o obj-$(CONFIG_SMP) += topology.o -obj-$(CONFIG_IOMMU_API) += iommu.o ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_MODULES) += module.o endif diff --git a/drivers/base/iommu.c b/drivers/base/iommu.c deleted file mode 100644 index 6e6b6a11b3ce..000000000000 --- a/drivers/base/iommu.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. - * Author: Joerg Roedel - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#include -#include -#include -#include -#include -#include - -static struct iommu_ops *iommu_ops; - -void register_iommu(struct iommu_ops *ops) -{ - if (iommu_ops) - BUG(); - - iommu_ops = ops; -} - -bool iommu_found(void) -{ - return iommu_ops != NULL; -} -EXPORT_SYMBOL_GPL(iommu_found); - -struct iommu_domain *iommu_domain_alloc(void) -{ - struct iommu_domain *domain; - int ret; - - domain = kmalloc(sizeof(*domain), GFP_KERNEL); - if (!domain) - return NULL; - - ret = iommu_ops->domain_init(domain); - if (ret) - goto out_free; - - return domain; - -out_free: - kfree(domain); - - return NULL; -} -EXPORT_SYMBOL_GPL(iommu_domain_alloc); - -void iommu_domain_free(struct iommu_domain *domain) -{ - iommu_ops->domain_destroy(domain); - kfree(domain); -} -EXPORT_SYMBOL_GPL(iommu_domain_free); - -int iommu_attach_device(struct iommu_domain *domain, struct device *dev) -{ - return iommu_ops->attach_dev(domain, dev); -} -EXPORT_SYMBOL_GPL(iommu_attach_device); - -void iommu_detach_device(struct iommu_domain *domain, struct device *dev) -{ - iommu_ops->detach_dev(domain, dev); -} -EXPORT_SYMBOL_GPL(iommu_detach_device); - -phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, - unsigned long iova) -{ - return iommu_ops->iova_to_phys(domain, iova); -} -EXPORT_SYMBOL_GPL(iommu_iova_to_phys); - -int iommu_domain_has_cap(struct iommu_domain *domain, - unsigned long cap) -{ - return iommu_ops->domain_has_cap(domain, cap); -} -EXPORT_SYMBOL_GPL(iommu_domain_has_cap); - -int iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, int gfp_order, int prot) -{ - unsigned long invalid_mask; - size_t size; - - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; - - BUG_ON((iova | paddr) & invalid_mask); - - return iommu_ops->map(domain, iova, paddr, gfp_order, prot); -} -EXPORT_SYMBOL_GPL(iommu_map); - -int iommu_unmap(struct iommu_domain *domain, unsigned long iova, int gfp_order) -{ - unsigned long invalid_mask; - size_t size; - - size = 0x1000UL << gfp_order; - invalid_mask = size - 1; - - BUG_ON(iova & invalid_mask); - - return iommu_ops->unmap(domain, iova, gfp_order); -} -EXPORT_SYMBOL_GPL(iommu_unmap); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig new file mode 100644 index 000000000000..2c5dfb48a22a --- /dev/null +++ b/drivers/iommu/Kconfig @@ -0,0 +1,3 @@ +# IOMMU_API always gets selected by whoever wants it. +config IOMMU_API + bool diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile new file mode 100644 index 000000000000..241ba4c46a19 --- /dev/null +++ b/drivers/iommu/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_IOMMU_API) += iommu.o diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c new file mode 100644 index 000000000000..6e6b6a11b3ce --- /dev/null +++ b/drivers/iommu/iommu.c @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. + * Author: Joerg Roedel + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include +#include +#include +#include +#include + +static struct iommu_ops *iommu_ops; + +void register_iommu(struct iommu_ops *ops) +{ + if (iommu_ops) + BUG(); + + iommu_ops = ops; +} + +bool iommu_found(void) +{ + return iommu_ops != NULL; +} +EXPORT_SYMBOL_GPL(iommu_found); + +struct iommu_domain *iommu_domain_alloc(void) +{ + struct iommu_domain *domain; + int ret; + + domain = kmalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) + return NULL; + + ret = iommu_ops->domain_init(domain); + if (ret) + goto out_free; + + return domain; + +out_free: + kfree(domain); + + return NULL; +} +EXPORT_SYMBOL_GPL(iommu_domain_alloc); + +void iommu_domain_free(struct iommu_domain *domain) +{ + iommu_ops->domain_destroy(domain); + kfree(domain); +} +EXPORT_SYMBOL_GPL(iommu_domain_free); + +int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + return iommu_ops->attach_dev(domain, dev); +} +EXPORT_SYMBOL_GPL(iommu_attach_device); + +void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + iommu_ops->detach_dev(domain, dev); +} +EXPORT_SYMBOL_GPL(iommu_detach_device); + +phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, + unsigned long iova) +{ + return iommu_ops->iova_to_phys(domain, iova); +} +EXPORT_SYMBOL_GPL(iommu_iova_to_phys); + +int iommu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + return iommu_ops->domain_has_cap(domain, cap); +} +EXPORT_SYMBOL_GPL(iommu_domain_has_cap); + +int iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, int gfp_order, int prot) +{ + unsigned long invalid_mask; + size_t size; + + size = 0x1000UL << gfp_order; + invalid_mask = size - 1; + + BUG_ON((iova | paddr) & invalid_mask); + + return iommu_ops->map(domain, iova, paddr, gfp_order, prot); +} +EXPORT_SYMBOL_GPL(iommu_map); + +int iommu_unmap(struct iommu_domain *domain, unsigned long iova, int gfp_order) +{ + unsigned long invalid_mask; + size_t size; + + size = 0x1000UL << gfp_order; + invalid_mask = size - 1; + + BUG_ON(iova & invalid_mask); + + return iommu_ops->unmap(domain, iova, gfp_order); +} +EXPORT_SYMBOL_GPL(iommu_unmap); -- cgit v1.2.2 From b10f127e1a4d8cac5414c6e2b152c205b66c9f16 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Thu, 2 Jun 2011 03:20:08 +0300 Subject: msm: iommu: move to drivers/iommu/ This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Compile-tested for MSM8X60. Signed-off-by: Ohad Ben-Cohen Acked-by: David Brown Signed-off-by: Joerg Roedel --- arch/arm/mach-msm/Kconfig | 16 - arch/arm/mach-msm/Makefile | 2 +- arch/arm/mach-msm/iommu.c | 731 ------------------------------------------ arch/arm/mach-msm/iommu_dev.c | 422 ------------------------ drivers/iommu/Kconfig | 16 + drivers/iommu/Makefile | 1 + drivers/iommu/msm_iommu.c | 731 ++++++++++++++++++++++++++++++++++++++++++ drivers/iommu/msm_iommu_dev.c | 422 ++++++++++++++++++++++++ 8 files changed, 1171 insertions(+), 1170 deletions(-) delete mode 100644 arch/arm/mach-msm/iommu.c delete mode 100644 arch/arm/mach-msm/iommu_dev.c create mode 100644 drivers/iommu/msm_iommu.c create mode 100644 drivers/iommu/msm_iommu_dev.c diff --git a/arch/arm/mach-msm/Kconfig b/arch/arm/mach-msm/Kconfig index efb7b7dfc20b..888e92502e15 100644 --- a/arch/arm/mach-msm/Kconfig +++ b/arch/arm/mach-msm/Kconfig @@ -148,22 +148,6 @@ config MACH_MSM8960_RUMI3 endmenu -config MSM_IOMMU - bool "MSM IOMMU Support" - depends on ARCH_MSM8X60 || ARCH_MSM8960 - select IOMMU_API - default n - help - Support for the IOMMUs found on certain Qualcomm SOCs. - These IOMMUs allow virtualization of the address space used by most - cores within the multimedia subsystem. - - If unsure, say N here. - -config IOMMU_PGTABLES_L2 - def_bool y - depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n - config MSM_DEBUG_UART int default 1 if MSM_DEBUG_UART1 diff --git a/arch/arm/mach-msm/Makefile b/arch/arm/mach-msm/Makefile index 9519fd28a025..b70658c5ae00 100644 --- a/arch/arm/mach-msm/Makefile +++ b/arch/arm/mach-msm/Makefile @@ -3,7 +3,7 @@ obj-y += clock.o obj-$(CONFIG_DEBUG_FS) += clock-debug.o obj-$(CONFIG_MSM_VIC) += irq-vic.o -obj-$(CONFIG_MSM_IOMMU) += iommu.o iommu_dev.o devices-iommu.o +obj-$(CONFIG_MSM_IOMMU) += devices-iommu.o obj-$(CONFIG_ARCH_MSM7X00A) += dma.o irq.o acpuclock-arm11.o obj-$(CONFIG_ARCH_MSM7X30) += dma.o diff --git a/arch/arm/mach-msm/iommu.c b/arch/arm/mach-msm/iommu.c deleted file mode 100644 index 1a584e077c61..000000000000 --- a/arch/arm/mach-msm/iommu.c +++ /dev/null @@ -1,731 +0,0 @@ -/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#define MRC(reg, processor, op1, crn, crm, op2) \ -__asm__ __volatile__ ( \ -" mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ -: "=r" (reg)) - -#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) -#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) - -static int msm_iommu_tex_class[4]; - -DEFINE_SPINLOCK(msm_iommu_lock); - -struct msm_priv { - unsigned long *pgtable; - struct list_head list_attached; -}; - -static int __enable_clocks(struct msm_iommu_drvdata *drvdata) -{ - int ret; - - ret = clk_enable(drvdata->pclk); - if (ret) - goto fail; - - if (drvdata->clk) { - ret = clk_enable(drvdata->clk); - if (ret) - clk_disable(drvdata->pclk); - } -fail: - return ret; -} - -static void __disable_clocks(struct msm_iommu_drvdata *drvdata) -{ - if (drvdata->clk) - clk_disable(drvdata->clk); - clk_disable(drvdata->pclk); -} - -static int __flush_iotlb(struct iommu_domain *domain) -{ - struct msm_priv *priv = domain->priv; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - int ret = 0; -#ifndef CONFIG_IOMMU_PGTABLES_L2 - unsigned long *fl_table = priv->pgtable; - int i; - - if (!list_empty(&priv->list_attached)) { - dmac_flush_range(fl_table, fl_table + SZ_16K); - - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { - void *sl_table = __va(fl_table[i] & - FL_BASE_MASK); - dmac_flush_range(sl_table, sl_table + SZ_4K); - } - } -#endif - - list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { - if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) - BUG(); - - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); - BUG_ON(!iommu_drvdata); - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); - __disable_clocks(iommu_drvdata); - } -fail: - return ret; -} - -static void __reset_context(void __iomem *base, int ctx) -{ - SET_BPRCOSH(base, ctx, 0); - SET_BPRCISH(base, ctx, 0); - SET_BPRCNSH(base, ctx, 0); - SET_BPSHCFG(base, ctx, 0); - SET_BPMTCFG(base, ctx, 0); - SET_ACTLR(base, ctx, 0); - SET_SCTLR(base, ctx, 0); - SET_FSRRESTORE(base, ctx, 0); - SET_TTBR0(base, ctx, 0); - SET_TTBR1(base, ctx, 0); - SET_TTBCR(base, ctx, 0); - SET_BFBCR(base, ctx, 0); - SET_PAR(base, ctx, 0); - SET_FAR(base, ctx, 0); - SET_CTX_TLBIALL(base, ctx, 0); - SET_TLBFLPTER(base, ctx, 0); - SET_TLBSLPTER(base, ctx, 0); - SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); -} - -static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) -{ - unsigned int prrr, nmrr; - __reset_context(base, ctx); - - /* Set up HTW mode */ - /* TLB miss configuration: perform HTW on miss */ - SET_TLBMCFG(base, ctx, 0x3); - - /* V2P configuration: HTW for access */ - SET_V2PCFG(base, ctx, 0x3); - - SET_TTBCR(base, ctx, 0); - SET_TTBR0_PA(base, ctx, (pgtable >> 14)); - - /* Invalidate the TLB for this context */ - SET_CTX_TLBIALL(base, ctx, 0); - - /* Set interrupt number to "secure" interrupt */ - SET_IRPTNDX(base, ctx, 0); - - /* Enable context fault interrupt */ - SET_CFEIE(base, ctx, 1); - - /* Stall access on a context fault and let the handler deal with it */ - SET_CFCFG(base, ctx, 1); - - /* Redirect all cacheable requests to L2 slave port. */ - SET_RCISH(base, ctx, 1); - SET_RCOSH(base, ctx, 1); - SET_RCNSH(base, ctx, 1); - - /* Turn on TEX Remap */ - SET_TRE(base, ctx, 1); - - /* Set TEX remap attributes */ - RCP15_PRRR(prrr); - RCP15_NMRR(nmrr); - SET_PRRR(base, ctx, prrr); - SET_NMRR(base, ctx, nmrr); - - /* Turn on BFB prefetch */ - SET_BFBDFE(base, ctx, 1); - -#ifdef CONFIG_IOMMU_PGTABLES_L2 - /* Configure page tables as inner-cacheable and shareable to reduce - * the TLB miss penalty. - */ - SET_TTBR0_SH(base, ctx, 1); - SET_TTBR1_SH(base, ctx, 1); - - SET_TTBR0_NOS(base, ctx, 1); - SET_TTBR1_NOS(base, ctx, 1); - - SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR0_IRGNL(base, ctx, 1); - - SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR1_IRGNL(base, ctx, 1); - - SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ - SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ -#endif - - /* Enable the MMU */ - SET_M(base, ctx, 1); -} - -static int msm_iommu_domain_init(struct iommu_domain *domain) -{ - struct msm_priv *priv = kzalloc(sizeof(*priv), GFP_KERNEL); - - if (!priv) - goto fail_nomem; - - INIT_LIST_HEAD(&priv->list_attached); - priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, - get_order(SZ_16K)); - - if (!priv->pgtable) - goto fail_nomem; - - memset(priv->pgtable, 0, SZ_16K); - domain->priv = priv; - return 0; - -fail_nomem: - kfree(priv); - return -ENOMEM; -} - -static void msm_iommu_domain_destroy(struct iommu_domain *domain) -{ - struct msm_priv *priv; - unsigned long flags; - unsigned long *fl_table; - int i; - - spin_lock_irqsave(&msm_iommu_lock, flags); - priv = domain->priv; - domain->priv = NULL; - - if (priv) { - fl_table = priv->pgtable; - - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) - free_page((unsigned long) __va(((fl_table[i]) & - FL_BASE_MASK))); - - free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); - priv->pgtable = NULL; - } - - kfree(priv); - spin_unlock_irqrestore(&msm_iommu_lock, flags); -} - -static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) -{ - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - struct msm_iommu_ctx_drvdata *tmp_drvdata; - int ret = 0; - unsigned long flags; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = domain->priv; - - if (!priv || !dev) { - ret = -EINVAL; - goto fail; - } - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { - ret = -EINVAL; - goto fail; - } - - if (!list_empty(&ctx_drvdata->attached_elm)) { - ret = -EBUSY; - goto fail; - } - - list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) - if (tmp_drvdata == ctx_drvdata) { - ret = -EBUSY; - goto fail; - } - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - __program_context(iommu_drvdata->base, ctx_dev->num, - __pa(priv->pgtable)); - - __disable_clocks(iommu_drvdata); - list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); - ret = __flush_iotlb(domain); - -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - return ret; -} - -static void msm_iommu_detach_dev(struct iommu_domain *domain, - struct device *dev) -{ - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - unsigned long flags; - int ret; - - spin_lock_irqsave(&msm_iommu_lock, flags); - priv = domain->priv; - - if (!priv || !dev) - goto fail; - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) - goto fail; - - ret = __flush_iotlb(domain); - if (ret) - goto fail; - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - __reset_context(iommu_drvdata->base, ctx_dev->num); - __disable_clocks(iommu_drvdata); - list_del_init(&ctx_drvdata->attached_elm); - -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); -} - -static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, - phys_addr_t pa, int order, int prot) -{ - struct msm_priv *priv; - unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - unsigned int pgprot; - size_t len = 0x1000UL << order; - int ret = 0, tex, sh; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; - tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; - - if (tex < 0 || tex > NUM_TEX_CLASS - 1) { - ret = -EINVAL; - goto fail; - } - - priv = domain->priv; - if (!priv) { - ret = -EINVAL; - goto fail; - } - - fl_table = priv->pgtable; - - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad size: %d\n", len); - ret = -EINVAL; - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - ret = -EINVAL; - goto fail; - } - - if (len == SZ_16M || len == SZ_1M) { - pgprot = sh ? FL_SHARED : 0; - pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? FL_TEX0 : 0; - } else { - pgprot = sh ? SL_SHARED : 0; - pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? SL_TEX0 : 0; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (len == SZ_16M) { - int i = 0; - for (i = 0; i < 16; i++) - *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | - FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | - FL_SHARED | FL_NG | pgprot; - } - - if (len == SZ_1M) - *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | - FL_TYPE_SECT | FL_SHARED | pgprot; - - /* Need a 2nd level table */ - if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { - unsigned long *sl; - sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, - get_order(SZ_4K)); - - if (!sl) { - pr_debug("Could not allocate second level table\n"); - ret = -ENOMEM; - goto fail; - } - - memset(sl, 0, SZ_4K); - *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); - } - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - - if (len == SZ_4K) - *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | - SL_SHARED | SL_TYPE_SMALL | pgprot; - - if (len == SZ_64K) { - int i; - - for (i = 0; i < 16; i++) - *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | - SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; - } - - ret = __flush_iotlb(domain); -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - return ret; -} - -static int msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, - int order) -{ - struct msm_priv *priv; - unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - size_t len = 0x1000UL << order; - int i, ret = 0; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = domain->priv; - - if (!priv) { - ret = -ENODEV; - goto fail; - } - - fl_table = priv->pgtable; - - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad length: %d\n", len); - ret = -EINVAL; - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - ret = -EINVAL; - goto fail; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (*fl_pte == 0) { - pr_debug("First level PTE is 0\n"); - ret = -ENODEV; - goto fail; - } - - /* Unmap supersection */ - if (len == SZ_16M) - for (i = 0; i < 16; i++) - *(fl_pte+i) = 0; - - if (len == SZ_1M) - *fl_pte = 0; - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - if (len == SZ_64K) { - for (i = 0; i < 16; i++) - *(sl_pte+i) = 0; - } - - if (len == SZ_4K) - *sl_pte = 0; - - if (len == SZ_4K || len == SZ_64K) { - int used = 0; - - for (i = 0; i < NUM_SL_PTE; i++) - if (sl_table[i]) - used = 1; - if (!used) { - free_page((unsigned long)sl_table); - *fl_pte = 0; - } - } - - ret = __flush_iotlb(domain); -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - return ret; -} - -static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, - unsigned long va) -{ - struct msm_priv *priv; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - unsigned int par; - unsigned long flags; - void __iomem *base; - phys_addr_t ret = 0; - int ctx; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = domain->priv; - if (list_empty(&priv->list_attached)) - goto fail; - - ctx_drvdata = list_entry(priv->list_attached.next, - struct msm_iommu_ctx_drvdata, attached_elm); - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); - - base = iommu_drvdata->base; - ctx = ctx_drvdata->num; - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - /* Invalidate context TLB */ - SET_CTX_TLBIALL(base, ctx, 0); - SET_V2PPR(base, ctx, va & V2Pxx_VA); - - par = GET_PAR(base, ctx); - - /* We are dealing with a supersection */ - if (GET_NOFAULT_SS(base, ctx)) - ret = (par & 0xFF000000) | (va & 0x00FFFFFF); - else /* Upper 20 bits from PAR, lower 12 from VA */ - ret = (par & 0xFFFFF000) | (va & 0x00000FFF); - - if (GET_FAULT(base, ctx)) - ret = 0; - - __disable_clocks(iommu_drvdata); -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - return ret; -} - -static int msm_iommu_domain_has_cap(struct iommu_domain *domain, - unsigned long cap) -{ - return 0; -} - -static void print_ctx_regs(void __iomem *base, int ctx) -{ - unsigned int fsr = GET_FSR(base, ctx); - pr_err("FAR = %08x PAR = %08x\n", - GET_FAR(base, ctx), GET_PAR(base, ctx)); - pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr, - (fsr & 0x02) ? "TF " : "", - (fsr & 0x04) ? "AFF " : "", - (fsr & 0x08) ? "APF " : "", - (fsr & 0x10) ? "TLBMF " : "", - (fsr & 0x20) ? "HTWDEEF " : "", - (fsr & 0x40) ? "HTWSEEF " : "", - (fsr & 0x80) ? "MHF " : "", - (fsr & 0x10000) ? "SL " : "", - (fsr & 0x40000000) ? "SS " : "", - (fsr & 0x80000000) ? "MULTI " : ""); - - pr_err("FSYNR0 = %08x FSYNR1 = %08x\n", - GET_FSYNR0(base, ctx), GET_FSYNR1(base, ctx)); - pr_err("TTBR0 = %08x TTBR1 = %08x\n", - GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); - pr_err("SCTLR = %08x ACTLR = %08x\n", - GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); - pr_err("PRRR = %08x NMRR = %08x\n", - GET_PRRR(base, ctx), GET_NMRR(base, ctx)); -} - -irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) -{ - struct msm_iommu_drvdata *drvdata = dev_id; - void __iomem *base; - unsigned int fsr; - int i, ret; - - spin_lock(&msm_iommu_lock); - - if (!drvdata) { - pr_err("Invalid device ID in context interrupt handler\n"); - goto fail; - } - - base = drvdata->base; - - pr_err("Unexpected IOMMU page fault!\n"); - pr_err("base = %08x\n", (unsigned int) base); - - ret = __enable_clocks(drvdata); - if (ret) - goto fail; - - for (i = 0; i < drvdata->ncb; i++) { - fsr = GET_FSR(base, i); - if (fsr) { - pr_err("Fault occurred in context %d.\n", i); - pr_err("Interesting registers:\n"); - print_ctx_regs(base, i); - SET_FSR(base, i, 0x4000000F); - } - } - __disable_clocks(drvd