From 74315cccd2104a953f493acca2c6b0519d6f5c6f Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 8 Jun 2011 17:29:11 -0400 Subject: [PATCH 01/16] iommu-api: Add missing header file If CONFIG_IOMMU_API is not defined some functions will just return -ENODEV. Add errno.h for the definition of ENODEV. Signed-off-by: Laura Abbott Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0a2ba4098996..9940319d6f9d 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -19,6 +19,8 @@ #ifndef __LINUX_IOMMU_H #define __LINUX_IOMMU_H +#include + #define IOMMU_READ (1) #define IOMMU_WRITE (2) #define IOMMU_CACHE (4) /* DMA cache coherency */ From 39c555460cbc6d35656878c89d4664fbd6c81551 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 11:52:27 +0200 Subject: [PATCH 02/16] x86/amd-iommu: Remove redundant device_flush_dte() calls Remove these function calls from places where the function has already been called by another function. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 7c3a95e54ec5..cc6f6da630e8 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -1798,7 +1798,6 @@ static int device_change_notifier(struct notifier_block *nb, goto out; } - device_flush_dte(dev); iommu_completion_wait(iommu); out: @@ -2605,7 +2604,6 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, if (!iommu) return; - device_flush_dte(dev); iommu_completion_wait(iommu); } From 8fa5f802abf3cd374b5f07418cea72c5d9d204cc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:24:45 +0200 Subject: [PATCH 03/16] x86/amd-iommu: Introduce global dev_data_list This list keeps all allocated iommu_dev_data structs in a list together. This is needed for instances that have no associated device. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 56 +++++++++++++++++++++----- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 4c9982995414..35520e31c1b4 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -310,6 +310,7 @@ struct protection_domain { */ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ + struct list_head dev_data_list; /* For global dev_data_list */ struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index cc6f6da630e8..8b8489084649 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -45,6 +45,10 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); static LIST_HEAD(iommu_pd_list); static DEFINE_SPINLOCK(iommu_pd_list_lock); +/* List of all available dev_data structures */ +static LIST_HEAD(dev_data_list); +static DEFINE_SPINLOCK(dev_data_list_lock); + /* * Domain for untranslated devices - only allocated * if iommu=pt passed on kernel cmd line. @@ -68,6 +72,35 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ +static struct iommu_dev_data *alloc_dev_data(void) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + if (!dev_data) + return NULL; + + atomic_set(&dev_data->bind, 0); + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_add_tail(&dev_data->dev_data_list, &dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static void free_dev_data(struct iommu_dev_data *dev_data) +{ + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_del(&dev_data->dev_data_list); + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + kfree(dev_data); +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -139,32 +172,28 @@ static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; struct pci_dev *pdev; - u16 devid, alias; + u16 alias; if (dev->archdata.iommu) return 0; - dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); + dev_data = alloc_dev_data(); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - devid = get_device_id(dev); - alias = amd_iommu_alias_table[devid]; + alias = amd_iommu_alias_table[get_device_id(dev)]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; else { - kfree(dev_data); + free_dev_data(dev_data); return -ENOTSUPP; } - atomic_set(&dev_data->bind, 0); - dev->archdata.iommu = dev_data; - return 0; } @@ -184,11 +213,16 @@ static void iommu_ignore_device(struct device *dev) static void iommu_uninit_device(struct device *dev) { - kfree(dev->archdata.iommu); + /* + * Nothing to do here - we keep dev_data around for unplugged devices + * and reuse it when the device is re-plugged - not doing so would + * introduce a ton of races. + */ } void __init amd_iommu_uninit_devices(void) { + struct iommu_dev_data *dev_data, *n; struct pci_dev *pdev = NULL; for_each_pci_dev(pdev) { @@ -198,6 +232,10 @@ void __init amd_iommu_uninit_devices(void) iommu_uninit_device(&pdev->dev); } + + /* Free all of our dev_data structures */ + list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) + free_dev_data(dev_data); } int __init amd_iommu_init_devices(void) From f62dda66b5601396c28b563cc18b481af04a91b3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:55:35 +0200 Subject: [PATCH 04/16] x86/amd-iommu: Store devid in dev_data This allows to use dev_data independent of struct device later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 + arch/x86/kernel/amd_iommu.c | 39 ++++++++++---------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 35520e31c1b4..2833862311b5 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -315,6 +315,7 @@ struct iommu_dev_data { struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ + u16 devid; /* PCI Device ID */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 8b8489084649..4e8b176297b5 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -72,7 +72,7 @@ static void update_domain(struct protection_domain *domain); * ****************************************************************************/ -static struct iommu_dev_data *alloc_dev_data(void) +static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; unsigned long flags; @@ -81,6 +81,7 @@ static struct iommu_dev_data *alloc_dev_data(void) if (!dev_data) return NULL; + dev_data->devid = devid; atomic_set(&dev_data->bind, 0); spin_lock_irqsave(&dev_data_list_lock, flags); @@ -177,13 +178,13 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(); + dev_data = alloc_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; dev_data->dev = dev; - alias = amd_iommu_alias_table[get_device_id(dev)]; + alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) dev_data->alias = &pdev->dev; @@ -714,16 +715,16 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) */ static int device_flush_dte(struct device *dev) { + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct pci_dev *pdev; - u16 devid; int ret; - pdev = to_pci_dev(dev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + pdev = to_pci_dev(dev); + dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; - ret = iommu_flush_dte(iommu, devid); + ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; @@ -1570,11 +1571,9 @@ static void do_attach(struct device *dev, struct protection_domain *domain) struct amd_iommu *iommu; struct pci_dev *pdev; bool ats = false; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); if (amd_iommu_iotlb_sup) @@ -1583,7 +1582,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) /* Update data structures */ dev_data->domain = domain; list_add(&dev_data->list, &domain->dev_list); - set_dte_entry(devid, domain, ats); + set_dte_entry(dev_data->devid, domain, ats); /* Do reference counting */ domain->dev_iommu[iommu->index] += 1; @@ -1597,11 +1596,9 @@ static void do_detach(struct device *dev) { struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - u16 devid; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; dev_data = get_dev_data(dev); + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1610,7 +1607,7 @@ static void do_detach(struct device *dev) /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(devid); + clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ device_flush_dte(dev); @@ -1760,9 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; unsigned long flags; - u16 devid; - devid = get_device_id(dev); dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1897,8 +1892,7 @@ static void update_device_table(struct protection_domain *domain) list_for_each_entry(dev_data, &domain->dev_list, list) { struct pci_dev *pdev = to_pci_dev(dev_data->dev); - u16 devid = get_device_id(dev_data->dev); - set_dte_entry(devid, domain, pci_ats_enabled(pdev)); + set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); } } @@ -2652,16 +2646,13 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, struct iommu_dev_data *dev_data; struct amd_iommu *iommu; int ret; - u16 devid; if (!check_device(dev)) return -EINVAL; dev_data = dev->archdata.iommu; - devid = get_device_id(dev); - - iommu = amd_iommu_rlookup_table[devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; if (!iommu) return -EINVAL; From ea61cddb9dcb9da9bcabd40f1620c24abaf645c9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 12:56:30 +0200 Subject: [PATCH 05/16] x86/amd-iommu: Store ATS state in dev_data This allows the low-level functions to operate on dev_data exclusivly later. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 4 +++ arch/x86/kernel/amd_iommu.c | 44 ++++++++++++++------------ 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 2833862311b5..9fc045ee2fcb 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -316,6 +316,10 @@ struct iommu_dev_data { struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ + struct { + bool enabled; + int qdep; + } ats; /* ATS state */ }; /* diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 4e8b176297b5..9e07ef65a016 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -695,17 +695,16 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) */ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; - u16 devid; int qdep; - qdep = pci_ats_queue_depth(pdev); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + dev_data = get_dev_data(dev); + qdep = dev_data->ats.qdep; + iommu = amd_iommu_rlookup_table[dev_data->devid]; - build_inv_iotlb_pages(&cmd, devid, qdep, address, size); + build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size); return iommu_queue_command(iommu, &cmd); } @@ -728,7 +727,7 @@ static int device_flush_dte(struct device *dev) if (ret) return ret; - if (pci_ats_enabled(pdev)) + if (dev_data->ats.enabled) ret = device_flush_iotlb(dev, 0, ~0UL); return ret; @@ -760,9 +759,8 @@ static void __domain_flush_pages(struct protection_domain *domain, } list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - if (!pci_ats_enabled(pdev)) + if (!dev_data->ats.enabled) continue; ret |= device_flush_iotlb(dev_data->dev, address, size); @@ -1576,8 +1574,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) iommu = amd_iommu_rlookup_table[dev_data->devid]; pdev = to_pci_dev(dev); - if (amd_iommu_iotlb_sup) - ats = pci_ats_enabled(pdev); + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1674,11 +1671,16 @@ static int attach_device(struct device *dev, struct protection_domain *domain) { struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; int ret; - if (amd_iommu_iotlb_sup) - pci_enable_ats(pdev, PAGE_SHIFT); + dev_data = get_dev_data(dev); + + if (amd_iommu_iotlb_sup && pci_enable_ats(pdev, PAGE_SHIFT) == 0) { + dev_data->ats.enabled = true; + dev_data->ats.qdep = pci_ats_queue_depth(pdev); + } write_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev, domain); @@ -1736,7 +1738,7 @@ static void __detach_device(struct device *dev) */ static void detach_device(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); + struct iommu_dev_data *dev_data; unsigned long flags; /* lock device table */ @@ -1744,8 +1746,12 @@ static void detach_device(struct device *dev) __detach_device(dev); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) - pci_disable_ats(pdev); + dev_data = get_dev_data(dev); + + if (dev_data->ats.enabled) { + pci_disable_ats(to_pci_dev(dev)); + dev_data->ats.enabled = false; + } } /* @@ -1890,10 +1896,8 @@ static void update_device_table(struct protection_domain *domain) { struct iommu_dev_data *dev_data; - list_for_each_entry(dev_data, &domain->dev_list, list) { - struct pci_dev *pdev = to_pci_dev(dev_data->dev); - set_dte_entry(dev_data->devid, domain, pci_ats_enabled(pdev)); - } + list_for_each_entry(dev_data, &domain->dev_list, list) + set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled); } static void update_domain(struct protection_domain *domain) From 6c542047937ad66db1d31e290fd2a3b290424ffa Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:07:31 +0200 Subject: [PATCH 06/16] x86/amd-iommu: Use only dev_data for dte and iotlb flushing routines This patch make the functions flushing the DTE and IOTLBs only take the dev_data structure instead of the struct device directly. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 9e07ef65a016..0a5b46504b5d 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -693,14 +693,13 @@ void iommu_flush_all_caches(struct amd_iommu *iommu) /* * Command send function for flushing on-device TLB */ -static int device_flush_iotlb(struct device *dev, u64 address, size_t size) +static int device_flush_iotlb(struct iommu_dev_data *dev_data, + u64 address, size_t size) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; struct iommu_cmd cmd; int qdep; - dev_data = get_dev_data(dev); qdep = dev_data->ats.qdep; iommu = amd_iommu_rlookup_table[dev_data->devid]; @@ -712,23 +711,19 @@ static int device_flush_iotlb(struct device *dev, u64 address, size_t size) /* * Command send function for invalidating a device table entry */ -static int device_flush_dte(struct device *dev) +static int device_flush_dte(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; int ret; - pdev = to_pci_dev(dev); - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; ret = iommu_flush_dte(iommu, dev_data->devid); if (ret) return ret; if (dev_data->ats.enabled) - ret = device_flush_iotlb(dev, 0, ~0UL); + ret = device_flush_iotlb(dev_data, 0, ~0UL); return ret; } @@ -763,7 +758,7 @@ static void __domain_flush_pages(struct protection_domain *domain, if (!dev_data->ats.enabled) continue; - ret |= device_flush_iotlb(dev_data->dev, address, size); + ret |= device_flush_iotlb(dev_data, address, size); } WARN_ON(ret); @@ -815,7 +810,7 @@ static void domain_flush_devices(struct protection_domain *domain) spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(dev_data, &domain->dev_list, list) - device_flush_dte(dev_data->dev); + device_flush_dte(dev_data); spin_unlock_irqrestore(&domain->lock, flags); } @@ -1586,7 +1581,7 @@ static void do_attach(struct device *dev, struct protection_domain *domain) domain->dev_cnt += 1; /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } static void do_detach(struct device *dev) @@ -1607,7 +1602,7 @@ static void do_detach(struct device *dev) clear_dte_entry(dev_data->devid); /* Flush the DTE entry */ - device_flush_dte(dev); + device_flush_dte(dev_data); } /* From ec9e79ef062272a119994c4fdd5e434f3e1bf352 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:25:50 +0200 Subject: [PATCH 07/16] x86/amd-iommu: Use only dev_data in low-level domain attach/detach functions With this patch the low-level attach/detach functions only work on dev_data structures. This allows to remove the dev_data->dev pointer. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 1 - arch/x86/kernel/amd_iommu.c | 61 ++++++++++---------------- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index 9fc045ee2fcb..fffbe3c9e6c2 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,6 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *dev; /* Device this data belong to */ struct device *alias; /* The Alias Device */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 0a5b46504b5d..a15a17231909 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -182,8 +182,6 @@ static int iommu_init_device(struct device *dev) if (!dev_data) return -ENOMEM; - dev_data->dev = dev; - alias = amd_iommu_alias_table[dev_data->devid]; pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); if (pdev) @@ -1558,18 +1556,14 @@ static void clear_dte_entry(u16 devid) amd_iommu_apply_erratum_63(devid); } -static void do_attach(struct device *dev, struct protection_domain *domain) +static void do_attach(struct iommu_dev_data *dev_data, + struct protection_domain *domain) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - struct pci_dev *pdev; - bool ats = false; + bool ats; - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; - pdev = to_pci_dev(dev); - - ats = dev_data->ats.enabled; + iommu = amd_iommu_rlookup_table[dev_data->devid]; + ats = dev_data->ats.enabled; /* Update data structures */ dev_data->domain = domain; @@ -1584,13 +1578,11 @@ static void do_attach(struct device *dev, struct protection_domain *domain) device_flush_dte(dev_data); } -static void do_detach(struct device *dev) +static void do_detach(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - dev_data = get_dev_data(dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; /* decrease reference counters */ dev_data->domain->dev_iommu[iommu->index] -= 1; @@ -1609,13 +1601,12 @@ static void do_detach(struct device *dev) * If a device is not yet associated with a domain, this function does * assigns it visible for the hardware */ -static int __attach_device(struct device *dev, +static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *alias_data; int ret; - dev_data = get_dev_data(dev); alias_data = get_dev_data(dev_data->alias); if (!alias_data) @@ -1635,16 +1626,15 @@ static int __attach_device(struct device *dev, goto out_unlock; /* Do real assignment */ - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (alias_data->domain == NULL) - do_attach(dev_data->alias, domain); + do_attach(alias_data, domain); atomic_inc(&alias_data->bind); } if (dev_data->domain == NULL) - do_attach(dev, domain); + do_attach(dev_data, domain); atomic_inc(&dev_data->bind); @@ -1678,7 +1668,7 @@ static int attach_device(struct device *dev, } write_lock_irqsave(&amd_iommu_devtable_lock, flags); - ret = __attach_device(dev, domain); + ret = __attach_device(dev_data, domain); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); /* @@ -1694,9 +1684,8 @@ static int attach_device(struct device *dev, /* * Removes a device from a protection domain (unlocked) */ -static void __detach_device(struct device *dev) +static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *dev_data = get_dev_data(dev); struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1707,14 +1696,14 @@ static void __detach_device(struct device *dev) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias != dev) { - alias_data = get_dev_data(dev_data->alias); + alias_data = get_dev_data(dev_data->alias); + if (dev_data->devid != alias_data->devid) { if (atomic_dec_and_test(&alias_data->bind)) - do_detach(dev_data->alias); + do_detach(alias_data); } if (atomic_dec_and_test(&dev_data->bind)) - do_detach(dev); + do_detach(dev_data); spin_unlock_irqrestore(&domain->lock, flags); @@ -1725,7 +1714,7 @@ static void __detach_device(struct device *dev) */ if (iommu_pass_through && (dev_data->domain == NULL && domain != pt_domain)) - __attach_device(dev, pt_domain); + __attach_device(dev_data, pt_domain); } /* @@ -1736,13 +1725,13 @@ static void detach_device(struct device *dev) struct iommu_dev_data *dev_data; unsigned long flags; + dev_data = get_dev_data(dev); + /* lock device table */ write_lock_irqsave(&amd_iommu_devtable_lock, flags); - __detach_device(dev); + __detach_device(dev_data); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - dev_data = get_dev_data(dev); - if (dev_data->ats.enabled) { pci_disable_ats(to_pci_dev(dev)); dev_data->ats.enabled = false; @@ -1768,7 +1757,7 @@ static struct protection_domain *domain_for_device(struct device *dev) dom = dev_data->domain; if (dom == NULL && alias_data->domain != NULL) { - __attach_device(dev, alias_data->domain); + __attach_device(dev_data, alias_data->domain); dom = alias_data->domain; } @@ -2527,9 +2516,7 @@ static void cleanup_domain(struct protection_domain *domain) write_lock_irqsave(&amd_iommu_devtable_lock, flags); list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) { - struct device *dev = dev_data->dev; - - __detach_device(dev); + __detach_device(dev_data); atomic_set(&dev_data->bind, 0); } From 2b02b091ab3d1685926bfc76486bf7a22d92979a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 17:48:39 +0200 Subject: [PATCH 08/16] x86/amd-iommu: Allow dev_data->alias to be NULL Let dev_data->alias be just NULL if the device has no alias. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 77 ++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index a15a17231909..2d914a49b19a 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -183,12 +183,14 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; alias = amd_iommu_alias_table[dev_data->devid]; - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { - free_dev_data(dev_data); - return -ENOTSUPP; + if (alias != dev_data->devid) { + pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); + if (pdev) + dev_data->alias = &pdev->dev; + else { + free_dev_data(dev_data); + return -ENOTSUPP; + } } dev->archdata.iommu = dev_data; @@ -1604,29 +1606,31 @@ static void do_detach(struct iommu_dev_data *dev_data) static int __attach_device(struct iommu_dev_data *dev_data, struct protection_domain *domain) { - struct iommu_dev_data *alias_data; int ret; - alias_data = get_dev_data(dev_data->alias); - - if (!alias_data) - return -EINVAL; - /* lock domain */ spin_lock(&domain->lock); - /* Some sanity checks */ - ret = -EBUSY; - if (alias_data->domain != NULL && - alias_data->domain != domain) - goto out_unlock; + if (dev_data->alias != NULL) { + struct iommu_dev_data *alias_data; - if (dev_data->domain != NULL && - dev_data->domain != domain) - goto out_unlock; + alias_data = get_dev_data(dev_data->alias); - /* Do real assignment */ - if (dev_data->devid != alias_data->devid) { + ret = -EINVAL; + if (!alias_data) + goto out_unlock; + + /* Some sanity checks */ + ret = -EBUSY; + if (alias_data->domain != NULL && + alias_data->domain != domain) + goto out_unlock; + + if (dev_data->domain != NULL && + dev_data->domain != domain) + goto out_unlock; + + /* Do real assignment */ if (alias_data->domain == NULL) do_attach(alias_data, domain); @@ -1696,8 +1700,8 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - alias_data = get_dev_data(dev_data->alias); - if (dev_data->devid != alias_data->devid) { + if (dev_data->alias) { + alias_data = get_dev_data(dev_data->alias); if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1744,25 +1748,26 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct protection_domain *dom; struct iommu_dev_data *dev_data, *alias_data; + struct protection_domain *dom = NULL; unsigned long flags; dev_data = get_dev_data(dev); - alias_data = get_dev_data(dev_data->alias); - if (!alias_data) - return NULL; - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - dom = dev_data->domain; - if (dom == NULL && - alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; + if (dev_data->domain) + return dev_data->domain; + + if (dev_data->alias != NULL) { + alias_data = get_dev_data(dev_data->alias); + + read_lock_irqsave(&amd_iommu_devtable_lock, flags); + if (alias_data->domain != NULL) { + __attach_device(dev_data, alias_data->domain); + dom = alias_data->domain; + } + read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - return dom; } From 3b03bb745eb9fcafc6ef80fcbd04f0f0512acdc9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 18:53:25 +0200 Subject: [PATCH 09/16] x86/amd-iommu: Search for existind dev_data before allocting a new one Search for existing dev_data first will allow to switch dev_data->alias to just store dev_data instead of struct device. Signed-off-by: Joerg Roedel --- arch/x86/kernel/amd_iommu.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 2d914a49b19a..f64799878d45 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -102,6 +102,37 @@ static void free_dev_data(struct iommu_dev_data *dev_data) kfree(dev_data); } +static struct iommu_dev_data *search_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + unsigned long flags; + + spin_lock_irqsave(&dev_data_list_lock, flags); + list_for_each_entry(dev_data, &dev_data_list, dev_data_list) { + if (dev_data->devid == devid) + goto out_unlock; + } + + dev_data = NULL; + +out_unlock: + spin_unlock_irqrestore(&dev_data_list_lock, flags); + + return dev_data; +} + +static struct iommu_dev_data *find_dev_data(u16 devid) +{ + struct iommu_dev_data *dev_data; + + dev_data = search_dev_data(devid); + + if (dev_data == NULL) + dev_data = alloc_dev_data(devid); + + return dev_data; +} + static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -178,7 +209,7 @@ static int iommu_init_device(struct device *dev) if (dev->archdata.iommu) return 0; - dev_data = alloc_dev_data(get_device_id(dev)); + dev_data = find_dev_data(get_device_id(dev)); if (!dev_data) return -ENOMEM; From 71f77580906b60eb43daff0efb99792e76a3d06d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 9 Jun 2011 19:03:15 +0200 Subject: [PATCH 10/16] x86/amd-iommu: Store device alias as dev_data pointer This finally allows PCI-Device-IDs to be handled by the IOMMU driver that have no corresponding struct device present in the system. Signed-off-by: Joerg Roedel --- arch/x86/include/asm/amd_iommu_types.h | 2 +- arch/x86/kernel/amd_iommu.c | 34 ++++++++++++-------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/amd_iommu_types.h b/arch/x86/include/asm/amd_iommu_types.h index fffbe3c9e6c2..5b9c5075e81a 100644 --- a/arch/x86/include/asm/amd_iommu_types.h +++ b/arch/x86/include/asm/amd_iommu_types.h @@ -311,7 +311,7 @@ struct protection_domain { struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct list_head dev_data_list; /* For global dev_data_list */ - struct device *alias; /* The Alias Device */ + struct iommu_dev_data *alias_data;/* The alias dev_data */ struct protection_domain *domain; /* Domain the device is bound to */ atomic_t bind; /* Domain attach reverent count */ u16 devid; /* PCI Device ID */ diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index f64799878d45..0ed1c940d76b 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c @@ -203,7 +203,6 @@ static bool check_device(struct device *dev) static int iommu_init_device(struct device *dev) { struct iommu_dev_data *dev_data; - struct pci_dev *pdev; u16 alias; if (dev->archdata.iommu) @@ -215,13 +214,16 @@ static int iommu_init_device(struct device *dev) alias = amd_iommu_alias_table[dev_data->devid]; if (alias != dev_data->devid) { - pdev = pci_get_bus_and_slot(PCI_BUS(alias), alias & 0xff); - if (pdev) - dev_data->alias = &pdev->dev; - else { + struct iommu_dev_data *alias_data; + + alias_data = find_dev_data(alias); + if (alias_data == NULL) { + pr_err("AMD-Vi: Warning: Unhandled device %s\n", + dev_name(dev)); free_dev_data(dev_data); return -ENOTSUPP; } + dev_data->alias_data = alias_data; } dev->archdata.iommu = dev_data; @@ -1642,14 +1644,8 @@ static int __attach_device(struct iommu_dev_data *dev_data, /* lock domain */ spin_lock(&domain->lock); - if (dev_data->alias != NULL) { - struct iommu_dev_data *alias_data; - - alias_data = get_dev_data(dev_data->alias); - - ret = -EINVAL; - if (!alias_data) - goto out_unlock; + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; /* Some sanity checks */ ret = -EBUSY; @@ -1721,7 +1717,6 @@ static int attach_device(struct device *dev, */ static void __detach_device(struct iommu_dev_data *dev_data) { - struct iommu_dev_data *alias_data; struct protection_domain *domain; unsigned long flags; @@ -1731,8 +1726,9 @@ static void __detach_device(struct iommu_dev_data *dev_data) spin_lock_irqsave(&domain->lock, flags); - if (dev_data->alias) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; + if (atomic_dec_and_test(&alias_data->bind)) do_detach(alias_data); } @@ -1779,7 +1775,7 @@ static void detach_device(struct device *dev) */ static struct protection_domain *domain_for_device(struct device *dev) { - struct iommu_dev_data *dev_data, *alias_data; + struct iommu_dev_data *dev_data; struct protection_domain *dom = NULL; unsigned long flags; @@ -1788,8 +1784,8 @@ static struct protection_domain *domain_for_device(struct device *dev) if (dev_data->domain) return dev_data->domain; - if (dev_data->alias != NULL) { - alias_data = get_dev_data(dev_data->alias); + if (dev_data->alias_data != NULL) { + struct iommu_dev_data *alias_data = dev_data->alias_data; read_lock_irqsave(&amd_iommu_devtable_lock, flags); if (alias_data->domain != NULL) { From ab493a0f0f55d28636ac860ea682d57b84257f10 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Thu, 2 Jun 2011 02:48:05 +0300 Subject: [PATCH 11/16] drivers: iommu: move to a dedicated folder Create a dedicated folder for iommu drivers, and move the base iommu implementation over there. Grouping the various iommu drivers in a single location will help finding similar problems shared by different platforms, so they could be solved once, in the iommu framework, instead of solved differently (or duplicated) in each driver. Signed-off-by: Ohad Ben-Cohen Signed-off-by: Joerg Roedel --- arch/arm/mach-msm/Kconfig | 3 --- arch/ia64/Kconfig | 3 --- arch/x86/Kconfig | 5 ++--- drivers/Kconfig | 2 ++ drivers/Makefile | 1 + drivers/base/Makefile | 1 - drivers/iommu/Kconfig | 3 +++ drivers/iommu/Makefile | 1 + drivers/{base => iommu}/iommu.c | 0 9 files changed, 9 insertions(+), 10 deletions(-) create mode 100644 drivers/iommu/Kconfig create mode 100644 drivers/iommu/Makefile rename drivers/{base => iommu}/iommu.c (100%) diff --git a/arch/arm/mach-msm/Kconfig b/arch/arm/mach-msm/Kconfig index 1516896e8d17..efb7b7dfc20b 100644 --- a/arch/arm/mach-msm/Kconfig +++ b/arch/arm/mach-msm/Kconfig @@ -205,9 +205,6 @@ config MSM_GPIOMUX config MSM_V2_TLMM bool -config IOMMU_API - bool - config MSM_SCM bool endif diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 38280ef4a2af..9929e4e11ea0 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -681,6 +681,3 @@ source "lib/Kconfig" config IOMMU_HELPER def_bool (IA64_HP_ZX1 || IA64_HP_ZX1_SWIOTLB || IA64_GENERIC || SWIOTLB) - -config IOMMU_API - def_bool (DMAR) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index da349723d411..460d57370016 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -685,6 +685,7 @@ config AMD_IOMMU select SWIOTLB select PCI_MSI select PCI_IOV + select IOMMU_API depends on X86_64 && PCI && ACPI ---help--- With this option you can enable support for AMD IOMMU hardware in @@ -720,9 +721,6 @@ config SWIOTLB config IOMMU_HELPER def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU) -config IOMMU_API - def_bool (AMD_IOMMU || DMAR) - config MAXSMP bool "Enable Maximum number of SMP Processors and NUMA Nodes" depends on X86_64 && SMP && DEBUG_KERNEL && EXPERIMENTAL @@ -1945,6 +1943,7 @@ config PCI_CNB20LE_QUIRK config DMAR bool "Support for DMA Remapping Devices (EXPERIMENTAL)" depends on PCI_MSI && ACPI && EXPERIMENTAL + select IOMMU_API help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. diff --git a/drivers/Kconfig b/drivers/Kconfig index 3bb154d8c8cc..9d513188b47a 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -126,4 +126,6 @@ source "drivers/hwspinlock/Kconfig" source "drivers/clocksource/Kconfig" +source "drivers/iommu/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 09f3232bcdcd..2f7a71a933de 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -122,3 +122,4 @@ obj-y += ieee802154/ obj-y += clk/ obj-$(CONFIG_HWSPINLOCK) += hwspinlock/ +obj-$(CONFIG_IOMMU_API) += iommu/ diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 4c5701c15f53..5ab0d07c4578 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -13,7 +13,6 @@ obj-$(CONFIG_FW_LOADER) += firmware_class.o obj-$(CONFIG_NUMA) += node.o obj-$(CONFIG_MEMORY_HOTPLUG_SPARSE) += memory.o obj-$(CONFIG_SMP) += topology.o -obj-$(CONFIG_IOMMU_API) += iommu.o ifeq ($(CONFIG_SYSFS),y) obj-$(CONFIG_MODULES) += module.o endif diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig new file mode 100644 index 000000000000..2c5dfb48a22a --- /dev/null +++ b/drivers/iommu/Kconfig @@ -0,0 +1,3 @@ +# IOMMU_API always gets selected by whoever wants it. +config IOMMU_API + bool diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile new file mode 100644 index 000000000000..241ba4c46a19 --- /dev/null +++ b/drivers/iommu/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_IOMMU_API) += iommu.o diff --git a/drivers/base/iommu.c b/drivers/iommu/iommu.c similarity index 100% rename from drivers/base/iommu.c rename to drivers/iommu/iommu.c From b10f127e1a4d8cac5414c6e2b152c205b66c9f16 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Thu, 2 Jun 2011 03:20:08 +0300 Subject: [PATCH 12/16] msm: iommu: move to drivers/iommu/ This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Compile-tested for MSM8X60. Signed-off-by: Ohad Ben-Cohen Acked-by: David Brown Signed-off-by: Joerg Roedel --- arch/arm/mach-msm/Kconfig | 16 ---------------- arch/arm/mach-msm/Makefile | 2 +- drivers/iommu/Kconfig | 16 ++++++++++++++++ drivers/iommu/Makefile | 1 + .../iommu.c => drivers/iommu/msm_iommu.c | 0 .../iommu_dev.c => drivers/iommu/msm_iommu_dev.c | 0 6 files changed, 18 insertions(+), 17 deletions(-) rename arch/arm/mach-msm/iommu.c => drivers/iommu/msm_iommu.c (100%) rename arch/arm/mach-msm/iommu_dev.c => drivers/iommu/msm_iommu_dev.c (100%) diff --git a/arch/arm/mach-msm/Kconfig b/arch/arm/mach-msm/Kconfig index efb7b7dfc20b..888e92502e15 100644 --- a/arch/arm/mach-msm/Kconfig +++ b/arch/arm/mach-msm/Kconfig @@ -148,22 +148,6 @@ config MACH_MSM8960_RUMI3 endmenu -config MSM_IOMMU - bool "MSM IOMMU Support" - depends on ARCH_MSM8X60 || ARCH_MSM8960 - select IOMMU_API - default n - help - Support for the IOMMUs found on certain Qualcomm SOCs. - These IOMMUs allow virtualization of the address space used by most - cores within the multimedia subsystem. - - If unsure, say N here. - -config IOMMU_PGTABLES_L2 - def_bool y - depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n - config MSM_DEBUG_UART int default 1 if MSM_DEBUG_UART1 diff --git a/arch/arm/mach-msm/Makefile b/arch/arm/mach-msm/Makefile index 9519fd28a025..b70658c5ae00 100644 --- a/arch/arm/mach-msm/Makefile +++ b/arch/arm/mach-msm/Makefile @@ -3,7 +3,7 @@ obj-y += clock.o obj-$(CONFIG_DEBUG_FS) += clock-debug.o obj-$(CONFIG_MSM_VIC) += irq-vic.o -obj-$(CONFIG_MSM_IOMMU) += iommu.o iommu_dev.o devices-iommu.o +obj-$(CONFIG_MSM_IOMMU) += devices-iommu.o obj-$(CONFIG_ARCH_MSM7X00A) += dma.o irq.o acpuclock-arm11.o obj-$(CONFIG_ARCH_MSM7X30) += dma.o diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 2c5dfb48a22a..21a80bfbdb52 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -1,3 +1,19 @@ # IOMMU_API always gets selected by whoever wants it. config IOMMU_API bool + +# MSM IOMMU support +config MSM_IOMMU + bool "MSM IOMMU Support" + depends on ARCH_MSM8X60 || ARCH_MSM8960 + select IOMMU_API + help + Support for the IOMMUs found on certain Qualcomm SOCs. + These IOMMUs allow virtualization of the address space used by most + cores within the multimedia subsystem. + + If unsure, say N here. + +config IOMMU_PGTABLES_L2 + def_bool y + depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 241ba4c46a19..1a71c82b1af2 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1 +1,2 @@ obj-$(CONFIG_IOMMU_API) += iommu.o +obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o diff --git a/arch/arm/mach-msm/iommu.c b/drivers/iommu/msm_iommu.c similarity index 100% rename from arch/arm/mach-msm/iommu.c rename to drivers/iommu/msm_iommu.c diff --git a/arch/arm/mach-msm/iommu_dev.c b/drivers/iommu/msm_iommu_dev.c similarity index 100% rename from arch/arm/mach-msm/iommu_dev.c rename to drivers/iommu/msm_iommu_dev.c From 29b68415e335ba9e0eb6057f9405aa4d9c23efe4 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Sun, 5 Jun 2011 18:22:18 +0300 Subject: [PATCH 13/16] x86: amd_iommu: move to drivers/iommu/ This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Compile-tested on x86_64. Signed-off-by: Ohad Ben-Cohen Signed-off-by: Joerg Roedel --- arch/x86/Kconfig | 28 ------------------ arch/x86/kernel/Makefile | 2 +- drivers/iommu/Kconfig | 29 +++++++++++++++++++ drivers/iommu/Makefile | 1 + .../x86/kernel => drivers/iommu}/amd_iommu.c | 0 5 files changed, 31 insertions(+), 29 deletions(-) rename {arch/x86/kernel => drivers/iommu}/amd_iommu.c (100%) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 460d57370016..1b6a2e212db4 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -680,34 +680,6 @@ config CALGARY_IOMMU_ENABLED_BY_DEFAULT Calgary anyway, pass 'iommu=calgary' on the kernel command line. If unsure, say Y. -config AMD_IOMMU - bool "AMD IOMMU support" - select SWIOTLB - select PCI_MSI - select PCI_IOV - select IOMMU_API - depends on X86_64 && PCI && ACPI - ---help--- - With this option you can enable support for AMD IOMMU hardware in - your system. An IOMMU is a hardware component which provides - remapping of DMA memory accesses from devices. With an AMD IOMMU you - can isolate the the DMA memory of different devices and protect the - system from misbehaving device drivers or hardware. - - You can find out if your system has an AMD IOMMU if you look into - your BIOS for an option to enable it or if you have an IVRS ACPI - table. - -config AMD_IOMMU_STATS - bool "Export AMD IOMMU statistics to debugfs" - depends on AMD_IOMMU - select DEBUG_FS - ---help--- - This option enables code in the AMD IOMMU driver to collect various - statistics about whats happening in the driver and exports that - information to userspace via debugfs. - If unsure, say N. - # need this always selected by IOMMU for the VIA workaround config SWIOTLB def_bool y if X86_64 diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 90b06d4daee2..aef02989c930 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -123,7 +123,7 @@ ifeq ($(CONFIG_X86_64),y) obj-$(CONFIG_GART_IOMMU) += amd_gart_64.o aperture_64.o obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o - obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o + obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o obj-y += vsmp_64.o diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 21a80bfbdb52..9246c5bf25af 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -17,3 +17,32 @@ config MSM_IOMMU config IOMMU_PGTABLES_L2 def_bool y depends on MSM_IOMMU && MMU && SMP && CPU_DCACHE_DISABLE=n + +# AMD IOMMU support +config AMD_IOMMU + bool "AMD IOMMU support" + select SWIOTLB + select PCI_MSI + select PCI_IOV + select IOMMU_API + depends on X86_64 && PCI && ACPI + ---help--- + With this option you can enable support for AMD IOMMU hardware in + your system. An IOMMU is a hardware component which provides + remapping of DMA memory accesses from devices. With an AMD IOMMU you + can isolate the the DMA memory of different devices and protect the + system from misbehaving device drivers or hardware. + + You can find out if your system has an AMD IOMMU if you look into + your BIOS for an option to enable it or if you have an IVRS ACPI + table. + +config AMD_IOMMU_STATS + bool "Export AMD IOMMU statistics to debugfs" + depends on AMD_IOMMU + select DEBUG_FS + ---help--- + This option enables code in the AMD IOMMU driver to collect various + statistics about whats happening in the driver and exports that + information to userspace via debugfs. + If unsure, say N. diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 1a71c82b1af2..4237eaf84609 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o +obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o diff --git a/arch/x86/kernel/amd_iommu.c b/drivers/iommu/amd_iommu.c similarity index 100% rename from arch/x86/kernel/amd_iommu.c rename to drivers/iommu/amd_iommu.c From 166e9278a3f98bab29ebb3d685a81cfb11b98be0 Mon Sep 17 00:00:00 2001 From: Ohad Ben-Cohen Date: Fri, 10 Jun 2011 21:42:27 +0300 Subject: [PATCH 14/16] x86/ia64: intel-iommu: move to drivers/iommu/ This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Note: to move intel-iommu.c, the declaration of pci_find_upstream_pcie_bridge() has to move from drivers/pci/pci.h to include/linux/pci.h. This is handled in this patch, too. As suggested, also drop DMAR's EXPERIMENTAL tag while we're at it. Compile-tested on x86_64. Signed-off-by: Ohad Ben-Cohen Signed-off-by: Joerg Roedel --- arch/ia64/Kconfig | 21 ----------- arch/x86/Kconfig | 50 ------------------------- drivers/iommu/Kconfig | 49 ++++++++++++++++++++++++ drivers/iommu/Makefile | 2 + drivers/{pci => iommu}/dmar.c | 0 drivers/{pci => iommu}/intel-iommu.c | 1 - drivers/{pci => iommu}/intr_remapping.c | 1 - drivers/{pci => iommu}/intr_remapping.h | 0 drivers/{pci => iommu}/iova.c | 0 drivers/pci/Makefile | 5 --- drivers/pci/pci.h | 2 - include/linux/pci.h | 11 ++++++ 12 files changed, 62 insertions(+), 80 deletions(-) rename drivers/{pci => iommu}/dmar.c (100%) rename drivers/{pci => iommu}/intel-iommu.c (99%) rename drivers/{pci => iommu}/intr_remapping.c (99%) rename drivers/{pci => iommu}/intr_remapping.h (100%) rename drivers/{pci => iommu}/iova.c (100%) diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 9929e4e11ea0..7336ba653b8f 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -627,27 +627,6 @@ source "drivers/pci/hotplug/Kconfig" source "drivers/pcmcia/Kconfig" -config DMAR - bool "Support for DMA Remapping Devices (EXPERIMENTAL)" - depends on IA64_GENERIC && ACPI && EXPERIMENTAL - help - DMA remapping (DMAR) devices support enables independent address - translations for Direct Memory Access (DMA) from devices. - These DMA remapping devices are reported via ACPI tables - and include PCI device scope covered by these DMA - remapping devices. - -config DMAR_DEFAULT_ON - def_bool y - prompt "Enable DMA Remapping Devices by default" - depends on DMAR - help - Selecting this option will enable a DMAR device at boot time if - one is found. If this option is not selected, DMAR support can - be enabled by passing intel_iommu=on to the kernel. It is - recommended you say N here while the DMAR code remains - experimental. - endmenu endif diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1b6a2e212db4..a169573c64fc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1912,56 +1912,6 @@ config PCI_CNB20LE_QUIRK You should say N unless you know you need this. -config DMAR - bool "Support for DMA Remapping Devices (EXPERIMENTAL)" - depends on PCI_MSI && ACPI && EXPERIMENTAL - select IOMMU_API - help - DMA remapping (DMAR) devices support enables independent address - translations for Direct Memory Access (DMA) from devices. - These DMA remapping devices are reported via ACPI tables - and include PCI device scope covered by these DMA - remapping devices. - -config DMAR_DEFAULT_ON - def_bool y - prompt "Enable DMA Remapping Devices by default" - depends on DMAR - help - Selecting this option will enable a DMAR device at boot time if - one is found. If this option is not selected, DMAR support can - be enabled by passing intel_iommu=on to the kernel. It is - recommended you say N here while the DMAR code remains - experimental. - -config DMAR_BROKEN_GFX_WA - bool "Workaround broken graphics drivers (going away soon)" - depends on DMAR && BROKEN - ---help--- - Current Graphics drivers tend to use physical address - for DMA and avoid using DMA APIs. Setting this config - option permits the IOMMU driver to set a unity map for - all the OS-visible memory. Hence the driver can continue - to use physical addresses for DMA, at least until this - option is removed in the 2.6.32 kernel. - -config DMAR_FLOPPY_WA - def_bool y - depends on DMAR - ---help--- - Floppy disk drivers are known to bypass DMA API calls - thereby failing to work when IOMMU is enabled. This - workaround will setup a 1:1 mapping for the first - 16MiB to make floppy (an ISA device) work. - -config INTR_REMAP - bool "Support for Interrupt Remapping (EXPERIMENTAL)" - depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL - ---help--- - Supports Interrupt remapping for IO-APIC and MSI devices. - To use x2apic mode in the CPU's which support x2APIC enhancements or - to support platforms with CPU's having > 8 bit APIC ID, say Y. - source "drivers/pci/pcie/Kconfig" source "drivers/pci/Kconfig" diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9246c5bf25af..e2a5f141ae2d 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -46,3 +46,52 @@ config AMD_IOMMU_STATS statistics about whats happening in the driver and exports that information to userspace via debugfs. If unsure, say N. + +# Intel IOMMU support +config DMAR + bool "Support for DMA Remapping Devices" + depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) + select IOMMU_API + help + DMA remapping (DMAR) devices support enables independent address + translations for Direct Memory Access (DMA) from devices. + These DMA remapping devices are reported via ACPI tables + and include PCI device scope covered by these DMA + remapping devices. + +config DMAR_DEFAULT_ON + def_bool y + prompt "Enable DMA Remapping Devices by default" + depends on DMAR + help + Selecting this option will enable a DMAR device at boot time if + one is found. If this option is not selected, DMAR support can + be enabled by passing intel_iommu=on to the kernel. + +config DMAR_BROKEN_GFX_WA + bool "Workaround broken graphics drivers (going away soon)" + depends on DMAR && BROKEN && X86 + ---help--- + Current Graphics drivers tend to use physical address + for DMA and avoid using DMA APIs. Setting this config + option permits the IOMMU driver to set a unity map for + all the OS-visible memory. Hence the driver can continue + to use physical addresses for DMA, at least until this + option is removed in the 2.6.32 kernel. + +config DMAR_FLOPPY_WA + def_bool y + depends on DMAR && X86 + ---help--- + Floppy disk drivers are known to bypass DMA API calls + thereby failing to work when IOMMU is enabled. This + workaround will setup a 1:1 mapping for the first + 16MiB to make floppy (an ISA device) work. + +config INTR_REMAP + bool "Support for Interrupt Remapping (EXPERIMENTAL)" + depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL + ---help--- + Supports Interrupt remapping for IO-APIC and MSI devices. + To use x2apic mode in the CPU's which support x2APIC enhancements or + to support platforms with CPU's having > 8 bit APIC ID, say Y. diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 4237eaf84609..49e9c0f46bd5 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,3 +1,5 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o +obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o +obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o diff --git a/drivers/pci/dmar.c b/drivers/iommu/dmar.c similarity index 100% rename from drivers/pci/dmar.c rename to drivers/iommu/dmar.c diff --git a/drivers/pci/intel-iommu.c b/drivers/iommu/intel-iommu.c similarity index 99% rename from drivers/pci/intel-iommu.c rename to drivers/iommu/intel-iommu.c index f02c34d26d1b..c621c98c99da 100644 --- a/drivers/pci/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -42,7 +42,6 @@ #include #include #include -#include "pci.h" #define ROOT_SIZE VTD_PAGE_SIZE #define CONTEXT_SIZE VTD_PAGE_SIZE diff --git a/drivers/pci/intr_remapping.c b/drivers/iommu/intr_remapping.c similarity index 99% rename from drivers/pci/intr_remapping.c rename to drivers/iommu/intr_remapping.c index 3607faf28a4d..1a89d4a2cadf 100644 --- a/drivers/pci/intr_remapping.c +++ b/drivers/iommu/intr_remapping.c @@ -13,7 +13,6 @@ #include "intr_remapping.h" #include #include -#include "pci.h" static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; static struct hpet_scope ir_hpet[MAX_HPET_TBS]; diff --git a/drivers/pci/intr_remapping.h b/drivers/iommu/intr_remapping.h similarity index 100% rename from drivers/pci/intr_remapping.h rename to drivers/iommu/intr_remapping.h diff --git a/drivers/pci/iova.c b/drivers/iommu/iova.c similarity index 100% rename from drivers/pci/iova.c rename to drivers/iommu/iova.c diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index 094308e41be5..825c02b40daa 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -29,11 +29,6 @@ obj-$(CONFIG_PCI_MSI) += msi.o # Build the Hypertransport interrupt support obj-$(CONFIG_HT_IRQ) += htirq.o -# Build Intel IOMMU support -obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o - -obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o - obj-$(CONFIG_PCI_IOV) += iov.o # diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 731e20265ace..b7bf11dd546a 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -184,8 +184,6 @@ pci_match_one_device(const struct pci_device_id *id, const struct pci_dev *dev) return NULL; } -struct pci_dev *pci_find_upstream_pcie_bridge(struct pci_dev *pdev); - /* PCI slot sysfs helper code */ #define to_pci_slot(s) container_of(s, struct pci_slot, kobj) diff --git a/include/linux/pci.h b/include/linux/pci.h index c446b5ca2d38..970bfe0941c3 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1589,5 +1589,16 @@ int pci_vpd_find_tag(const u8 *buf, unsigned int off, unsigned int len, u8 rdt); int pci_vpd_find_info_keyword(const u8 *buf, unsigned int off, unsigned int len, const char *kw); +/** + * pci_find_upstream_pcie_bridge - find upstream PCIe-to-PCI bridge of a device + * @pdev: the PCI device + * + * if the device is PCIE, return NULL + * if the device isn't connected to a PCIe bridge (that is its parent is a + * legacy PCI bridge and the bridge is directly connected to bus 0), return its + * parent + */ +struct pci_dev *pci_find_upstream_pcie_bridge(struct pci_dev *pdev); + #endif /* __KERNEL__ */ #endif /* LINUX_PCI_H */ From 68255b628776dfafa7f67ca3afd66bd4ba377307 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 14 Jun 2011 15:51:54 +0200 Subject: [PATCH 15/16] iommu: Move iommu Kconfig entries to submenu For better navigation this patch moves the drivers/iommu drivers into its own submenu in Kconfig. Signed-off-by: Joerg Roedel --- drivers/iommu/Kconfig | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index e2a5f141ae2d..b57b3fa492f3 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -2,6 +2,17 @@ config IOMMU_API bool +menuconfig IOMMU_SUPPORT + bool "IOMMU Hardware Support" + default y + ---help--- + Say Y here if you want to compile device drivers for IO Memory + Management Units into the kernel. These devices usually allow to + remap DMA requests and/or remap interrupts from other devices on the + system. + +if IOMMU_SUPPORT + # MSM IOMMU support config MSM_IOMMU bool "MSM IOMMU Support" @@ -95,3 +106,5 @@ config INTR_REMAP Supports Interrupt remapping for IO-APIC and MSI devices. To use x2apic mode in the CPU's which support x2APIC enhancements or to support platforms with CPU's having > 8 bit APIC ID, say Y. + +endif # IOMMU_SUPPORT From 403f81d8ee532c976d50a5e1051f14ec78ae8db3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 14 Jun 2011 16:44:25 +0200 Subject: [PATCH 16/16] iommu/amd: Move missing parts to drivers/iommu A few parts of the driver were missing in drivers/iommu. Move them there to have the complete driver in that directory. Signed-off-by: Joerg Roedel --- arch/x86/kernel/Makefile | 1 - drivers/iommu/Makefile | 2 +- drivers/iommu/amd_iommu.c | 7 ++++--- {arch/x86/kernel => drivers/iommu}/amd_iommu_init.c | 8 +++++--- {arch/x86/include/asm => drivers/iommu}/amd_iommu_proto.h | 2 +- {arch/x86/include/asm => drivers/iommu}/amd_iommu_types.h | 0 .../include/asm/amd_iommu.h => include/linux/amd-iommu.h | 0 7 files changed, 11 insertions(+), 9 deletions(-) rename {arch/x86/kernel => drivers/iommu}/amd_iommu_init.c (99%) rename {arch/x86/include/asm => drivers/iommu}/amd_iommu_proto.h (98%) rename {arch/x86/include/asm => drivers/iommu}/amd_iommu_types.h (100%) rename arch/x86/include/asm/amd_iommu.h => include/linux/amd-iommu.h (100%) diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index aef02989c930..11817ff85399 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -123,7 +123,6 @@ ifeq ($(CONFIG_X86_64),y) obj-$(CONFIG_GART_IOMMU) += amd_gart_64.o aperture_64.o obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o - obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o obj-y += vsmp_64.o diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 49e9c0f46bd5..4d4d77df7cac 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o -obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o +obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 7c3a95e54ec5..5aa12eaabd21 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -27,13 +27,14 @@ #include #include #include +#include #include #include #include #include -#include -#include -#include + +#include "amd_iommu_proto.h" +#include "amd_iommu_types.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) diff --git a/arch/x86/kernel/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c similarity index 99% rename from arch/x86/kernel/amd_iommu_init.c rename to drivers/iommu/amd_iommu_init.c index bfc8453bd98d..82d2410f4205 100644 --- a/arch/x86/kernel/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -24,14 +24,16 @@ #include #include #include +#include #include -#include -#include -#include #include #include #include #include + +#include "amd_iommu_proto.h" +#include "amd_iommu_types.h" + /* * definitions for the ACPI scanning code */ diff --git a/arch/x86/include/asm/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h similarity index 98% rename from arch/x86/include/asm/amd_iommu_proto.h rename to drivers/iommu/amd_iommu_proto.h index 55d95eb789b3..7ffaa64410b0 100644 --- a/arch/x86/include/asm/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -19,7 +19,7 @@ #ifndef _ASM_X86_AMD_IOMMU_PROTO_H #define _ASM_X86_AMD_IOMMU_PROTO_H -#include +#include "amd_iommu_types.h" extern int amd_iommu_init_dma_ops(void); extern int amd_iommu_init_passthrough(void); diff --git a/arch/x86/include/asm/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h similarity index 100% rename from arch/x86/include/asm/amd_iommu_types.h rename to drivers/iommu/amd_iommu_types.h diff --git a/arch/x86/include/asm/amd_iommu.h b/include/linux/amd-iommu.h similarity index 100% rename from arch/x86/include/asm/amd_iommu.h rename to include/linux/amd-iommu.h