From 0bb6e243d7fbb39fced5bd4a4c83eb49c6e820ce Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:40 +0200 Subject: [PATCH] iommu/amd: Support IOMMU_DOMAIN_DMA type allocation This enables allocation of DMA-API default domains from the IOMMU core and switches allocation of domain dma-api domain to the IOMMU core too. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 315 ++++++++------------------------ drivers/iommu/amd_iommu_types.h | 3 - 2 files changed, 75 insertions(+), 243 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0f8776940bf5..27300aece203 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -64,10 +64,6 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - /* List of all available dev_data structures */ static LIST_HEAD(dev_data_list); static DEFINE_SPINLOCK(dev_data_list_lock); @@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) } /* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device + * This function actually applies the mapping to the page table of the + * dma_ops domain. */ -static struct dma_ops_domain *find_protection_domain(u16 devid) +static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) { - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; + u64 addr; - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); } +} - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); +/* + * Inits the unity mappings required for a specific device + */ +static void init_unity_mappings_for_device(struct device *dev, + struct dma_ops_domain *dma_dom) +{ + struct unity_map_entry *e; + u16 devid; - return ret; + devid = get_device_id(dev); + + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + alloc_unity_mapping(dma_dom, e); + } } /* @@ -290,11 +293,23 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { + struct dma_ops_domain *dma_domain; + struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) - iommu_group_put(group); + if (IS_ERR(group)) + return; + + domain = iommu_group_default_domain(group); + if (!domain) + goto out; + + dma_domain = to_pdomain(domain)->priv; + + init_unity_mappings_for_device(dev, dma_domain); +out: + iommu_group_put(group); } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) @@ -1414,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops @@ -2324,42 +2251,9 @@ static void detach_device(struct device *dev) dev_data->ats.enabled = false; } -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct protection_domain *dom = NULL; - unsigned long flags; - - dev_data = get_dev_data(dev); - - if (dev_data->domain) - return dev_data->domain; - - if (dev_data->alias_data != NULL) { - struct iommu_dev_data *alias_data = dev_data->alias_data; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - } - - return dom; -} - static int amd_iommu_add_device(struct device *dev) { - struct dma_ops_domain *dma_domain; - struct protection_domain *domain; - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - unsigned long flags; u16 devid; int ret; @@ -2376,35 +2270,6 @@ static int amd_iommu_add_device(struct device *dev) } init_iommu_group(dev); - dev_data = get_dev_data(dev); - - if (iommu_pass_through || dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(dev, pt_domain); - goto out; - } - - domain = domain_for_device(dev); - - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - init_unity_mappings_for_device(dma_domain, devid); - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } - - attach_device(dev, &dma_domain->domain); - dev->archdata.dma_ops = &amd_iommu_dma_ops; out: @@ -2445,34 +2310,19 @@ static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; struct iommu_domain *io_domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); if (!check_device(dev)) return ERR_PTR(-EINVAL); io_domain = iommu_get_domain_for_dev(dev); - if (io_domain) { - domain = to_pdomain(io_domain); - return domain; - } + if (!io_domain) + return NULL; - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) + domain = to_pdomain(io_domain); + if (!dma_ops_domain(domain)) return ERR_PTR(-EBUSY); - if (domain != NULL) - return domain; - - /* Device not bound yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); - - return &dma_dom->domain; + return domain; } static void update_device_table(struct protection_domain *domain) @@ -3014,23 +2864,7 @@ void __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } + int unhandled; iommu_detected = 1; swiotlb = 0; @@ -3050,14 +2884,6 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; - -free_domains: - - for_each_iommu(iommu) { - dma_ops_domain_free(iommu->default_dom); - } - - return ret; } /***************************************************************************** @@ -3142,30 +2968,39 @@ static int alloc_passthrough_domain(void) static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { struct protection_domain *pdomain; + struct dma_ops_domain *dma_domain; - /* We only support unmanaged domains for now */ - if (type != IOMMU_DOMAIN_UNMANAGED) + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; + + pdomain->mode = PAGE_MODE_3_LEVEL; + pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pdomain->pt_root) { + protection_domain_free(pdomain); + return NULL; + } + + pdomain->domain.geometry.aperture_start = 0; + pdomain->domain.geometry.aperture_end = ~0ULL; + pdomain->domain.geometry.force_aperture = true; + + break; + case IOMMU_DOMAIN_DMA: + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) { + pr_err("AMD-Vi: Failed to allocate\n"); + return NULL; + } + pdomain = &dma_domain->domain; + break; + default: return NULL; - - pdomain = protection_domain_alloc(); - if (!pdomain) - goto out_free; - - pdomain->mode = PAGE_MODE_3_LEVEL; - pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pdomain->pt_root) - goto out_free; - - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; + } return &pdomain->domain; - -out_free: - protection_domain_free(pdomain); - - return NULL; } static void amd_iommu_domain_free(struct iommu_domain *dom) diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 05030e523771..fe796cfc62ca 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -552,9 +552,6 @@ struct amd_iommu { /* if one, we need to send a completion wait command */ bool need_sync; - /* default dma_ops domain for that IOMMU */ - struct dma_ops_domain *default_dom; - /* IOMMU sysfs device */ struct device *iommu_dev;