iommu/amd: Support IOMMU_DOMAIN_DMA type allocation

This enables allocation of DMA-API default domains from the
IOMMU core and switches allocation of domain dma-api domain
to the IOMMU core too.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
Joerg Roedel 2015-05-28 18:41:40 +02:00
parent aafd8ba0ca
commit 0bb6e243d7
2 changed files with 75 additions and 243 deletions

View File

@ -64,10 +64,6 @@
static DEFINE_RWLOCK(amd_iommu_devtable_lock);
/* A list of preallocated protection domains */
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);
/* List of all available dev_data structures */
static LIST_HEAD(dev_data_list);
static DEFINE_SPINLOCK(dev_data_list_lock);
@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum)
}
/*
* In this function the list of preallocated protection domains is traversed to
* find the domain for a specific device
* This function actually applies the mapping to the page table of the
* dma_ops domain.
*/
static struct dma_ops_domain *find_protection_domain(u16 devid)
static void alloc_unity_mapping(struct dma_ops_domain *dma_dom,
struct unity_map_entry *e)
{
struct dma_ops_domain *entry, *ret = NULL;
unsigned long flags;
u16 alias = amd_iommu_alias_table[devid];
u64 addr;
if (list_empty(&iommu_pd_list))
return NULL;
spin_lock_irqsave(&iommu_pd_list_lock, flags);
list_for_each_entry(entry, &iommu_pd_list, list) {
if (entry->target_dev == devid ||
entry->target_dev == alias) {
ret = entry;
break;
}
for (addr = e->address_start; addr < e->address_end;
addr += PAGE_SIZE) {
if (addr < dma_dom->aperture_size)
__set_bit(addr >> PAGE_SHIFT,
dma_dom->aperture[0]->bitmap);
}
}
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
/*
* Inits the unity mappings required for a specific device
*/
static void init_unity_mappings_for_device(struct device *dev,
struct dma_ops_domain *dma_dom)
{
struct unity_map_entry *e;
u16 devid;
return ret;
devid = get_device_id(dev);
list_for_each_entry(e, &amd_iommu_unity_map, list) {
if (!(devid >= e->devid_start && devid <= e->devid_end))
continue;
alloc_unity_mapping(dma_dom, e);
}
}
/*
@ -290,11 +293,23 @@ static bool check_device(struct device *dev)
static void init_iommu_group(struct device *dev)
{
struct dma_ops_domain *dma_domain;
struct iommu_domain *domain;
struct iommu_group *group;
group = iommu_group_get_for_dev(dev);
if (!IS_ERR(group))
iommu_group_put(group);
if (IS_ERR(group))
return;
domain = iommu_group_default_domain(group);
if (!domain)
goto out;
dma_domain = to_pdomain(domain)->priv;
init_unity_mappings_for_device(dev, dma_domain);
out:
iommu_group_put(group);
}
static int __last_alias(struct pci_dev *pdev, u16 alias, void *data)
@ -1414,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom,
return unmapped;
}
/*
* This function checks if a specific unity mapping entry is needed for
* this specific IOMMU.
*/
static int iommu_for_unity_map(struct amd_iommu *iommu,
struct unity_map_entry *entry)
{
u16 bdf, i;
for (i = entry->devid_start; i <= entry->devid_end; ++i) {
bdf = amd_iommu_alias_table[i];
if (amd_iommu_rlookup_table[bdf] == iommu)
return 1;
}
return 0;
}
/*
* This function actually applies the mapping to the page table of the
* dma_ops domain.
*/
static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
struct unity_map_entry *e)
{
u64 addr;
int ret;
for (addr = e->address_start; addr < e->address_end;
addr += PAGE_SIZE) {
ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
PAGE_SIZE);
if (ret)
return ret;
/*
* if unity mapping is in aperture range mark the page
* as allocated in the aperture
*/
if (addr < dma_dom->aperture_size)
__set_bit(addr >> PAGE_SHIFT,
dma_dom->aperture[0]->bitmap);
}
return 0;
}
/*
* Init the unity mappings for a specific IOMMU in the system
*
* Basically iterates over all unity mapping entries and applies them to
* the default domain DMA of that IOMMU if necessary.
*/
static int iommu_init_unity_mappings(struct amd_iommu *iommu)
{
struct unity_map_entry *entry;
int ret;
list_for_each_entry(entry, &amd_iommu_unity_map, list) {
if (!iommu_for_unity_map(iommu, entry))
continue;
ret = dma_ops_unity_map(iommu->default_dom, entry);
if (ret)
return ret;
}
return 0;
}
/*
* Inits the unity mappings required for a specific device
*/
static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
u16 devid)
{
struct unity_map_entry *e;
int ret;
list_for_each_entry(e, &amd_iommu_unity_map, list) {
if (!(devid >= e->devid_start && devid <= e->devid_end))
continue;
ret = dma_ops_unity_map(dma_dom, e);
if (ret)
return ret;
}
return 0;
}
/****************************************************************************
*
* The next functions belong to the address allocator for the dma_ops
@ -2324,42 +2251,9 @@ static void detach_device(struct device *dev)
dev_data->ats.enabled = false;
}
/*
* Find out the protection domain structure for a given PCI device. This
* will give us the pointer to the page table root for example.
*/
static struct protection_domain *domain_for_device(struct device *dev)
{
struct iommu_dev_data *dev_data;
struct protection_domain *dom = NULL;
unsigned long flags;
dev_data = get_dev_data(dev);
if (dev_data->domain)
return dev_data->domain;
if (dev_data->alias_data != NULL) {
struct iommu_dev_data *alias_data = dev_data->alias_data;
read_lock_irqsave(&amd_iommu_devtable_lock, flags);
if (alias_data->domain != NULL) {
__attach_device(dev_data, alias_data->domain);
dom = alias_data->domain;
}
read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
}
return dom;
}
static int amd_iommu_add_device(struct device *dev)
{
struct dma_ops_domain *dma_domain;
struct protection_domain *domain;
struct iommu_dev_data *dev_data;
struct amd_iommu *iommu;
unsigned long flags;
u16 devid;
int ret;
@ -2376,35 +2270,6 @@ static int amd_iommu_add_device(struct device *dev)
}
init_iommu_group(dev);
dev_data = get_dev_data(dev);
if (iommu_pass_through || dev_data->iommu_v2) {
/* Make sure passthrough domain is allocated */
alloc_passthrough_domain();
dev_data->passthrough = true;
attach_device(dev, pt_domain);
goto out;
}
domain = domain_for_device(dev);
/* allocate a protection domain if a device is added */
dma_domain = find_protection_domain(devid);
if (!dma_domain) {
dma_domain = dma_ops_domain_alloc();
if (!dma_domain)
goto out;
dma_domain->target_dev = devid;
init_unity_mappings_for_device(dma_domain, devid);
spin_lock_irqsave(&iommu_pd_list_lock, flags);
list_add_tail(&dma_domain->list, &iommu_pd_list);
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
}
attach_device(dev, &dma_domain->domain);
dev->archdata.dma_ops = &amd_iommu_dma_ops;
out:
@ -2445,34 +2310,19 @@ static struct protection_domain *get_domain(struct device *dev)
{
struct protection_domain *domain;
struct iommu_domain *io_domain;
struct dma_ops_domain *dma_dom;
u16 devid = get_device_id(dev);
if (!check_device(dev))
return ERR_PTR(-EINVAL);
io_domain = iommu_get_domain_for_dev(dev);
if (io_domain) {
domain = to_pdomain(io_domain);
return domain;
}
if (!io_domain)
return NULL;
domain = domain_for_device(dev);
if (domain != NULL && !dma_ops_domain(domain))
domain = to_pdomain(io_domain);
if (!dma_ops_domain(domain))
return ERR_PTR(-EBUSY);
if (domain != NULL)
return domain;
/* Device not bound yet - bind it */
dma_dom = find_protection_domain(devid);
if (!dma_dom)
dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
attach_device(dev, &dma_dom->domain);
DUMP_printk("Using protection domain %d for device %s\n",
dma_dom->domain.id, dev_name(dev));
return &dma_dom->domain;
return domain;
}
static void update_device_table(struct protection_domain *domain)
@ -3014,23 +2864,7 @@ void __init amd_iommu_init_api(void)
int __init amd_iommu_init_dma_ops(void)
{
struct amd_iommu *iommu;
int ret, unhandled;
/*
* first allocate a default protection domain for every IOMMU we
* found in the system. Devices not assigned to any other
* protection domain will be assigned to the default one.
*/
for_each_iommu(iommu) {
iommu->default_dom = dma_ops_domain_alloc();
if (iommu->default_dom == NULL)
return -ENOMEM;
iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
ret = iommu_init_unity_mappings(iommu);
if (ret)
goto free_domains;
}
int unhandled;
iommu_detected = 1;
swiotlb = 0;
@ -3050,14 +2884,6 @@ int __init amd_iommu_init_dma_ops(void)
pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n");
return 0;
free_domains:
for_each_iommu(iommu) {
dma_ops_domain_free(iommu->default_dom);
}
return ret;
}
/*****************************************************************************
@ -3142,30 +2968,39 @@ static int alloc_passthrough_domain(void)
static struct iommu_domain *amd_iommu_domain_alloc(unsigned type)
{
struct protection_domain *pdomain;
struct dma_ops_domain *dma_domain;
/* We only support unmanaged domains for now */
if (type != IOMMU_DOMAIN_UNMANAGED)
switch (type) {
case IOMMU_DOMAIN_UNMANAGED:
pdomain = protection_domain_alloc();
if (!pdomain)
return NULL;
pdomain->mode = PAGE_MODE_3_LEVEL;
pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
if (!pdomain->pt_root) {
protection_domain_free(pdomain);
return NULL;
}
pdomain->domain.geometry.aperture_start = 0;
pdomain->domain.geometry.aperture_end = ~0ULL;
pdomain->domain.geometry.force_aperture = true;
break;
case IOMMU_DOMAIN_DMA:
dma_domain = dma_ops_domain_alloc();
if (!dma_domain) {
pr_err("AMD-Vi: Failed to allocate\n");
return NULL;
}
pdomain = &dma_domain->domain;
break;
default:
return NULL;
pdomain = protection_domain_alloc();
if (!pdomain)
goto out_free;
pdomain->mode = PAGE_MODE_3_LEVEL;
pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
if (!pdomain->pt_root)
goto out_free;
pdomain->domain.geometry.aperture_start = 0;
pdomain->domain.geometry.aperture_end = ~0ULL;
pdomain->domain.geometry.force_aperture = true;
}
return &pdomain->domain;
out_free:
protection_domain_free(pdomain);
return NULL;
}
static void amd_iommu_domain_free(struct iommu_domain *dom)

View File

@ -552,9 +552,6 @@ struct amd_iommu {
/* if one, we need to send a completion wait command */
bool need_sync;
/* default dma_ops domain for that IOMMU */
struct dma_ops_domain *default_dom;
/* IOMMU sysfs device */
struct device *iommu_dev;