OpenE2K
/
linux
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

iommu/amd: Add per-domain timer to flush per-cpu queues 

Add a timer to each dma_ops domain so that we flush unused
IOTLB entries regularily, even if the queues don't get full
all the time.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
Joerg Roedel 2017-06-02 18:13:37 +02:00
parent a6e3f6f030
commit fca6af6a59
1 changed files with 67 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
drivers/iommu/amd_iommu.c
View File

@ -182,6 +182,13 @@ struct dma_ops_domain {
* flushed out of the IOTLB of the domain.
*/
atomic64_t flush_finish_cnt;
/*
* Timer to make sure we don't keep IOVAs around unflushed
* for too long
*/
struct timer_list flush_timer;
atomic_t flush_timer_on;
};
static struct iova_domain reserved_iova_ranges;
@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
return 0;
}
static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
{
atomic64_inc(&dom->flush_start_cnt);
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
atomic64_inc(&dom->flush_finish_cnt);
}
static inline bool queue_ring_full(struct flush_queue *queue)
{
assert_spin_locked(&queue->lock);
@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue)
queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
}
static void queue_add(struct dma_ops_domain *dom,
unsigned long address, unsigned long pages)
static void queue_ring_free_flushed(struct dma_ops_domain *dom,
struct flush_queue *queue)
{
struct flush_queue *queue;
unsigned long flags;
u64 counter;
u64 counter = atomic64_read(&dom->flush_finish_cnt);
int idx;
pages = __roundup_pow_of_two(pages);
address >>= PAGE_SHIFT;
counter = atomic64_read(&dom->flush_finish_cnt);
queue = get_cpu_ptr(dom->flush_queue);
spin_lock_irqsave(&queue->lock, flags);
queue_ring_for_each(idx, queue) {
/*
* This assumes that counter values in the ring-buffer are
@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom,
queue_ring_remove_head(queue);
}
}
static void queue_add(struct dma_ops_domain *dom,
unsigned long address, unsigned long pages)
{
struct flush_queue *queue;
unsigned long flags;
int idx;
pages = __roundup_pow_of_two(pages);
address >>= PAGE_SHIFT;
queue = get_cpu_ptr(dom->flush_queue);
spin_lock_irqsave(&queue->lock, flags);
queue_ring_free_flushed(dom, queue);
if (queue_ring_full(queue)) {
atomic64_inc(&dom->flush_start_cnt);
domain_flush_tlb(&dom->domain);
domain_flush_complete(&dom->domain);
atomic64_inc(&dom->flush_finish_cnt);
dma_ops_domain_flush_tlb(dom);
queue_release(dom, queue);
}
@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom,
queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
spin_unlock_irqrestore(&queue->lock, flags);
if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
put_cpu_ptr(dom->flush_queue);
}
static void queue_flush_timeout(unsigned long data)
{
struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
int cpu;
atomic_set(&dom->flush_timer_on, 0);
dma_ops_domain_flush_tlb(dom);
for_each_possible_cpu(cpu) {
struct flush_queue *queue;
unsigned long flags;
queue = per_cpu_ptr(dom->flush_queue, cpu);
spin_lock_irqsave(&queue->lock, flags);
queue_ring_free_flushed(dom, queue);
spin_unlock_irqrestore(&queue->lock, flags);
}
}
/*
* Free a domain, only used if something went wrong in the
* allocation path and we need to free an already allocated page table
@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
del_domain_from_list(&dom->domain);
if (timer_pending(&dom->flush_timer))
del_timer(&dom->flush_timer);
dma_ops_domain_free_flush_queue(dom);
put_iova_domain(&dom->iovad);
@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (dma_ops_domain_alloc_flush_queue(dma_dom))
goto free_dma_dom;
setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
(unsigned long)dma_dom);
atomic_set(&dma_dom->flush_timer_on, 0);
add_domain_to_list(&dma_dom->domain);
return dma_dom;