OpenE2K
/
linux
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

swiotlb: Add support for DMA_ATTR_SKIP_CPU_SYNC 

As a first step to making DMA_ATTR_SKIP_CPU_SYNC apply to architectures
beyond just ARM I need to make it so that the swiotlb will respect the
flag.  In order to do that I also need to update the swiotlb-xen since it
heavily makes use of the functionality.

Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad@kernel.org>
This commit is contained in:
Alexander Duyck 2016-11-02 07:13:02 -04:00 committed by Konrad Rzeszutek Wilk
parent 7641842164
commit 0443fa003f
3 changed files with 49 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
drivers/xen/swiotlb-xen.c
View File

@ -405,7 +405,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
*/ */
trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir); map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, size, dir,
attrs);
if (map == SWIOTLB_MAP_ERROR) if (map == SWIOTLB_MAP_ERROR)
return DMA_ERROR_CODE; return DMA_ERROR_CODE;
@ -419,7 +420,8 @@ dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page,
if (dma_capable(dev, dev_addr, size)) if (dma_capable(dev, dev_addr, size))
return dev_addr; return dev_addr;
swiotlb_tbl_unmap_single(dev, map, size, dir); swiotlb_tbl_unmap_single(dev, map, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC);
return DMA_ERROR_CODE; return DMA_ERROR_CODE;
} }
@ -445,7 +447,7 @@ static void xen_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
/* NOTE: We use dev_addr here, not paddr! */ /* NOTE: We use dev_addr here, not paddr! */
if (is_xen_swiotlb_buffer(dev_addr)) { if (is_xen_swiotlb_buffer(dev_addr)) {
swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
return; return;
} }
@ -558,11 +560,12 @@ xen_swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
start_dma_addr, start_dma_addr,
sg_phys(sg), sg_phys(sg),
sg->length, sg->length,
dir); dir, attrs);
if (map == SWIOTLB_MAP_ERROR) { if (map == SWIOTLB_MAP_ERROR) {
dev_warn(hwdev, "swiotlb buffer is full\n"); dev_warn(hwdev, "swiotlb buffer is full\n");
/* Don't panic here, we expect map_sg users /* Don't panic here, we expect map_sg users
to do proper error handling. */ to do proper error handling. */
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir, xen_swiotlb_unmap_sg_attrs(hwdev, sgl, i, dir,
attrs); attrs);
sg_dma_len(sgl) = 0; sg_dma_len(sgl) = 0;

6
include/linux/swiotlb.h
View File

@ -44,11 +44,13 @@ enum dma_sync_target {
extern phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, extern phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
dma_addr_t tbl_dma_addr, dma_addr_t tbl_dma_addr,
phys_addr_t phys, size_t size, phys_addr_t phys, size_t size,
enum dma_data_direction dir); enum dma_data_direction dir,
unsigned long attrs);
extern void swiotlb_tbl_unmap_single(struct device *hwdev, extern void swiotlb_tbl_unmap_single(struct device *hwdev,
phys_addr_t tlb_addr, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir); size_t size, enum dma_data_direction dir,
unsigned long attrs);
extern void swiotlb_tbl_sync_single(struct device *hwdev, extern void swiotlb_tbl_sync_single(struct device *hwdev,
phys_addr_t tlb_addr, phys_addr_t tlb_addr,

59
lib/swiotlb.c
View File

@ -425,7 +425,8 @@ static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
dma_addr_t tbl_dma_addr, dma_addr_t tbl_dma_addr,
phys_addr_t orig_addr, size_t size, phys_addr_t orig_addr, size_t size,
enum dma_data_direction dir) enum dma_data_direction dir,
unsigned long attrs)
{ {
unsigned long flags; unsigned long flags;
phys_addr_t tlb_addr; phys_addr_t tlb_addr;
@ -526,7 +527,8 @@ found:
*/ */
for (i = 0; i < nslots; i++) for (i = 0; i < nslots; i++)
io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT); io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE); swiotlb_bounce(orig_addr, tlb_addr, size, DMA_TO_DEVICE);
return tlb_addr; return tlb_addr;
@ -539,18 +541,20 @@ EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
static phys_addr_t static phys_addr_t
map_single(struct device *hwdev, phys_addr_t phys, size_t size, map_single(struct device *hwdev, phys_addr_t phys, size_t size,
enum dma_data_direction dir) enum dma_data_direction dir, unsigned long attrs)
{ {
dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start); dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start);
return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, dir); return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
dir, attrs);
} }
/* /*
* dma_addr is the kernel virtual address of the bounce buffer to unmap. * dma_addr is the kernel virtual address of the bounce buffer to unmap.
*/ */
void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir) size_t size, enum dma_data_direction dir,
unsigned long attrs)
{ {
unsigned long flags; unsigned long flags;
int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT; int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
@ -561,6 +565,7 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
* First, sync the memory before unmapping the entry * First, sync the memory before unmapping the entry
*/ */
if (orig_addr != INVALID_PHYS_ADDR && if (orig_addr != INVALID_PHYS_ADDR &&
!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))) ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE); swiotlb_bounce(orig_addr, tlb_addr, size, DMA_FROM_DEVICE);
@ -654,7 +659,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
* GFP_DMA memory; fall back on map_single(), which * GFP_DMA memory; fall back on map_single(), which
* will grab memory from the lowest available address range. * will grab memory from the lowest available address range.
*/ */
phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE); phys_addr_t paddr = map_single(hwdev, 0, size,
DMA_FROM_DEVICE, 0);
if (paddr == SWIOTLB_MAP_ERROR) if (paddr == SWIOTLB_MAP_ERROR)
goto err_warn; goto err_warn;
@ -667,9 +673,13 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
(unsigned long long)dma_mask, (unsigned long long)dma_mask,
(unsigned long long)dev_addr); (unsigned long long)dev_addr);
/* DMA_TO_DEVICE to avoid memcpy in unmap_single */ /*
* DMA_TO_DEVICE to avoid memcpy in unmap_single.
* The DMA_ATTR_SKIP_CPU_SYNC is optional.
*/
swiotlb_tbl_unmap_single(hwdev, paddr, swiotlb_tbl_unmap_single(hwdev, paddr,
size, DMA_TO_DEVICE); size, DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
goto err_warn; goto err_warn;
} }
} }
@ -698,8 +708,12 @@ swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
if (!is_swiotlb_buffer(paddr)) if (!is_swiotlb_buffer(paddr))
free_pages((unsigned long)vaddr, get_order(size)); free_pages((unsigned long)vaddr, get_order(size));
else else
/* DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single */ /*
swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE); * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
* DMA_ATTR_SKIP_CPU_SYNC is optional.
*/
swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE,
DMA_ATTR_SKIP_CPU_SYNC);
} }
EXPORT_SYMBOL(swiotlb_free_coherent); EXPORT_SYMBOL(swiotlb_free_coherent);
@ -755,7 +769,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force);
/* Oh well, have to allocate and map a bounce buffer. */ /* Oh well, have to allocate and map a bounce buffer. */
map = map_single(dev, phys, size, dir); map = map_single(dev, phys, size, dir, attrs);
if (map == SWIOTLB_MAP_ERROR) { if (map == SWIOTLB_MAP_ERROR) {
swiotlb_full(dev, size, dir, 1); swiotlb_full(dev, size, dir, 1);
return phys_to_dma(dev, io_tlb_overflow_buffer); return phys_to_dma(dev, io_tlb_overflow_buffer);
@ -764,12 +778,13 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
dev_addr = phys_to_dma(dev, map); dev_addr = phys_to_dma(dev, map);
/* Ensure that the address returned is DMA'ble */ /* Ensure that the address returned is DMA'ble */
if (!dma_capable(dev, dev_addr, size)) { if (dma_capable(dev, dev_addr, size))
swiotlb_tbl_unmap_single(dev, map, size, dir); return dev_addr;
return phys_to_dma(dev, io_tlb_overflow_buffer);
}
return dev_addr; swiotlb_tbl_unmap_single(dev, map, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC);
return phys_to_dma(dev, io_tlb_overflow_buffer);
} }
EXPORT_SYMBOL_GPL(swiotlb_map_page); EXPORT_SYMBOL_GPL(swiotlb_map_page);
@ -782,14 +797,15 @@ EXPORT_SYMBOL_GPL(swiotlb_map_page);
* whatever the device wrote there. * whatever the device wrote there.
*/ */
static void unmap_single(struct device *hwdev, dma_addr_t dev_addr, static void unmap_single(struct device *hwdev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir) size_t size, enum dma_data_direction dir,
unsigned long attrs)
{ {
phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
BUG_ON(dir == DMA_NONE); BUG_ON(dir == DMA_NONE);
if (is_swiotlb_buffer(paddr)) { if (is_swiotlb_buffer(paddr)) {
swiotlb_tbl_unmap_single(hwdev, paddr, size, dir); swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs);
return; return;
} }
@ -809,7 +825,7 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir, size_t size, enum dma_data_direction dir,
unsigned long attrs) unsigned long attrs)
{ {
unmap_single(hwdev, dev_addr, size, dir); unmap_single(hwdev, dev_addr, size, dir, attrs);
} }
EXPORT_SYMBOL_GPL(swiotlb_unmap_page); EXPORT_SYMBOL_GPL(swiotlb_unmap_page);
@ -891,7 +907,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
if (swiotlb_force || if (swiotlb_force ||
!dma_capable(hwdev, dev_addr, sg->length)) { !dma_capable(hwdev, dev_addr, sg->length)) {
phys_addr_t map = map_single(hwdev, sg_phys(sg), phys_addr_t map = map_single(hwdev, sg_phys(sg),
sg->length, dir); sg->length, dir, attrs);
if (map == SWIOTLB_MAP_ERROR) { if (map == SWIOTLB_MAP_ERROR) {
/* Don't panic here, we expect map_sg users /* Don't panic here, we expect map_sg users
to do proper error handling. */ to do proper error handling. */
@ -925,7 +941,8 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
BUG_ON(dir == DMA_NONE); BUG_ON(dir == DMA_NONE);
for_each_sg(sgl, sg, nelems, i) for_each_sg(sgl, sg, nelems, i)
unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir); unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir,
attrs);
} }
EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);