From 3e3a1823281fc342ac5d99d853167e009bb86258 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 16 Dec 2012 13:32:03 +0100 Subject: [PATCH 1/9] ARM: iommu: Include linux/kref.h in asm/dma-iommu.h The dma_iommu_mapping structure defined in asm/dma-iommu.h embeds a struct kref, include the appropriate header file. Signed-off-by: Laurent Pinchart Signed-off-by: Marek Szyprowski --- arch/arm/include/asm/dma-iommu.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h index 799b09409fad..666d8a8c3a2b 100644 --- a/arch/arm/include/asm/dma-iommu.h +++ b/arch/arm/include/asm/dma-iommu.h @@ -7,6 +7,7 @@ #include #include #include +#include struct dma_iommu_mapping { /* iommu specific data */ From d09e1333ec6c8c6fe2875dc2763ebbac84efe8d7 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Tue, 29 Jan 2013 14:57:07 +0200 Subject: [PATCH 2/9] ARM: dma-mapping: Set arm_dma_set_mask() for iommu->set_dma_mask() struct dma_map_ops iommu_ops doesn't have ->set_dma_mask, which causes crash when dma_set_mask() is called from some driver. Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index dda3904dc64c..d91c4880f559 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1732,6 +1732,8 @@ struct dma_map_ops iommu_ops = { .unmap_sg = arm_iommu_unmap_sg, .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, .sync_sg_for_device = arm_iommu_sync_sg_for_device, + + .set_dma_mask = arm_dma_set_mask, }; struct dma_map_ops iommu_coherent_ops = { @@ -1745,6 +1747,8 @@ struct dma_map_ops iommu_coherent_ops = { .map_sg = arm_coherent_iommu_map_sg, .unmap_sg = arm_coherent_iommu_unmap_sg, + + .set_dma_mask = arm_dma_set_mask, }; /** From fab112a394b9cf5e3bf888af6ff5076ba9fa0f06 Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Thu, 24 Jan 2013 15:16:56 +0200 Subject: [PATCH 3/9] ARM: dma-mapping: Add macro to_dma_iommu_mapping() This can be built without CONFIG_ARM_DMA_USE_IOMMU. Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/include/asm/device.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index b69c0d3285f8..dc662fca9230 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -27,4 +27,10 @@ struct pdev_archdata { #endif }; +#ifdef CONFIG_ARM_DMA_USE_IOMMU +#define to_dma_iommu_mapping(dev) ((dev)->archdata.mapping) +#else +#define to_dma_iommu_mapping(dev) NULL +#endif + #endif From 6fe367580339a3adf77273d4366652665319bdeb Mon Sep 17 00:00:00 2001 From: Hiroshi Doyu Date: Thu, 24 Jan 2013 15:16:57 +0200 Subject: [PATCH 4/9] ARM: dma-mapping: Add arm_iommu_detach_device() A counter part of arm_iommu_attach_device(). Signed-off-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski --- arch/arm/include/asm/dma-iommu.h | 1 + arch/arm/mm/dma-mapping.c | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h index 666d8a8c3a2b..a8c56acc8c98 100644 --- a/arch/arm/include/asm/dma-iommu.h +++ b/arch/arm/include/asm/dma-iommu.h @@ -30,6 +30,7 @@ void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping); int arm_iommu_attach_device(struct device *dev, struct dma_iommu_mapping *mapping); +void arm_iommu_detach_device(struct device *dev); #endif /* __KERNEL__ */ #endif diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index d91c4880f559..6563e3872659 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1848,4 +1848,29 @@ int arm_iommu_attach_device(struct device *dev, return 0; } +/** + * arm_iommu_detach_device + * @dev: valid struct device pointer + * + * Detaches the provided device from a previously attached map. + * This voids the dma operations (dma_map_ops pointer) + */ +void arm_iommu_detach_device(struct device *dev) +{ + struct dma_iommu_mapping *mapping; + + mapping = to_dma_iommu_mapping(dev); + if (!mapping) { + dev_warn(dev, "Not attached\n"); + return; + } + + iommu_detach_device(mapping->domain, dev); + kref_put(&mapping->kref, release_iommu_mapping); + mapping = NULL; + set_dma_ops(dev, NULL); + + pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); +} + #endif From 18177d12c0cee5646c7c2045ea90ddf882011c97 Mon Sep 17 00:00:00 2001 From: Prathyush K Date: Fri, 4 Jan 2013 06:22:42 -0500 Subject: [PATCH 5/9] arm: dma mapping: export arm iommu functions This patch adds EXPORT_SYMBOL_GPL calls to the three arm iommu functions - arm_iommu_create_mapping, arm_iommu_free_mapping and arm_iommu_attach_device. These three functions are arm specific wrapper functions for creating/freeing/using an iommu mapping and they are called by various drivers. If any of these drivers need to be built as dynamic modules, these functions need to be exported. Changelog v2: using EXPORT_SYMBOL_GPL as suggested by Marek. Signed-off-by: Prathyush K [m.szyprowski: extended with recently introduced EXPORT_SYMBOL_GPL(arm_iommu_detach_device)] Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 6563e3872659..6ab882a05050 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1803,6 +1803,7 @@ err2: err: return ERR_PTR(err); } +EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); static void release_iommu_mapping(struct kref *kref) { @@ -1819,6 +1820,7 @@ void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) if (mapping) kref_put(&mapping->kref, release_iommu_mapping); } +EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); /** * arm_iommu_attach_device @@ -1847,6 +1849,7 @@ int arm_iommu_attach_device(struct device *dev, pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); return 0; } +EXPORT_SYMBOL_GPL(arm_iommu_attach_device); /** * arm_iommu_detach_device @@ -1872,5 +1875,6 @@ void arm_iommu_detach_device(struct device *dev) pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); } +EXPORT_SYMBOL_GPL(arm_iommu_detach_device); #endif From 9848e48f4c316ccb64d6f29ff0ed85f11d7bf532 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Wed, 16 Jan 2013 15:38:44 +0100 Subject: [PATCH 6/9] ARM: dma-mapping: add support for CMA regions placed in highmem zone This patch adds missing pieces to correctly support memory pages served from CMA regions placed in high memory zones. Please note that the default global CMA area is still put into lowmem and is limited by optional architecture specific DMA zone. One can however put device specific CMA regions in high memory zone to reduce lowmem usage. Signed-off-by: Marek Szyprowski Signed-off-by: Kyungmin Park Acked-by: Michal Nazarewicz --- arch/arm/mm/dma-mapping.c | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 6ab882a05050..94d7359074c2 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -186,13 +186,24 @@ static u64 get_coherent_dma_mask(struct device *dev) static void __dma_clear_buffer(struct page *page, size_t size) { - void *ptr; /* * Ensure that the allocated pages are zeroed, and that any data * lurking in the kernel direct-mapped region is invalidated. */ - ptr = page_address(page); - if (ptr) { + if (PageHighMem(page)) { + phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); + phys_addr_t end = base + size; + while (size > 0) { + void *ptr = kmap_atomic(page); + memset(ptr, 0, PAGE_SIZE); + dmac_flush_range(ptr, ptr + PAGE_SIZE); + kunmap_atomic(ptr); + page++; + size -= PAGE_SIZE; + } + outer_flush_range(base, end); + } else { + void *ptr = page_address(page); memset(ptr, 0, size); dmac_flush_range(ptr, ptr + size); outer_flush_range(__pa(ptr), __pa(ptr) + size); @@ -243,7 +254,8 @@ static void __dma_free_buffer(struct page *page, size_t size) #endif static void *__alloc_from_contiguous(struct device *dev, size_t size, - pgprot_t prot, struct page **ret_page); + pgprot_t prot, struct page **ret_page, + const void *caller); static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, pgprot_t prot, struct page **ret_page, @@ -346,10 +358,11 @@ static int __init atomic_pool_init(void) goto no_pages; if (IS_ENABLED(CONFIG_CMA)) - ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page); + ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page, + atomic_pool_init); else ptr = __alloc_remap_buffer(NULL, pool->size, GFP_KERNEL, prot, - &page, NULL); + &page, atomic_pool_init); if (ptr) { int i; @@ -542,27 +555,41 @@ static int __free_from_pool(void *start, size_t size) } static void *__alloc_from_contiguous(struct device *dev, size_t size, - pgprot_t prot, struct page **ret_page) + pgprot_t prot, struct page **ret_page, + const void *caller) { unsigned long order = get_order(size); size_t count = size >> PAGE_SHIFT; struct page *page; + void *ptr; page = dma_alloc_from_contiguous(dev, count, order); if (!page) return NULL; __dma_clear_buffer(page, size); - __dma_remap(page, size, prot); + if (PageHighMem(page)) { + ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); + if (!ptr) { + dma_release_from_contiguous(dev, page, count); + return NULL; + } + } else { + __dma_remap(page, size, prot); + ptr = page_address(page); + } *ret_page = page; - return page_address(page); + return ptr; } static void __free_from_contiguous(struct device *dev, struct page *page, - size_t size) + void *cpu_addr, size_t size) { - __dma_remap(page, size, pgprot_kernel); + if (PageHighMem(page)) + __dma_free_remap(cpu_addr, size); + else + __dma_remap(page, size, pgprot_kernel); dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } @@ -583,9 +610,9 @@ static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) #define __get_dma_pgprot(attrs, prot) __pgprot(0) #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL #define __alloc_from_pool(size, ret_page) NULL -#define __alloc_from_contiguous(dev, size, prot, ret) NULL +#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL #define __free_from_pool(cpu_addr, size) 0 -#define __free_from_contiguous(dev, page, size) do { } while (0) +#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0) #define __dma_free_remap(cpu_addr, size) do { } while (0) #endif /* CONFIG_MMU */ @@ -645,7 +672,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, else if (!IS_ENABLED(CONFIG_CMA)) addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); else - addr = __alloc_from_contiguous(dev, size, prot, &page); + addr = __alloc_from_contiguous(dev, size, prot, &page, caller); if (addr) *handle = pfn_to_dma(dev, page_to_pfn(page)); @@ -739,7 +766,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, * Non-atomic allocations cannot be freed with IRQs disabled */ WARN_ON(irqs_disabled()); - __free_from_contiguous(dev, page, size); + __free_from_contiguous(dev, page, cpu_addr, size); } } From f8669bef11fadfe811a5d7d59cb327499edac088 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Wed, 16 Jan 2013 15:41:02 +0100 Subject: [PATCH 7/9] ARM: dma-mapping: use himem for DMA buffers for IOMMU-mapped devices IOMMU can provide access to any memory page, so there is no point in limiting the allocated pages only to lowmem, once other parts of dma-mapping subsystem correctly supports himem pages. Signed-off-by: Marek Szyprowski --- arch/arm/mm/dma-mapping.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 94d7359074c2..2163af4b31b8 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1095,12 +1095,17 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, return pages; } + /* + * IOMMU can map any pages, so himem can also be used here + */ + gfp |= __GFP_NOWARN | __GFP_HIGHMEM; + while (count) { int j, order = __fls(count); - pages[i] = alloc_pages(gfp | __GFP_NOWARN, order); + pages[i] = alloc_pages(gfp, order); while (!pages[i] && order) - pages[i] = alloc_pages(gfp | __GFP_NOWARN, --order); + pages[i] = alloc_pages(gfp, --order); if (!pages[i]) goto error; From 60460abffc71523d65774f43ce1252972eeb0629 Mon Sep 17 00:00:00 2001 From: Seung-Woo Kim Date: Wed, 6 Feb 2013 13:21:14 +0900 Subject: [PATCH 8/9] ARM: dma-mapping: Add maximum alignment order for dma iommu buffers Alignment order for a dma iommu buffer is set by buffer size. For large buffer, it is a waste of iommu address space. So configurable parameter to limit maximum alignment order can reduce the waste. Signed-off-by: Seung-Woo Kim Signed-off-by: Kyungmin.park Signed-off-by: Marek Szyprowski --- arch/arm/Kconfig | 21 +++++++++++++++++++++ arch/arm/mm/dma-mapping.c | 3 +++ 2 files changed, 24 insertions(+) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 67874b82a4ed..4e89112b57e0 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -75,6 +75,27 @@ config ARM_DMA_USE_IOMMU select ARM_HAS_SG_CHAIN select NEED_SG_DMA_LENGTH +if ARM_DMA_USE_IOMMU + +config ARM_DMA_IOMMU_ALIGNMENT + int "Maximum PAGE_SIZE order of alignment for DMA IOMMU buffers" + range 4 9 + default 8 + help + DMA mapping framework by default aligns all buffers to the smallest + PAGE_SIZE order which is greater than or equal to the requested buffer + size. This works well for buffers up to a few hundreds kilobytes, but + for larger buffers it just a waste of address space. Drivers which has + relatively small addressing window (like 64Mib) might run out of + virtual space with just a few allocations. + + With this parameter you can specify the maximum PAGE_SIZE order for + DMA IOMMU buffers. Larger buffers will be aligned only to this + specified order. The order is expressed as a power of two multiplied + by the PAGE_SIZE. + +endif + config HAVE_PWM bool diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 2163af4b31b8..6e2511561c3e 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1029,6 +1029,9 @@ static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, unsigned int count, start; unsigned long flags; + if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) + order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; + count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) + (1 << mapping->order) - 1) >> mapping->order; From d589829107c5528164a9b7dfe50d0001780865ed Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Fri, 8 Feb 2013 10:54:48 +0100 Subject: [PATCH 9/9] ARM: DMA-mapping: fix memory leak in IOMMU dma-mapping implementation This patch removes page_address() usage in IOMMU-aware dma-mapping implementation and replaced it with direct use of the cpu virtual address provided by the caller. page_address() returned incorrect address for pages remapped in atomic pool, what caused memory leak. Reported-by: Hiroshi Doyu Signed-off-by: Marek Szyprowski Tested-by: Hiroshi Doyu --- arch/arm/mm/dma-mapping.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 6e2511561c3e..c7e3759f16d3 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1292,11 +1292,11 @@ err_mapping: return NULL; } -static void __iommu_free_atomic(struct device *dev, struct page **pages, +static void __iommu_free_atomic(struct device *dev, void *cpu_addr, dma_addr_t handle, size_t size) { __iommu_remove_mapping(dev, handle, size); - __free_from_pool(page_address(pages[0]), size); + __free_from_pool(cpu_addr, size); } static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, @@ -1379,7 +1379,7 @@ void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, } if (__in_atomic_pool(cpu_addr, size)) { - __iommu_free_atomic(dev, pages, handle, size); + __iommu_free_atomic(dev, cpu_addr, handle, size); return; }