OpenE2K
/
linux
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'x86-dma-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull x86 dma mapping updates from Ingo Molnar:
 "This tree, by Christoph Hellwig, switches over the x86 architecture to
  the generic dma-direct and swiotlb code, and also unifies more of the
  dma-direct code between architectures. The now unused x86-only
  primitives are removed"

* 'x86-dma-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  dma-mapping: Don't clear GFP_ZERO in dma_alloc_attrs
  swiotlb: Make swiotlb_{alloc,free}_buffer depend on CONFIG_DMA_DIRECT_OPS
  dma/swiotlb: Remove swiotlb_{alloc,free}_coherent()
  dma/direct: Handle force decryption for DMA coherent buffers in common code
  dma/direct: Handle the memory encryption bit in common code
  dma/swiotlb: Remove swiotlb_set_mem_attributes()
  set_memory.h: Provide set_memory_{en,de}crypted() stubs
  x86/dma: Remove dma_alloc_coherent_gfp_flags()
  iommu/intel-iommu: Enable CONFIG_DMA_DIRECT_OPS=y and clean up intel_{alloc,free}_coherent()
  iommu/amd_iommu: Use CONFIG_DMA_DIRECT_OPS=y and dma_direct_{alloc,free}()
  x86/dma/amd_gart: Use dma_direct_{alloc,free}()
  x86/dma/amd_gart: Look at dev->coherent_dma_mask instead of GFP_DMA
  x86/dma: Use generic swiotlb_ops
  x86/dma: Use DMA-direct (CONFIG_DMA_DIRECT_OPS=y)
  x86/dma: Remove dma_alloc_coherent_mask()
This commit is contained in:
Linus Torvalds 2018-04-02 17:18:45 -07:00
parent ce6eba3dba e89f5b3701
commit 2fcd2b306a
30 changed files with 181 additions and 557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
arch/arm/include/asm/dma-direct.h
View File

@ -2,13 +2,13 @@
#ifndef ASM_ARM_DMA_DIRECT_H #ifndef ASM_ARM_DMA_DIRECT_H
#define ASM_ARM_DMA_DIRECT_H 1 #define ASM_ARM_DMA_DIRECT_H 1
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
unsigned int offset = paddr & ~PAGE_MASK; unsigned int offset = paddr & ~PAGE_MASK;
return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset; return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
} }
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
{ {
unsigned int offset = dev_addr & ~PAGE_MASK; unsigned int offset = dev_addr & ~PAGE_MASK;
return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset; return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;

10
arch/mips/cavium-octeon/dma-octeon.c
View File

@ -10,7 +10,7 @@
* IP32 changes by Ilya. * IP32 changes by Ilya.
* Copyright (C) 2010 Cavium Networks, Inc. * Copyright (C) 2010 Cavium Networks, Inc.
*/ */
#include <linux/dma-mapping.h> #include <linux/dma-direct.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/export.h> #include <linux/export.h>
@ -182,7 +182,7 @@ struct octeon_dma_map_ops {
phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr); phys_addr_t (*dma_to_phys)(struct device *dev, dma_addr_t daddr);
}; };
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
struct octeon_dma_map_ops, struct octeon_dma_map_ops,
@ -190,9 +190,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
return ops->phys_to_dma(dev, paddr); return ops->phys_to_dma(dev, paddr);
} }
EXPORT_SYMBOL(phys_to_dma); EXPORT_SYMBOL(__phys_to_dma);
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
{ {
struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev), struct octeon_dma_map_ops *ops = container_of(get_dma_ops(dev),
struct octeon_dma_map_ops, struct octeon_dma_map_ops,
@ -200,7 +200,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
return ops->dma_to_phys(dev, daddr); return ops->dma_to_phys(dev, daddr);
} }
EXPORT_SYMBOL(dma_to_phys); EXPORT_SYMBOL(__dma_to_phys);
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = { static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
.dma_map_ops = { .dma_map_ops = {

4
arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
View File

@ -69,8 +69,8 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
return addr + size - 1 <= *dev->dma_mask; return addr + size - 1 <= *dev->dma_mask;
} }
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
struct dma_map_ops; struct dma_map_ops;
extern const struct dma_map_ops *octeon_pci_dma_map_ops; extern const struct dma_map_ops *octeon_pci_dma_map_ops;

10
arch/mips/include/asm/mach-loongson64/dma-coherence.h
View File

@ -25,13 +25,13 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
return addr + size - 1 <= *dev->dma_mask; return addr + size - 1 <= *dev->dma_mask;
} }
extern dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); extern dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
extern phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); extern phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr, static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr,
size_t size) size_t size)
{ {
#ifdef CONFIG_CPU_LOONGSON3 #ifdef CONFIG_CPU_LOONGSON3
return phys_to_dma(dev, virt_to_phys(addr)); return __phys_to_dma(dev, virt_to_phys(addr));
#else #else
return virt_to_phys(addr) | 0x80000000; return virt_to_phys(addr) | 0x80000000;
#endif #endif
@ -41,7 +41,7 @@ static inline dma_addr_t plat_map_dma_mem_page(struct device *dev,
struct page *page) struct page *page)
{ {
#ifdef CONFIG_CPU_LOONGSON3 #ifdef CONFIG_CPU_LOONGSON3
return phys_to_dma(dev, page_to_phys(page)); return __phys_to_dma(dev, page_to_phys(page));
#else #else
return page_to_phys(page) | 0x80000000; return page_to_phys(page) | 0x80000000;
#endif #endif
@ -51,7 +51,7 @@ static inline unsigned long plat_dma_addr_to_phys(struct device *dev,
dma_addr_t dma_addr) dma_addr_t dma_addr)
{ {
#if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT) #if defined(CONFIG_CPU_LOONGSON3) && defined(CONFIG_64BIT)
return dma_to_phys(dev, dma_addr); return __dma_to_phys(dev, dma_addr);
#elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT) #elif defined(CONFIG_CPU_LOONGSON2F) && defined(CONFIG_64BIT)
return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff); return (dma_addr > 0x8fffffff) ? dma_addr : (dma_addr & 0x0fffffff);
#else #else

4
arch/mips/loongson64/common/dma-swiotlb.c
View File

@ -63,7 +63,7 @@ static int loongson_dma_supported(struct device *dev, u64 mask)
return swiotlb_dma_supported(dev, mask); return swiotlb_dma_supported(dev, mask);
} }
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
long nid; long nid;
#ifdef CONFIG_PHYS48_TO_HT40 #ifdef CONFIG_PHYS48_TO_HT40
@ -75,7 +75,7 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
return paddr; return paddr;
} }
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
{ {
long nid; long nid;
#ifdef CONFIG_PHYS48_TO_HT40 #ifdef CONFIG_PHYS48_TO_HT40

4
arch/powerpc/include/asm/dma-direct.h
View File

@ -17,12 +17,12 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
return addr + size - 1 <= *dev->dma_mask; return addr + size - 1 <= *dev->dma_mask;
} }
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
return paddr + get_dma_offset(dev); return paddr + get_dma_offset(dev);
} }
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
{ {
return daddr - get_dma_offset(dev); return daddr - get_dma_offset(dev);
} }

3
arch/x86/Kconfig
View File

@ -54,7 +54,6 @@ config X86
select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_FORTIFY_SOURCE
select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_GCOV_PROFILE_ALL
select ARCH_HAS_KCOV if X86_64 select ARCH_HAS_KCOV if X86_64
select ARCH_HAS_PHYS_TO_DMA
select ARCH_HAS_MEMBARRIER_SYNC_CORE select ARCH_HAS_MEMBARRIER_SYNC_CORE
select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_PMEM_API if X86_64
select ARCH_HAS_REFCOUNT select ARCH_HAS_REFCOUNT
@ -83,6 +82,7 @@ config X86
select CLOCKSOURCE_VALIDATE_LAST_CYCLE select CLOCKSOURCE_VALIDATE_LAST_CYCLE
select CLOCKSOURCE_WATCHDOG select CLOCKSOURCE_WATCHDOG
select DCACHE_WORD_ACCESS select DCACHE_WORD_ACCESS
select DMA_DIRECT_OPS
select EDAC_ATOMIC_SCRUB select EDAC_ATOMIC_SCRUB
select EDAC_SUPPORT select EDAC_SUPPORT
select GENERIC_CLOCKEVENTS select GENERIC_CLOCKEVENTS
@ -680,6 +680,7 @@ config X86_SUPPORTS_MEMORY_FAILURE
config STA2X11 config STA2X11
bool "STA2X11 Companion Chip Support" bool "STA2X11 Companion Chip Support"
depends on X86_32_NON_STANDARD && PCI depends on X86_32_NON_STANDARD && PCI
select ARCH_HAS_PHYS_TO_DMA
select X86_DEV_DMA_OPS select X86_DEV_DMA_OPS
select X86_DMA_REMAP select X86_DMA_REMAP
select SWIOTLB select SWIOTLB

3
arch/x86/include/asm/device.h
View File

@ -6,6 +6,9 @@ struct dev_archdata {
#if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU) #if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU)
void *iommu; /* hook for IOMMU specific extension */ void *iommu; /* hook for IOMMU specific extension */
#endif #endif
#ifdef CONFIG_STA2X11
bool is_sta2x11;
#endif
}; };
#if defined(CONFIG_X86_DEV_DMA_OPS) && defined(CONFIG_PCI_DOMAINS) #if defined(CONFIG_X86_DEV_DMA_OPS) && defined(CONFIG_PCI_DOMAINS)

25
arch/x86/include/asm/dma-direct.h
View File

@ -2,29 +2,8 @@
#ifndef ASM_X86_DMA_DIRECT_H #ifndef ASM_X86_DMA_DIRECT_H
#define ASM_X86_DMA_DIRECT_H 1 #define ASM_X86_DMA_DIRECT_H 1
#include <linux/mem_encrypt.h>
#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */
bool dma_capable(struct device *dev, dma_addr_t addr, size_t size); bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr);
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr); phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr);
#else
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
{
if (!dev->dma_mask)
return 0;
return addr + size - 1 <= *dev->dma_mask;
}
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
{
return __sme_set(paddr);
}
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
{
return __sme_clr(daddr);
}
#endif /* CONFIG_X86_DMA_REMAP */
#endif /* ASM_X86_DMA_DIRECT_H */ #endif /* ASM_X86_DMA_DIRECT_H */

33
arch/x86/include/asm/dma-mapping.h
View File

@ -36,37 +36,4 @@ int arch_dma_supported(struct device *dev, u64 mask);
bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp); bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp);
#define arch_dma_alloc_attrs arch_dma_alloc_attrs #define arch_dma_alloc_attrs arch_dma_alloc_attrs
extern void *dma_generic_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_addr, gfp_t flag,
unsigned long attrs);
extern void dma_generic_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs);
static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
gfp_t gfp)
{
unsigned long dma_mask = 0;
dma_mask = dev->coherent_dma_mask;
if (!dma_mask)
dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32);
return dma_mask;
}
static inline gfp_t dma_alloc_coherent_gfp_flags(struct device *dev, gfp_t gfp)
{
unsigned long dma_mask = dma_alloc_coherent_mask(dev, gfp);
if (dma_mask <= DMA_BIT_MASK(24))
gfp |= GFP_DMA;
#ifdef CONFIG_X86_64
if (dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA))
gfp |= GFP_DMA32;
#endif
return gfp;
}
#endif #endif

3
arch/x86/include/asm/iommu.h
View File

@ -2,13 +2,10 @@
#ifndef _ASM_X86_IOMMU_H #ifndef _ASM_X86_IOMMU_H
#define _ASM_X86_IOMMU_H #define _ASM_X86_IOMMU_H
extern const struct dma_map_ops nommu_dma_ops;
extern int force_iommu, no_iommu; extern int force_iommu, no_iommu;
extern int iommu_detected; extern int iommu_detected;
extern int iommu_pass_through; extern int iommu_pass_through;
int x86_dma_supported(struct device *dev, u64 mask);
/* 10 seconds */ /* 10 seconds */
#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) #define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)

2
arch/x86/include/asm/mem_encrypt.h
View File

@ -49,8 +49,6 @@ int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
/* Architecture __weak replacement functions */ /* Architecture __weak replacement functions */
void __init mem_encrypt_init(void); void __init mem_encrypt_init(void);
void swiotlb_set_mem_attributes(void *vaddr, unsigned long size);
bool sme_active(void); bool sme_active(void);
bool sev_active(void); bool sev_active(void);

8
arch/x86/include/asm/swiotlb.h
View File

@ -27,12 +27,4 @@ static inline void pci_swiotlb_late_init(void)
{ {
} }
#endif #endif
extern void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs);
extern void x86_swiotlb_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs);
#endif /* _ASM_X86_SWIOTLB_H */ #endif /* _ASM_X86_SWIOTLB_H */

2
arch/x86/kernel/Makefile
View File

@ -57,7 +57,7 @@ obj-$(CONFIG_X86_ESPFIX64) += espfix_64.o
obj-$(CONFIG_SYSFS) += ksysfs.o obj-$(CONFIG_SYSFS) += ksysfs.o
obj-y += bootflag.o e820.o obj-y += bootflag.o e820.o
obj-y += pci-dma.o quirks.o topology.o kdebugfs.o obj-y += pci-dma.o quirks.o topology.o kdebugfs.o
obj-y += alternative.o i8253.o pci-nommu.o hw_breakpoint.o obj-y += alternative.o i8253.o hw_breakpoint.o
obj-y += tsc.o tsc_msr.o io_delay.o rtc.o obj-y += tsc.o tsc_msr.o io_delay.o rtc.o
obj-y += pci-iommu_table.o obj-y += pci-iommu_table.o
obj-y += resource.o obj-y += resource.o

39
arch/x86/kernel/amd_gart_64.c
View File

@ -480,30 +480,21 @@ static void *
gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
gfp_t flag, unsigned long attrs) gfp_t flag, unsigned long attrs)
{ {
dma_addr_t paddr; void *vaddr;
unsigned long align_mask;
struct page *page;
if (force_iommu && !(flag & GFP_DMA)) { vaddr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); if (!vaddr ||
page = alloc_pages(flag | __GFP_ZERO, get_order(size)); !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))
if (!page) return vaddr;
return NULL;
align_mask = (1UL << get_order(size)) - 1;
paddr = dma_map_area(dev, page_to_phys(page), size,
DMA_BIDIRECTIONAL, align_mask);
flush_gart();
if (paddr != bad_dma_addr) {
*dma_addr = paddr;
return page_address(page);
}
__free_pages(page, get_order(size));
} else
return dma_generic_alloc_coherent(dev, size, dma_addr, flag,
attrs);
*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
flush_gart();
if (unlikely(*dma_addr == bad_dma_addr))
goto out_free;
return vaddr;
out_free:
dma_direct_free(dev, size, vaddr, *dma_addr, attrs);
return NULL; return NULL;
} }
@ -513,7 +504,7 @@ gart_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0);
dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs); dma_direct_free(dev, size, vaddr, dma_addr, attrs);
} }
static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr) static int gart_mapping_error(struct device *dev, dma_addr_t dma_addr)
@ -705,7 +696,7 @@ static const struct dma_map_ops gart_dma_ops = {
.alloc = gart_alloc_coherent, .alloc = gart_alloc_coherent,
.free = gart_free_coherent, .free = gart_free_coherent,
.mapping_error = gart_mapping_error, .mapping_error = gart_mapping_error,
.dma_supported = x86_dma_supported, .dma_supported = dma_direct_supported,
}; };
static void gart_iommu_shutdown(void) static void gart_iommu_shutdown(void)

5
arch/x86/kernel/pci-calgary_64.c
View File

@ -33,6 +33,7 @@
#include <linux/string.h> #include <linux/string.h>
#include <linux/crash_dump.h> #include <linux/crash_dump.h>
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/dma-direct.h>
#include <linux/bitmap.h> #include <linux/bitmap.h>
#include <linux/pci_ids.h> #include <linux/pci_ids.h>
#include <linux/pci.h> #include <linux/pci.h>
@ -445,8 +446,6 @@ static void* calgary_alloc_coherent(struct device *dev, size_t size,
npages = size >> PAGE_SHIFT; npages = size >> PAGE_SHIFT;
order = get_order(size); order = get_order(size);
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
/* alloc enough pages (and possibly more) */ /* alloc enough pages (and possibly more) */
ret = (void *)__get_free_pages(flag, order); ret = (void *)__get_free_pages(flag, order);
if (!ret) if (!ret)
@ -493,7 +492,7 @@ static const struct dma_map_ops calgary_dma_ops = {
.map_page = calgary_map_page, .map_page = calgary_map_page,
.unmap_page = calgary_unmap_page, .unmap_page = calgary_unmap_page,
.mapping_error = calgary_mapping_error, .mapping_error = calgary_mapping_error,
.dma_supported = x86_dma_supported, .dma_supported = dma_direct_supported,
}; };
static inline void __iomem * busno_to_bbar(unsigned char num) static inline void __iomem * busno_to_bbar(unsigned char num)

70
arch/x86/kernel/pci-dma.c
View File

@ -18,7 +18,7 @@
static int forbid_dac __read_mostly; static int forbid_dac __read_mostly;
const struct dma_map_ops *dma_ops = &nommu_dma_ops; const struct dma_map_ops *dma_ops = &dma_direct_ops;
EXPORT_SYMBOL(dma_ops); EXPORT_SYMBOL(dma_ops);
static int iommu_sac_force __read_mostly; static int iommu_sac_force __read_mostly;
@ -76,70 +76,12 @@ void __init pci_iommu_alloc(void)
} }
} }
} }
void *dma_generic_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_addr, gfp_t flag,
unsigned long attrs)
{
unsigned long dma_mask;
struct page *page;
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
dma_addr_t addr;
dma_mask = dma_alloc_coherent_mask(dev, flag);
again:
page = NULL;
/* CMA can be used only in the context which permits sleeping */
if (gfpflags_allow_blocking(flag)) {
page = dma_alloc_from_contiguous(dev, count, get_order(size),
flag);
if (page) {
addr = phys_to_dma(dev, page_to_phys(page));
if (addr + size > dma_mask) {
dma_release_from_contiguous(dev, page, count);
page = NULL;
}
}
}
/* fallback */
if (!page)
page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
if (!page)
return NULL;
addr = phys_to_dma(dev, page_to_phys(page));
if (addr + size > dma_mask) {
__free_pages(page, get_order(size));
if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
flag = (flag & ~GFP_DMA32) | GFP_DMA;
goto again;
}
return NULL;
}
memset(page_address(page), 0, size);
*dma_addr = addr;
return page_address(page);
}
void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs)
{
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
struct page *page = virt_to_page(vaddr);
if (!dma_release_from_contiguous(dev, page, count))
free_pages((unsigned long)vaddr, get_order(size));
}
bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp) bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
{ {
if (!*dev) if (!*dev)
*dev = &x86_dma_fallback_dev; *dev = &x86_dma_fallback_dev;
*gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
if (!is_device_dma_capable(*dev)) if (!is_device_dma_capable(*dev))
return false; return false;
return true; return true;
@ -245,16 +187,6 @@ int arch_dma_supported(struct device *dev, u64 mask)
} }
EXPORT_SYMBOL(arch_dma_supported); EXPORT_SYMBOL(arch_dma_supported);
int x86_dma_supported(struct device *dev, u64 mask)
{
/* Copied from i386. Doesn't make much sense, because it will
only work for pci_alloc_coherent.
The caller just has to use GFP_DMA in this case. */
if (mask < DMA_BIT_MASK(24))
return 0;
return 1;
}
static int __init pci_iommu_init(void) static int __init pci_iommu_init(void)
{ {
struct iommu_table_entry *p; struct iommu_table_entry *p;

48
arch/x86/kernel/pci-swiotlb.c
View File

@ -17,52 +17,6 @@
int swiotlb __read_mostly; int swiotlb __read_mostly;
void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
void *vaddr;
/*
* Don't print a warning when the first allocation attempt fails.
* swiotlb_alloc_coherent() will print a warning when the DMA
* memory allocation ultimately failed.
*/
flags |= __GFP_NOWARN;
vaddr = dma_generic_alloc_coherent(hwdev, size, dma_handle, flags,
attrs);
if (vaddr)
return vaddr;
return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags);
}
void x86_swiotlb_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_addr,
unsigned long attrs)
{
if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
swiotlb_free_coherent(dev, size, vaddr, dma_addr);
else
dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
}
static const struct dma_map_ops x86_swiotlb_dma_ops = {
.mapping_error = swiotlb_dma_mapping_error,
.alloc = x86_swiotlb_alloc_coherent,
.free = x86_swiotlb_free_coherent,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.dma_supported = NULL,
};
/* /*
* pci_swiotlb_detect_override - set swiotlb to 1 if necessary * pci_swiotlb_detect_override - set swiotlb to 1 if necessary
* *
@ -112,7 +66,7 @@ void __init pci_swiotlb_init(void)
{ {
if (swiotlb) { if (swiotlb) {
swiotlb_init(0); swiotlb_init(0);
dma_ops = &x86_swiotlb_dma_ops; dma_ops = &swiotlb_dma_ops;
} }
} }

90
arch/x86/mm/mem_encrypt.c
View File

@ -195,67 +195,6 @@ void __init sme_early_init(void)
swiotlb_force = SWIOTLB_FORCE; swiotlb_force = SWIOTLB_FORCE;
} }
static void *sev_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
gfp_t gfp, unsigned long attrs)
{
unsigned long dma_mask;
unsigned int order;
struct page *page;
void *vaddr = NULL;
dma_mask = dma_alloc_coherent_mask(dev, gfp);
order = get_order(size);
/*
* Memory will be memset to zero after marking decrypted, so don't
* bother clearing it before.
*/
gfp &= ~__GFP_ZERO;
page = alloc_pages_node(dev_to_node(dev), gfp, order);
if (page) {
dma_addr_t addr;
/*
* Since we will be clearing the encryption bit, check the
* mask with it already cleared.
*/
addr = __sme_clr(phys_to_dma(dev, page_to_phys(page)));
if ((addr + size) > dma_mask) {
__free_pages(page, get_order(size));
} else {
vaddr = page_address(page);
*dma_handle = addr;
}
}
if (!vaddr)
vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
if (!vaddr)
return NULL;
/* Clear the SME encryption bit for DMA use if not swiotlb area */
if (!is_swiotlb_buffer(dma_to_phys(dev, *dma_handle))) {
set_memory_decrypted((unsigned long)vaddr, 1 << order);
memset(vaddr, 0, PAGE_SIZE << order);
*dma_handle = __sme_clr(*dma_handle);
}
return vaddr;
}
static void sev_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs)
{
/* Set the SME encryption bit for re-use if not swiotlb area */
if (!is_swiotlb_buffer(dma_to_phys(dev, dma_handle)))
set_memory_encrypted((unsigned long)vaddr,
1 << get_order(size));
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc) static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
{ {
pgprot_t old_prot, new_prot; pgprot_t old_prot, new_prot;
@ -408,20 +347,6 @@ bool sev_active(void)
} }
EXPORT_SYMBOL(sev_active); EXPORT_SYMBOL(sev_active);
static const struct dma_map_ops sev_dma_ops = {
.alloc = sev_alloc,
.free = sev_free,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.mapping_error = swiotlb_dma_mapping_error,
};
/* Architecture __weak replacement functions */ /* Architecture __weak replacement functions */
void __init mem_encrypt_init(void) void __init mem_encrypt_init(void)
{ {
@ -432,12 +357,11 @@ void __init mem_encrypt_init(void)
swiotlb_update_mem_attributes(); swiotlb_update_mem_attributes();
/* /*
* With SEV, DMA operations cannot use encryption. New DMA ops * With SEV, DMA operations cannot use encryption, we need to use
* are required in order to mark the DMA areas as decrypted or * SWIOTLB to bounce buffer DMA operation.
* to use bounce buffers.
*/ */
if (sev_active()) if (sev_active())
dma_ops = &sev_dma_ops; dma_ops = &swiotlb_dma_ops;
/* /*
* With SEV, we need to unroll the rep string I/O instructions. * With SEV, we need to unroll the rep string I/O instructions.
@ -450,11 +374,3 @@ void __init mem_encrypt_init(void)
: "Secure Memory Encryption (SME)"); : "Secure Memory Encryption (SME)");
} }
void swiotlb_set_mem_attributes(void *vaddr, unsigned long size)
{
WARN(PAGE_ALIGN(size) != size,
"size is not page-aligned (%#lx)\n", size);
/* Make the SWIOTLB buffer area decrypted */
set_memory_decrypted((unsigned long)vaddr, size >> PAGE_SHIFT);
}

52
arch/x86/pci/sta2x11-fixup.c
View File

@ -159,43 +159,6 @@ static dma_addr_t a2p(dma_addr_t a, struct pci_dev *pdev)
return p; return p;
} }
/**
* sta2x11_swiotlb_alloc_coherent - Allocate swiotlb bounce buffers
* returns virtual address. This is the only "special" function here.
* @dev: PCI device
* @size: Size of the buffer
* @dma_handle: DMA address
* @flags: memory flags
*/
static void *sta2x11_swiotlb_alloc_coherent(struct device *dev,
size_t size,
dma_addr_t *dma_handle,
gfp_t flags,
unsigned long attrs)
{
void *vaddr;
vaddr = x86_swiotlb_alloc_coherent(dev, size, dma_handle, flags, attrs);
*dma_handle = p2a(*dma_handle, to_pci_dev(dev));
return vaddr;
}
/* We have our own dma_ops: the same as swiotlb but from alloc (above) */
static const struct dma_map_ops sta2x11_dma_ops = {
.alloc = sta2x11_swiotlb_alloc_coherent,
.free = x86_swiotlb_free_coherent,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = swiotlb_map_sg_attrs,
.unmap_sg = swiotlb_unmap_sg_attrs,
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
.sync_single_for_device = swiotlb_sync_single_for_device,
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
.sync_sg_for_device = swiotlb_sync_sg_for_device,
.mapping_error = swiotlb_dma_mapping_error,
.dma_supported = x86_dma_supported,
};
/* At setup time, we use our own ops if the device is a ConneXt one */ /* At setup time, we use our own ops if the device is a ConneXt one */
static void sta2x11_setup_pdev(struct pci_dev *pdev) static void sta2x11_setup_pdev(struct pci_dev *pdev)
{ {
@ -205,7 +168,8 @@ static void sta2x11_setup_pdev(struct pci_dev *pdev)
return; return;
pci_set_consistent_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); pci_set_consistent_dma_mask(pdev, STA2X11_AMBA_SIZE - 1);
pci_set_dma_mask(pdev, STA2X11_AMBA_SIZE - 1); pci_set_dma_mask(pdev, STA2X11_AMBA_SIZE - 1);
pdev->dev.dma_ops = &sta2x11_dma_ops; pdev->dev.dma_ops = &swiotlb_dma_ops;
pdev->dev.archdata.is_sta2x11 = true;
/* We must enable all devices as master, for audio DMA to work */ /* We must enable all devices as master, for audio DMA to work */
pci_set_master(pdev); pci_set_master(pdev);
@ -225,7 +189,7 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
{ {
struct sta2x11_mapping *map; struct sta2x11_mapping *map;
if (dev->dma_ops != &sta2x11_dma_ops) { if (!dev->archdata.is_sta2x11) {
if (!dev->dma_mask) if (!dev->dma_mask)
return false; return false;
return addr + size - 1 <= *dev->dma_mask; return addr + size - 1 <= *dev->dma_mask;
@ -243,13 +207,13 @@ bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
} }
/** /**
* phys_to_dma - Return the DMA AMBA address used for this STA2x11 device * __phys_to_dma - Return the DMA AMBA address used for this STA2x11 device
* @dev: device for a PCI device * @dev: device for a PCI device
* @paddr: Physical address * @paddr: Physical address
*/ */
dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
if (dev->dma_ops != &sta2x11_dma_ops) if (!dev->archdata.is_sta2x11)
return paddr; return paddr;
return p2a(paddr, to_pci_dev(dev)); return p2a(paddr, to_pci_dev(dev));
} }
@ -259,9 +223,9 @@ dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
* @dev: device for a PCI device * @dev: device for a PCI device
* @daddr: STA2x11 AMBA DMA address * @daddr: STA2x11 AMBA DMA address
*/ */
phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr)
{ {
if (dev->dma_ops != &sta2x11_dma_ops) if (!dev->archdata.is_sta2x11)
return daddr; return daddr;
return a2p(daddr, to_pci_dev(dev)); return a2p(daddr, to_pci_dev(dev));
} }

2
drivers/iommu/Kconfig
View File

@ -107,6 +107,7 @@ config IOMMU_PGTABLES_L2
# AMD IOMMU support # AMD IOMMU support
config AMD_IOMMU config AMD_IOMMU
bool "AMD IOMMU support" bool "AMD IOMMU support"
select DMA_DIRECT_OPS
select SWIOTLB select SWIOTLB
select PCI_MSI select PCI_MSI
select PCI_ATS select PCI_ATS
@ -142,6 +143,7 @@ config DMAR_TABLE
config INTEL_IOMMU config INTEL_IOMMU
bool "Support for Intel IOMMU using DMA Remapping Devices" bool "Support for Intel IOMMU using DMA Remapping Devices"
depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC)
select DMA_DIRECT_OPS
select IOMMU_API select IOMMU_API
select IOMMU_IOVA select IOMMU_IOVA
select DMAR_TABLE select DMAR_TABLE

75
drivers/iommu/amd_iommu.c
View File

@ -28,6 +28,7 @@
#include <linux/debugfs.h> #include <linux/debugfs.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/dma-direct.h>
#include <linux/iommu-helper.h> #include <linux/iommu-helper.h>
#include <linux/iommu.h> #include <linux/iommu.h>
#include <linux/delay.h> #include <linux/delay.h>
@ -2193,7 +2194,7 @@ static int amd_iommu_add_device(struct device *dev)
dev_name(dev)); dev_name(dev));
iommu_ignore_device(dev); iommu_ignore_device(dev);
dev->dma_ops = &nommu_dma_ops; dev->dma_ops = &dma_direct_ops;
goto out; goto out;
} }
init_iommu_group(dev); init_iommu_group(dev);
@ -2599,51 +2600,32 @@ static void *alloc_coherent(struct device *dev, size_t size,
unsigned long attrs) unsigned long attrs)
{ {
u64 dma_mask = dev->coherent_dma_mask; u64 dma_mask = dev->coherent_dma_mask;
struct protection_domain *domain; struct protection_domain *domain = get_domain(dev);
struct dma_ops_domain *dma_dom; bool is_direct = false;
struct page *page; void *virt_addr;
domain = get_domain(dev); if (IS_ERR(domain)) {
if (PTR_ERR(domain) == -EINVAL) { if (PTR_ERR(domain) != -EINVAL)
page = alloc_pages(flag, get_order(size));
*dma_addr = page_to_phys(page);
return page_address(page);
} else if (IS_ERR(domain))
return NULL;
dma_dom = to_dma_ops_domain(domain);
size = PAGE_ALIGN(size);
dma_mask = dev->coherent_dma_mask;
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
flag |= __GFP_ZERO;
page = alloc_pages(flag | __GFP_NOWARN, get_order(size));
if (!page) {
if (!gfpflags_allow_blocking(flag))
return NULL;
page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
get_order(size), flag);
if (!page)
return NULL; return NULL;
is_direct = true;
} }
virt_addr = dma_direct_alloc(dev, size, dma_addr, flag, attrs);
if (!virt_addr || is_direct)
return virt_addr;
if (!dma_mask) if (!dma_mask)
dma_mask = *dev->dma_mask; dma_mask = *dev->dma_mask;
*dma_addr = __map_single(dev, dma_dom, page_to_phys(page), *dma_addr = __map_single(dev, to_dma_ops_domain(domain),
size, DMA_BIDIRECTIONAL, dma_mask); virt_to_phys(virt_addr), PAGE_ALIGN(size),
DMA_BIDIRECTIONAL, dma_mask);
if (*dma_addr == AMD_IOMMU_MAPPING_ERROR) if (*dma_addr == AMD_IOMMU_MAPPING_ERROR)
goto out_free; goto out_free;
return virt_addr;
return page_address(page);
out_free: out_free:
dma_direct_free(dev, size, virt_addr, *dma_addr, attrs);
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, get_order(size));
return NULL; return NULL;
} }
@ -2654,24 +2636,17 @@ static void free_coherent(struct device *dev, size_t size,
void *virt_addr, dma_addr_t dma_addr, void *virt_addr, dma_addr_t dma_addr,
unsigned long attrs) unsigned long attrs)
{ {
struct protection_domain *domain; struct protection_domain *domain = get_domain(dev);
struct dma_ops_domain *dma_dom;
struct page *page;
page = virt_to_page(virt_addr);
size = PAGE_ALIGN(size); size = PAGE_ALIGN(size);
domain = get_domain(dev); if (!IS_ERR(domain)) {
if (IS_ERR(domain)) struct dma_ops_domain *dma_dom = to_dma_ops_domain(domain);
goto free_mem;
dma_dom = to_dma_ops_domain(domain); __unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL);
}
__unmap_single(dma_dom, dma_addr, size, DMA_BIDIRECTIONAL); dma_direct_free(dev, size, virt_addr, dma_addr, attrs);
free_mem:
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, get_order(size));
} }
/* /*
@ -2680,7 +2655,7 @@ free_mem:
*/ */
static int amd_iommu_dma_supported(struct device *dev, u64 mask) static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{ {
if (!x86_dma_supported(dev, mask)) if (!dma_direct_supported(dev, mask))
return 0; return 0;
return check_device(dev); return check_device(dev);
} }
@ -2794,7 +2769,7 @@ int __init amd_iommu_init_dma_ops(void)
* continue to be SWIOTLB. * continue to be SWIOTLB.
*/ */
if (!swiotlb) if (!swiotlb)
dma_ops = &nommu_dma_ops; dma_ops = &dma_direct_ops;
if (amd_iommu_unmap_flush) if (amd_iommu_unmap_flush)
pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n"); pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n");

65
drivers/iommu/intel-iommu.c
View File

@ -31,6 +31,7 @@
#include <linux/pci.h> #include <linux/pci.h>
#include <linux/dmar.h> #include <linux/dmar.h>
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/dma-direct.h>
#include <linux/mempool.h> #include <linux/mempool.h>
#include <linux/memory.h> #include <linux/memory.h>
#include <linux/cpu.h> #include <linux/cpu.h>
@ -45,6 +46,7 @@
#include <linux/pci-ats.h> #include <linux/pci-ats.h>
#include <linux/memblock.h> #include <linux/memblock.h>
#include <linux/dma-contiguous.h> #include <linux/dma-contiguous.h>
#include <linux/dma-direct.h>
#include <linux/crash_dump.h> #include <linux/crash_dump.h>
#include <asm/irq_remapping.h> #include <asm/irq_remapping.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
@ -3707,61 +3709,30 @@ static void *intel_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags, dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs) unsigned long attrs)
{ {
struct page *page = NULL; void *vaddr;
int order;
size = PAGE_ALIGN(size); vaddr = dma_direct_alloc(dev, size, dma_handle, flags, attrs);
order = get_order(size); if (iommu_no_mapping(dev) || !vaddr)
return vaddr;
if (!iommu_no_mapping(dev)) *dma_handle = __intel_map_single(dev, virt_to_phys(vaddr),
flags &= ~(GFP_DMA | GFP_DMA32); PAGE_ALIGN(size), DMA_BIDIRECTIONAL,
else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) { dev->coherent_dma_mask);
if (dev->coherent_dma_mask < DMA_BIT_MASK(32)) if (!*dma_handle)
flags |= GFP_DMA; goto out_free_pages;
else return vaddr;
flags |= GFP_DMA32;
}
if (gfpflags_allow_blocking(flags)) {
unsigned int count = size >> PAGE_SHIFT;
page = dma_alloc_from_contiguous(dev, count, order, flags);
if (page && iommu_no_mapping(dev) &&
page_to_phys(page) + size > dev->coherent_dma_mask) {
dma_release_from_contiguous(dev, page, count);
page = NULL;
}
}
if (!page)
page = alloc_pages(flags, order);
if (!page)
return NULL;
memset(page_address(page), 0, size);
*dma_handle = __intel_map_single(dev, page_to_phys(page), size,
DMA_BIDIRECTIONAL,
dev->coherent_dma_mask);
if (*dma_handle)
return page_address(page);
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, order);
out_free_pages:
dma_direct_free(dev, size, vaddr, *dma_handle, attrs);
return NULL; return NULL;
} }
static void intel_free_coherent(struct device *dev, size_t size, void *vaddr, static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs) dma_addr_t dma_handle, unsigned long attrs)
{ {
int order; if (!iommu_no_mapping(dev))
struct page *page = virt_to_page(vaddr); intel_unmap(dev, dma_handle, PAGE_ALIGN(size));
dma_direct_free(dev, size, vaddr, dma_handle, attrs);
size = PAGE_ALIGN(size);
order = get_order(size);
intel_unmap(dev, dma_handle, size);
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, order);
} }
static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist, static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
@ -3871,7 +3842,7 @@ const struct dma_map_ops intel_dma_ops = {
.unmap_page = intel_unmap_page, .unmap_page = intel_unmap_page,
.mapping_error = intel_mapping_error, .mapping_error = intel_mapping_error,
#ifdef CONFIG_X86 #ifdef CONFIG_X86
.dma_supported = x86_dma_supported, .dma_supported = dma_direct_supported,
#endif #endif
}; };

16
drivers/xen/swiotlb-xen.c
View File

@ -53,20 +53,6 @@
* API. * API.
*/ */
#ifndef CONFIG_X86
static unsigned long dma_alloc_coherent_mask(struct device *dev,
gfp_t gfp)
{
unsigned long dma_mask = 0;
dma_mask = dev->coherent_dma_mask;
if (!dma_mask)
dma_mask = (gfp & GFP_DMA) ? DMA_BIT_MASK(24) : DMA_BIT_MASK(32);
return dma_mask;
}
#endif
#define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0) #define XEN_SWIOTLB_ERROR_CODE (~(dma_addr_t)0x0)
static char *xen_io_tlb_start, *xen_io_tlb_end; static char *xen_io_tlb_start, *xen_io_tlb_end;
@ -328,7 +314,7 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
return ret; return ret;
if (hwdev && hwdev->coherent_dma_mask) if (hwdev && hwdev->coherent_dma_mask)
dma_mask = dma_alloc_coherent_mask(hwdev, flags); dma_mask = hwdev->coherent_dma_mask;
/* At this point dma_handle is the physical address, next we are /* At this point dma_handle is the physical address, next we are
* going to set it to the machine address. * going to set it to the machine address.

21
include/linux/dma-direct.h
View File

@ -3,18 +3,19 @@
#define _LINUX_DMA_DIRECT_H 1 #define _LINUX_DMA_DIRECT_H 1
#include <linux/dma-mapping.h> #include <linux/dma-mapping.h>
#include <linux/mem_encrypt.h>
#ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA #ifdef CONFIG_ARCH_HAS_PHYS_TO_DMA
#include <asm/dma-direct.h> #include <asm/dma-direct.h>
#else #else
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{ {
dma_addr_t dev_addr = (dma_addr_t)paddr; dma_addr_t dev_addr = (dma_addr_t)paddr;
return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT); return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
} }
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) static inline phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t dev_addr)
{ {
phys_addr_t paddr = (phys_addr_t)dev_addr; phys_addr_t paddr = (phys_addr_t)dev_addr;
@ -30,6 +31,22 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
} }
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */ #endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
/*
* If memory encryption is supported, phys_to_dma will set the memory encryption
* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
* and __dma_to_phys versions should only be used on non-encrypted memory for
* special occasions like DMA coherent buffers.
*/
static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
{
return __sme_set(__phys_to_dma(dev, paddr));
}
static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
{
return __sme_clr(__dma_to_phys(dev, daddr));
}
#ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN #ifdef CONFIG_ARCH_HAS_DMA_MARK_CLEAN
void dma_mark_clean(void *addr, size_t size); void dma_mark_clean(void *addr, size_t size);
#else #else

8
include/linux/dma-mapping.h
View File

@ -518,12 +518,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size,
if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
return cpu_addr; return cpu_addr;
/* /* let the implementation decide on the zone to allocate from: */
* Let the implementation decide on the zone to allocate from, and flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
* decide on the way of zeroing the memory given that the memory
* returned should always be zeroed.
*/
flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_ZERO);
if (!arch_dma_alloc_attrs(&dev, &flag)) if (!arch_dma_alloc_attrs(&dev, &flag))
return NULL; return NULL;

12
include/linux/set_memory.h
View File

@ -17,4 +17,16 @@ static inline int set_memory_x(unsigned long addr, int numpages) { return 0; }
static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; }
#endif #endif
#ifndef CONFIG_ARCH_HAS_MEM_ENCRYPT
static inline int set_memory_encrypted(unsigned long addr, int numpages)
{
return 0;
}
static inline int set_memory_decrypted(unsigned long addr, int numpages)
{
return 0;
}
#endif /* CONFIG_ARCH_HAS_MEM_ENCRYPT */
#endif /* _LINUX_SET_MEMORY_H_ */ #endif /* _LINUX_SET_MEMORY_H_ */

8
include/linux/swiotlb.h
View File

@ -72,14 +72,6 @@ void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
void swiotlb_free(struct device *dev, size_t size, void *vaddr, void swiotlb_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs); dma_addr_t dma_addr, unsigned long attrs);
extern void
*swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags);
extern void
swiotlb_free_coherent(struct device *hwdev, size_t size,
void *vaddr, dma_addr_t dma_handle);
extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, extern dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size, unsigned long offset, size_t size,
enum dma_data_direction dir, enum dma_data_direction dir,

35
lib/dma-direct.c
View File

@ -9,6 +9,7 @@
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/dma-contiguous.h> #include <linux/dma-contiguous.h>
#include <linux/pfn.h> #include <linux/pfn.h>
#include <linux/set_memory.h>
#define DIRECT_MAPPING_ERROR 0 #define DIRECT_MAPPING_ERROR 0
@ -20,6 +21,14 @@
#define ARCH_ZONE_DMA_BITS 24 #define ARCH_ZONE_DMA_BITS 24
#endif #endif
/*
* For AMD SEV all DMA must be to unencrypted addresses.
*/
static inline bool force_dma_unencrypted(void)
{
return sev_active();
}
static bool static bool
check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
const char *caller) const char *caller)
@ -37,7 +46,9 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size,
static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
{ {
return phys_to_dma(dev, phys) + size - 1 <= dev->coherent_dma_mask; dma_addr_t addr = force_dma_unencrypted() ?
__phys_to_dma(dev, phys) : phys_to_dma(dev, phys);
return addr + size - 1 <= dev->coherent_dma_mask;
} }
void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
@ -46,6 +57,10 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
int page_order = get_order(size); int page_order = get_order(size);
struct page *page = NULL; struct page *page = NULL;
void *ret;
/* we always manually zero the memory once we are done: */
gfp &= ~__GFP_ZERO;
/* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */
if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS))
@ -78,10 +93,15 @@ again:
if (!page) if (!page)
return NULL; return NULL;
ret = page_address(page);
*dma_handle = phys_to_dma(dev, page_to_phys(page)); if (force_dma_unencrypted()) {
memset(page_address(page), 0, size); set_memory_decrypted((unsigned long)ret, 1 << page_order);
return page_address(page); *dma_handle = __phys_to_dma(dev, page_to_phys(page));
} else {
*dma_handle = phys_to_dma(dev, page_to_phys(page));
}
memset(ret, 0, size);
return ret;
} }
/* /*
@ -92,9 +112,12 @@ void dma_direct_free(struct device *dev, size_t size, void *cpu_addr,
dma_addr_t dma_addr, unsigned long attrs) dma_addr_t dma_addr, unsigned long attrs)
{ {
unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
unsigned int page_order = get_order(size);
if (force_dma_unencrypted())
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count)) if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count))
free_pages((unsigned long)cpu_addr, get_order(size)); free_pages((unsigned long)cpu_addr, page_order);
} }
static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,

77
lib/swiotlb.c
View File

@ -31,6 +31,7 @@
#include <linux/gfp.h> #include <linux/gfp.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/mem_encrypt.h> #include <linux/mem_encrypt.h>
#include <linux/set_memory.h>
#include <asm/io.h> #include <asm/io.h>
#include <asm/dma.h> #include <asm/dma.h>
@ -156,22 +157,6 @@ unsigned long swiotlb_size_or_default(void)
return size ? size : (IO_TLB_DEFAULT_SIZE); return size ? size : (IO_TLB_DEFAULT_SIZE);
} }
void __weak swiotlb_set_mem_attributes(void *vaddr, unsigned long size) { }
/* For swiotlb, clear memory encryption mask from dma addresses */
static dma_addr_t swiotlb_phys_to_dma(struct device *hwdev,
phys_addr_t address)
{
return __sme_clr(phys_to_dma(hwdev, address));
}
/* Note that this doesn't work with highmem page */
static dma_addr_t swiotlb_virt_to_bus(struct device *hwdev,
volatile void *address)
{
return phys_to_dma(hwdev, virt_to_phys(address));
}
static bool no_iotlb_memory; static bool no_iotlb_memory;
void swiotlb_print_info(void) void swiotlb_print_info(void)
@ -209,12 +194,12 @@ void __init swiotlb_update_mem_attributes(void)
vaddr = phys_to_virt(io_tlb_start); vaddr = phys_to_virt(io_tlb_start);
bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT); bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT);
swiotlb_set_mem_attributes(vaddr, bytes); set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
memset(vaddr, 0, bytes); memset(vaddr, 0, bytes);
vaddr = phys_to_virt(io_tlb_overflow_buffer); vaddr = phys_to_virt(io_tlb_overflow_buffer);
bytes = PAGE_ALIGN(io_tlb_overflow); bytes = PAGE_ALIGN(io_tlb_overflow);
swiotlb_set_mem_attributes(vaddr, bytes); set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
memset(vaddr, 0, bytes); memset(vaddr, 0, bytes);
} }
@ -355,7 +340,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
io_tlb_start = virt_to_phys(tlb); io_tlb_start = virt_to_phys(tlb);
io_tlb_end = io_tlb_start + bytes; io_tlb_end = io_tlb_start + bytes;
swiotlb_set_mem_attributes(tlb, bytes); set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
memset(tlb, 0, bytes); memset(tlb, 0, bytes);
/* /*
@ -366,7 +351,8 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
if (!v_overflow_buffer) if (!v_overflow_buffer)
goto cleanup2; goto cleanup2;
swiotlb_set_mem_attributes(v_overflow_buffer, io_tlb_overflow); set_memory_decrypted((unsigned long)v_overflow_buffer,
io_tlb_overflow >> PAGE_SHIFT);
memset(v_overflow_buffer, 0, io_tlb_overflow); memset(v_overflow_buffer, 0, io_tlb_overflow);
io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer); io_tlb_overflow_buffer = virt_to_phys(v_overflow_buffer);
@ -622,7 +608,7 @@ map_single(struct device *hwdev, phys_addr_t phys, size_t size,
return SWIOTLB_MAP_ERROR; return SWIOTLB_MAP_ERROR;
} }
start_dma_addr = swiotlb_phys_to_dma(hwdev, io_tlb_start); start_dma_addr = __phys_to_dma(hwdev, io_tlb_start);
return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size,
dir, attrs); dir, attrs);
} }
@ -706,6 +692,7 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
} }
} }
#ifdef CONFIG_DMA_DIRECT_OPS
static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
size_t size) size_t size)
{ {
@ -726,12 +713,12 @@ swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
goto out_warn; goto out_warn;
phys_addr = swiotlb_tbl_map_single(dev, phys_addr = swiotlb_tbl_map_single(dev,
swiotlb_phys_to_dma(dev, io_tlb_start), __phys_to_dma(dev, io_tlb_start),
0, size, DMA_FROM_DEVICE, 0); 0, size, DMA_FROM_DEVICE, 0);
if (phys_addr == SWIOTLB_MAP_ERROR) if (phys_addr == SWIOTLB_MAP_ERROR)
goto out_warn; goto out_warn;
*dma_handle = swiotlb_phys_to_dma(dev, phys_addr); *dma_handle = __phys_to_dma(dev, phys_addr);
if (dma_coherent_ok(dev, *dma_handle, size)) if (dma_coherent_ok(dev, *dma_handle, size))
goto out_unmap; goto out_unmap;
@ -759,28 +746,6 @@ out_warn:
return NULL; return NULL;
} }
void *
swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags)
{
int order = get_order(size);
unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;
void *ret;
ret = (void *)__get_free_pages(flags, order);
if (ret) {
*dma_handle = swiotlb_virt_to_bus(hwdev, ret);
if (dma_coherent_ok(hwdev, *dma_handle, size)) {
memset(ret, 0, size);
return ret;
}
free_pages((unsigned long)ret, order);
}
return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);
}
EXPORT_SYMBOL(swiotlb_alloc_coherent);
static bool swiotlb_free_buffer(struct device *dev, size_t size, static bool swiotlb_free_buffer(struct device *dev, size_t size,
dma_addr_t dma_addr) dma_addr_t dma_addr)
{ {
@ -799,15 +764,7 @@ static bool swiotlb_free_buffer(struct device *dev, size_t size,
DMA_ATTR_SKIP_CPU_SYNC); DMA_ATTR_SKIP_CPU_SYNC);
return true; return true;
} }
#endif
void
swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
dma_addr_t dev_addr)
{
if (!swiotlb_free_buffer(hwdev, size, dev_addr))
free_pages((unsigned long)vaddr, get_order(size));
}
EXPORT_SYMBOL(swiotlb_free_coherent);
static void static void
swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir, swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir,
@ -867,10 +824,10 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
map = map_single(dev, phys, size, dir, attrs); map = map_single(dev, phys, size, dir, attrs);
if (map == SWIOTLB_MAP_ERROR) { if (map == SWIOTLB_MAP_ERROR) {
swiotlb_full(dev, size, dir, 1); swiotlb_full(dev, size, dir, 1);
return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); return __phys_to_dma(dev, io_tlb_overflow_buffer);
} }
dev_addr = swiotlb_phys_to_dma(dev, map); dev_addr = __phys_to_dma(dev, map);
/* Ensure that the address returned is DMA'ble */ /* Ensure that the address returned is DMA'ble */
if (dma_capable(dev, dev_addr, size)) if (dma_capable(dev, dev_addr, size))
@ -879,7 +836,7 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
attrs |= DMA_ATTR_SKIP_CPU_SYNC; attrs |= DMA_ATTR_SKIP_CPU_SYNC;
swiotlb_tbl_unmap_single(dev, map, size, dir, attrs); swiotlb_tbl_unmap_single(dev, map, size, dir, attrs);
return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); return __phys_to_dma(dev, io_tlb_overflow_buffer);
} }
/* /*
@ -1009,7 +966,7 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
sg_dma_len(sgl) = 0; sg_dma_len(sgl) = 0;
return 0; return 0;
} }
sg->dma_address = swiotlb_phys_to_dma(hwdev, map); sg->dma_address = __phys_to_dma(hwdev, map);
} else } else
sg->dma_address = dev_addr; sg->dma_address = dev_addr;
sg_dma_len(sg) = sg->length; sg_dma_len(sg) = sg->length;
@ -1073,7 +1030,7 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
int int
swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
{ {
return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer)); return (dma_addr == __phys_to_dma(hwdev, io_tlb_overflow_buffer));
} }
/* /*
@ -1085,7 +1042,7 @@ swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
int int
swiotlb_dma_supported(struct device *hwdev, u64 mask) swiotlb_dma_supported(struct device *hwdev, u64 mask)
{ {
return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; return __phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
} }
#ifdef CONFIG_DMA_DIRECT_OPS #ifdef CONFIG_DMA_DIRECT_OPS