KVM: arm/arm64: Split dcache/icache flushing
As we're about to introduce opportunistic invalidation of the icache, let's split dcache and icache flushing. Acked-by: Christoffer Dall <cdall@linaro.org> Signed-off-by: Marc Zyngier <marc.zyngier@arm.com> Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
This commit is contained in:
parent
d68119864e
commit
a15f693935
|
@ -126,21 +126,12 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
|
|||
return (vcpu_cp15(vcpu, c1_SCTLR) & 0b101) == 0b101;
|
||||
}
|
||||
|
||||
static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
static inline void __clean_dcache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
/*
|
||||
* If we are going to insert an instruction page and the icache is
|
||||
* either VIPT or PIPT, there is a potential problem where the host
|
||||
* (or another VM) may have used the same page as this guest, and we
|
||||
* read incorrect data from the icache. If we're using a PIPT cache,
|
||||
* we can invalidate just that page, but if we are using a VIPT cache
|
||||
* we need to invalidate the entire icache - damn shame - as written
|
||||
* in the ARM ARM (DDI 0406C.b - Page B3-1393).
|
||||
*
|
||||
* VIVT caches are tagged using both the ASID and the VMID and doesn't
|
||||
* need any kind of flushing (DDI 0406C.b - Page B3-1392).
|
||||
* Clean the dcache to the Point of Coherency.
|
||||
*
|
||||
* We need to do this through a kernel mapping (using the
|
||||
* user-space mapping has proved to be the wrong
|
||||
|
@ -155,19 +146,52 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
|
|||
|
||||
kvm_flush_dcache_to_poc(va, PAGE_SIZE);
|
||||
|
||||
if (icache_is_pipt())
|
||||
__cpuc_coherent_user_range((unsigned long)va,
|
||||
(unsigned long)va + PAGE_SIZE);
|
||||
|
||||
size -= PAGE_SIZE;
|
||||
pfn++;
|
||||
|
||||
kunmap_atomic(va);
|
||||
}
|
||||
}
|
||||
|
||||
if (!icache_is_pipt() && !icache_is_vivt_asid_tagged()) {
|
||||
static inline void __invalidate_icache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
/*
|
||||
* If we are going to insert an instruction page and the icache is
|
||||
* either VIPT or PIPT, there is a potential problem where the host
|
||||
* (or another VM) may have used the same page as this guest, and we
|
||||
* read incorrect data from the icache. If we're using a PIPT cache,
|
||||
* we can invalidate just that page, but if we are using a VIPT cache
|
||||
* we need to invalidate the entire icache - damn shame - as written
|
||||
* in the ARM ARM (DDI 0406C.b - Page B3-1393).
|
||||
*
|
||||
* VIVT caches are tagged using both the ASID and the VMID and doesn't
|
||||
* need any kind of flushing (DDI 0406C.b - Page B3-1392).
|
||||
*/
|
||||
|
||||
VM_BUG_ON(size & ~PAGE_MASK);
|
||||
|
||||
if (icache_is_vivt_asid_tagged())
|
||||
return;
|
||||
|
||||
if (!icache_is_pipt()) {
|
||||
/* any kind of VIPT cache */
|
||||
__flush_icache_all();
|
||||
return;
|
||||
}
|
||||
|
||||
/* PIPT cache. As for the d-side, use a temporary kernel mapping. */
|
||||
while (size) {
|
||||
void *va = kmap_atomic_pfn(pfn);
|
||||
|
||||
__cpuc_coherent_user_range((unsigned long)va,
|
||||
(unsigned long)va + PAGE_SIZE);
|
||||
|
||||
size -= PAGE_SIZE;
|
||||
pfn++;
|
||||
|
||||
kunmap_atomic(va);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -230,19 +230,26 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
|
|||
return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
|
||||
}
|
||||
|
||||
static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
static inline void __clean_dcache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
void *va = page_address(pfn_to_page(pfn));
|
||||
|
||||
kvm_flush_dcache_to_poc(va, size);
|
||||
}
|
||||
|
||||
static inline void __invalidate_icache_guest_page(struct kvm_vcpu *vcpu,
|
||||
kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
if (icache_is_aliasing()) {
|
||||
/* any kind of VIPT cache */
|
||||
__flush_icache_all();
|
||||
} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
|
||||
/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
|
||||
void *va = page_address(pfn_to_page(pfn));
|
||||
|
||||
flush_icache_range((unsigned long)va,
|
||||
(unsigned long)va + size);
|
||||
}
|
||||
|
|
|
@ -1257,10 +1257,16 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
|
|||
kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
|
||||
}
|
||||
|
||||
static void coherent_cache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
static void clean_dcache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
__coherent_cache_guest_page(vcpu, pfn, size);
|
||||
__clean_dcache_guest_page(vcpu, pfn, size);
|
||||
}
|
||||
|
||||
static void invalidate_icache_guest_page(struct kvm_vcpu *vcpu, kvm_pfn_t pfn,
|
||||
unsigned long size)
|
||||
{
|
||||
__invalidate_icache_guest_page(vcpu, pfn, size);
|
||||
}
|
||||
|
||||
static void kvm_send_hwpoison_signal(unsigned long address,
|
||||
|
@ -1391,7 +1397,9 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
|
|||
new_pmd = kvm_s2pmd_mkwrite(new_pmd);
|
||||
kvm_set_pfn_dirty(pfn);
|
||||
}
|
||||
coherent_cache_guest_page(vcpu, pfn, PMD_SIZE);
|
||||
clean_dcache_guest_page(vcpu, pfn, PMD_SIZE);
|
||||
invalidate_icache_guest_page(vcpu, pfn, PMD_SIZE);
|
||||
|
||||
ret = stage2_set_pmd_huge(kvm, memcache, fault_ipa, &new_pmd);
|
||||
} else {
|
||||
pte_t new_pte = pfn_pte(pfn, mem_type);
|
||||
|
@ -1401,7 +1409,9 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
|
|||
kvm_set_pfn_dirty(pfn);
|
||||
mark_page_dirty(kvm, gfn);
|
||||
}
|
||||
coherent_cache_guest_page(vcpu, pfn, PAGE_SIZE);
|
||||
clean_dcache_guest_page(vcpu, pfn, PAGE_SIZE);
|
||||
invalidate_icache_guest_page(vcpu, pfn, PAGE_SIZE);
|
||||
|
||||
ret = stage2_set_pte(kvm, memcache, fault_ipa, &new_pte, flags);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue