x86/kvm/mmu: introduce guest_mmu
When EPT is used for nested guest we need to re-init MMU as shadow EPT MMU (nested_ept_init_mmu_context() does that). When we return back from L2 to L1 kvm_mmu_reset_context() in nested_vmx_load_cr3() resets MMU back to normal TDP mode. Add a special 'guest_mmu' so we can use separate root caches; the improved hit rate is not very important for single vCPU performance, but it avoids contention on the mmu_lock for many vCPUs. On the nested CPUID benchmark, with 16 vCPUs, an L2->L1->L2 vmexit goes from 42k to 26k cycles. Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com> Reviewed-by: Sean Christopherson <sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
This commit is contained in:
parent
6a82cd1c7b
commit
14c07ad89f
|
@ -548,6 +548,9 @@ struct kvm_vcpu_arch {
|
||||||
/* Non-nested MMU for L1 */
|
/* Non-nested MMU for L1 */
|
||||||
struct kvm_mmu root_mmu;
|
struct kvm_mmu root_mmu;
|
||||||
|
|
||||||
|
/* L1 MMU when running nested */
|
||||||
|
struct kvm_mmu guest_mmu;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Paging state of an L2 guest (used for nested npt)
|
* Paging state of an L2 guest (used for nested npt)
|
||||||
*
|
*
|
||||||
|
|
|
@ -4832,7 +4832,10 @@ EXPORT_SYMBOL_GPL(kvm_init_shadow_mmu);
|
||||||
static union kvm_mmu_page_role
|
static union kvm_mmu_page_role
|
||||||
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty)
|
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty)
|
||||||
{
|
{
|
||||||
union kvm_mmu_page_role role = vcpu->arch.mmu->base_role;
|
union kvm_mmu_page_role role;
|
||||||
|
|
||||||
|
/* Role is inherited from root_mmu */
|
||||||
|
role.word = vcpu->arch.root_mmu.base_role.word;
|
||||||
|
|
||||||
role.level = PT64_ROOT_4LEVEL;
|
role.level = PT64_ROOT_4LEVEL;
|
||||||
role.direct = false;
|
role.direct = false;
|
||||||
|
@ -4982,8 +4985,10 @@ EXPORT_SYMBOL_GPL(kvm_mmu_load);
|
||||||
|
|
||||||
void kvm_mmu_unload(struct kvm_vcpu *vcpu)
|
void kvm_mmu_unload(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOTS_ALL);
|
kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
|
||||||
WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa));
|
WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));
|
||||||
|
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
|
||||||
|
WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
|
EXPORT_SYMBOL_GPL(kvm_mmu_unload);
|
||||||
|
|
||||||
|
@ -5422,13 +5427,18 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu)
|
||||||
|
|
||||||
vcpu->arch.mmu = &vcpu->arch.root_mmu;
|
vcpu->arch.mmu = &vcpu->arch.root_mmu;
|
||||||
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
|
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
|
||||||
|
|
||||||
vcpu->arch.root_mmu.root_hpa = INVALID_PAGE;
|
vcpu->arch.root_mmu.root_hpa = INVALID_PAGE;
|
||||||
vcpu->arch.root_mmu.translate_gpa = translate_gpa;
|
vcpu->arch.root_mmu.translate_gpa = translate_gpa;
|
||||||
vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
|
|
||||||
|
|
||||||
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
|
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
|
||||||
vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
|
vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
|
||||||
|
|
||||||
|
vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE;
|
||||||
|
vcpu->arch.guest_mmu.translate_gpa = translate_gpa;
|
||||||
|
for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
|
||||||
|
vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;
|
||||||
|
|
||||||
|
vcpu->arch.nested_mmu.translate_gpa = translate_nested_gpa;
|
||||||
return alloc_mmu_pages(vcpu);
|
return alloc_mmu_pages(vcpu);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -8423,8 +8423,10 @@ static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
|
||||||
vmcs_write64(VMCS_LINK_POINTER, -1ull);
|
vmcs_write64(VMCS_LINK_POINTER, -1ull);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
|
static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
|
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||||
|
|
||||||
if (vmx->nested.current_vmptr == -1ull)
|
if (vmx->nested.current_vmptr == -1ull)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -8438,10 +8440,12 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
|
||||||
vmx->nested.posted_intr_nv = -1;
|
vmx->nested.posted_intr_nv = -1;
|
||||||
|
|
||||||
/* Flush VMCS12 to guest memory */
|
/* Flush VMCS12 to guest memory */
|
||||||
kvm_vcpu_write_guest_page(&vmx->vcpu,
|
kvm_vcpu_write_guest_page(vcpu,
|
||||||
vmx->nested.current_vmptr >> PAGE_SHIFT,
|
vmx->nested.current_vmptr >> PAGE_SHIFT,
|
||||||
vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
|
vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
|
||||||
|
|
||||||
|
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
|
||||||
|
|
||||||
vmx->nested.current_vmptr = -1ull;
|
vmx->nested.current_vmptr = -1ull;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8449,8 +8453,10 @@ static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
|
||||||
* Free whatever needs to be freed from vmx->nested when L1 goes down, or
|
* Free whatever needs to be freed from vmx->nested when L1 goes down, or
|
||||||
* just stops using VMX.
|
* just stops using VMX.
|
||||||
*/
|
*/
|
||||||
static void free_nested(struct vcpu_vmx *vmx)
|
static void free_nested(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
|
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
||||||
|
|
||||||
if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
|
if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -8483,6 +8489,8 @@ static void free_nested(struct vcpu_vmx *vmx)
|
||||||
vmx->nested.pi_desc = NULL;
|
vmx->nested.pi_desc = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
|
||||||
|
|
||||||
free_loaded_vmcs(&vmx->nested.vmcs02);
|
free_loaded_vmcs(&vmx->nested.vmcs02);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8491,7 +8499,7 @@ static int handle_vmoff(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
if (!nested_vmx_check_permission(vcpu))
|
if (!nested_vmx_check_permission(vcpu))
|
||||||
return 1;
|
return 1;
|
||||||
free_nested(to_vmx(vcpu));
|
free_nested(vcpu);
|
||||||
return nested_vmx_succeed(vcpu);
|
return nested_vmx_succeed(vcpu);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8517,7 +8525,7 @@ static int handle_vmclear(struct kvm_vcpu *vcpu)
|
||||||
VMXERR_VMCLEAR_VMXON_POINTER);
|
VMXERR_VMCLEAR_VMXON_POINTER);
|
||||||
|
|
||||||
if (vmptr == vmx->nested.current_vmptr)
|
if (vmptr == vmx->nested.current_vmptr)
|
||||||
nested_release_vmcs12(vmx);
|
nested_release_vmcs12(vcpu);
|
||||||
|
|
||||||
kvm_vcpu_write_guest(vcpu,
|
kvm_vcpu_write_guest(vcpu,
|
||||||
vmptr + offsetof(struct vmcs12, launch_state),
|
vmptr + offsetof(struct vmcs12, launch_state),
|
||||||
|
@ -8872,7 +8880,8 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu)
|
||||||
VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
|
VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
|
||||||
}
|
}
|
||||||
|
|
||||||
nested_release_vmcs12(vmx);
|
nested_release_vmcs12(vcpu);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Load VMCS12 from guest memory since it is not already
|
* Load VMCS12 from guest memory since it is not already
|
||||||
* cached.
|
* cached.
|
||||||
|
@ -10928,12 +10937,10 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
|
||||||
*/
|
*/
|
||||||
static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
|
static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
struct vcpu_vmx *vmx = to_vmx(vcpu);
|
vcpu_load(vcpu);
|
||||||
|
vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);
|
||||||
vcpu_load(vcpu);
|
free_nested(vcpu);
|
||||||
vmx_switch_vmcs(vcpu, &vmx->vmcs01);
|
vcpu_put(vcpu);
|
||||||
free_nested(vmx);
|
|
||||||
vcpu_put(vcpu);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
|
static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
|
||||||
|
@ -11300,6 +11307,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
WARN_ON(mmu_is_nested(vcpu));
|
WARN_ON(mmu_is_nested(vcpu));
|
||||||
|
|
||||||
|
vcpu->arch.mmu = &vcpu->arch.guest_mmu;
|
||||||
kvm_init_shadow_ept_mmu(vcpu,
|
kvm_init_shadow_ept_mmu(vcpu,
|
||||||
to_vmx(vcpu)->nested.msrs.ept_caps &
|
to_vmx(vcpu)->nested.msrs.ept_caps &
|
||||||
VMX_EPT_EXECUTE_ONLY_BIT,
|
VMX_EPT_EXECUTE_ONLY_BIT,
|
||||||
|
@ -11315,6 +11323,7 @@ static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
|
||||||
|
|
||||||
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
|
static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
|
||||||
{
|
{
|
||||||
|
vcpu->arch.mmu = &vcpu->arch.root_mmu;
|
||||||
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
|
vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -13731,7 +13740,7 @@ static void vmx_leave_nested(struct kvm_vcpu *vcpu)
|
||||||
to_vmx(vcpu)->nested.nested_run_pending = 0;
|
to_vmx(vcpu)->nested.nested_run_pending = 0;
|
||||||
nested_vmx_vmexit(vcpu, -1, 0, 0);
|
nested_vmx_vmexit(vcpu, -1, 0, 0);
|
||||||
}
|
}
|
||||||
free_nested(to_vmx(vcpu));
|
free_nested(vcpu);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int vmx_check_intercept(struct kvm_vcpu *vcpu,
|
static int vmx_check_intercept(struct kvm_vcpu *vcpu,
|
||||||
|
|
Loading…
Reference in New Issue