diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6f414969f096f120fd38a4af9d864608dba09a5a..678572e64635d2e550b68bec3a308e869482617e 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -400,6 +400,7 @@ struct kvm_mmu { u64 *pae_root; u64 *lm_root; + u64 *pml5_root; /* * check zero bits on shadow page table entries, these @@ -679,7 +680,6 @@ struct kvm_vcpu_arch { unsigned long cr3_lm_rsvd_bits; int maxphyaddr; - int max_tdp_level; /* emulate context */ @@ -1699,8 +1699,8 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, bool skip_mmu_sync); -void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, - int tdp_huge_page_level); +void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, + int tdp_max_root_level, int tdp_huge_page_level); static inline u16 kvm_read_ldt(void) { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b873445ee58de6d574aeb42a59cdb9d38b0fdb7e..5d31133645a363dec03c3b4dc4d0d8e56a0c9736 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -96,6 +96,7 @@ module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); bool tdp_enabled = false; static int max_huge_page_level __read_mostly; +static int tdp_root_level __read_mostly; static int max_tdp_level __read_mostly; enum { @@ -3359,7 +3360,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * the shadow page table may be a PAE or a long mode page table. */ pm_mask = PT_PRESENT_MASK; - if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { + if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; /* @@ -3404,7 +3405,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * on demand, as running a 32-bit L1 VMM is very rare. The PDP is * handled above (to share logic with PAE), deal with the PML4 here. */ - if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { + if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { if (vcpu->arch.mmu->lm_root == NULL) { u64 *lm_root; @@ -3417,7 +3418,24 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) vcpu->arch.mmu->lm_root = lm_root; } - vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); + if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_5LEVEL) { + if (vcpu->arch.mmu->pml5_root == NULL) { + u64 *pml5_root; + + pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + if (!pml5_root) + return -ENOMEM; + + pml5_root[0] = __pa(vcpu->arch.mmu->lm_root) | pm_mask; + + vcpu->arch.mmu->pml5_root = pml5_root; + } + } + + if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_5LEVEL) + vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pml5_root); + else + vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); } set_root_pgd: @@ -4542,6 +4560,10 @@ static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { + /* tdp_root_level is architecture forced level, use it if nonzero */ + if (tdp_root_level) + return tdp_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; @@ -5259,10 +5281,11 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) } EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); -void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, - int tdp_huge_page_level) +void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, + int tdp_max_root_level, int tdp_huge_page_level) { tdp_enabled = enable_tdp; + tdp_root_level = tdp_forced_root_level; max_tdp_level = tdp_max_root_level; /* @@ -5357,6 +5380,7 @@ static void free_mmu_pages(struct kvm_mmu *mmu) { free_page((unsigned long)mmu->pae_root); free_page((unsigned long)mmu->lm_root); + free_page((unsigned long)mmu->pml5_root); } static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 329518c40fa12e221804beb52b5ca80a357bcbb5..a77335e358a8724e05bd2c8c3579a7fa905a4f0d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -281,7 +281,7 @@ static inline void invlpga(unsigned long addr, u32 asid) static int get_max_npt_level(void) { #ifdef CONFIG_X86_64 - return PT64_ROOT_4LEVEL; + return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; #else return PT32E_ROOT_LEVEL; #endif @@ -1024,7 +1024,9 @@ static __init int svm_hardware_setup(void) if (npt_enabled && !npt) npt_enabled = false; - kvm_configure_mmu(npt_enabled, get_max_npt_level(), PG_LEVEL_1G); + /* Force VM NPT level equal to the host's max NPT level */ + kvm_configure_mmu(npt_enabled, get_max_npt_level(), + get_max_npt_level(), PG_LEVEL_1G); pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); if (nrips) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a20b5e704283f4dc92b17ffdb879b1fb04877f91..b72da001c270a97e40599eb0429053db705f05e1 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8889,7 +8889,8 @@ static __init int hardware_setup(void) ept_lpage_level = PG_LEVEL_2M; else ept_lpage_level = PG_LEVEL_4K; - kvm_configure_mmu(enable_ept, vmx_get_max_tdp_level(), ept_lpage_level); + kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), + ept_lpage_level); /* * Only enable PML when hardware supports PML feature, and both EPT