From 007134de250e064759302032399b22eaa0cf19ec Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 18 Aug 2021 11:55:47 -0500 Subject: [PATCH 1/3] KVM: x86: Allow CPU to force vendor-specific TDP level ANBZ: #44795 commit 746700d21fd52399c97aeb7791584bbf5426983c upstream. AMD future CPUs will require a 5-level NPT if host CR4.LA57 is set. To prevent kvm_mmu_get_tdp_level() from incorrectly changing NPT level on behalf of CPUs, add a new parameter in kvm_configure_mmu() to force a fixed TDP level. [backport note] Context-only adjustment: hunk offsets differ because 5.10 still has cr3_lm_rsvd_bits in struct kvm_vcpu_arch and the older kvm_mmu_new_pgd() prototype, and svm_hardware_setup() guards npt_enabled differently. The functional change is identical to upstream. Prerequisite of the 5-level NPT series; without it kvm_amd refuses to load on hosts running with CR4.LA57 set. Signed-off-by: Wei Huang Message-Id: <20210818165549.3771014-2-wei.huang2@amd.com> Signed-off-by: Paolo Bonzini Signed-off-by: Yuanhe Shu --- arch/x86/include/asm/kvm_host.h | 5 ++--- arch/x86/kvm/mmu/mmu.c | 10 ++++++++-- arch/x86/kvm/svm/svm.c | 4 +++- arch/x86/kvm/vmx/vmx.c | 3 ++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6f414969f096..8a0eb22d0403 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -679,7 +679,6 @@ struct kvm_vcpu_arch { unsigned long cr3_lm_rsvd_bits; int maxphyaddr; - int max_tdp_level; /* emulate context */ @@ -1699,8 +1698,8 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, bool skip_mmu_sync); -void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, - int tdp_huge_page_level); +void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, + int tdp_max_root_level, int tdp_huge_page_level); static inline u16 kvm_read_ldt(void) { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b873445ee58d..e2a0a9068132 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -96,6 +96,7 @@ module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); bool tdp_enabled = false; static int max_huge_page_level __read_mostly; +static int tdp_root_level __read_mostly; static int max_tdp_level __read_mostly; enum { @@ -4542,6 +4543,10 @@ static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { + /* tdp_root_level is architecture forced level, use it if nonzero */ + if (tdp_root_level) + return tdp_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; @@ -5259,10 +5264,11 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) } EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); -void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, - int tdp_huge_page_level) +void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level, + int tdp_max_root_level, int tdp_huge_page_level) { tdp_enabled = enable_tdp; + tdp_root_level = tdp_forced_root_level; max_tdp_level = tdp_max_root_level; /* diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 329518c40fa1..1be27e5fb577 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1024,7 +1024,9 @@ static __init int svm_hardware_setup(void) if (npt_enabled && !npt) npt_enabled = false; - kvm_configure_mmu(npt_enabled, get_max_npt_level(), PG_LEVEL_1G); + /* Force VM NPT level equal to the host's max NPT level */ + kvm_configure_mmu(npt_enabled, get_max_npt_level(), + get_max_npt_level(), PG_LEVEL_1G); pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); if (nrips) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a20b5e704283..b72da001c270 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8889,7 +8889,8 @@ static __init int hardware_setup(void) ept_lpage_level = PG_LEVEL_2M; else ept_lpage_level = PG_LEVEL_4K; - kvm_configure_mmu(enable_ept, vmx_get_max_tdp_level(), ept_lpage_level); + kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), + ept_lpage_level); /* * Only enable PML when hardware supports PML feature, and both EPT -- Gitee From afb9e6399fe983dccebc777f49abb11e482c2827 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 18 Aug 2021 11:55:48 -0500 Subject: [PATCH 2/3] KVM: x86/mmu: Support shadowing NPT when 5-level paging is enabled in host ANBZ: #44795 commit cb0f722aff6e9ba970a9fee9263c7821bbe811de upstream. When the 5-level page table CPU flag is set in the host, but the guest has CR4.LA57=0 (including the case of a 32-bit guest), the top level of the shadow NPT page tables will be fixed, consisting of one pointer to a lower-level table and 511 non-present entries. Extend the existing code that creates the fixed PML4 or PDP table, to provide a fixed PML5 table if needed. This is not needed on EPT because the number of layers in the tables is specified in the EPTP instead of depending on the host CR4. [backport note] Reworked for 5.10, which has neither mmu_alloc_special_roots() nor the pml4_root rename: - mmu->lm_root plays the role of upstream mmu->pml4_root, so pml5_root is added next to it in struct kvm_mmu and freed in free_mmu_pages(). - The PML5 page is allocated lazily inside mmu_alloc_shadow_roots(), the same place where 5.10 already allocates lm_root on demand, instead of in the upstream helper. pml5_root[0] points at lm_root with the same pm_mask, and root_hpa is set from pml5_root when shadow_root_level is PT64_ROOT_5LEVEL. - The two shadow_root_level comparisons against PT64_ROOT_4LEVEL become >= so that the 5-level case takes the same path, matching upstream. Suggested-by: Paolo Bonzini Signed-off-by: Wei Huang Message-Id: <20210818165549.3771014-3-wei.huang2@amd.com> Signed-off-by: Paolo Bonzini Signed-off-by: Yuanhe Shu --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/mmu/mmu.c | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 8a0eb22d0403..678572e64635 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -400,6 +400,7 @@ struct kvm_mmu { u64 *pae_root; u64 *lm_root; + u64 *pml5_root; /* * check zero bits on shadow page table entries, these diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index e2a0a9068132..5d31133645a3 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3360,7 +3360,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * the shadow page table may be a PAE or a long mode page table. */ pm_mask = PT_PRESENT_MASK; - if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { + if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; /* @@ -3405,7 +3405,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * on demand, as running a 32-bit L1 VMM is very rare. The PDP is * handled above (to share logic with PAE), deal with the PML4 here. */ - if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_4LEVEL) { + if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { if (vcpu->arch.mmu->lm_root == NULL) { u64 *lm_root; @@ -3418,7 +3418,24 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) vcpu->arch.mmu->lm_root = lm_root; } - vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); + if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_5LEVEL) { + if (vcpu->arch.mmu->pml5_root == NULL) { + u64 *pml5_root; + + pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + if (!pml5_root) + return -ENOMEM; + + pml5_root[0] = __pa(vcpu->arch.mmu->lm_root) | pm_mask; + + vcpu->arch.mmu->pml5_root = pml5_root; + } + } + + if (vcpu->arch.mmu->shadow_root_level == PT64_ROOT_5LEVEL) + vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pml5_root); + else + vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); } set_root_pgd: @@ -5363,6 +5380,7 @@ static void free_mmu_pages(struct kvm_mmu *mmu) { free_page((unsigned long)mmu->pae_root); free_page((unsigned long)mmu->lm_root); + free_page((unsigned long)mmu->pml5_root); } static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) -- Gitee From 3d51ba44d0d77b42cd01c776399cdd160db24568 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 18 Aug 2021 11:55:49 -0500 Subject: [PATCH 3/3] KVM: SVM: Add 5-level page table support for SVM ANBZ: #44795 commit 43e540cc9f2ca12a2364ddf64e5ef929a546550d upstream. When the 5-level page table is enabled on host OS, the nested page table for guest VMs must use 5-level as well. Update get_npt_level() function to reflect this requirement. In the meanwhile, remove the code that prevents kvm-amd driver from being loaded when 5-level page table is detected. [backport note] Context-only adjustment: in 5.10 the pgtable_l5_enabled() bail-out lives in has_svm() rather than svm_hardware_setup(), so it is dropped from there. The functional change is identical to upstream. Signed-off-by: Wei Huang Message-Id: <20210818165549.3771014-4-wei.huang2@amd.com> [Tweak condition as suggested by Sean. - Paolo] Signed-off-by: Paolo Bonzini Signed-off-by: Yuanhe Shu --- arch/x86/kvm/svm/svm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 1be27e5fb577..a77335e358a8 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -281,7 +281,7 @@ static inline void invlpga(unsigned long addr, u32 asid) static int get_max_npt_level(void) { #ifdef CONFIG_X86_64 - return PT64_ROOT_4LEVEL; + return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; #else return PT32E_ROOT_LEVEL; #endif -- Gitee