diff --git a/tee_blackbox/guest/0001-add-cvm-base-feature.patch b/tee_blackbox/guest/0001-add-cvm-base-feature.patch new file mode 100644 index 0000000000000000000000000000000000000000..3966ccdfe0a050a2bb2e4310314bb0da5b69281d --- /dev/null +++ b/tee_blackbox/guest/0001-add-cvm-base-feature.patch @@ -0,0 +1,3393 @@ +From c4c2c42dfe869dfc74ea323b4081254e820348b5 Mon Sep 17 00:00:00 2001 +From: Jingxian He +Date: Sat, 3 Aug 2024 19:42:44 +0800 +Subject: [PATCH 1/6] add cvm base feature + +--- + arch/arm64/Kconfig | 8 + + arch/arm64/configs/openeuler_defconfig | 5 +- + arch/arm64/include/asm/kvm_emulate.h | 18 + + arch/arm64/include/asm/kvm_host.h | 15 +- + arch/arm64/include/asm/kvm_tmi.h | 333 ++++++++ + arch/arm64/include/asm/kvm_tmm.h | 30 +- + arch/arm64/include/asm/set_memory.h | 1 + + arch/arm64/include/asm/virtcca_cvm_guest.h | 39 + + arch/arm64/include/uapi/asm/kvm.h | 4 + + arch/arm64/kernel/Makefile | 1 + + arch/arm64/kernel/virtcca_cvm_guest.c | 93 +++ + arch/arm64/kvm/Kconfig | 6 +- + arch/arm64/kvm/Makefile | 3 + + arch/arm64/kvm/arch_timer.c | 93 +++ + arch/arm64/kvm/arm.c | 134 +++- + arch/arm64/kvm/guest.c | 5 + + arch/arm64/kvm/mmio.c | 14 +- + arch/arm64/kvm/mmu.c | 10 + + arch/arm64/kvm/pmu-emul.c | 9 + + arch/arm64/kvm/psci.c | 10 +- + arch/arm64/kvm/reset.c | 11 + + arch/arm64/kvm/tmi.c | 141 ++++ + arch/arm64/kvm/vgic/vgic-v3.c | 14 +- + arch/arm64/kvm/vgic/vgic.c | 52 +- + arch/arm64/kvm/virtcca_cvm.c | 889 +++++++++++++++++++++ + arch/arm64/kvm/virtcca_cvm_exit.c | 221 +++++ + arch/arm64/mm/init.c | 3 + + arch/arm64/mm/mmu.c | 5 +- + arch/arm64/mm/pageattr.c | 3 + + drivers/irqchip/irq-gic-v3-its.c | 231 +++++- + drivers/perf/arm_pmu.c | 17 + + include/kvm/arm_arch_timer.h | 4 + + include/linux/kvm_host.h | 22 + + include/linux/perf/arm_pmu.h | 4 + + include/uapi/linux/kvm.h | 13 + + kernel/dma/swiotlb.c | 16 + + virt/kvm/kvm_main.c | 4 + + 37 files changed, 2423 insertions(+), 58 deletions(-) + create mode 100644 arch/arm64/include/asm/kvm_tmi.h + create mode 100644 arch/arm64/include/asm/virtcca_cvm_guest.h + create mode 100644 arch/arm64/kernel/virtcca_cvm_guest.c + create mode 100644 arch/arm64/kvm/tmi.c + create mode 100644 arch/arm64/kvm/virtcca_cvm.c + create mode 100644 arch/arm64/kvm/virtcca_cvm_exit.c + +diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig +index a162b83bd731..501ec560a939 100644 +--- a/arch/arm64/Kconfig ++++ b/arch/arm64/Kconfig +@@ -2576,6 +2576,14 @@ config COMPAT + def_bool y + depends on AARCH32_EL0 || ARM64_ILP32 + ++config HISI_VIRTCCA_GUEST ++ bool "Enable cvm guest run" ++ depends on DMA_RESTRICTED_POOL ++ help ++ Support VIRTCCA CVM guest based on S-EL2 ++ ++ If unsure, say N. ++ + menu "Power management options" + + source "kernel/power/Kconfig" +diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig +index 17338d53a87e..072d26fcc238 100644 +--- a/arch/arm64/configs/openeuler_defconfig ++++ b/arch/arm64/configs/openeuler_defconfig +@@ -595,6 +595,7 @@ CONFIG_DMI=y + # end of Boot options + + CONFIG_COMPAT=y ++CONFIG_HISI_VIRTCCA_GUEST=y + + # + # Power management options +@@ -758,7 +759,7 @@ CONFIG_KVM_GENERIC_HARDWARE_ENABLING=y + CONFIG_KVM_HISI_VIRT=y + CONFIG_VIRTUALIZATION=y + CONFIG_KVM=y +-CONFIG_CVM_HOST=y ++CONFIG_HISI_VIRTCCA_HOST=y + # CONFIG_NVHE_EL2_DEBUG is not set + CONFIG_KVM_ARM_MULTI_LPI_TRANSLATE_CACHE=y + CONFIG_ARCH_VCPU_STAT=y +@@ -7821,7 +7822,7 @@ CONFIG_ARCH_HAS_DMA_PREP_COHERENT=y + CONFIG_SWIOTLB=y + # CONFIG_SWIOTLB_DYNAMIC is not set + CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC=y +-# CONFIG_DMA_RESTRICTED_POOL is not set ++CONFIG_DMA_RESTRICTED_POOL=y + CONFIG_DMA_NONCOHERENT_MMAP=y + CONFIG_DMA_COHERENT_POOL=y + CONFIG_DMA_DIRECT_REMAP=y +diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h +index 589ee66e64cf..2293caab9f8f 100644 +--- a/arch/arm64/include/asm/kvm_emulate.h ++++ b/arch/arm64/include/asm/kvm_emulate.h +@@ -639,4 +639,22 @@ static __always_inline void kvm_reset_cptr_el2(struct kvm_vcpu *vcpu) + + kvm_write_cptr_el2(val); + } ++ ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++static inline bool kvm_is_virtcca_cvm(struct kvm *kvm) ++{ ++ if (static_branch_unlikely(&virtcca_cvm_is_available)) ++ return kvm->arch.is_virtcca_cvm; ++ return false; ++} ++ ++static inline enum virtcca_cvm_state virtcca_cvm_state(struct kvm *kvm) ++{ ++ struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; ++ ++ if (!virtcca_cvm) ++ return 0; ++ return READ_ONCE(virtcca_cvm->state); ++} ++#endif + #endif /* __ARM64_KVM_EMULATE_H__ */ +diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h +index 66c0bb96f007..ac8115098ec9 100644 +--- a/arch/arm64/include/asm/kvm_host.h ++++ b/arch/arm64/include/asm/kvm_host.h +@@ -27,7 +27,7 @@ + #include + #include + #include +-#ifdef CONFIG_CVM_HOST ++#ifdef CONFIG_HISI_VIRTCCA_HOST + #include + #endif + +@@ -292,9 +292,12 @@ struct kvm_arch { + u64 tlbi_dvmbm; + #endif + +-#ifdef CONFIG_CVM_HOST +- struct cvm cvm; +- bool is_cvm; ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ union { ++ struct cvm cvm; ++ struct virtcca_cvm *virtcca_cvm; ++ }; ++ bool is_virtcca_cvm; + #endif + }; + +@@ -622,8 +625,8 @@ struct kvm_vcpu_arch { + cpumask_var_t pre_sched_cpus; + #endif + +-#ifdef CONFIG_CVM_HOST +- struct cvm_tec tec; ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ struct virtcca_cvm_tec tec; + #endif + }; + +diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h +new file mode 100644 +index 000000000000..fc4fe9a71169 +--- /dev/null ++++ b/arch/arm64/include/asm/kvm_tmi.h +@@ -0,0 +1,333 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Copyright (c) 2024, The Linux Foundation. All rights reserved. ++ */ ++#ifndef __TMM_TMI_H ++#define __TMM_TMI_H ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++#include ++#include ++#include ++#include ++#include ++ ++#define GRANULE_SIZE 4096 ++ ++#define NO_NUMA 0 /* numa bitmap */ ++ ++#define TMM_TTT_LEVEL_2 2 ++#define TMM_TTT_LEVEL_3 3 ++ ++/* TMI error codes. */ ++#define TMI_SUCCESS 0 ++#define TMI_ERROR_INPUT 1 ++#define TMI_ERROR_MEMORY 2 ++#define TMI_ERROR_ALIAS 3 ++#define TMI_ERROR_IN_USE 4 ++#define TMI_ERROR_CVM_STATE 5 ++#define TMI_ERROR_OWNER 6 ++#define TMI_ERROR_TEC 7 ++#define TMI_ERROR_TTT_WALK 8 ++#define TMI_ERROR_TTT_ENTRY 9 ++#define TMI_ERROR_NOT_SUPPORTED 10 ++#define TMI_ERROR_INTERNAL 11 ++#define TMI_ERROR_CVM_POWEROFF 12 ++#define TMI_ERROR_TTT_CREATED 13 ++ ++#define TMI_RETURN_STATUS(ret) ((ret) & 0xFF) ++#define TMI_RETURN_INDEX(ret) (((ret) >> 8) & 0xFF) ++ ++#define TMI_FEATURE_REGISTER_0_S2SZ GENMASK(7, 0) ++#define TMI_FEATURE_REGISTER_0_LPA2 BIT(8) ++#define TMI_FEATURE_REGISTER_0_SVE_EN BIT(9) ++#define TMI_FEATURE_REGISTER_0_SVE_VL GENMASK(13, 10) ++#define TMI_FEATURE_REGISTER_0_NUM_BPS GENMASK(17, 14) ++#define TMI_FEATURE_REGISTER_0_NUM_WPS GENMASK(21, 18) ++#define TMI_FEATURE_REGISTER_0_PMU_EN BIT(22) ++#define TMI_FEATURE_REGISTER_0_PMU_NUM_CTRS GENMASK(27, 23) ++#define TMI_FEATURE_REGISTER_0_HASH_SHA_256 BIT(28) ++#define TMI_FEATURE_REGISTER_0_HASH_SHA_512 BIT(29) ++ ++#define TMI_CVM_PARAM_FLAG_LPA2 BIT(0) ++#define TMI_CVM_PARAM_FLAG_SVE BIT(1) ++#define TMI_CVM_PARAM_FLAG_PMU BIT(2) ++ ++#define TMI_NOT_RUNNABLE 0 ++#define TMI_RUNNABLE 1 ++ ++/* ++ * The number of GPRs (starting from X0) that are ++ * configured by the host when a TEC is created. ++ */ ++#define TEC_CREATE_NR_GPRS (8U) ++ ++struct tmi_tec_params { ++ uint64_t gprs[TEC_CREATE_NR_GPRS]; ++ uint64_t pc; ++ uint64_t flags; ++ uint64_t ram_size; ++}; ++ ++#define TEC_ENTRY_FLAG_EMUL_MMIO (1UL << 0U) ++#define TEC_ENTRY_FLAG_INJECT_SEA (1UL << 1U) ++#define TEC_ENTRY_FLAG_TRAP_WFI (1UL << 2U) ++#define TEC_ENTRY_FLAG_TRAP_WFE (1UL << 3U) ++ ++#define TMI_EXIT_SYNC 0 ++#define TMI_EXIT_IRQ 1 ++#define TMI_EXIT_FIQ 2 ++#define TMI_EXIT_PSCI 3 ++#define TMI_EXIT_HOST_CALL 5 ++#define TMI_EXIT_SERROR 6 ++ ++/* ++ * The number of GPRs (starting from X0) per voluntary exit context. ++ * Per SMCCC. ++ */ ++ #define TEC_EXIT_NR_GPRS (31U) ++ ++/* Maximum number of Interrupt Controller List Registers. */ ++#define TEC_GIC_NUM_LRS (16U) ++ ++struct tmi_tec_entry { ++ uint64_t flags; ++ uint64_t gprs[TEC_EXIT_NR_GPRS]; ++ uint64_t gicv3_lrs[TEC_GIC_NUM_LRS]; ++ uint64_t gicv3_hcr; ++}; ++ ++struct tmi_tec_exit { ++ uint64_t exit_reason; ++ uint64_t esr; ++ uint64_t far; ++ uint64_t hpfar; ++ uint64_t gprs[TEC_EXIT_NR_GPRS]; ++ uint64_t gicv3_hcr; ++ uint64_t gicv3_lrs[TEC_GIC_NUM_LRS]; ++ uint64_t gicv3_misr; ++ uint64_t gicv3_vmcr; ++ uint64_t cntv_ctl; ++ uint64_t cntv_cval; ++ uint64_t cntp_ctl; ++ uint64_t cntp_cval; ++ uint64_t imm; ++ uint64_t pmu_ovf_status; ++}; ++ ++struct tmi_tec_run { ++ struct tmi_tec_entry tec_entry; ++ struct tmi_tec_exit tec_exit; ++}; ++ ++#define TMI_FNUM_MIN_VALUE U(0x150) ++#define TMI_FNUM_MAX_VALUE U(0x18F) ++ ++/****************************************************************************** ++ * Bit definitions inside the function id as per the SMC calling convention ++ ******************************************************************************/ ++#define FUNCID_TYPE_SHIFT 31 ++#define FUNCID_CC_SHIFT 30 ++#define FUNCID_OEN_SHIFT 24 ++#define FUNCID_NUM_SHIFT 0 ++ ++#define FUNCID_TYPE_MASK 0x1 ++#define FUNCID_CC_MASK 0x1 ++#define FUNCID_OEN_MASK 0x3f ++#define FUNCID_NUM_MASK 0xffff ++ ++#define FUNCID_TYPE_WIDTH 1 ++#define FUNCID_CC_WIDTH 1 ++#define FUNCID_OEN_WIDTH 6 ++#define FUNCID_NUM_WIDTH 16 ++ ++#define SMC_64 1 ++#define SMC_32 0 ++#define SMC_TYPE_FAST 1 ++#define SMC_TYPE_STD 0 ++ ++/***************************************************************************** ++ * Owning entity number definitions inside the function id as per the SMC ++ * calling convention ++ *****************************************************************************/ ++#define OEN_ARM_START 0 ++#define OEN_ARM_END 0 ++#define OEN_CPU_START 1 ++#define OEN_CPU_END 1 ++#define OEN_SIP_START 2 ++#define OEN_SIP_END 2 ++#define OEN_OEM_START 3 ++#define OEN_OEM_END 3 ++#define OEN_STD_START 4 /* Standard Calls */ ++#define OEN_STD_END 4 ++#define OEN_TAP_START 48 /* Trusted Applications */ ++#define OEN_TAP_END 49 ++#define OEN_TOS_START 50 /* Trusted OS */ ++#define OEN_TOS_END 63 ++#define OEN_LIMIT 64 ++ ++/* Get TMI fastcall std FID from function number */ ++#define TMI_FID(smc_cc, func_num) \ ++ ((SMC_TYPE_FAST << FUNCID_TYPE_SHIFT) | \ ++ ((smc_cc) << FUNCID_CC_SHIFT) | \ ++ (OEN_STD_START << FUNCID_OEN_SHIFT) | \ ++ ((func_num) << FUNCID_NUM_SHIFT)) ++ ++#define U(_x) (_x##U) ++ ++#define TMI_NO_MEASURE_CONTENT U(0) ++#define TMI_MEASURE_CONTENT U(1) ++ ++#define CVM_IPA_MAX_VAL (1UL << 48) ++ ++/* ++ * SMC_TMM_INIT_COMPLETE is the only function in the TMI that originates from ++ * the CVM world and is handled by the SPMD. The remaining functions are ++ * always invoked by the Normal world, forward by SPMD and handled by the ++ * TMM. ++ */ ++#define TMI_FNUM_VERSION_REQ U(0x260) ++#define TMI_FNUM_MEM_INFO_SHOW U(0x261) ++#define TMI_FNUM_DATA_CREATE U(0x262) ++#define TMI_FNUM_DATA_DESTROY U(0x263) ++#define TMI_FNUM_CVM_ACTIVATE U(0x264) ++#define TMI_FNUM_CVM_CREATE U(0x265) ++#define TMI_FNUM_CVM_DESTROY U(0x266) ++#define TMI_FNUM_TEC_CREATE U(0x267) ++#define TMI_FNUM_TEC_DESTROY U(0x268) ++#define TMI_FNUM_TEC_ENTER U(0x269) ++#define TMI_FNUM_TTT_CREATE U(0x26A) ++#define TMI_FNUM_PSCI_COMPLETE U(0x26B) ++#define TMI_FNUM_FEATURES U(0x26C) ++#define TMI_FNUM_TTT_MAP_RANGE U(0x26D) ++#define TMI_FNUM_TTT_UNMAP_RANGE U(0x26E) ++#define TMI_FNUM_INF_TEST U(0x270) ++ ++/* TMI SMC64 PIDs handled by the SPMD */ ++#define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) ++#define TMI_TMM_DATA_CREATE TMI_FID(SMC_64, TMI_FNUM_DATA_CREATE) ++#define TMI_TMM_DATA_DESTROY TMI_FID(SMC_64, TMI_FNUM_DATA_DESTROY) ++#define TMI_TMM_CVM_ACTIVATE TMI_FID(SMC_64, TMI_FNUM_CVM_ACTIVATE) ++#define TMI_TMM_CVM_CREATE TMI_FID(SMC_64, TMI_FNUM_CVM_CREATE) ++#define TMI_TMM_CVM_DESTROY TMI_FID(SMC_64, TMI_FNUM_CVM_DESTROY) ++#define TMI_TMM_TEC_CREATE TMI_FID(SMC_64, TMI_FNUM_TEC_CREATE) ++#define TMI_TMM_TEC_DESTROY TMI_FID(SMC_64, TMI_FNUM_TEC_DESTROY) ++#define TMI_TMM_TEC_ENTER TMI_FID(SMC_64, TMI_FNUM_TEC_ENTER) ++#define TMI_TMM_TTT_CREATE TMI_FID(SMC_64, TMI_FNUM_TTT_CREATE) ++#define TMI_TMM_PSCI_COMPLETE TMI_FID(SMC_64, TMI_FNUM_PSCI_COMPLETE) ++#define TMI_TMM_FEATURES TMI_FID(SMC_64, TMI_FNUM_FEATURES) ++#define TMI_TMM_MEM_INFO_SHOW TMI_FID(SMC_64, TMI_FNUM_MEM_INFO_SHOW) ++#define TMI_TMM_TTT_MAP_RANGE TMI_FID(SMC_64, TMI_FNUM_TTT_MAP_RANGE) ++#define TMI_TMM_TTT_UNMAP_RANGE TMI_FID(SMC_64, TMI_FNUM_TTT_UNMAP_RANGE) ++#define TMI_TMM_INF_TEST TMI_FID(SMC_64, TMI_FNUM_INF_TEST) ++ ++#define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) ++#define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) ++ ++#define TMI_ABI_VERSION_MAJOR U(0x1) ++ ++/* KVM_CAP_ARM_TMM on VM fd */ ++#define KVM_CAP_ARM_TMM_CONFIG_CVM_HOST 0 ++#define KVM_CAP_ARM_TMM_CREATE_RD 1 ++#define KVM_CAP_ARM_TMM_POPULATE_CVM 2 ++#define KVM_CAP_ARM_TMM_ACTIVATE_CVM 3 ++ ++#define KVM_CAP_ARM_TMM_MEASUREMENT_ALGO_SHA256 0 ++#define KVM_CAP_ARM_TMM_MEASUREMENT_ALGO_SHA512 1 ++ ++#define KVM_CAP_ARM_TMM_RPV_SIZE 64 ++ ++/* List of configuration items accepted for KVM_CAP_ARM_TMM_CONFIG_CVM_HOST */ ++#define KVM_CAP_ARM_TMM_CFG_RPV 0 ++#define KVM_CAP_ARM_TMM_CFG_HASH_ALGO 1 ++#define KVM_CAP_ARM_TMM_CFG_SVE 2 ++#define KVM_CAP_ARM_TMM_CFG_DBG 3 ++#define KVM_CAP_ARM_TMM_CFG_PMU 4 ++ ++DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_available); ++DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_enable); ++ ++struct kvm_cap_arm_tmm_config_item { ++ __u32 cfg; ++ union { ++ /* cfg == KVM_CAP_ARM_TMM_CFG_RPV */ ++ struct { ++ __u8 rpv[KVM_CAP_ARM_TMM_RPV_SIZE]; ++ }; ++ ++ /* cfg == KVM_CAP_ARM_TMM_CFG_HASH_ALGO */ ++ struct { ++ __u32 hash_algo; ++ }; ++ ++ /* cfg == KVM_CAP_ARM_TMM_CFG_SVE */ ++ struct { ++ __u32 sve_vq; ++ }; ++ ++ /* cfg == KVM_CAP_ARM_TMM_CFG_DBG */ ++ struct { ++ __u32 num_brps; ++ __u32 num_wrps; ++ }; ++ ++ /* cfg == KVM_CAP_ARM_TMM_CFG_PMU */ ++ struct { ++ __u32 num_pmu_cntrs; ++ }; ++ /* Fix the size of the union */ ++ __u8 reserved[256]; ++ }; ++}; ++ ++#define KVM_ARM_TMM_POPULATE_FLAGS_MEASURE (1U << 0) ++struct kvm_cap_arm_tmm_populate_region_args { ++ __u64 populate_ipa_base1; ++ __u64 populate_ipa_size1; ++ __u64 populate_ipa_base2; ++ __u64 populate_ipa_size2; ++ __u32 flags; ++ __u32 reserved[3]; ++}; ++ ++static inline bool tmm_is_addr_ttt_level_aligned(uint64_t addr, int level) ++{ ++ uint64_t mask = (1 << (12 + 9 * (3 - level))) - 1; ++ ++ return (addr & mask) == 0; ++} ++ ++#define ID_AA64PFR0_SEL2_MASK ULL(0xf) ++#define ID_AA64PFR0_SEL2_SHIFT 36 ++ ++static inline bool is_armv8_4_sel2_present(void) ++{ ++ return ((read_sysreg(id_aa64pfr0_el1) >> ID_AA64PFR0_SEL2_SHIFT) & ++ ID_AA64PFR0_SEL2_MASK) == 1UL; ++} ++ ++u64 tmi_version(void); ++u64 tmi_data_create(u64 data, u64 rd, u64 map_addr, u64 src, u64 level); ++u64 tmi_data_destroy(u64 rd, u64 map_addr, u64 level); ++u64 tmi_cvm_activate(u64 rd); ++u64 tmi_cvm_create(u64 params_ptr, u64 numa_set); ++u64 tmi_cvm_destroy(u64 rd); ++u64 tmi_tec_create(u64 numa_set, u64 rd, u64 mpidr, u64 params_ptr); ++u64 tmi_tec_destroy(u64 tec); ++u64 tmi_tec_enter(u64 tec, u64 run_ptr); ++u64 tmi_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level); ++u64 tmi_psci_complete(u64 calling_tec, u64 target_tec); ++u64 tmi_features(u64 index); ++u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_node); ++u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id); ++u64 tmi_mem_info_show(u64 mem_info_addr); ++ ++void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); ++int kvm_load_user_data(struct kvm *kvm, unsigned long arg); ++unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, ++ unsigned long target_affinity, unsigned long lowest_affinity_level); ++int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, ++ bool serror_pending, bool ext_dabt_pending); ++int kvm_init_cvm_vm(struct kvm *kvm); ++int kvm_enable_virtcca_cvm(struct kvm *kvm); ++#endif ++#endif +diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h +index f70d73be034b..96532ca8fdcc 100644 +--- a/arch/arm64/include/asm/kvm_tmm.h ++++ b/arch/arm64/include/asm/kvm_tmm.h +@@ -7,8 +7,8 @@ + + #include + +-enum cvm_state { +- CVM_STATE_NONE, ++enum virtcca_cvm_state { ++ CVM_STATE_NONE = 1, + CVM_STATE_NEW, + CVM_STATE_ACTIVE, + CVM_STATE_DYING +@@ -36,7 +36,7 @@ struct tmi_cvm_params { + }; + + struct cvm { +- enum cvm_state state; ++ enum virtcca_cvm_state state; + u32 cvm_vmid; + u64 rd; + u64 loader_start; +@@ -48,10 +48,23 @@ struct cvm { + bool is_cvm; + }; + ++struct virtcca_cvm { ++ enum virtcca_cvm_state state; ++ u32 cvm_vmid; ++ u64 rd; ++ u64 loader_start; ++ u64 image_end; ++ u64 initrd_start; ++ u64 dtb_end; ++ u64 ram_size; ++ struct kvm_numa_info numa_info; ++ struct tmi_cvm_params *params; ++}; ++ + /* + * struct cvm_tec - Additional per VCPU data for a CVM + */ +-struct cvm_tec { ++struct virtcca_cvm_tec { + u64 tec; + bool tec_created; + void *tec_run; +@@ -59,22 +72,19 @@ struct cvm_tec { + + int kvm_init_tmm(void); + int kvm_cvm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap); +-int kvm_init_cvm_vm(struct kvm *kvm); + void kvm_destroy_cvm(struct kvm *kvm); +-int kvm_create_tec(struct kvm_vcpu *vcpu); ++int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu); + void kvm_destroy_tec(struct kvm_vcpu *vcpu); + int kvm_tec_enter(struct kvm_vcpu *vcpu); + int handle_cvm_exit(struct kvm_vcpu *vcpu, int rec_run_status); + int kvm_arm_create_cvm(struct kvm *kvm); + void kvm_free_rd(struct kvm *kvm); +-int cvm_create_rd(struct kvm *kvm); + int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target); +-int kvm_arch_tec_init(struct kvm_vcpu *vcpu); + + void kvm_cvm_unmap_destroy_range(struct kvm *kvm); + +-#define CVM_TTT_BLOCK_LEVEL 2 +-#define CVM_TTT_MAX_LEVEL 3 ++#define CVM_TTT_BLOCK_LEVEL 2 ++#define CVM_TTT_MAX_LEVEL 3 + + #define CVM_PAGE_SHIFT 12 + #define CVM_PAGE_SIZE BIT(CVM_PAGE_SHIFT) +diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h +index 0f740b781187..2031b31c05f9 100644 +--- a/arch/arm64/include/asm/set_memory.h ++++ b/arch/arm64/include/asm/set_memory.h +@@ -4,6 +4,7 @@ + #define _ASM_ARM64_SET_MEMORY_H + + #include ++#include + + bool can_set_direct_map(void); + #define can_set_direct_map can_set_direct_map +diff --git a/arch/arm64/include/asm/virtcca_cvm_guest.h b/arch/arm64/include/asm/virtcca_cvm_guest.h +new file mode 100644 +index 000000000000..2a68626a4f81 +--- /dev/null ++++ b/arch/arm64/include/asm/virtcca_cvm_guest.h +@@ -0,0 +1,39 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef __VIRTCCA_CVM_GUEST_H ++#define __VIRTCCA_CVM_GUEST_H ++ ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++struct device; ++ ++extern int set_cvm_memory_encrypted(unsigned long addr, int numpages); ++ ++extern int set_cvm_memory_decrypted(unsigned long addr, int numpages); ++ ++extern bool is_virtcca_cvm_world(void); ++ ++extern void __init swiotlb_cvm_update_mem_attributes(void); ++ ++#else ++ ++static inline int set_cvm_memory_encrypted(unsigned long addr, int numpages) ++{ ++ return 0; ++} ++ ++static inline int set_cvm_memory_decrypted(unsigned long addr, int numpages) ++{ ++ return 0; ++} ++ ++static inline bool is_virtcca_cvm_world(void) ++{ ++ return false; ++} ++ ++static inline void __init swiotlb_cvm_update_mem_attributes(void) {} ++ ++#endif /* CONFIG_HISI_VIRTCCA_GUEST */ ++#endif /* __VIRTCCA_CVM_GUEST_H */ +diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h +index f7ddd73a8c0f..97941e582d83 100644 +--- a/arch/arm64/include/uapi/asm/kvm.h ++++ b/arch/arm64/include/uapi/asm/kvm.h +@@ -110,6 +110,7 @@ struct kvm_regs { + #define KVM_ARM_VCPU_PTRAUTH_ADDRESS 5 /* VCPU uses address authentication */ + #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ + #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ ++#define KVM_ARM_VCPU_TEC 8 /* VCPU TEC state as part of cvm */ + + struct kvm_vcpu_init { + __u32 target; +@@ -415,6 +416,9 @@ enum { + #define KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES 3 + #define KVM_DEV_ARM_ITS_CTRL_RESET 4 + ++#define KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA256 0 ++#define KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA512 1 ++ + /* Device Control API on vcpu fd */ + #define KVM_ARM_VCPU_PMU_V3_CTRL 0 + #define KVM_ARM_VCPU_PMU_V3_IRQ 0 +diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile +index 21ef9c21a400..cccf8332ed29 100644 +--- a/arch/arm64/kernel/Makefile ++++ b/arch/arm64/kernel/Makefile +@@ -81,6 +81,7 @@ obj-$(CONFIG_COMPAT_VDSO) += vdso32-wrap.o + obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ + obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o + obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o ++obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o + CFLAGS_patch-scs.o += -mbranch-protection=none + + # Force dependency (vdso*-wrap.S includes vdso.so through incbin) +diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c +new file mode 100644 +index 000000000000..6c49875467ce +--- /dev/null ++++ b/arch/arm64/kernel/virtcca_cvm_guest.c +@@ -0,0 +1,93 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++#define CVM_PTE_NS_BIT 5 ++#define CVM_PTE_NS_MASK (1 << CVM_PTE_NS_BIT) ++ ++static bool cvm_guest_enable __read_mostly; ++ ++/* please use 'virtcca_cvm_guest=1' to enable cvm guest feature */ ++static int __init setup_cvm_guest(char *str) ++{ ++ int ret; ++ unsigned int val; ++ ++ if (!str) ++ return 0; ++ ++ ret = kstrtouint(str, 10, &val); ++ if (ret) { ++ pr_warn("Unable to parse cvm_guest.\n"); ++ } else { ++ if (val) ++ cvm_guest_enable = true; ++ } ++ return ret; ++} ++early_param("virtcca_cvm_guest", setup_cvm_guest); ++ ++bool is_virtcca_cvm_world(void) ++{ ++ return cvm_guest_enable; ++} ++ ++static int change_page_range_cvm(pte_t *ptep, unsigned long addr, void *data) ++{ ++ bool encrypt = (bool)data; ++ pte_t pte = READ_ONCE(*ptep); ++ ++ if (encrypt) { ++ if (!(pte.pte & CVM_PTE_NS_MASK)) ++ return 0; ++ pte.pte = pte.pte & (~CVM_PTE_NS_MASK); ++ } else { ++ if (pte.pte & CVM_PTE_NS_MASK) ++ return 0; ++ /* Set NS BIT */ ++ pte.pte = pte.pte | CVM_PTE_NS_MASK; ++ } ++ set_pte(ptep, pte); ++ ++ return 0; ++} ++ ++static int __change_memory_common_cvm(unsigned long start, unsigned long size, bool encrypt) ++{ ++ int ret; ++ ++ ret = apply_to_page_range(&init_mm, start, size, change_page_range_cvm, (void *)encrypt); ++ flush_tlb_kernel_range(start, start + size); ++ return ret; ++} ++ ++static int __set_memory_encrypted(unsigned long addr, ++ int numpages, ++ bool encrypt) ++{ ++ if (!is_virtcca_cvm_world()) ++ return 0; ++ ++ WARN_ON(!__is_lm_address(addr)); ++ return __change_memory_common_cvm(addr, PAGE_SIZE * numpages, encrypt); ++} ++ ++int set_cvm_memory_encrypted(unsigned long addr, int numpages) ++{ ++ return __set_memory_encrypted(addr, numpages, true); ++} ++ ++int set_cvm_memory_decrypted(unsigned long addr, int numpages) ++{ ++ return __set_memory_encrypted(addr, numpages, false); ++} +diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig +index 1fa6fba6077c..52edbd7f6340 100644 +--- a/arch/arm64/kvm/Kconfig ++++ b/arch/arm64/kvm/Kconfig +@@ -49,11 +49,11 @@ menuconfig KVM + + If unsure, say N. + +-config CVM_HOST +- bool "Enable cvm host feature" ++config HISI_VIRTCCA_HOST ++ bool "Enable virtcca cvm host feature" + depends on KVM + help +- Support CVM based on S-EL2 ++ Support VIRTCCA CVM based on S-EL2 + + If unsure, say N. + +diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile +index 952eee572e23..eadf41417ffa 100644 +--- a/arch/arm64/kvm/Makefile ++++ b/arch/arm64/kvm/Makefile +@@ -24,6 +24,9 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o pvsched.o \ + + kvm-$(CONFIG_VIRT_PLAT_DEV) += vgic/shadow_dev.o + kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o ++kvm-$(CONFIG_HISI_VIRTCCA_HOST) += tmi.o ++kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm.o ++kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_exit.o + obj-$(CONFIG_KVM_HISI_VIRT) += hisilicon/ + + always-y := hyp_constants.h hyp-constants.s +diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c +index 43957fce50f6..3f1129d3df6a 100644 +--- a/arch/arm64/kvm/arch_timer.c ++++ b/arch/arm64/kvm/arch_timer.c +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -175,8 +176,77 @@ static void timer_set_cval(struct arch_timer_context *ctxt, u64 cval) + } + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++static bool cvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) ++{ ++ return timer_ctx && ++ ((timer_get_ctl(timer_ctx) & ++ (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); ++} ++ ++void kvm_cvm_timers_update(struct kvm_vcpu *vcpu) ++{ ++ int i; ++ u64 cval, now; ++ bool status, level; ++ struct arch_timer_context *timer; ++ struct arch_timer_cpu *arch_timer = &vcpu->arch.timer_cpu; ++ ++ for (i = 0; i < NR_KVM_TIMERS; i++) { ++ timer = &arch_timer->timers[i]; ++ ++ if (!timer->loaded) { ++ if (!cvm_timer_irq_can_fire(timer)) ++ continue; ++ cval = timer_get_cval(timer); ++ now = kvm_phys_timer_read() - timer_get_offset(timer); ++ level = (cval <= now); ++ kvm_timer_update_irq(vcpu, level, timer); ++ } else { ++ status = timer_get_ctl(timer) & ARCH_TIMER_CTRL_IT_STAT; ++ level = cvm_timer_irq_can_fire(timer) && status; ++ if (level != timer->irq.level) ++ kvm_timer_update_irq(vcpu, level, timer); ++ } ++ } ++} ++ ++static void set_cvm_timers_loaded(struct kvm_vcpu *vcpu, bool loaded) ++{ ++ int i; ++ struct arch_timer_cpu *arch_timer = &vcpu->arch.timer_cpu; ++ ++ for (i = 0; i < NR_KVM_TIMERS; i++) { ++ struct arch_timer_context *timer = &arch_timer->timers[i]; ++ ++ timer->loaded = loaded; ++ } ++} ++ ++static void kvm_timer_blocking(struct kvm_vcpu *vcpu); ++static void kvm_timer_unblocking(struct kvm_vcpu *vcpu); ++ ++static inline void cvm_vcpu_load_timer_callback(struct kvm_vcpu *vcpu) ++{ ++ kvm_cvm_timers_update(vcpu); ++ kvm_timer_unblocking(vcpu); ++ set_cvm_timers_loaded(vcpu, true); ++} ++ ++static inline void cvm_vcpu_put_timer_callback(struct kvm_vcpu *vcpu) ++{ ++ set_cvm_timers_loaded(vcpu, false); ++ if (rcuwait_active(kvm_arch_vcpu_get_wait(vcpu))) ++ kvm_timer_blocking(vcpu); ++} ++#endif ++ + static void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) + { ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (kvm_is_virtcca_cvm(ctxt->vcpu->kvm)) ++ return; ++#endif + if (!ctxt->offset.vm_offset) { + WARN(offset, "timer %ld\n", arch_timer_ctx_index(ctxt)); + return; +@@ -883,6 +953,13 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ cvm_vcpu_load_timer_callback(vcpu); ++ return; ++ } ++#endif ++ + if (unlikely(!timer->enabled)) + return; + +@@ -981,6 +1058,13 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) + struct arch_timer_cpu *timer = vcpu_timer(vcpu); + struct timer_map map; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ cvm_vcpu_put_timer_callback(vcpu); ++ return; ++ } ++#endif ++ + if (unlikely(!timer->enabled)) + return; + +@@ -1766,6 +1850,15 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) + return -EINVAL; + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ /* ++ * We don't use mapped IRQs for CVM because the TMI doesn't allow ++ * us setting the LR.HW bit in the VGIC. ++ */ ++ if (vcpu_is_tec(vcpu)) ++ return 0; ++#endif ++ + get_timer_map(vcpu, &map); + + #ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c +index 333c65ced849..fe3add87ef52 100644 +--- a/arch/arm64/kvm/arm.c ++++ b/arch/arm64/kvm/arm.c +@@ -40,6 +40,8 @@ + #include + #include + #include ++#include ++#include + + #include + #include +@@ -132,6 +134,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, + } + mutex_unlock(&kvm->slots_lock); + break; ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ case KVM_CAP_ARM_TMM: ++ if (static_branch_unlikely(&virtcca_cvm_is_available)) ++ r = kvm_cvm_enable_cap(kvm, cap); ++ break; ++#endif + default: + r = -EINVAL; + break; +@@ -153,6 +161,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) + { + int ret; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (kvm_arm_cvm_type(type)) { ++ ret = kvm_enable_virtcca_cvm(kvm); ++ if (ret) ++ return ret; ++ } ++#endif ++ + ret = kvm_sched_affinity_vm_init(kvm); + if (ret) + return ret; +@@ -199,8 +215,20 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) + + bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (kvm_arm_cvm_type(type)) { ++ ret = kvm_init_cvm_vm(kvm); ++ if (ret) ++ goto out_free_stage2_pgd; ++ } ++#endif ++ + return 0; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++out_free_stage2_pgd: ++ kvm_free_stage2_pgd(&kvm->arch.mmu); ++#endif + err_free_cpumask: + free_cpumask_var(kvm->arch.supported_cpus); + err_unshare_kvm: +@@ -235,6 +263,10 @@ void kvm_arch_destroy_vm(struct kvm *kvm) + kvm_unshare_hyp(kvm, kvm + 1); + + kvm_arm_teardown_hypercalls(kvm); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (kvm_is_virtcca_cvm(kvm)) ++ kvm_destroy_cvm(kvm); ++#endif + } + + int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +@@ -306,7 +338,12 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) + r = system_supports_mte(); + break; + case KVM_CAP_STEAL_TIME: +- r = kvm_arm_pvtime_supported(); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (kvm && kvm_is_virtcca_cvm(kvm)) ++ r = 0; ++ else ++#endif ++ r = kvm_arm_pvtime_supported(); + break; + case KVM_CAP_ARM_EL1_32BIT: + r = cpus_have_const_cap(ARM64_HAS_32BIT_EL1); +@@ -346,6 +383,15 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) + case KVM_CAP_ARM_VIRT_MSI_BYPASS: + r = sdev_enable; + break; ++#endif ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ case KVM_CAP_ARM_TMM: ++ if (!is_armv8_4_sel2_present()) { ++ r = -ENXIO; ++ break; ++ } ++ r = static_key_enabled(&virtcca_cvm_is_available); ++ break; + #endif + default: + r = 0; +@@ -499,8 +545,23 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) + + vcpu->cpu = cpu; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ if (single_task_running()) ++ vcpu_clear_wfx_traps(vcpu); ++ else ++ vcpu_set_wfx_traps(vcpu); ++ } ++#endif + kvm_vgic_load(vcpu); + kvm_timer_vcpu_load(vcpu); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) ++ kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); ++ return; ++ } ++#endif + if (has_vhe()) + kvm_vcpu_load_sysregs_vhe(vcpu); + kvm_arch_vcpu_load_fp(vcpu); +@@ -531,6 +592,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) + + void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) + { ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ kvm_cvm_vcpu_put(vcpu); ++ return; ++ } ++#endif + kvm_arch_vcpu_put_debug_state_flags(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + if (has_vhe()) +@@ -705,6 +772,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) + * Tell the rest of the code that there are userspace irqchip + * VMs in the wild. + */ ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (!kvm_is_virtcca_cvm(kvm)) ++#endif + static_branch_inc(&userspace_irqchip_in_use); + } + +@@ -959,6 +1029,18 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret) + xfer_to_guest_mode_work_pending(); + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++static inline void update_pmu_phys_irq(struct kvm_vcpu *vcpu, bool *pmu_stopped) ++{ ++ struct kvm_pmu *pmu = &vcpu->arch.pmu; ++ ++ if (pmu->irq_level) { ++ *pmu_stopped = true; ++ arm_pmu_set_phys_irq(false); ++ } ++} ++#endif ++ + /* + * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while + * the vCPU is running. +@@ -1011,6 +1093,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) + run->exit_reason = KVM_EXIT_UNKNOWN; + run->flags = 0; + while (ret > 0) { ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ bool pmu_stopped = false; ++#endif + /* + * Check conditions before entering the guest + */ +@@ -1038,6 +1123,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) + kvm_arm_vmid_update(&vcpu->arch.hw_mmu->vmid); + + kvm_pmu_flush_hwstate(vcpu); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ update_pmu_phys_irq(vcpu, &pmu_stopped); ++#endif + + local_irq_disable(); + +@@ -1076,7 +1165,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) + trace_kvm_entry(*vcpu_pc(vcpu)); + guest_timing_enter_irqoff(); + +- ret = kvm_arm_vcpu_enter_exit(vcpu); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ ret = kvm_tec_enter(vcpu); ++ else ++#endif ++ ret = kvm_arm_vcpu_enter_exit(vcpu); + + vcpu->mode = OUTSIDE_GUEST_MODE; + vcpu->stat.exits++; +@@ -1130,13 +1224,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) + + local_irq_enable(); + +- trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (!vcpu_is_tec(vcpu)) { ++#endif ++ trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); + +- /* Exit types that need handling before we can be preempted */ +- handle_exit_early(vcpu, ret); ++ /* Exit types that need handling before we can be preempted */ ++ handle_exit_early(vcpu, ret); + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ } ++#endif + preempt_enable(); + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (pmu_stopped) ++ arm_pmu_set_phys_irq(true); ++#endif + /* + * The ARMv8 architecture doesn't give the hypervisor + * a mechanism to prevent a guest from dropping to AArch32 EL0 +@@ -1156,7 +1260,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) + ret = ARM_EXCEPTION_IL; + } + +- ret = handle_exit(vcpu, ret); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ ret = handle_cvm_exit(vcpu, ret); ++ else ++#endif ++ ret = handle_exit(vcpu, ret); + #ifdef CONFIG_ARCH_VCPU_STAT + update_vcpu_stat_time(&vcpu->stat); + #endif +@@ -1669,6 +1778,11 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) + struct kvm_device_attr attr; + + switch (ioctl) { ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ case KVM_LOAD_USER_DATA: { ++ return kvm_load_user_data(kvm, arg); ++ } ++#endif + case KVM_CREATE_IRQCHIP: { + int ret; + if (!vgic_present) +@@ -2555,6 +2669,14 @@ static __init int kvm_arm_init(void) + + in_hyp_mode = is_kernel_in_hyp_mode(); + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (static_branch_unlikely(&virtcca_cvm_is_enable) && in_hyp_mode) { ++ err = kvm_init_tmm(); ++ if (err) ++ return err; ++ } ++#endif ++ + if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || + cpus_have_final_cap(ARM64_WORKAROUND_1508412)) + kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \ +diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c +index d3161a683838..105f4e00ec8b 100644 +--- a/arch/arm64/kvm/guest.c ++++ b/arch/arm64/kvm/guest.c +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + + #include "trace.h" + +@@ -875,6 +876,10 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, + bool has_esr = events->exception.serror_has_esr; + bool ext_dabt_pending = events->exception.ext_dabt_pending; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ return kvm_cvm_vcpu_set_events(vcpu, serror_pending, ext_dabt_pending); ++#endif + if (serror_pending && has_esr) { + if (!cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) + return -EINVAL; +diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c +index 3dd38a151d2a..40d6d056e261 100644 +--- a/arch/arm64/kvm/mmio.c ++++ b/arch/arm64/kvm/mmio.c +@@ -6,6 +6,7 @@ + + #include + #include ++#include + #include + + #include "trace.h" +@@ -109,6 +110,12 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu) + &data); + data = vcpu_data_host_to_guest(vcpu, data, len); + vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), data); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)-> ++ tec_entry.gprs[0] = data; ++ } ++#endif + } + + /* +@@ -178,7 +185,12 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) + run->mmio.phys_addr = fault_ipa; + run->mmio.len = len; + vcpu->mmio_needed = 1; +- ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)->tec_entry.flags |= ++ TEC_ENTRY_FLAG_EMUL_MMIO; ++ } ++#endif + if (!ret) { + /* We handled the access successfully in the kernel. */ + if (!is_write) +diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c +index 121a3d90240d..f3ab2c39f764 100644 +--- a/arch/arm64/kvm/mmu.c ++++ b/arch/arm64/kvm/mmu.c +@@ -871,7 +871,11 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t + u64 mmfr0, mmfr1; + u32 phys_shift; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if ((type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) && (!kvm_is_virtcca_cvm(kvm))) ++#else + if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) ++#endif + return -EINVAL; + + phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type); +@@ -1414,6 +1418,12 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, + + fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); + write_fault = kvm_is_write_fault(vcpu); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ write_fault = true; ++ prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; ++ } ++#endif + exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu); + VM_BUG_ON(write_fault && exec_fault); + vcpu->stat.mabt_exit_stat++; +diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c +index 6b066e04dc5d..cdc023f845b7 100644 +--- a/arch/arm64/kvm/pmu-emul.c ++++ b/arch/arm64/kvm/pmu-emul.c +@@ -15,6 +15,7 @@ + #include + #include + #include ++#include + + #define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0) + +@@ -324,6 +325,14 @@ static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) + { + u64 reg = 0; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ reg = run->tec_exit.pmu_ovf_status; ++ return reg; ++ } ++#endif + if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) { + reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0); + reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); +diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c +index 1f69b667332b..b544418b68ed 100644 +--- a/arch/arm64/kvm/psci.c ++++ b/arch/arm64/kvm/psci.c +@@ -12,6 +12,7 @@ + + #include + #include ++#include + + #include + #include +@@ -79,6 +80,10 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) + return PSCI_RET_INVALID_PARAMS; + + spin_lock(&vcpu->arch.mp_state_lock); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ cvm_psci_complete(source_vcpu, vcpu); ++#endif + if (!kvm_arm_vcpu_stopped(vcpu)) { + if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) + ret = PSCI_RET_ALREADY_ON; +@@ -141,7 +146,10 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) + + /* Ignore other bits of target affinity */ + target_affinity &= target_affinity_mask; +- ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ return cvm_psci_vcpu_affinity_info(vcpu, target_affinity, lowest_affinity_level); ++#endif + /* + * If one or more VCPU matching target affinity are running + * then ON else OFF +diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c +index 7a65a35ee4ac..d38e74db97c2 100644 +--- a/arch/arm64/kvm/reset.c ++++ b/arch/arm64/kvm/reset.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + + /* Maximum phys_shift supported for any VM on this host */ + static u32 __ro_after_init kvm_ipa_limit; +@@ -139,6 +140,12 @@ int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) + return -EPERM; + + return kvm_vcpu_finalize_sve(vcpu); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ case KVM_ARM_VCPU_TEC: ++ if (!kvm_is_virtcca_cvm(vcpu->kvm)) ++ return -EINVAL; ++ return kvm_finalize_vcpu_tec(vcpu); ++#endif + } + + return -EINVAL; +@@ -162,6 +169,10 @@ void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu) + kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu)); + kfree(sve_state); + kfree(vcpu->arch.ccsidr); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ kvm_destroy_tec(vcpu); ++#endif + } + + static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu) +diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c +new file mode 100644 +index 000000000000..c1f22139d786 +--- /dev/null ++++ b/arch/arm64/kvm/tmi.c +@@ -0,0 +1,141 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (c) 2024, The Linux Foundation. All rights reserved. ++ */ ++#include ++#include ++#include ++ ++u64 tmi_version(void) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_VERSION_REQ, &res); ++ return res.a1; ++} ++ ++u64 tmi_data_create(u64 numa_set, u64 rd, u64 map_addr, u64 src, u64 level) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_DATA_CREATE, numa_set, rd, map_addr, src, level, &res); ++ return res.a1; ++} ++ ++u64 tmi_data_destroy(u64 rd, u64 map_addr, u64 level) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_DATA_DESTROY, rd, map_addr, level, &res); ++ return res.a1; ++} ++ ++u64 tmi_cvm_activate(u64 rd) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_CVM_ACTIVATE, rd, &res); ++ return res.a1; ++} ++ ++u64 tmi_cvm_create(u64 params_ptr, u64 numa_set) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_CVM_CREATE, params_ptr, numa_set, &res); ++ return res.a1; ++} ++ ++u64 tmi_cvm_destroy(u64 rd) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_CVM_DESTROY, rd, &res); ++ return res.a1; ++} ++ ++u64 tmi_tec_create(u64 numa_set, u64 rd, u64 mpidr, u64 params_ptr) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TEC_CREATE, numa_set, rd, mpidr, params_ptr, &res); ++ return res.a1; ++} ++ ++u64 tmi_tec_destroy(u64 tec) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TEC_DESTROY, tec, &res); ++ return res.a1; ++} ++ ++u64 tmi_tec_enter(u64 tec, u64 run_ptr) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TEC_ENTER, tec, run_ptr, &res); ++ return res.a1; ++} ++ ++u64 tmi_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TTT_CREATE, numa_set, rd, map_addr, level, &res); ++ return res.a1; ++} ++ ++u64 tmi_psci_complete(u64 calling_tec, u64 target_tec) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_PSCI_COMPLETE, calling_tec, target_tec, &res); ++ return res.a1; ++} ++ ++u64 tmi_features(u64 index) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_FEATURES, index, &res); ++ return res.a1; ++} ++ ++u64 tmi_mem_info_show(u64 mem_info_addr) ++{ ++ struct arm_smccc_res res; ++ u64 pa_addr = __pa(mem_info_addr); ++ ++ arm_smccc_1_1_smc(TMI_TMM_MEM_INFO_SHOW, pa_addr, &res); ++ return res.a1; ++} ++EXPORT_SYMBOL_GPL(tmi_mem_info_show); ++ ++u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_node) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TTT_MAP_RANGE, rd, map_addr, size, cur_node, target_node, &res); ++ return res.a1; ++} ++ ++u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id) ++{ ++ struct arm_smccc_res res; ++ ++ arm_smccc_1_1_smc(TMI_TMM_TTT_UNMAP_RANGE, rd, map_addr, size, node_id, &res); ++ return res.a1; ++} ++ ++u64 tmi_tmm_inf_test(u64 x1, u64 x2, u64 x3, u64 x4, u64 x5) ++{ ++ struct arm_smccc_res res; ++ u64 vttbr_el2_pa = __pa(x2); ++ u64 cvm_params_pa = __pa(x3); ++ u64 tec_params_pa = __pa(x4); ++ ++ arm_smccc_1_1_smc(TMI_TMM_INF_TEST, x1, vttbr_el2_pa, cvm_params_pa, tec_params_pa, x5, &res); ++ return res.a1; ++} ++EXPORT_SYMBOL_GPL(tmi_tmm_inf_test); +\ No newline at end of file +diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c +index 69ca111e349d..dab599e857b5 100644 +--- a/arch/arm64/kvm/vgic/vgic-v3.c ++++ b/arch/arm64/kvm/vgic/vgic-v3.c +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + + #include "vgic.h" + +@@ -680,7 +681,10 @@ int vgic_v3_probe(const struct gic_kvm_info *info) + (unsigned long long)info->vcpu.start); + } else if (kvm_get_mode() != KVM_MODE_PROTECTED) { + kvm_vgic_global_state.vcpu_base = info->vcpu.start; +- kvm_vgic_global_state.can_emulate_gicv2 = true; ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (!static_branch_unlikely(&virtcca_cvm_is_available)) ++#endif ++ kvm_vgic_global_state.can_emulate_gicv2 = true; + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); + if (ret) { + kvm_err("Cannot register GICv2 KVM device.\n"); +@@ -760,7 +764,13 @@ void vgic_v3_load(struct kvm_vcpu *vcpu) + void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) + { + struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; +- ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) { ++ cpu_if->vgic_vmcr = ++ ((struct tmi_tec_run *)vcpu->arch.tec.tec_run)->tec_exit.gicv3_vmcr; ++ return; ++ } ++#endif + if (likely(cpu_if->vgic_sre)) + cpu_if->vgic_vmcr = kvm_call_hyp_ret(__vgic_v3_read_vmcr); + } +diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c +index 2459b0adea08..ec110006acf5 100644 +--- a/arch/arm64/kvm/vgic/vgic.c ++++ b/arch/arm64/kvm/vgic/vgic.c +@@ -11,6 +11,7 @@ + #include + + #include ++#include + + #include "vgic.h" + +@@ -897,12 +898,44 @@ static inline bool can_access_vgic_from_kernel(void) + return !static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif) || has_vhe(); + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++static inline void vgic_tmm_save_state(struct kvm_vcpu *vcpu) ++{ ++ int i; ++ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; ++ struct tmi_tec_run *tec_run = vcpu->arch.tec.tec_run; ++ ++ for (i = 0; i < kvm_vgic_global_state.nr_lr; ++i) { ++ cpu_if->vgic_lr[i] = tec_run->tec_exit.gicv3_lrs[i]; ++ tec_run->tec_entry.gicv3_lrs[i] = 0; ++ } ++} ++ ++static inline void vgic_tmm_restore_state(struct kvm_vcpu *vcpu) ++{ ++ int i; ++ struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; ++ struct tmi_tec_run *tec_run = vcpu->arch.tec.tec_run; ++ ++ for (i = 0; i < kvm_vgic_global_state.nr_lr; ++i) { ++ tec_run->tec_entry.gicv3_lrs[i] = cpu_if->vgic_lr[i]; ++ tec_run->tec_exit.gicv3_lrs[i] = cpu_if->vgic_lr[i]; ++ } ++} ++#endif ++ + static inline void vgic_save_state(struct kvm_vcpu *vcpu) + { + if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + vgic_v2_save_state(vcpu); + else +- __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ vgic_tmm_save_state(vcpu); ++ else ++#endif ++ __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); ++ + } + + /* Sync back the hardware VGIC state into our emulation after a guest's run. */ +@@ -932,7 +965,12 @@ static inline void vgic_restore_state(struct kvm_vcpu *vcpu) + if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + vgic_v2_restore_state(vcpu); + else +- __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ vgic_tmm_restore_state(vcpu); ++ else ++#endif ++ __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); + } + + /* Flush our emulation state into the GIC hardware before entering the guest. */ +@@ -973,7 +1011,10 @@ void kvm_vgic_load(struct kvm_vcpu *vcpu) + { + if (unlikely(!vgic_initialized(vcpu->kvm))) + return; +- ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ return; ++#endif + if (kvm_vgic_global_state.type == VGIC_V2) + vgic_v2_load(vcpu); + else +@@ -984,7 +1025,10 @@ void kvm_vgic_put(struct kvm_vcpu *vcpu) + { + if (unlikely(!vgic_initialized(vcpu->kvm))) + return; +- ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ return; ++#endif + if (kvm_vgic_global_state.type == VGIC_V2) + vgic_v2_put(vcpu); + else +diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c +new file mode 100644 +index 000000000000..367bbf4fa3e6 +--- /dev/null ++++ b/arch/arm64/kvm/virtcca_cvm.c +@@ -0,0 +1,889 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (c) 2024, The Linux Foundation. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++/* Protects access to cvm_vmid_bitmap */ ++static DEFINE_SPINLOCK(cvm_vmid_lock); ++static unsigned long *cvm_vmid_bitmap; ++DEFINE_STATIC_KEY_FALSE(virtcca_cvm_is_available); ++DEFINE_STATIC_KEY_FALSE(virtcca_cvm_is_enable); ++#define SIMD_PAGE_SIZE 0x3000 ++ ++int kvm_enable_virtcca_cvm(struct kvm *kvm) ++{ ++ if (!static_key_enabled(&virtcca_cvm_is_available)) ++ return -EFAULT; ++ ++ kvm->arch.is_virtcca_cvm = true; ++ return 0; ++} ++ ++static int __init setup_virtcca_cvm_host(char *str) ++{ ++ int ret; ++ unsigned int val; ++ ++ if (!str) ++ return 0; ++ ++ ret = kstrtouint(str, 10, &val); ++ if (ret) { ++ pr_warn("Unable to parse cvm_guest.\n"); ++ } else { ++ if (val) ++ static_branch_enable(&virtcca_cvm_is_enable); ++ } ++ return ret; ++} ++early_param("virtcca_cvm_host", setup_virtcca_cvm_host); ++ ++static int cvm_vmid_init(void) ++{ ++ unsigned int vmid_count = 1 << kvm_get_vmid_bits(); ++ ++ cvm_vmid_bitmap = bitmap_zalloc(vmid_count, GFP_KERNEL); ++ if (!cvm_vmid_bitmap) { ++ kvm_err("%s: Couldn't allocate cvm vmid bitmap\n", __func__); ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++static unsigned long tmm_feat_reg0; ++ ++static bool tmm_supports(unsigned long feature) ++{ ++ return !!u64_get_bits(tmm_feat_reg0, feature); ++} ++ ++bool kvm_cvm_supports_sve(void) ++{ ++ return tmm_supports(TMI_FEATURE_REGISTER_0_SVE_EN); ++} ++ ++bool kvm_cvm_supports_pmu(void) ++{ ++ return tmm_supports(TMI_FEATURE_REGISTER_0_PMU_EN); ++} ++ ++u32 kvm_cvm_ipa_limit(void) ++{ ++ return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_S2SZ); ++} ++ ++u32 kvm_cvm_get_num_brps(void) ++{ ++ return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_NUM_BPS); ++} ++ ++u32 kvm_cvm_get_num_wrps(void) ++{ ++ return u64_get_bits(tmm_feat_reg0, TMI_FEATURE_REGISTER_0_NUM_WPS); ++} ++ ++static int cvm_vmid_reserve(void) ++{ ++ int ret; ++ unsigned int vmid_count = 1 << kvm_get_vmid_bits(); ++ ++ spin_lock(&cvm_vmid_lock); ++ ret = bitmap_find_free_region(cvm_vmid_bitmap, vmid_count, 0); ++ spin_unlock(&cvm_vmid_lock); ++ ++ return ret; ++} ++ ++static void cvm_vmid_release(unsigned int vmid) ++{ ++ spin_lock(&cvm_vmid_lock); ++ bitmap_release_region(cvm_vmid_bitmap, vmid, 0); ++ spin_unlock(&cvm_vmid_lock); ++} ++ ++static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) ++{ ++ u64 shift = ARM64_HW_PGTABLE_LEVEL_SHIFT(pgt->start_level - 1); ++ u64 mask = BIT(pgt->ia_bits) - 1; ++ ++ return (addr & mask) >> shift; ++} ++ ++static u32 kvm_pgd_pages(u32 ia_bits, u32 start_level) ++{ ++ struct kvm_pgtable pgt = { ++ .ia_bits = ia_bits, ++ .start_level = start_level, ++ }; ++ return __kvm_pgd_page_idx(&pgt, -1ULL) + 1; ++} ++ ++/* ++ * the configurable physical numa range in QEMU is 0-127, ++ * but in real scenarios, 0-63 is sufficient. ++ */ ++static u64 kvm_get_host_numa_set_by_vcpu(u64 vcpu, struct kvm *kvm) ++{ ++ int64_t i; ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_numa_info *numa_info = &cvm->numa_info; ++ ++ for (i = 0; i < numa_info->numa_cnt && i < MAX_NUMA_NODE; i++) { ++ if (test_bit(vcpu, (unsigned long *)numa_info->numa_nodes[i].cpu_id)) ++ return numa_info->numa_nodes[i].host_numa_nodes[0]; ++ } ++ return NO_NUMA; ++} ++ ++static u64 kvm_get_first_binded_numa_set(struct kvm *kvm) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_numa_info *numa_info = &cvm->numa_info; ++ ++ if (numa_info->numa_cnt > 0) ++ return numa_info->numa_nodes[0].host_numa_nodes[0]; ++ return NO_NUMA; ++} ++ ++int kvm_arm_create_cvm(struct kvm *kvm) ++{ ++ int ret; ++ struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; ++ unsigned int pgd_sz; ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ /* get affine host numa set by default vcpu 0 */ ++ u64 numa_set = kvm_get_host_numa_set_by_vcpu(0, kvm); ++ ++ if (!kvm_is_virtcca_cvm(kvm) || virtcca_cvm_state(kvm) != CVM_STATE_NONE) ++ return 0; ++ ++ if (!cvm->params) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ ret = cvm_vmid_reserve(); ++ if (ret < 0) ++ goto out; ++ ++ cvm->cvm_vmid = ret; ++ ++ pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level); ++ ++ cvm->params->ttt_level_start = kvm->arch.mmu.pgt->start_level; ++ cvm->params->ttt_num_start = pgd_sz; ++ cvm->params->s2sz = VTCR_EL2_IPA(kvm->arch.vtcr); ++ cvm->params->vmid = cvm->cvm_vmid; ++ cvm->params->ns_vtcr = kvm->arch.vtcr; ++ cvm->params->vttbr_el2 = kvm->arch.mmu.pgd_phys; ++ memcpy(cvm->params->rpv, &cvm->cvm_vmid, sizeof(cvm->cvm_vmid)); ++ cvm->rd = tmi_cvm_create(__pa(cvm->params), numa_set); ++ if (!cvm->rd) { ++ kvm_err("KVM creates cVM failed: %d\n", cvm->cvm_vmid); ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ WRITE_ONCE(cvm->state, CVM_STATE_NEW); ++ ret = 0; ++out: ++ kfree(cvm->params); ++ cvm->params = NULL; ++ if (ret < 0) { ++ kfree(cvm); ++ kvm->arch.virtcca_cvm = NULL; ++ } ++ return ret; ++} ++ ++void kvm_destroy_cvm(struct kvm *kvm) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ uint32_t cvm_vmid; ++ ++ if (!cvm) ++ return; ++ ++ cvm_vmid = cvm->cvm_vmid; ++ kfree(cvm->params); ++ cvm->params = NULL; ++ ++ if (virtcca_cvm_state(kvm) == CVM_STATE_NONE) ++ return; ++ ++ cvm_vmid_release(cvm_vmid); ++ ++ WRITE_ONCE(cvm->state, CVM_STATE_DYING); ++ ++ if (!tmi_cvm_destroy(cvm->rd)) ++ kvm_info("KVM has destroyed cVM: %d\n", cvm->cvm_vmid); ++ ++ kfree(cvm); ++ kvm->arch.virtcca_cvm = NULL; ++} ++ ++static int kvm_cvm_ttt_create(struct virtcca_cvm *cvm, ++ unsigned long addr, ++ int level, ++ u64 numa_set) ++{ ++ addr = ALIGN_DOWN(addr, cvm_ttt_level_mapsize(level - 1)); ++ return tmi_ttt_create(numa_set, cvm->rd, addr, level); ++} ++ ++int kvm_cvm_create_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, ++ unsigned long ipa, ++ int level, ++ int max_level, ++ struct kvm_mmu_memory_cache *mc) ++{ ++ int ret = 0; ++ if (WARN_ON(level == max_level)) ++ return 0; ++ ++ while (level++ < max_level) { ++ u64 numa_set = kvm_get_first_binded_numa_set(kvm); ++ ++ ret = kvm_cvm_ttt_create(cvm, ipa, level, numa_set); ++ if (ret) ++ return -ENXIO; ++ } ++ ++ return 0; ++} ++ ++static int kvm_cvm_create_protected_data_page(struct kvm *kvm, struct virtcca_cvm *cvm, ++ unsigned long ipa, int level, struct page *src_page, u64 numa_set) ++{ ++ phys_addr_t src_phys = 0; ++ int ret; ++ ++ if (src_page) ++ src_phys = page_to_phys(src_page); ++ ret = tmi_data_create(numa_set, cvm->rd, ipa, src_phys, level); ++ ++ if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { ++ /* Create missing RTTs and retry */ ++ int level_fault = TMI_RETURN_INDEX(ret); ++ ++ ret = kvm_cvm_create_ttt_levels(kvm, cvm, ipa, level_fault, ++ level, NULL); ++ if (ret) ++ goto err; ++ ret = tmi_data_create(numa_set, cvm->rd, ipa, src_phys, level); ++ } ++ if (ret) ++ goto err; ++ ++ return 0; ++ ++err: ++ kvm_err("Cvm create protected data page fail:%d\n", ret); ++ return ret; ++} ++ ++static u64 cvm_granule_size(u32 level) ++{ ++ return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); ++} ++ ++static bool is_data_create_region(phys_addr_t ipa_base, ++ struct kvm_cap_arm_tmm_populate_region_args *args) ++{ ++ if ((ipa_base >= args->populate_ipa_base1 && ++ ipa_base < args->populate_ipa_base1 + args->populate_ipa_size1) || ++ (ipa_base >= args->populate_ipa_base2 && ++ ipa_base < args->populate_ipa_base2 + args->populate_ipa_size2)) ++ return true; ++ return false; ++} ++ ++int kvm_cvm_populate_par_region(struct kvm *kvm, u64 numa_set, ++ phys_addr_t ipa_base, phys_addr_t ipa_end, ++ struct kvm_cap_arm_tmm_populate_region_args *args) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_memory_slot *memslot; ++ gfn_t base_gfn, end_gfn; ++ int idx; ++ phys_addr_t ipa; ++ int ret = 0; ++ int level = TMM_TTT_LEVEL_3; ++ unsigned long map_size = cvm_granule_size(level); ++ ++ base_gfn = gpa_to_gfn(ipa_base); ++ end_gfn = gpa_to_gfn(ipa_end); ++ ++ idx = srcu_read_lock(&kvm->srcu); ++ memslot = gfn_to_memslot(kvm, base_gfn); ++ if (!memslot) { ++ ret = -EFAULT; ++ goto out; ++ } ++ ++ /* We require the region to be contained within a single memslot */ ++ if (memslot->base_gfn + memslot->npages < end_gfn) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ mmap_read_lock(current->mm); ++ ++ ipa = ipa_base; ++ while (ipa < ipa_end) { ++ struct page *page = NULL; ++ kvm_pfn_t pfn = 0; ++ ++ /* ++ * FIXME: This causes over mapping, but there's no good ++ * solution here with the ABI as it stands ++ */ ++ ipa = ALIGN_DOWN(ipa, map_size); ++ ++ if (is_data_create_region(ipa, args)) { ++ pfn = gfn_to_pfn_memslot(memslot, gpa_to_gfn(ipa)); ++ if (is_error_pfn(pfn)) { ++ ret = -EFAULT; ++ break; ++ } ++ ++ page = pfn_to_page(pfn); ++ } ++ ++ ret = kvm_cvm_create_protected_data_page(kvm, cvm, ipa, level, page, numa_set); ++ if (ret) ++ goto err_release_pfn; ++ ++ ipa += map_size; ++ if (pfn) ++ kvm_release_pfn_dirty(pfn); ++err_release_pfn: ++ if (ret) { ++ if (pfn) ++ kvm_release_pfn_clean(pfn); ++ break; ++ } ++ } ++ ++ mmap_read_unlock(current->mm); ++out: ++ srcu_read_unlock(&kvm->srcu, idx); ++ return ret; ++} ++ ++int kvm_finalize_vcpu_tec(struct kvm_vcpu *vcpu) ++{ ++ int ret = 0; ++ int i; ++ u64 numa_set; ++ struct tmi_tec_params *params_ptr = NULL; ++ struct user_pt_regs *vcpu_regs = vcpu_gp_regs(vcpu); ++ u64 mpidr = kvm_vcpu_get_mpidr_aff(vcpu); ++ struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; ++ struct virtcca_cvm_tec *tec = &vcpu->arch.tec; ++ ++ mutex_lock(&vcpu->kvm->lock); ++ tec->tec_run = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); ++ if (!tec->tec_run) { ++ ret = -ENOMEM; ++ goto tec_free; ++ } ++ params_ptr = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); ++ if (!params_ptr) { ++ ret = -ENOMEM; ++ goto tec_free; ++ } ++ ++ for (i = 0; i < TEC_CREATE_NR_GPRS; ++i) ++ params_ptr->gprs[i] = vcpu_regs->regs[i]; ++ ++ params_ptr->pc = vcpu_regs->pc; ++ ++ if (vcpu->vcpu_id == 0) ++ params_ptr->flags = TMI_RUNNABLE; ++ else ++ params_ptr->flags = TMI_NOT_RUNNABLE; ++ params_ptr->ram_size = cvm->ram_size; ++ numa_set = kvm_get_host_numa_set_by_vcpu(vcpu->vcpu_id, vcpu->kvm); ++ tec->tec = tmi_tec_create(numa_set, cvm->rd, mpidr, __pa(params_ptr)); ++ ++ tec->tec_created = true; ++ kfree(params_ptr); ++ mutex_unlock(&vcpu->kvm->lock); ++ return ret; ++ ++tec_free: ++ kfree(tec->tec_run); ++ kfree(params_ptr); ++ mutex_unlock(&vcpu->kvm->lock); ++ return ret; ++} ++ ++static int config_cvm_hash_algo(struct tmi_cvm_params *params, ++ struct kvm_cap_arm_tmm_config_item *cfg) ++{ ++ switch (cfg->hash_algo) { ++ case KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA256: ++ if (!tmm_supports(TMI_FEATURE_REGISTER_0_HASH_SHA_256)) ++ return -EINVAL; ++ break; ++ case KVM_CAP_ARM_RME_MEASUREMENT_ALGO_SHA512: ++ if (!tmm_supports(TMI_FEATURE_REGISTER_0_HASH_SHA_512)) ++ return -EINVAL; ++ break; ++ default: ++ return -EINVAL; ++ } ++ params->measurement_algo = cfg->hash_algo; ++ return 0; ++} ++ ++static int config_cvm_sve(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct tmi_cvm_params *params; ++ int max_sve_vq; ++ ++ params = cvm->params; ++ max_sve_vq = u64_get_bits(tmm_feat_reg0, ++ TMI_FEATURE_REGISTER_0_SVE_VL); ++ ++ if (!kvm_cvm_supports_sve()) ++ return -EINVAL; ++ ++ if (cfg->sve_vq > max_sve_vq) ++ return -EINVAL; ++ ++ params->sve_vl = cfg->sve_vq; ++ params->flags |= TMI_CVM_PARAM_FLAG_SVE; ++ ++ return 0; ++} ++ ++static int config_cvm_pmu(struct kvm *kvm, struct kvm_cap_arm_tmm_config_item *cfg) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct tmi_cvm_params *params; ++ int max_pmu_num_ctrs; ++ ++ params = cvm->params; ++ max_pmu_num_ctrs = u64_get_bits(tmm_feat_reg0, ++ TMI_FEATURE_REGISTER_0_PMU_NUM_CTRS); ++ ++ if (!kvm_cvm_supports_pmu()) ++ return -EINVAL; ++ ++ if (cfg->num_pmu_cntrs > max_pmu_num_ctrs) ++ return -EINVAL; ++ ++ params->pmu_num_cnts = cfg->num_pmu_cntrs; ++ params->flags |= TMI_CVM_PARAM_FLAG_PMU; ++ ++ return 0; ++} ++ ++static int kvm_tmm_config_cvm(struct kvm *kvm, struct kvm_enable_cap *cap) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_cap_arm_tmm_config_item cfg; ++ int r = 0; ++ ++ if (virtcca_cvm_state(kvm) != CVM_STATE_NONE) ++ return -EBUSY; ++ ++ if (copy_from_user(&cfg, (void __user *)cap->args[1], sizeof(cfg))) ++ return -EFAULT; ++ ++ switch (cfg.cfg) { ++ case KVM_CAP_ARM_TMM_CFG_SVE: ++ r = config_cvm_sve(kvm, &cfg); ++ break; ++ case KVM_CAP_ARM_TMM_CFG_PMU: ++ r = config_cvm_pmu(kvm, &cfg); ++ break; ++ case KVM_CAP_ARM_TMM_CFG_HASH_ALGO: ++ r = config_cvm_hash_algo(cvm->params, &cfg); ++ break; ++ default: ++ r = -EINVAL; ++ } ++ ++ return r; ++} ++ ++static int kvm_cvm_map_range(struct kvm *kvm) ++{ ++ int ret; ++ u64 curr_numa_set; ++ int idx; ++ u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_numa_info *numa_info = &cvm->numa_info; ++ gpa_t gpa; ++ ++ curr_numa_set = kvm_get_first_binded_numa_set(kvm); ++ gpa = round_up(cvm->dtb_end, l2_granule); ++ for (idx = 0; idx < numa_info->numa_cnt; idx++) { ++ struct kvm_numa_node *numa_node = &numa_info->numa_nodes[idx]; ++ ++ if (idx) ++ gpa = numa_node->ipa_start; ++ if (gpa >= numa_node->ipa_start && ++ gpa < numa_node->ipa_start + numa_node->ipa_size) { ++ ret = tmi_ttt_map_range(cvm->rd, gpa, ++ numa_node->ipa_size - gpa + numa_node->ipa_start, ++ curr_numa_set, numa_node->host_numa_nodes[0]); ++ if (ret) { ++ kvm_err("tmi_ttt_map_range failed: %d.\n", ret); ++ return ret; ++ } ++ } ++ } ++ ++ return ret; ++} ++ ++static int kvm_activate_cvm(struct kvm *kvm) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ ++ if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) ++ return -EINVAL; ++ ++ if (kvm_cvm_map_range(kvm)) ++ return -EFAULT; ++ ++ if (tmi_cvm_activate(cvm->rd)) { ++ kvm_err("tmi_cvm_activate failed!\n"); ++ return -ENXIO; ++ } ++ ++ WRITE_ONCE(cvm->state, CVM_STATE_ACTIVE); ++ kvm_info("cVM%d is activated!\n", cvm->cvm_vmid); ++ return 0; ++} ++ ++static int kvm_populate_ram_region(struct kvm *kvm, u64 map_size, ++ phys_addr_t ipa_base, phys_addr_t ipa_end, ++ struct kvm_cap_arm_tmm_populate_region_args *args) ++{ ++ phys_addr_t gpa; ++ u64 numa_set = kvm_get_first_binded_numa_set(kvm); ++ ++ for (gpa = ipa_base; gpa < ipa_end; gpa += map_size) { ++ if (kvm_cvm_populate_par_region(kvm, numa_set, gpa, gpa + map_size, args)) { ++ kvm_err("kvm_cvm_populate_par_region failed: %d\n", -EFAULT); ++ return -EFAULT; ++ } ++ } ++ return 0; ++} ++ ++static int kvm_populate_ipa_cvm_range(struct kvm *kvm, ++ struct kvm_cap_arm_tmm_populate_region_args *args) ++{ ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ u64 l2_granule = cvm_granule_size(TMM_TTT_LEVEL_2); ++ phys_addr_t ipa_base1, ipa_end2; ++ ++ if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) ++ return -EINVAL; ++ if (!IS_ALIGNED(args->populate_ipa_base1, PAGE_SIZE) || ++ !IS_ALIGNED(args->populate_ipa_size1, PAGE_SIZE) || ++ !IS_ALIGNED(args->populate_ipa_base2, PAGE_SIZE) || ++ !IS_ALIGNED(args->populate_ipa_size2, PAGE_SIZE)) ++ return -EINVAL; ++ ++ if (args->populate_ipa_base1 < cvm->loader_start || ++ args->populate_ipa_base2 < args->populate_ipa_base1 + args->populate_ipa_size1 || ++ cvm->dtb_end < args->populate_ipa_base2 + args->populate_ipa_size2) ++ return -EINVAL; ++ ++ if (args->flags & ~TMI_MEASURE_CONTENT) ++ return -EINVAL; ++ ipa_base1 = round_down(args->populate_ipa_base1, l2_granule); ++ ipa_end2 = round_up(args->populate_ipa_base2 + args->populate_ipa_size2, l2_granule); ++ ++ return kvm_populate_ram_region(kvm, l2_granule, ipa_base1, ipa_end2, args); ++} ++ ++int kvm_cvm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) ++{ ++ int r = 0; ++ ++ mutex_lock(&kvm->lock); ++ switch (cap->args[0]) { ++ case KVM_CAP_ARM_TMM_CONFIG_CVM_HOST: ++ r = kvm_tmm_config_cvm(kvm, cap); ++ break; ++ case KVM_CAP_ARM_TMM_CREATE_RD: ++ r = kvm_arm_create_cvm(kvm); ++ break; ++ case KVM_CAP_ARM_TMM_POPULATE_CVM: { ++ struct kvm_cap_arm_tmm_populate_region_args args; ++ void __user *argp = u64_to_user_ptr(cap->args[1]); ++ ++ if (copy_from_user(&args, argp, sizeof(args))) { ++ r = -EFAULT; ++ break; ++ } ++ r = kvm_populate_ipa_cvm_range(kvm, &args); ++ break; ++ } ++ case KVM_CAP_ARM_TMM_ACTIVATE_CVM: ++ r = kvm_activate_cvm(kvm); ++ break; ++ default: ++ r = -EINVAL; ++ break; ++ } ++ mutex_unlock(&kvm->lock); ++ ++ return r; ++} ++ ++void kvm_destroy_tec(struct kvm_vcpu *vcpu) ++{ ++ struct virtcca_cvm_tec *tec = &vcpu->arch.tec; ++ ++ if (!vcpu_is_tec(vcpu)) ++ return; ++ ++ if (tmi_tec_destroy(tec->tec) != 0) ++ kvm_err("%s vcpu id : %d failed!\n", __func__, vcpu->vcpu_id); ++ ++ tec->tec = 0; ++ kfree(tec->tec_run); ++} ++ ++static int tmi_check_version(void) ++{ ++ u64 res; ++ int version_major; ++ int version_minor; ++ ++ res = tmi_version(); ++ if (res == SMCCC_RET_NOT_SUPPORTED) ++ return -ENXIO; ++ ++ version_major = TMI_ABI_VERSION_GET_MAJOR(res); ++ version_minor = TMI_ABI_VERSION_GET_MINOR(res); ++ ++ if (version_major != TMI_ABI_VERSION_MAJOR) { ++ kvm_err("Unsupported TMI_ABI (version %d %d)\n", version_major, ++ version_minor); ++ return -ENXIO; ++ } ++ ++ kvm_info("TMI ABI version %d,%d\n", version_major, version_minor); ++ return 0; ++} ++ ++int kvm_tec_enter(struct kvm_vcpu *vcpu) ++{ ++ struct tmi_tec_run *run; ++ struct virtcca_cvm_tec *tec = &vcpu->arch.tec; ++ struct virtcca_cvm *cvm = vcpu->kvm->arch.virtcca_cvm; ++ ++ if (READ_ONCE(cvm->state) != CVM_STATE_ACTIVE) ++ return -EINVAL; ++ ++ run = tec->tec_run; ++ /* set/clear TWI TWE flags */ ++ if (vcpu->arch.hcr_el2 & HCR_TWI) ++ run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFI; ++ else ++ run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFI; ++ ++ if (vcpu->arch.hcr_el2 & HCR_TWE) ++ run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFE; ++ else ++ run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFE; ++ ++ return tmi_tec_enter(tec->tec, __pa(run)); ++} ++ ++int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target) ++{ ++ int ret; ++ struct virtcca_cvm_tec *calling_tec = &calling->arch.tec; ++ struct virtcca_cvm_tec *target_tec = &target->arch.tec; ++ ++ ret = tmi_psci_complete(calling_tec->tec, target_tec->tec); ++ if (ret) ++ return -EINVAL; ++ return 0; ++} ++ ++int kvm_init_tmm(void) ++{ ++ int ret; ++ ++ if (PAGE_SIZE != SZ_4K) ++ return 0; ++ ++ if (tmi_check_version()) ++ return 0; ++ ++ ret = cvm_vmid_init(); ++ if (ret) ++ return ret; ++ ++ tmm_feat_reg0 = tmi_features(0); ++ kvm_info("TMM feature0: 0x%lx\n", tmm_feat_reg0); ++ ++ static_branch_enable(&virtcca_cvm_is_available); ++ ++ return 0; ++} ++ ++static bool is_numa_ipa_range_valid(struct kvm_numa_info *numa_info) ++{ ++ unsigned long i; ++ struct kvm_numa_node *numa_node, *prev_numa_node; ++ ++ prev_numa_node = NULL; ++ for (i = 0; i < numa_info->numa_cnt; i++) { ++ numa_node = &numa_info->numa_nodes[i]; ++ if (numa_node->ipa_start + numa_node->ipa_size < numa_node->ipa_start) ++ return false; ++ if (prev_numa_node && ++ numa_node->ipa_start < prev_numa_node->ipa_start + prev_numa_node->ipa_size) ++ return false; ++ prev_numa_node = numa_node; ++ } ++ if (numa_node->ipa_start + numa_node->ipa_size > CVM_IPA_MAX_VAL) ++ return false; ++ return true; ++} ++ ++int kvm_load_user_data(struct kvm *kvm, unsigned long arg) ++{ ++ struct kvm_user_data user_data; ++ void __user *argp = (void __user *)arg; ++ struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; ++ struct kvm_numa_info *numa_info; ++ ++ if (!kvm_is_virtcca_cvm(kvm)) ++ return -EFAULT; ++ ++ if (copy_from_user(&user_data, argp, sizeof(user_data))) ++ return -EINVAL; ++ ++ numa_info = &user_data.numa_info; ++ if (numa_info->numa_cnt > MAX_NUMA_NODE) ++ return -EINVAL; ++ ++ if (numa_info->numa_cnt > 0) { ++ unsigned long i, total_size = 0; ++ struct kvm_numa_node *numa_node = &numa_info->numa_nodes[0]; ++ unsigned long ipa_end = numa_node->ipa_start + numa_node->ipa_size; ++ ++ if (!is_numa_ipa_range_valid(numa_info)) ++ return -EINVAL; ++ if (user_data.loader_start < numa_node->ipa_start || ++ user_data.dtb_end > ipa_end) ++ return -EINVAL; ++ for (i = 0; i < numa_info->numa_cnt; i++) ++ total_size += numa_info->numa_nodes[i].ipa_size; ++ if (total_size != user_data.ram_size) ++ return -EINVAL; ++ } ++ ++ if (user_data.image_end <= user_data.loader_start || ++ user_data.initrd_start < user_data.image_end || ++ user_data.dtb_end < user_data.initrd_start || ++ user_data.ram_size < user_data.dtb_end - user_data.loader_start) ++ return -EINVAL; ++ ++ cvm->loader_start = user_data.loader_start; ++ cvm->image_end = user_data.image_end; ++ cvm->initrd_start = user_data.initrd_start; ++ cvm->dtb_end = user_data.dtb_end; ++ cvm->ram_size = user_data.ram_size; ++ memcpy(&cvm->numa_info, numa_info, sizeof(struct kvm_numa_info)); ++ ++ return 0; ++} ++ ++void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu) ++{ ++ kvm_timer_vcpu_put(vcpu); ++ kvm_vgic_put(vcpu); ++ vcpu->cpu = -1; ++} ++ ++unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, ++ unsigned long target_affinity, unsigned long lowest_affinity_level) ++{ ++ struct kvm_vcpu *target_vcpu; ++ ++ if (lowest_affinity_level != 0) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ target_vcpu = kvm_mpidr_to_vcpu(vcpu->kvm, target_affinity); ++ if (!target_vcpu) ++ return PSCI_RET_INVALID_PARAMS; ++ ++ cvm_psci_complete(vcpu, target_vcpu); ++ return PSCI_RET_SUCCESS; ++} ++ ++int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, ++ bool serror_pending, bool ext_dabt_pending) ++{ ++ struct virtcca_cvm_tec *tec = &vcpu->arch.tec; ++ ++ if (serror_pending) ++ return -EINVAL; ++ ++ if (ext_dabt_pending) { ++ if (!(((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags & ++ TEC_ENTRY_FLAG_EMUL_MMIO)) ++ return -EINVAL; ++ ++ ((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags ++ &= ~TEC_ENTRY_FLAG_EMUL_MMIO; ++ ((struct tmi_tec_run *)tec->tec_run)->tec_entry.flags ++ |= TEC_ENTRY_FLAG_INJECT_SEA; ++ } ++ return 0; ++} ++ ++int kvm_init_cvm_vm(struct kvm *kvm) ++{ ++ struct tmi_cvm_params *params; ++ struct virtcca_cvm *cvm; ++ ++ if (kvm->arch.virtcca_cvm) { ++ kvm_info("cvm already create.\n"); ++ return 0; ++ } ++ ++ cvm = (struct virtcca_cvm *)kzalloc(sizeof(struct virtcca_cvm), GFP_KERNEL_ACCOUNT); ++ if (!cvm) ++ return -ENOMEM; ++ ++ kvm->arch.virtcca_cvm = cvm; ++ params = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); ++ if (!params) { ++ kfree(kvm->arch.virtcca_cvm); ++ kvm->arch.virtcca_cvm = NULL; ++ return -ENOMEM; ++ } ++ ++ cvm->params = params; ++ WRITE_ONCE(cvm->state, CVM_STATE_NONE); ++ ++ return 0; ++} +diff --git a/arch/arm64/kvm/virtcca_cvm_exit.c b/arch/arm64/kvm/virtcca_cvm_exit.c +new file mode 100644 +index 000000000000..9654375a9c8c +--- /dev/null ++++ b/arch/arm64/kvm/virtcca_cvm_exit.c +@@ -0,0 +1,221 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (c) 2024, The Linux Foundation. All rights reserved. ++ */ ++#include ++#include ++#include ++ ++#include ++#include ++#include ++ ++typedef int (*exit_handler_fn)(struct kvm_vcpu *vcpu); ++ ++static void update_arch_timer_irq_lines(struct kvm_vcpu *vcpu, bool unmask_ctl) ++{ ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) = run->tec_exit.cntv_ctl; ++ __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0) = run->tec_exit.cntv_cval; ++ __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) = run->tec_exit.cntp_ctl; ++ __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = run->tec_exit.cntp_cval; ++ ++ /* Because the timer mask is tainted by TMM, we don't know the ++ * true intent of the guest. Here, we assume mask is always ++ * cleared during WFI. ++ */ ++ if (unmask_ctl) { ++ __vcpu_sys_reg(vcpu, CNTV_CTL_EL0) &= ~ARCH_TIMER_CTRL_IT_MASK; ++ __vcpu_sys_reg(vcpu, CNTP_CTL_EL0) &= ~ARCH_TIMER_CTRL_IT_MASK; ++ } ++ ++ kvm_cvm_timers_update(vcpu); ++} ++ ++static int tec_exit_reason_notimpl(struct kvm_vcpu *vcpu) ++{ ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ pr_err("[vcpu %d] Unhandled exit reason from cvm (ESR: %#llx)\n", ++ vcpu->vcpu_id, run->tec_exit.esr); ++ return -ENXIO; ++} ++ ++/* The process is the same as kvm_handle_wfx, ++ * except the tracing and updating operation for pc, ++ * we copy kvm_handle_wfx process here ++ * to avoid changing kvm_handle_wfx function. ++ */ ++static int tec_exit_wfx(struct kvm_vcpu *vcpu) ++{ ++ u64 esr = kvm_vcpu_get_esr(vcpu); ++ ++ if (esr & ESR_ELx_WFx_ISS_WFE) { ++ vcpu->stat.wfe_exit_stat++; ++ } else { ++ vcpu->stat.wfi_exit_stat++; ++ } ++ ++ if (esr & ESR_ELx_WFx_ISS_WFxT) { ++ if (esr & ESR_ELx_WFx_ISS_RV) { ++ u64 val, now; ++ ++ now = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_TIMER_CNT); ++ val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); ++ ++ if (now >= val) ++ goto out; ++ } else { ++ /* Treat WFxT as WFx if RN is invalid */ ++ esr &= ~ESR_ELx_WFx_ISS_WFxT; ++ } ++ } ++ ++ if (esr & ESR_ELx_WFx_ISS_WFE) { ++ kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); ++ } else { ++ if (esr & ESR_ELx_WFx_ISS_WFxT) ++ vcpu_set_flag(vcpu, IN_WFIT); ++ ++ kvm_vcpu_wfi(vcpu); ++ } ++ ++out: ++ return 1; ++} ++ ++static int tec_exit_sys_reg(struct kvm_vcpu *vcpu) ++{ ++ int ret; ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ unsigned long esr = kvm_vcpu_get_esr(vcpu); ++ int rt = kvm_vcpu_sys_get_rt(vcpu); ++ bool is_write = !(esr & 1); ++ ++ if (is_write) ++ vcpu_set_reg(vcpu, rt, run->tec_exit.gprs[0]); ++ ++ ret = kvm_handle_sys_reg(vcpu); ++ ++ if (ret >= 0 && !is_write) ++ run->tec_entry.gprs[0] = vcpu_get_reg(vcpu, rt); ++ ++ return ret; ++} ++ ++static int tec_exit_sync_dabt(struct kvm_vcpu *vcpu) ++{ ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ if (kvm_vcpu_dabt_iswrite(vcpu) && kvm_vcpu_dabt_isvalid(vcpu)) { ++ vcpu_set_reg(vcpu, kvm_vcpu_dabt_get_rd(vcpu), ++ run->tec_exit.gprs[0]); ++ } ++ return kvm_handle_guest_abort(vcpu); ++} ++ ++static int tec_exit_sync_iabt(struct kvm_vcpu *vcpu) ++{ ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ pr_err("[vcpu %d] Unhandled instruction abort (ESR: %#llx).\n", ++ vcpu->vcpu_id, run->tec_exit.esr); ++ ++ return -ENXIO; ++} ++ ++static exit_handler_fn tec_exit_handlers[] = { ++ [0 ... ESR_ELx_EC_MAX] = tec_exit_reason_notimpl, ++ [ESR_ELx_EC_WFx] = tec_exit_wfx, ++ [ESR_ELx_EC_SYS64] = tec_exit_sys_reg, ++ [ESR_ELx_EC_DABT_LOW] = tec_exit_sync_dabt, ++ [ESR_ELx_EC_IABT_LOW] = tec_exit_sync_iabt ++}; ++ ++static int tec_exit_psci(struct kvm_vcpu *vcpu) ++{ ++ int i; ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) ++ vcpu_set_reg(vcpu, i, run->tec_exit.gprs[i]); ++ ++ return kvm_psci_call(vcpu); ++} ++ ++static int tec_exit_host_call(struct kvm_vcpu *vcpu) ++{ ++ int ret, i; ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ ++ vcpu->stat.hvc_exit_stat++; ++ ++ for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) ++ vcpu_set_reg(vcpu, i, run->tec_exit.gprs[i]); ++ ++ ret = kvm_smccc_call_handler(vcpu); ++ ++ if (ret < 0) { ++ vcpu_set_reg(vcpu, 0, ~0UL); ++ ret = 1; ++ } ++ for (i = 0; i < TEC_EXIT_NR_GPRS; ++i) ++ run->tec_entry.gprs[i] = vcpu_get_reg(vcpu, i); ++ ++ return ret; ++} ++ ++/* ++ * Return > 0 to return to guest, < 0 on error, 0(and set exit_reason) on ++ * proper exit to userspace ++ */ ++ ++int handle_cvm_exit(struct kvm_vcpu *vcpu, int tec_run_ret) ++{ ++ unsigned long status; ++ struct tmi_tec_run *run = vcpu->arch.tec.tec_run; ++ u8 esr_ec = ESR_ELx_EC(run->tec_exit.esr); ++ bool is_wfx; ++ ++ status = TMI_RETURN_STATUS(tec_run_ret); ++ ++ if (status == TMI_ERROR_CVM_POWEROFF) { ++ vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; ++ vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; ++ return 0; ++ } ++ ++ if (status == TMI_ERROR_CVM_STATE) { ++ vcpu->run->exit_reason = KVM_EXIT_UNKNOWN; ++ return 0; ++ } ++ ++ if (tec_run_ret) ++ return -ENXIO; ++ ++ vcpu->arch.fault.esr_el2 = run->tec_exit.esr; ++ vcpu->arch.fault.far_el2 = run->tec_exit.far; ++ vcpu->arch.fault.hpfar_el2 = run->tec_exit.hpfar; ++ ++ is_wfx = (run->tec_exit.exit_reason == TMI_EXIT_SYNC) && (esr_ec == ESR_ELx_EC_WFx); ++ update_arch_timer_irq_lines(vcpu, is_wfx); ++ ++ run->tec_entry.flags = 0; ++ ++ switch (run->tec_exit.exit_reason) { ++ case TMI_EXIT_FIQ: ++ case TMI_EXIT_IRQ: ++ return 1; ++ case TMI_EXIT_PSCI: ++ return tec_exit_psci(vcpu); ++ case TMI_EXIT_SYNC: ++ return tec_exit_handlers[esr_ec](vcpu); ++ case TMI_EXIT_HOST_CALL: ++ return tec_exit_host_call(vcpu); ++ } ++ ++ kvm_pr_unimpl("Unsupported exit reason : 0x%llx\n", ++ run->tec_exit.exit_reason); ++ return 0; ++} +diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c +index 8c8d7653ba84..66a7fff9f373 100644 +--- a/arch/arm64/mm/init.c ++++ b/arch/arm64/mm/init.c +@@ -45,6 +45,7 @@ + #include + #include + #include ++#include + + #include "internal.h" + +@@ -610,6 +611,8 @@ void __init mem_init(void) + + swiotlb_init(swiotlb, SWIOTLB_VERBOSE); + ++ swiotlb_cvm_update_mem_attributes(); ++ + /* this will put all unused low memory onto the freelists */ + memblock_free_all(); + +diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c +index 4142a75a414e..31f04f19b635 100644 +--- a/arch/arm64/mm/mmu.c ++++ b/arch/arm64/mm/mmu.c +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include + + #define NO_BLOCK_MAPPINGS BIT(0) + #define NO_CONT_MAPPINGS BIT(1) +@@ -589,7 +590,7 @@ static void __init map_mem(pgd_t *pgdp) + + early_kfence_pool = arm64_kfence_alloc_pool(); + +- if (can_set_direct_map()) ++ if (can_set_direct_map() || is_virtcca_cvm_world()) + flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + + /* +@@ -1343,7 +1344,7 @@ int arch_add_memory(int nid, u64 start, u64 size, + + VM_BUG_ON(!mhp_range_allowed(start, size, true)); + +- if (can_set_direct_map()) ++ if (can_set_direct_map() || is_virtcca_cvm_world()) + flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; + + __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), +diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c +index 0e270a1c51e6..06e81d1dbc1e 100644 +--- a/arch/arm64/mm/pageattr.c ++++ b/arch/arm64/mm/pageattr.c +@@ -195,6 +195,9 @@ int set_direct_map_default_noflush(struct page *page) + #ifdef CONFIG_DEBUG_PAGEALLOC + void __kernel_map_pages(struct page *page, int numpages, int enable) + { ++ if (is_virtcca_cvm_world()) ++ return; ++ + if (!can_set_direct_map()) + return; + +diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c +index b93ef18b9c98..89b31199d005 100644 +--- a/drivers/irqchip/irq-gic-v3-its.c ++++ b/drivers/irqchip/irq-gic-v3-its.c +@@ -29,6 +29,10 @@ + #include + #include + #include ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++#include ++#include ++#endif + + #include + #include +@@ -399,6 +403,93 @@ static int build_devid_pools(void) + } + #endif + ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ ++static struct device cvm_alloc_device; ++static LIST_HEAD(cvm_its_nodes); ++static raw_spinlock_t cvm_its_lock; ++ ++struct its_device_order { ++ struct its_device *dev; ++ struct list_head entry; ++ int itt_order; ++}; ++ ++static inline struct page *its_alloc_shared_pages_node(int node, gfp_t gfp, ++ unsigned int order) ++{ ++ return swiotlb_alloc(&cvm_alloc_device, (1 << order) * PAGE_SIZE); ++} ++ ++static inline struct page *its_alloc_shared_pages(gfp_t gfp, unsigned int order) ++{ ++ return its_alloc_shared_pages_node(NUMA_NO_NODE, gfp, order); ++} ++ ++static void its_free_shared_pages(void *addr, int order) ++{ ++ if (order < 0) ++ return; ++ ++ swiotlb_free(&cvm_alloc_device, (struct page *)addr, (1 << order) * PAGE_SIZE); ++} ++ ++static int add_its_device_order(struct its_device *dev, int itt_order) ++{ ++ struct its_device_order *new; ++ unsigned long flags; ++ ++ new = kmalloc(sizeof(struct its_device_order), GFP_KERNEL); ++ if (!new) ++ return -ENOMEM; ++ new->dev = dev; ++ new->itt_order = itt_order; ++ raw_spin_lock_irqsave(&cvm_its_lock, flags); ++ list_add_tail(&new->entry, &cvm_its_nodes); ++ raw_spin_unlock_irqrestore(&cvm_its_lock, flags); ++ return 0; ++} ++ ++/* get its device order and then free its device order */ ++static int get_its_device_order(struct its_device *dev) ++{ ++ struct its_device_order *pos, *tmp; ++ unsigned long flags; ++ int itt_order = -1; ++ ++ raw_spin_lock_irqsave(&cvm_its_lock, flags); ++ list_for_each_entry_safe(pos, tmp, &cvm_its_nodes, entry) { ++ if (pos->dev == dev) { ++ itt_order = pos->itt_order; ++ list_del(&pos->entry); ++ kfree(pos); ++ goto found; ++ } ++ } ++found: ++ raw_spin_unlock_irqrestore(&cvm_its_lock, flags); ++ return itt_order; ++} ++ ++static void *its_alloc_shared_page_address(struct its_device *dev, ++ struct its_node *its, int sz) ++{ ++ struct page *page; ++ int itt_order; ++ ++ itt_order = get_order(sz); ++ if (add_its_device_order(dev, itt_order)) ++ return NULL; ++ ++ page = its_alloc_shared_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, ++ itt_order); ++ if (!page) ++ return NULL; ++ return (void *)page_address(page); ++} ++ ++#endif ++ + /* + * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we + * always have vSGIs mapped. +@@ -2421,7 +2512,13 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags) + { + struct page *prop_page; + +- prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ prop_page = its_alloc_shared_pages(gfp_flags, ++ get_order(LPI_PROPBASE_SZ)); ++ else ++#endif ++ prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); + if (!prop_page) + return NULL; + +@@ -2432,8 +2529,14 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags) + + static void its_free_prop_table(struct page *prop_page) + { +- free_pages((unsigned long)page_address(prop_page), +- get_order(LPI_PROPBASE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(page_address(prop_page), ++ get_order(LPI_PROPBASE_SZ)); ++ else ++#endif ++ free_pages((unsigned long)page_address(prop_page), ++ get_order(LPI_PROPBASE_SZ)); + } + + static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) +@@ -2555,7 +2658,13 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, + order = get_order(GITS_BASER_PAGES_MAX * psz); + } + +- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ page = its_alloc_shared_pages_node(its->numa_node, ++ GFP_KERNEL | __GFP_ZERO, order); ++ else ++#endif ++ page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + +@@ -2568,7 +2677,12 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, + /* 52bit PA is supported only when PageSize=64K */ + if (psz != SZ_64K) { + pr_err("ITS: no 52bit PA support when psz=%d\n", psz); +- free_pages((unsigned long)base, order); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(base, order); ++ else ++#endif ++ free_pages((unsigned long)base, order); + return -ENXIO; + } + +@@ -2624,7 +2738,12 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, + pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", + &its->phys_base, its_base_type_string[type], + val, tmp); +- free_pages((unsigned long)base, order); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(base, order); ++ else ++#endif ++ free_pages((unsigned long)base, order); + return -ENXIO; + } + +@@ -2763,8 +2882,14 @@ static void its_free_tables(struct its_node *its) + + for (i = 0; i < GITS_BASER_NR_REGS; i++) { + if (its->tables[i].base) { +- free_pages((unsigned long)its->tables[i].base, +- its->tables[i].order); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(its->tables[i].base, ++ its->tables[i].order); ++ else ++#endif ++ free_pages((unsigned long)its->tables[i].base, ++ its->tables[i].order); + its->tables[i].base = NULL; + } + } +@@ -3034,7 +3159,13 @@ static bool allocate_vpe_l2_table(int cpu, u32 id) + + /* Allocate memory for 2nd level table */ + if (!table[idx]) { +- page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ page = its_alloc_shared_pages(GFP_KERNEL | __GFP_ZERO, ++ get_order(psz)); ++ else ++#endif ++ page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); + if (!page) + return false; + +@@ -3153,7 +3284,13 @@ static int allocate_vpe_l1_table(void) + + pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", + np, npg, psz, epp, esz); +- page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ page = its_alloc_shared_pages(GFP_ATOMIC | __GFP_ZERO, ++ get_order(np * PAGE_SIZE)); ++ else ++#endif ++ page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); + if (!page) + return -ENOMEM; + +@@ -3199,8 +3336,14 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags) + { + struct page *pend_page; + +- pend_page = alloc_pages(gfp_flags | __GFP_ZERO, +- get_order(LPI_PENDBASE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ pend_page = its_alloc_shared_pages(gfp_flags | __GFP_ZERO, ++ get_order(LPI_PENDBASE_SZ)); ++ else ++#endif ++ pend_page = alloc_pages(gfp_flags | __GFP_ZERO, ++ get_order(LPI_PENDBASE_SZ)); + if (!pend_page) + return NULL; + +@@ -3212,7 +3355,13 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags) + + static void its_free_pending_table(struct page *pt) + { +- free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(page_address(pt), ++ get_order(LPI_PENDBASE_SZ)); ++ else ++#endif ++ free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); + } + + /* +@@ -3547,8 +3696,15 @@ static bool its_alloc_table_entry(struct its_node *its, + + /* Allocate memory for 2nd level table */ + if (!table[idx]) { +- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, +- get_order(baser->psz)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ page = its_alloc_shared_pages_node(its->numa_node, ++ GFP_KERNEL | __GFP_ZERO, ++ get_order(baser->psz)); ++ else ++#endif ++ page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, ++ get_order(baser->psz)); + if (!page) + return false; + +@@ -3651,7 +3807,12 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, + nr_ites = max(2, nvecs); + sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); + sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; +- itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ itt = its_alloc_shared_page_address(dev, its, sz); ++ else ++#endif ++ itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); + if (alloc_lpis) { + lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); + if (lpi_map) +@@ -3665,7 +3826,12 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, + + if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { + kfree(dev); +- kfree(itt); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(itt, get_order(sz)); ++ else ++#endif ++ kfree(itt); + bitmap_free(lpi_map); + kfree(col_map); + return NULL; +@@ -3702,7 +3868,12 @@ static void its_free_device(struct its_device *its_dev) + list_del(&its_dev->entry); + raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); + kfree(its_dev->event_map.col_map); +- kfree(its_dev->itt); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(its_dev->itt, get_its_device_order(its_dev)); ++ else ++#endif ++ kfree(its_dev->itt); + + #ifdef CONFIG_VIRT_PLAT_DEV + if (its_dev->is_vdev) { +@@ -5445,8 +5616,15 @@ static int __init its_probe_one(struct its_node *its) + } + } + +- page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, +- get_order(ITS_CMD_QUEUE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ page = its_alloc_shared_pages_node(its->numa_node, ++ GFP_KERNEL | __GFP_ZERO, ++ get_order(ITS_CMD_QUEUE_SZ)); ++ else ++#endif ++ page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, ++ get_order(ITS_CMD_QUEUE_SZ)); + if (!page) { + err = -ENOMEM; + goto out_unmap_sgir; +@@ -5510,7 +5688,12 @@ static int __init its_probe_one(struct its_node *its) + out_free_tables: + its_free_tables(its); + out_free_cmd: +- free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) ++ its_free_shared_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); ++ else ++#endif ++ free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); + out_unmap_sgir: + if (its->sgir_base) + iounmap(its->sgir_base); +@@ -5999,6 +6182,12 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, + #endif + int err; + ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (is_virtcca_cvm_world()) { ++ device_initialize(&cvm_alloc_device); ++ raw_spin_lock_init(&cvm_its_lock); ++ } ++#endif + gic_rdists = rdists; + + its_parent = parent_domain; +diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c +index d712a19e47ac..3f6a9986670a 100644 +--- a/drivers/perf/arm_pmu.c ++++ b/drivers/perf/arm_pmu.c +@@ -736,6 +736,23 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) + return 0; + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++void arm_pmu_set_phys_irq(bool enable) ++{ ++ int cpu = get_cpu(); ++ struct arm_pmu *pmu = per_cpu(cpu_armpmu, cpu); ++ int irq; ++ ++ irq = armpmu_get_cpu_irq(pmu, cpu); ++ if (irq && !enable) ++ per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq); ++ else if (irq && enable) ++ per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq); ++ ++ put_cpu(); ++} ++#endif ++ + #ifdef CONFIG_CPU_PM + static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) + { +diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h +index fee20f66da52..9d3f034bd885 100644 +--- a/include/kvm/arm_arch_timer.h ++++ b/include/kvm/arm_arch_timer.h +@@ -172,4 +172,8 @@ static inline bool has_cntpoff(void) + return (has_vhe() && cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)); + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++/* Needed for S-EL2 */ ++void kvm_cvm_timers_update(struct kvm_vcpu *vcpu); ++#endif + #endif +diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h +index 4557f4735316..000ae1ba0635 100644 +--- a/include/linux/kvm_host.h ++++ b/include/linux/kvm_host.h +@@ -543,6 +543,28 @@ static __always_inline void guest_state_exit_irqoff(void) + instrumentation_end(); + } + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ ++#define KVM_TYPE_CVM_BIT 8 ++#define CVM_MAX_HALT_POLL_NS 100000 ++ ++DECLARE_STATIC_KEY_FALSE(virtcca_cvm_is_available); ++ ++static __always_inline bool vcpu_is_tec(struct kvm_vcpu *vcpu) ++{ ++ if (static_branch_unlikely(&virtcca_cvm_is_available)) ++ return vcpu->arch.tec.tec_run; ++ ++ return false; ++} ++ ++static inline bool kvm_arm_cvm_type(unsigned long type) ++{ ++ return type & (1UL << KVM_TYPE_CVM_BIT); ++} ++ ++#endif ++ + static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) + { + /* +diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h +index 143fbc10ecfe..c50f29236390 100644 +--- a/include/linux/perf/arm_pmu.h ++++ b/include/linux/perf/arm_pmu.h +@@ -167,6 +167,10 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn); + static inline int arm_pmu_acpi_probe(armpmu_init_fn init_fn) { return 0; } + #endif + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++void arm_pmu_set_phys_irq(bool enable); ++#endif ++ + #ifdef CONFIG_KVM + void kvm_host_pmu_init(struct arm_pmu *pmu); + #else +diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h +index cb63d56d8b0e..15012e713205 100644 +--- a/include/uapi/linux/kvm.h ++++ b/include/uapi/linux/kvm.h +@@ -1509,6 +1509,19 @@ struct kvm_numa_info { + #define KVM_SET_TSS_ADDR _IO(KVMIO, 0x47) + #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO, 0x48, __u64) + ++#define KVM_LOAD_USER_DATA _IOW(KVMIO, 0x49, struct kvm_user_data) ++ ++#define KVM_CAP_ARM_TMM 300 /* FIXME: Large number to prevent conflicts */ ++ ++struct kvm_user_data { ++ __u64 loader_start; ++ __u64 image_end; ++ __u64 initrd_start; ++ __u64 dtb_end; ++ __u64 ram_size; ++ struct kvm_numa_info numa_info; ++}; ++ + /* enable ucontrol for s390 */ + struct kvm_s390_ucas_mapping { + __u64 user_addr; +diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c +index a7d5fb473b32..5c10d1b7c39f 100644 +--- a/kernel/dma/swiotlb.c ++++ b/kernel/dma/swiotlb.c +@@ -1734,4 +1734,20 @@ static int __init rmem_swiotlb_setup(struct reserved_mem *rmem) + } + + RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup); ++ ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++void __init swiotlb_cvm_update_mem_attributes(void) ++{ ++ void *vaddr; ++ unsigned long bytes; ++ ++ if (!is_virtcca_cvm_world() || !is_swiotlb_allocated()) ++ return; ++ vaddr = phys_to_virt(io_tlb_default_mem.defpool.start); ++ bytes = PAGE_ALIGN(io_tlb_default_mem.defpool.nslabs << IO_TLB_SHIFT); ++ set_cvm_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT); ++ memset(vaddr, 0, bytes); ++ io_tlb_default_mem.for_alloc = true; ++} ++#endif + #endif /* CONFIG_DMA_RESTRICTED_POOL */ +diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c +index 62f19b4b9605..3ccb5f2bc39a 100644 +--- a/virt/kvm/kvm_main.c ++++ b/virt/kvm/kvm_main.c +@@ -3550,6 +3550,10 @@ static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu) + { + struct kvm *kvm = vcpu->kvm; + ++#ifdef CONFIG_HISI_VIRTCCA_HOST ++ if (vcpu_is_tec(vcpu)) ++ return CVM_MAX_HALT_POLL_NS; ++#endif + if (kvm->override_halt_poll_ns) { + /* + * Ensure kvm->max_halt_poll_ns is not read before +-- +2.34.1 + diff --git a/tee_blackbox/guest/0002-cvm-guest-base-commit.patch b/tee_blackbox/guest/0002-cvm-guest-base-commit.patch new file mode 100644 index 0000000000000000000000000000000000000000..49a0ddaf45342c39162402b530a57af883d3c9e7 --- /dev/null +++ b/tee_blackbox/guest/0002-cvm-guest-base-commit.patch @@ -0,0 +1,264 @@ +From 67d6651e526e501360187b79772a2f63f6231614 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Fri, 10 Jan 2025 11:38:43 +0800 +Subject: [PATCH 2/6] cvm guest base commit + +--- + arch/arm64/kernel/virtcca_cvm_guest.c | 29 +++++++++++++++++++++++++++ + drivers/base/core.c | 2 ++ + drivers/block/Kconfig | 2 +- + drivers/irqchip/irq-gic-v3-its.c | 2 ++ + drivers/net/Kconfig | 2 +- + drivers/virtio/Kconfig | 4 ++-- + drivers/virtio/virtio_mmio.c | 3 ++- + drivers/virtio/virtio_pci_common.c | 2 ++ + drivers/virtio/virtio_pci_common.h | 1 + + drivers/virtio/virtio_ring.c | 3 +++ + include/linux/virtcca_cvm_domain.h | 29 +++++++++++++++++++++++++++ + 11 files changed, 74 insertions(+), 5 deletions(-) + create mode 100644 include/linux/virtcca_cvm_domain.h + +diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c +index 2d4a26a297b6..e5368d3e1bfe 100644 +--- a/arch/arm64/kernel/virtcca_cvm_guest.c ++++ b/arch/arm64/kernel/virtcca_cvm_guest.c +@@ -7,6 +7,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -69,6 +70,7 @@ bool is_virtcca_cvm_world(void) + { + return cvm_guest_enable; + } ++EXPORT_SYMBOL_GPL(is_virtcca_cvm_world); + + static int change_page_range_cvm(pte_t *ptep, unsigned long addr, void *data) + { +@@ -120,3 +122,30 @@ int set_cvm_memory_decrypted(unsigned long addr, int numpages) + { + return __set_memory_encrypted(addr, numpages, false); + } ++ ++/* ++ * struct io_tlb_no_swiotlb_mem - whether use the ++ * bounce buffer mechanism or not ++ * @for_alloc: %true if the pool is used for memory allocation. ++ * Here it is set to %false, to force devices to use direct dma operations. ++ * ++ * @force_bounce: %true if swiotlb bouncing is forced. ++ * Here it is set to %false, to force devices to use direct dma operations. ++ */ ++static struct io_tlb_mem io_tlb_no_swiotlb_mem = { ++ .for_alloc = false, ++ .force_bounce = false, ++}; ++ ++void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable) ++{ ++ if (!is_virtcca_cvm_world()) ++ return; ++ ++ if (enable) ++ swiotlb_dev_init(dev); ++ else ++ dev->dma_io_tlb_mem = &io_tlb_no_swiotlb_mem; ++} ++EXPORT_SYMBOL_GPL(enable_swiotlb_for_cvm_dev); ++ +diff --git a/drivers/base/core.c b/drivers/base/core.c +index cb323700e952..66d7dc60fa76 100644 +--- a/drivers/base/core.c ++++ b/drivers/base/core.c +@@ -32,6 +32,7 @@ + #include + #include + #include /* for dma_default_coherent */ ++#include + + #include "base.h" + #include "physical_location.h" +@@ -3137,6 +3138,7 @@ void device_initialize(struct device *dev) + dev->dma_coherent = dma_default_coherent; + #endif + swiotlb_dev_init(dev); ++ enable_swiotlb_for_cvm_dev(dev, false); + } + EXPORT_SYMBOL_GPL(device_initialize); + +diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig +index 5b9d4aaebb81..8a124d79888c 100644 +--- a/drivers/block/Kconfig ++++ b/drivers/block/Kconfig +@@ -347,7 +347,7 @@ config XEN_BLKDEV_BACKEND + + + config VIRTIO_BLK +- tristate "Virtio block driver" ++ bool "Virtio block driver" + depends on VIRTIO + select SG_POOL + help +diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c +index 252c9cc5dc40..d8eef29e540c 100644 +--- a/drivers/irqchip/irq-gic-v3-its.c ++++ b/drivers/irqchip/irq-gic-v3-its.c +@@ -32,6 +32,7 @@ + #ifdef CONFIG_HISI_VIRTCCA_GUEST + #include + #include ++#include + #endif + + #include +@@ -6191,6 +6192,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, + #ifdef CONFIG_HISI_VIRTCCA_GUEST + if (is_virtcca_cvm_world()) { + device_initialize(&cvm_alloc_device); ++ enable_swiotlb_for_cvm_dev(&cvm_alloc_device, true); + raw_spin_lock_init(&cvm_its_lock); + } + #endif +diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig +index 3531558c7b51..44b2108dafea 100644 +--- a/drivers/net/Kconfig ++++ b/drivers/net/Kconfig +@@ -438,7 +438,7 @@ config VETH + versa. + + config VIRTIO_NET +- tristate "Virtio network driver" ++ bool "Virtio network driver" + depends on VIRTIO + select NET_FAILOVER + help +diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig +index 0a53a61231c2..29eb6cea187d 100644 +--- a/drivers/virtio/Kconfig ++++ b/drivers/virtio/Kconfig +@@ -11,7 +11,7 @@ config VIRTIO + or CONFIG_S390_GUEST. + + config VIRTIO_PCI_LIB +- tristate ++ bool + help + Modern PCI device implementation. This module implements the + basic probe and control for devices which are based on modern +@@ -48,7 +48,7 @@ config VIRTIO_HARDEN_NOTIFICATION + If unsure, say N. + + config VIRTIO_PCI +- tristate "PCI driver for virtio devices" ++ bool "PCI driver for virtio devices" + depends on PCI + select VIRTIO_PCI_LIB + select VIRTIO +diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c +index 59892a31cf76..fb4abd2afc46 100644 +--- a/drivers/virtio/virtio_mmio.c ++++ b/drivers/virtio/virtio_mmio.c +@@ -70,7 +70,7 @@ + #include + #include + #include +- ++#include + + + /* The alignment to use between consumer and producer parts of vring. +@@ -619,6 +619,7 @@ static int virtio_mmio_probe(struct platform_device *pdev) + unsigned long magic; + int rc; + ++ enable_swiotlb_for_cvm_dev(&pdev->dev, true); + vm_dev = kzalloc(sizeof(*vm_dev), GFP_KERNEL); + if (!vm_dev) + return -ENOMEM; +diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c +index 64dfa54d702f..95d9a3aed68c 100644 +--- a/drivers/virtio/virtio_pci_common.c ++++ b/drivers/virtio/virtio_pci_common.c +@@ -525,6 +525,8 @@ static int virtio_pci_probe(struct pci_dev *pci_dev, + struct virtio_pci_device *vp_dev, *reg_dev = NULL; + int rc; + ++ enable_swiotlb_for_cvm_dev(&pci_dev->dev, true); ++ + /* allocate our structure and fill it out */ + vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL); + if (!vp_dev) +diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h +index 4b773bd7c58c..0f0e19f94d30 100644 +--- a/drivers/virtio/virtio_pci_common.h ++++ b/drivers/virtio/virtio_pci_common.h +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + + struct virtio_pci_vq_info { + /* the actual virtqueue */ +diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c +index 80669e05bf0e..2b702392d20b 100644 +--- a/drivers/virtio/virtio_ring.c ++++ b/drivers/virtio/virtio_ring.c +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + #include + + #ifdef DEBUG +@@ -294,6 +295,8 @@ static bool vring_use_dma_api(const struct virtio_device *vdev) + if (xen_domain()) + return true; + ++ if (virtcca_cvm_domain()) ++ return true; + return false; + } + +diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h +new file mode 100644 +index 000000000000..41b2d15082ab +--- /dev/null ++++ b/include/linux/virtcca_cvm_domain.h +@@ -0,0 +1,29 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++/* ++ * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++ ++#ifndef __VIRTCCA_CVM_DOMAIN_H ++#define __VIRTCCA_CVM_DOMAIN_H ++ ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ ++#include ++static inline bool virtcca_cvm_domain(void) ++{ ++ return is_virtcca_cvm_world(); ++} ++ ++extern void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable); ++ ++#else ++static inline bool virtcca_cvm_domain(void) ++{ ++ return false; ++} ++ ++static inline void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable) {} ++ ++#endif ++ ++#endif /* __VIRTCCA_CVM_DOMAIN_H */ +-- +2.34.1 + diff --git a/tee_blackbox/guest/0003-improve-nfs-workqueue-bind-method.patch b/tee_blackbox/guest/0003-improve-nfs-workqueue-bind-method.patch new file mode 100644 index 0000000000000000000000000000000000000000..040e7b31c0ce8555f14a78da56318c6feec23347 --- /dev/null +++ b/tee_blackbox/guest/0003-improve-nfs-workqueue-bind-method.patch @@ -0,0 +1,70 @@ +From 9cd63793447bfaa054253f815fd73942f40e2140 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Thu, 2 Jan 2025 17:28:28 +0800 +Subject: [PATCH 3/6] improve nfs workqueue bind method + +--- + fs/nfs/inode.c | 6 ++++++ + net/sunrpc/sched.c | 11 +++++++++++ + 2 files changed, 17 insertions(+) + +diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c +index 56bbf59bda3c..d05298529c5d 100644 +--- a/fs/nfs/inode.c ++++ b/fs/nfs/inode.c +@@ -41,6 +41,7 @@ + #include + #include + #include ++#include + + #include "nfs4_fs.h" + #include "callback.h" +@@ -2411,6 +2412,11 @@ static int nfsiod_start(void) + { + struct workqueue_struct *wq; + dprintk("RPC: creating workqueue nfsiod\n"); ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (virtcca_cvm_domain()) ++ wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM, 0); ++ else ++#endif + wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (wq == NULL) + return -ENOMEM; +diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c +index cef623ea1506..f63af54baa58 100644 +--- a/net/sunrpc/sched.c ++++ b/net/sunrpc/sched.c +@@ -21,6 +21,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -1282,10 +1283,20 @@ static int rpciod_start(void) + /* + * Create the rpciod thread and wait for it to start. + */ ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (virtcca_cvm_domain()) ++ wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 0); ++ else ++#endif + wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (!wq) + goto out_failed; + rpciod_workqueue = wq; ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (virtcca_cvm_domain()) ++ wq = alloc_workqueue("xprtiod", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); ++ else ++#endif + wq = alloc_workqueue("xprtiod", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!wq) + goto free_rpciod; +-- +2.34.1 + diff --git a/tee_blackbox/guest/0004-improve-virtio-get-queue-method.patch b/tee_blackbox/guest/0004-improve-virtio-get-queue-method.patch new file mode 100644 index 0000000000000000000000000000000000000000..0c965b1ec3a54dd3f1294041b0c32e9d350df252 --- /dev/null +++ b/tee_blackbox/guest/0004-improve-virtio-get-queue-method.patch @@ -0,0 +1,37 @@ +From 8fdc2470ec26a9f9dc9c530bc6a67d88cb2161d7 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Thu, 2 Jan 2025 17:30:56 +0800 +Subject: [PATCH 4/6] improve virtio get queue method + +--- + net/core/dev.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/net/core/dev.c b/net/core/dev.c +index 75e94b327b31..588917fd7b72 100644 +--- a/net/core/dev.c ++++ b/net/core/dev.c +@@ -148,6 +148,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -4230,8 +4231,11 @@ u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, + if (queue_index < 0 || skb->ooo_okay || + queue_index >= dev->real_num_tx_queues) { + int new_index = get_xps_queue(dev, sb_dev, skb); +- ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (new_index < 0 || virtcca_cvm_domain()) ++#else + if (new_index < 0) ++#endif + new_index = skb_tx_hash(dev, sb_dev, skb); + + if (queue_index != new_index && sk && +-- +2.34.1 + diff --git a/tee_blackbox/guest/0005-add-ipi-direct-inject-method.patch b/tee_blackbox/guest/0005-add-ipi-direct-inject-method.patch new file mode 100644 index 0000000000000000000000000000000000000000..9ba0a64047fd17147b34ea585b34731f4ea804e2 --- /dev/null +++ b/tee_blackbox/guest/0005-add-ipi-direct-inject-method.patch @@ -0,0 +1,294 @@ +From 78c920eda2a609030e765a98f41a40f6ed300ad3 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Fri, 3 Jan 2025 10:47:14 +0800 +Subject: [PATCH 5/6] add ipi direct inject method + +--- + arch/arm64/include/asm/virtcca_cvm_guest.h | 8 +++ + arch/arm64/kernel/smp.c | 17 +++++- + arch/arm64/kernel/virtcca_cvm_guest.c | 52 ++++++++++++++++++ + include/linux/virtcca_cvm_domain.h | 61 +++++++++++++++++++++- + kernel/sched/idle.c | 30 +++++++++++ + 5 files changed, 165 insertions(+), 3 deletions(-) + +diff --git a/arch/arm64/include/asm/virtcca_cvm_guest.h b/arch/arm64/include/asm/virtcca_cvm_guest.h +index c1ac1a9cd537..17447af7bfcd 100644 +--- a/arch/arm64/include/asm/virtcca_cvm_guest.h ++++ b/arch/arm64/include/asm/virtcca_cvm_guest.h +@@ -8,6 +8,14 @@ + #ifdef CONFIG_HISI_VIRTCCA_GUEST + struct device; + ++extern struct cpumask cvm_spin_cpumask; ++DECLARE_PER_CPU(unsigned int, cvm_unpark_idle_notify); ++DECLARE_PER_CPU(unsigned int, cvm_park_idle_state); ++ ++extern void smp_cross_call(const struct cpumask *target, unsigned int ipinr); ++ ++extern void cvm_send_call_function_single_ipi(int cpu, int ipi_type); ++ + extern int set_cvm_memory_encrypted(unsigned long addr, int numpages); + + extern int set_cvm_memory_decrypted(unsigned long addr, int numpages); +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 50f6576f1b31..7819ad96f17f 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -34,6 +34,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -805,7 +806,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { + [IPI_WAKEUP] = "CPU wake-up interrupts", + }; + +-static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); ++void smp_cross_call(const struct cpumask *target, unsigned int ipinr); + + unsigned long irq_err_count; + +@@ -832,6 +833,12 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) + + void arch_send_call_function_single_ipi(int cpu) + { ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (virtcca_cvm_domain()) { ++ cvm_send_call_function_single_ipi(cpu, IPI_CALL_FUNC); ++ return; ++ } ++#endif + smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); + } + +@@ -958,7 +965,7 @@ static irqreturn_t ipi_handler(int irq, void *data) + return IRQ_HANDLED; + } + +-static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) ++void smp_cross_call(const struct cpumask *target, unsigned int ipinr) + { + trace_ipi_raise(target, ipi_types[ipinr]); + __ipi_send_mask(ipi_desc[ipinr], target); +@@ -1027,6 +1034,12 @@ void __init set_smp_ipi_range(int ipi_base, int n) + + void arch_smp_send_reschedule(int cpu) + { ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (virtcca_cvm_domain()) { ++ cvm_send_call_function_single_ipi(cpu, IPI_RESCHEDULE); ++ return; ++ } ++#endif + smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); + } + +diff --git a/arch/arm64/kernel/virtcca_cvm_guest.c b/arch/arm64/kernel/virtcca_cvm_guest.c +index e5368d3e1bfe..4990bfe5292e 100644 +--- a/arch/arm64/kernel/virtcca_cvm_guest.c ++++ b/arch/arm64/kernel/virtcca_cvm_guest.c +@@ -8,6 +8,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -72,6 +73,57 @@ bool is_virtcca_cvm_world(void) + } + EXPORT_SYMBOL_GPL(is_virtcca_cvm_world); + ++struct cpumask cvm_spin_cpumask; ++static DEFINE_SPINLOCK(ipi_passthrough_lock); ++DEFINE_PER_CPU(unsigned int, cvm_unpark_idle_notify); ++DEFINE_PER_CPU(unsigned int, cvm_park_idle_state); ++ ++void cvm_send_call_function_single_ipi(int cpu, int ipi_type) { ++ if (cvm_spin_cpumask_test_cpu(cpu) && is_cvm_in_park_idle_state(cpu)) ++ cvm_set_unpark_idle_notify(cpu); ++ else ++ smp_cross_call(cpumask_of(cpu), ipi_type); ++} ++ ++static ssize_t soft_ipi_passthrough_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ spin_lock(&ipi_passthrough_lock); ++ if (cpumask_parse(buf, &cvm_spin_cpumask) < 0) { ++ return -EINVAL; ++ } ++ spin_unlock(&ipi_passthrough_lock); ++ ++ return count; ++} ++ ++static ssize_t soft_ipi_passthrough_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int ret; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(&cvm_spin_cpumask)); ++ ++ return ret; ++} ++ ++static struct kobj_attribute soft_ipi_passthrough_attr = __ATTR_RW(soft_ipi_passthrough); ++ ++static int __init soft_ipi_passthrough_init(void) ++{ ++ unsigned int cpu; ++ unsigned int max_nr_cpus = num_possible_cpus(); ++ ++ cpumask_clear(&cvm_spin_cpumask); ++ for (cpu = 0; cpu < max_nr_cpus; cpu++) { ++ cvm_clear_unpark_idle_notify(cpu); ++ } ++ ++ return sysfs_create_file(kernel_kobj, &soft_ipi_passthrough_attr.attr); ++} ++late_initcall(soft_ipi_passthrough_init); ++ + static int change_page_range_cvm(pte_t *ptep, unsigned long addr, void *data) + { + bool encrypt = (bool)data; +diff --git a/include/linux/virtcca_cvm_domain.h b/include/linux/virtcca_cvm_domain.h +index 41b2d15082ab..b14a49730fd5 100644 +--- a/include/linux/virtcca_cvm_domain.h ++++ b/include/linux/virtcca_cvm_domain.h +@@ -16,13 +16,72 @@ static inline bool virtcca_cvm_domain(void) + + extern void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable); + ++static inline bool cvm_spin_cpumask_test_cpu(int cpu) ++{ ++ return cpumask_test_cpu(cpu, &cvm_spin_cpumask); ++} ++ ++enum cvm_park_idle_state { ++ CVM_PARK_RUNNING, ++ CVM_PARK_IDLE ++}; ++ ++static inline bool is_cvm_in_park_idle_state(int cpu) ++{ ++ return (per_cpu(cvm_park_idle_state, cpu) == CVM_PARK_IDLE); ++} ++ ++static inline void cvm_set_park_idle_state(int cpu, ++ enum cvm_park_idle_state state) ++{ ++ per_cpu(cvm_park_idle_state, cpu) = state; ++} ++ ++static inline bool is_cvm_unpark_idle_notify_set(int cpu) ++{ ++ return (per_cpu(cvm_unpark_idle_notify, cpu) == 1); ++} ++ ++static inline void cvm_set_unpark_idle_notify(int cpu) ++{ ++ per_cpu(cvm_unpark_idle_notify, cpu) = 1; ++} ++ ++static inline void cvm_clear_unpark_idle_notify(int cpu) ++{ ++ per_cpu(cvm_unpark_idle_notify, cpu) = 0; ++} ++ + #else + static inline bool virtcca_cvm_domain(void) + { + return false; + } + +-static inline void enable_swiotlb_for_cvm_dev(struct device *dev, bool enable) {} ++static inline void enable_swiotlb_for_cvm_dev(struct device *dev, ++ bool enable) {} ++ ++static inline bool cvm_spin_cpumask_test_cpu(int cpu) ++{ ++ return false; ++} ++ ++static inline bool is_cvm_in_park_idle_state(int cpu) ++{ ++ return false; ++} ++ ++static inline void cvm_set_park_idle_state(int cpu, ++ enum cvm_park_idle_state state) {} ++ ++static inline bool is_cvm_unpark_idle_notify_set(int cpu) ++{ ++ return false; ++} ++ ++static inline void cvm_set_unpark_idle_notify(int cpu) {} ++ ++static inline void cvm_clear_unpark_idle_notify(int cpu) {} + + #endif + +diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c +index 5007b25c5bc6..2397b06fac26 100644 +--- a/kernel/sched/idle.c ++++ b/kernel/sched/idle.c +@@ -7,6 +7,8 @@ + * tasks which are handled in sched/fair.c ) + */ + ++#include ++ + /* Linker adds these: start and end of __cpuidle functions */ + extern char __cpuidle_text_start[], __cpuidle_text_end[]; + +@@ -229,6 +231,28 @@ static void cpuidle_idle_call(void) + local_irq_enable(); + } + ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++static noinline int cvm_cpu_idle_poll(void) ++{ ++ unsigned int cpu = smp_processor_id(); ++ ++ if (!virtcca_cvm_domain() || !cvm_spin_cpumask_test_cpu(cpu)) { ++ return 0; ++ } ++ trace_cpu_idle(0, smp_processor_id()); ++ local_irq_enable(); ++ ++ cvm_set_park_idle_state(cpu, CVM_PARK_IDLE); ++ while (!tif_need_resched() && !is_cvm_unpark_idle_notify_set(cpu)) { ++ cpu_relax(); ++ } ++ cvm_clear_unpark_idle_notify(cpu); ++ cvm_set_park_idle_state(cpu, CVM_PARK_RUNNING); ++ trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id()); ++ ++ return 1; ++} ++#endif + /* + * Generic idle loop implementation + * +@@ -269,6 +293,12 @@ static void do_idle(void) + arch_cpu_idle_enter(); + rcu_nocb_flush_deferred_wakeup(); + ++#ifdef CONFIG_HISI_VIRTCCA_GUEST ++ if (cvm_cpu_idle_poll()) { ++ arch_cpu_idle_exit(); ++ break; ++ } ++#endif + /* + * In poll mode we reenable interrupts and spin. Also if we + * detected in the wakeup from idle path that the tick +-- +2.34.1 + diff --git a/tee_blackbox/guest/0006-add-poll-method-for-nfs.patch b/tee_blackbox/guest/0006-add-poll-method-for-nfs.patch new file mode 100644 index 0000000000000000000000000000000000000000..82c5df627172e4e66b581bc33b3153870ab7a187 --- /dev/null +++ b/tee_blackbox/guest/0006-add-poll-method-for-nfs.patch @@ -0,0 +1,500 @@ +From 3efe291ecdcea8276d6778aa54762046c45c2dd4 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Fri, 10 Jan 2025 09:40:18 +0800 +Subject: [PATCH 6/6] add poll method for nfs + +--- + drivers/char/Kconfig | 7 + + drivers/char/Makefile | 1 + + drivers/char/poll_cq_kthread.c | 322 ++++++++++++++++++ + drivers/char/poll_cq_kthread.h | 17 + + drivers/infiniband/core/cq.c | 42 +++ + .../infiniband/core/ib_core_poll_kthread.h | 22 ++ + 6 files changed, 411 insertions(+) + create mode 100644 drivers/char/poll_cq_kthread.c + create mode 100644 drivers/char/poll_cq_kthread.h + create mode 100644 drivers/infiniband/core/ib_core_poll_kthread.h + +diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig +index ea7ace87a9df..367b1d5ec455 100644 +--- a/drivers/char/Kconfig ++++ b/drivers/char/Kconfig +@@ -442,4 +442,11 @@ config ADI + and SSM (Silicon Secured Memory). Intended consumers of this + driver include crash and makedumpfile. + ++config NFS_POLL_CQ_KTHREAD ++ tristate "Improve NFS using poll method" ++ default m ++ help ++ A Network File System read/write acceleration method ++ based on dynamic polling of partner kernel threads. ++ + endmenu +diff --git a/drivers/char/Makefile b/drivers/char/Makefile +index 109af71c5416..9ec7b00548f6 100644 +--- a/drivers/char/Makefile ++++ b/drivers/char/Makefile +@@ -46,3 +46,4 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o + obj-$(CONFIG_XILLYBUS_CLASS) += xillybus/ + obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o + obj-$(CONFIG_ADI) += adi.o ++obj-$(CONFIG_NFS_POLL_CQ_KTHREAD) += poll_cq_kthread.o +diff --git a/drivers/char/poll_cq_kthread.c b/drivers/char/poll_cq_kthread.c +new file mode 100644 +index 000000000000..9437cf819cd8 +--- /dev/null ++++ b/drivers/char/poll_cq_kthread.c +@@ -0,0 +1,322 @@ ++#include ++#include ++ ++#include ++#include ++ ++#include ++#include ++#include ++ ++#include "./poll_cq_kthread.h" ++#include "../infiniband/core/ib_core_poll_kthread.h" ++ ++#define INVALID_IB_POLL_CORE -1 ++/* A lock for concurrent access. */ ++static DEFINE_SPINLOCK(polling_kthread_spinlock); ++unsigned int poll_core; ++ ++struct task_struct *poll_cq_thread; ++struct task_struct *waker_polling_thread; ++struct polling_kthread ib_cq_polling_kthread = { ++ .use_polling_kthread = 0, ++ .debug_cq_poll_stat = 0, ++ .cqe_polling_cnt = 0, ++}; ++ ++LIST_HEAD(ib_cq_poll_list); ++/* A lock for concurrent access. */ ++static DEFINE_SPINLOCK(cq_list_lock); ++ ++void add_cq_to_poll_list(void *cq) ++{ ++ unsigned long flags; ++ cq_poll_node_t *cq_node; ++ ++ cq_node = (cq_poll_node_t *)kmalloc(sizeof(cq_poll_node_t), GFP_ATOMIC); ++ if (!cq_node) { ++ pr_err("add cq to poll list failed: kmalloc for cq_node failed.\n"); ++ return; ++ } ++ cq_node->cq = cq; ++ cq_node->time_used_ns = 0; ++ cq_node->poll_cq_cnt = 0; ++ cq_node->max_time_ns = 0; ++ ++ spin_lock_irqsave(&cq_list_lock, flags); ++ list_add_tail(&cq_node->list, &ib_cq_poll_list); ++ spin_unlock_irqrestore(&cq_list_lock, flags); ++} ++ ++void del_cq_from_poll_list(void *del_cq) ++{ ++ cq_poll_node_t *poll_node_entry, *poll_node_next; ++ void *curr_cq; ++ void *cq = del_cq; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&cq_list_lock, flags); ++ list_for_each_entry_safe(poll_node_entry, poll_node_next, ++ &ib_cq_poll_list, list) { ++ curr_cq = poll_node_entry->cq; ++ if (curr_cq == cq) { ++ list_del(&poll_node_entry->list); ++ kfree(poll_node_entry); ++ break; ++ } ++ } ++ spin_unlock_irqrestore(&cq_list_lock, flags); ++} ++ ++void clear_cq_poll_list(void) ++{ ++ unsigned long flags; ++ cq_poll_node_t *entry, *next; ++ ++ spin_lock_irqsave(&cq_list_lock, flags); ++ list_for_each_entry_safe(entry, next, &ib_cq_poll_list, list) { ++ list_del(&entry->list); ++ kfree(entry); ++ } ++ spin_unlock_irqrestore(&cq_list_lock, flags); ++ ++ INIT_LIST_HEAD(&ib_cq_poll_list); ++} ++ ++void cq_polling(void *data) ++{ ++ void *cq; ++ int completed = 0; ++ unsigned long flags; ++ u64 time_interval; ++ ktime_t start_time_stamp, end_time_stamp; ++ cq_poll_node_t *poll_node_entry, *poll_node_next; ++ ++ spin_lock_irqsave(&cq_list_lock, flags); ++ list_for_each_entry_safe(poll_node_entry, poll_node_next, ++ &ib_cq_poll_list, list) { ++ cq = poll_node_entry->cq; ++ if (!cq) { ++ WARN_ONCE(1, "got NULL CQ 0x%p in poll list\n", cq); ++ continue; ++ } ++ start_time_stamp = ktime_get(); ++ completed = ib_poll_cq_thread(cq); ++ end_time_stamp = ktime_get(); ++ if (ib_cq_polling_kthread.debug_cq_poll_stat && completed) { ++ time_interval = ktime_to_ns(ktime_sub(end_time_stamp, start_time_stamp)); ++ poll_node_entry->time_used_ns += time_interval; ++ poll_node_entry->poll_cq_cnt++; ++ if (poll_node_entry->max_time_ns < time_interval) { ++ poll_node_entry->max_time_ns = time_interval; ++ } ++ } ++ } ++ spin_unlock_irqrestore(&cq_list_lock, flags); ++} ++ ++void wakeup_and_poll(struct task_struct *awakened_thread) ++{ ++ wake_up_process(awakened_thread); ++ ++ cq_polling(NULL); ++} ++ ++int polling_thread(void *data) ++{ ++ while(1) { ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ wakeup_and_poll(waker_polling_thread); ++ } ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (unlikely(kthread_should_stop())) { ++ set_current_state(TASK_RUNNING); ++ break; ++ } ++ schedule(); ++ } ++ ++ return 0; ++} ++ ++int polling_awaken_thread(void *data) ++{ ++ while(1) { ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ wakeup_and_poll(poll_cq_thread); ++ } ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (unlikely(kthread_should_stop())) { ++ set_current_state(TASK_RUNNING); ++ break; ++ } ++ schedule(); ++ } ++ ++ return 0; ++} ++ ++/* sysfs ui */ ++ ++static ssize_t ib_core_poll_cpu_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int ret; ++ ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ ret = sprintf(buf, "Current polling core is: %u\n", poll_core); ++ } else { ++ ret = sprintf(buf, "Current polling thread is down\n"); ++ } ++ ++ return ret; ++} ++ ++static ssize_t ib_core_poll_cpu_store(struct kobject *kobj, ++ struct kobj_attribute *attr, const char *buf, ++ size_t count) ++{ ++ int input_poll_core; ++ unsigned int max_nr_cores = num_possible_cpus(); ++ ++ if (sscanf(buf, "%d", &input_poll_core) < 0) { ++ pr_err("Invalid input, input format: e.g. 8\n"); ++ return -EINVAL; ++ } ++ spin_lock(&polling_kthread_spinlock); ++ if (input_poll_core == INVALID_IB_POLL_CORE) { ++ ib_cq_polling_kthread.use_polling_kthread = 0; ++ set_kthread_polling_ctx(&ib_cq_polling_kthread); ++ clear_cq_poll_list(); ++ goto out; ++ } ++ if (input_poll_core < 0 || input_poll_core >= max_nr_cores) { ++ pr_err("Invalid CPU core ID. Valid range is 0 to %u.\n", max_nr_cores - 1); ++ goto err_inval; ++ } ++ ib_cq_polling_kthread.use_polling_kthread = 1; ++ set_kthread_polling_ctx(&ib_cq_polling_kthread); ++ poll_core = (unsigned int)input_poll_core; ++ set_cpus_allowed_ptr(poll_cq_thread, cpumask_of(poll_core)); ++ set_cpus_allowed_ptr(waker_polling_thread, cpumask_of(poll_core)); ++ wake_up_process(waker_polling_thread); ++ ++out: ++ spin_unlock(&polling_kthread_spinlock); ++ return count; ++ ++err_inval: ++ spin_unlock(&polling_kthread_spinlock); ++ return -EINVAL; ++} ++ ++static ssize_t ib_core_poll_stat_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int ret; ++ unsigned long flags; ++ unsigned long avg_poll_time; ++ cq_poll_node_t *poll_node_entry, *poll_node_next; ++ ++ ib_cq_polling_kthread = get_kthread_polling_ctx(); ++ ret = sprintf(buf, "kthread polled cqes cnt: %lu\n", ++ ib_cq_polling_kthread.cqe_polling_cnt); ++ ++ spin_lock_irqsave(&cq_list_lock, flags); ++ printk("cq\t\tpoll cnt\tavg poll time(ns)\tmax poll time(ns)\n"); ++ list_for_each_entry_safe(poll_node_entry, poll_node_next, ++ &ib_cq_poll_list, list) { ++ avg_poll_time = poll_node_entry->time_used_ns / poll_node_entry->poll_cq_cnt; ++ printk("%p\t%lu\t\t%lu\t\t\t%lu", poll_node_entry->cq, ++ poll_node_entry->poll_cq_cnt, avg_poll_time, ++ poll_node_entry->max_time_ns); ++ } ++ spin_unlock_irqrestore(&cq_list_lock, flags); ++ ++ return ret; ++} ++ ++static ssize_t ib_core_poll_stat_store(struct kobject *kobj, ++ struct kobj_attribute *attr, const char *buf, ++ size_t count) ++{ ++ unsigned int input_debug_cq_poll_stat; ++ ++ if (sscanf(buf, "%u", &input_debug_cq_poll_stat) < 0) { ++ pr_err("Invalid input, input format: <0/1> for on/off\n"); ++ return -EINVAL; ++ } ++ spin_lock(&polling_kthread_spinlock); ++ ib_cq_polling_kthread.debug_cq_poll_stat = input_debug_cq_poll_stat; ++ set_kthread_polling_ctx(&ib_cq_polling_kthread); ++ spin_unlock(&polling_kthread_spinlock); ++ ++ return count; ++} ++ ++static struct kobj_attribute ib_core_poll_cpu_attr = __ATTR_RW(ib_core_poll_cpu); ++static struct kobj_attribute ib_core_poll_stat_attr = __ATTR_RW(ib_core_poll_stat); ++ ++static int __init mod_poll_cq_kthread_init(void) ++{ ++ int ret; ++ ++ ret = sysfs_create_file(kernel_kobj, &ib_core_poll_cpu_attr.attr); ++ if (ret) { ++ return ret; ++ } ++ ret = sysfs_create_file(kernel_kobj, &ib_core_poll_stat_attr.attr); ++ if (ret) { ++ return ret; ++ } ++ ++ /* init poll thread */ ++ if (!poll_cq_thread) ++ poll_cq_thread = kthread_create(polling_thread, NULL, "polling_thread"); ++ if (IS_ERR(poll_cq_thread)) { ++ ret = PTR_ERR(poll_cq_thread); ++ poll_cq_thread = NULL; ++ goto out; ++ } ++ if (!waker_polling_thread) ++ waker_polling_thread = kthread_create(polling_awaken_thread, ++ NULL, "polling_awaken_thread"); ++ if (IS_ERR(waker_polling_thread)) { ++ ret = PTR_ERR(waker_polling_thread); ++ waker_polling_thread = NULL; ++ goto out; ++ } ++ ib_cq_polling_kthread.add_to_poll_list = add_cq_to_poll_list; ++ ib_cq_polling_kthread.del_from_poll_list = del_cq_from_poll_list; ++ set_kthread_polling_ctx(&ib_cq_polling_kthread); ++ ++out: ++ return 0; ++} ++late_initcall(mod_poll_cq_kthread_init); ++ ++static void mod_poll_cq_kthread_exit(void) ++{ ++ if (poll_cq_thread) { ++ kthread_stop(poll_cq_thread); ++ } ++ if (waker_polling_thread) { ++ kthread_stop(waker_polling_thread); ++ } ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ ib_cq_polling_kthread.use_polling_kthread = 0; ++ ib_cq_polling_kthread.debug_cq_poll_stat = 0; ++ set_kthread_polling_ctx(&ib_cq_polling_kthread); ++ clear_cq_poll_list(); ++ } ++ ++ sysfs_remove_file(kernel_kobj, &ib_core_poll_cpu_attr.attr); ++ sysfs_remove_file(kernel_kobj, &ib_core_poll_stat_attr.attr); ++} ++module_exit(mod_poll_cq_kthread_exit); ++ ++MODULE_DESCRIPTION("cq polling kthread init and config"); ++MODULE_LICENSE("GPL v2"); ++MODULE_AUTHOR("Chen Haotian "); +\ No newline at end of file +diff --git a/drivers/char/poll_cq_kthread.h b/drivers/char/poll_cq_kthread.h +new file mode 100644 +index 000000000000..df5b6e161819 +--- /dev/null ++++ b/drivers/char/poll_cq_kthread.h +@@ -0,0 +1,17 @@ ++#ifndef __POLL_CQ_KTHREAD_H ++#define __POLL_CQ_KTHREAD_H ++ ++typedef struct cq_poll_node ++{ ++ struct list_head list; ++ void *cq; ++ unsigned long time_used_ns; ++ unsigned long poll_cq_cnt; ++ unsigned long max_time_ns; ++} cq_poll_node_t; ++ ++extern void add_cq_to_poll_list(void *cq); ++extern void del_cq_from_poll_list(void *del_cq); ++extern void clear_cq_poll_list(void); ++ ++#endif /* __POLL_CQ_KTHREAD_H */ +\ No newline at end of file +diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c +index a70876a0a231..aed5cdc47e69 100644 +--- a/drivers/infiniband/core/cq.c ++++ b/drivers/infiniband/core/cq.c +@@ -7,6 +7,7 @@ + #include + + #include "core_priv.h" ++#include "./ib_core_poll_kthread.h" + + #include + /* Max size for shared CQ, may require tuning */ +@@ -195,6 +196,39 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) + queue_work(cq->comp_wq, &cq->work); + } + ++static struct polling_kthread ib_cq_polling_kthread = { ++ .use_polling_kthread = 0, ++ .debug_cq_poll_stat = 0, ++ .cqe_polling_cnt = 0, ++}; ++ ++struct polling_kthread get_kthread_polling_ctx(void) ++{ ++ return ib_cq_polling_kthread; ++} ++EXPORT_SYMBOL(get_kthread_polling_ctx); ++ ++void set_kthread_polling_ctx(struct polling_kthread *polling_ctx) ++{ ++ memcpy(&ib_cq_polling_kthread, polling_ctx, sizeof(struct polling_kthread)); ++} ++EXPORT_SYMBOL(set_kthread_polling_ctx); ++ ++int ib_poll_cq_thread(void *data) { ++ int completed; ++ struct ib_wc wcs[IB_POLL_BATCH]; ++ struct ib_cq *cq = (struct ib_cq *)data; ++ ++ memcpy(wcs, cq->wc, sizeof(wcs)); ++ completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, wcs, ++ IB_POLL_BATCH); ++ if (ib_cq_polling_kthread.debug_cq_poll_stat) { ++ ib_cq_polling_kthread.cqe_polling_cnt += completed; ++ } ++ return completed; ++} ++EXPORT_SYMBOL(ib_poll_cq_thread); ++ + /** + * __ib_alloc_cq - allocate a completion queue + * @dev: device to allocate the CQ for +@@ -243,6 +277,10 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe, + + rdma_dim_init(cq); + ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ cq->poll_ctx = IB_POLL_DIRECT; ++ ib_cq_polling_kthread.add_to_poll_list(cq); ++ } + switch (cq->poll_ctx) { + case IB_POLL_DIRECT: + cq->comp_handler = ib_cq_completion_direct; +@@ -338,6 +376,10 @@ void ib_free_cq(struct ib_cq *cq) + WARN_ON_ONCE(1); + } + ++ if (ib_cq_polling_kthread.use_polling_kthread) { ++ ib_cq_polling_kthread.del_from_poll_list(cq); ++ } ++ + rdma_dim_destroy(cq); + trace_cq_free(cq); + ret = cq->device->ops.destroy_cq(cq, NULL); +diff --git a/drivers/infiniband/core/ib_core_poll_kthread.h b/drivers/infiniband/core/ib_core_poll_kthread.h +new file mode 100644 +index 000000000000..5717e0e94015 +--- /dev/null ++++ b/drivers/infiniband/core/ib_core_poll_kthread.h +@@ -0,0 +1,22 @@ ++#ifndef __IB_CORE_POLL_KTHREAD_H ++#define __IB_CORE_POLL_KTHREAD_H ++ ++struct polling_kthread ++{ ++ /* configs */ ++ int use_polling_kthread; ++ unsigned int debug_cq_poll_stat; ++ ++ /* vars */ ++ unsigned long cqe_polling_cnt; ++ ++ /* ops */ ++ void (*add_to_poll_list)(void *new_entry); ++ void (*del_from_poll_list)(void *deleted_entry); ++}; ++ ++extern int ib_poll_cq_thread(void *data); ++extern struct polling_kthread get_kthread_polling_ctx(void); ++extern void set_kthread_polling_ctx(struct polling_kthread *polling_ctx); ++ ++#endif /* __IB_CORE_POLL_KTHREAD_H */ +-- +2.34.1 + diff --git a/tee_blackbox/host/0001-cvm-dynamic-wfx-config.patch b/tee_blackbox/host/0001-cvm-dynamic-wfx-config.patch new file mode 100644 index 0000000000000000000000000000000000000000..b184a6ced7564370751e377290f08243ff87a435 --- /dev/null +++ b/tee_blackbox/host/0001-cvm-dynamic-wfx-config.patch @@ -0,0 +1,75 @@ +From c782ca58c9b66917baf43e737bbf256645543177 Mon Sep 17 00:00:00 2001 +From: Haotian Chen +Date: Wed, 8 Jan 2025 14:24:52 +0800 +Subject: [PATCH] cvm dynamic wfx config + +--- + arch/arm64/kvm/virtcca_cvm.c | 41 ++++++++++++++++++++++++++++++++++-- + 1 file changed, 39 insertions(+), 2 deletions(-) + +diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c +index 43f61e5be40b..be66d6c7b25f 100644 +--- a/arch/arm64/kvm/virtcca_cvm.c ++++ b/arch/arm64/kvm/virtcca_cvm.c +@@ -726,6 +726,41 @@ static int tmi_check_version(void) + return 0; + } + ++struct cpumask cvm_wfx_no_trap_mask; ++static DEFINE_SPINLOCK(cvm_wfx_config_lock); ++static ssize_t cvm_wfx_trap_config_store(struct kobject *kobj, ++ struct kobj_attribute *attr, ++ const char *buf, size_t count) ++{ ++ spin_lock(&cvm_wfx_config_lock); ++ if (cpumask_parse(buf, &cvm_wfx_no_trap_mask) < 0) { ++ return -EINVAL; ++ } ++ spin_unlock(&cvm_wfx_config_lock); ++ ++ return count; ++} ++ ++static ssize_t cvm_wfx_trap_config_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ int ret; ++ ++ ret = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(&cvm_wfx_no_trap_mask)); ++ ++ return ret; ++} ++ ++static struct kobj_attribute cvm_wfx_trap_config_attr = __ATTR_RW(cvm_wfx_trap_config); ++ ++static int __init cvm_wfx_trap_config_init(void) ++{ ++ cpumask_clear(&cvm_wfx_no_trap_mask); ++ ++ return sysfs_create_file(kernel_kobj, &cvm_wfx_trap_config_attr.attr); ++} ++late_initcall(cvm_wfx_trap_config_init); ++ + int kvm_tec_enter(struct kvm_vcpu *vcpu) + { + struct tmi_tec_run *run; +@@ -742,12 +777,14 @@ int kvm_tec_enter(struct kvm_vcpu *vcpu) + + run = tec->tec_run; + /* set/clear TWI TWE flags */ +- if (vcpu->arch.hcr_el2 & HCR_TWI) ++ if (vcpu->arch.hcr_el2 & HCR_TWI && ++ !cpumask_test_cpu(vcpu->vcpu_id, &cvm_wfx_no_trap_mask)) + run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFI; + else + run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFI; + +- if (vcpu->arch.hcr_el2 & HCR_TWE) ++ if (vcpu->arch.hcr_el2 & HCR_TWE && ++ !cpumask_test_cpu(vcpu->vcpu_id, &cvm_wfx_no_trap_mask)) + run->tec_entry.flags |= TEC_ENTRY_FLAG_TRAP_WFE; + else + run->tec_entry.flags &= ~TEC_ENTRY_FLAG_TRAP_WFE; +-- +2.34.1 +