From 3985ab2b7c57ac94b263da6aa0bb78cd93ed0607 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Tue, 10 Sep 2024 15:06:14 +0800 Subject: [PATCH 01/12] virtcca feature: vfio attach the device list to secure world virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Vfio attach the device list to secure world Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 1 + drivers/iommu/iommu.c | 27 +++++++++++++++++++ drivers/vfio/vfio_iommu_type1.c | 25 +++++++++++++++++ include/linux/iommu.h | 5 ++++ include/uapi/linux/vfio.h | 1 + 5 files changed, 59 insertions(+) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c index d247168c68f7..e824adbaeb65 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -1172,6 +1172,7 @@ int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device return ret; } +EXPORT_SYMBOL_GPL(virtcca_smmu_secure_dev_operator); /** * virtcca_smmu_device_init - Initialize the smmu security features diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 28f63ad432de..80bfc4b0e815 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2473,6 +2473,33 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, return ret; } +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_attach_secure_dev - Attach the device of iommu + * group to confidential virtual machine + * @domain: The handle of iommu domain + * @group: Iommu group + * + * Returns: + * %0 if attach the all devices success + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) +{ + struct group_device *gdev; + int ret = 0; + + mutex_lock(&group->mutex); + for_each_group_device(group, gdev) + ret = virtcca_smmu_secure_dev_operator(domain, gdev->dev); + mutex_unlock(&group->mutex); + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); +#endif + int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 9c4adf11dbbe..29cb28863195 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -77,6 +77,9 @@ struct vfio_iommu { bool dirty_page_tracking; struct list_head emulated_iommu_groups; bool dirty_log_get_no_clear; +#ifdef CONFIG_HISI_VIRTCCA_HOST + bool secure; +#endif }; struct vfio_domain { @@ -2454,10 +2457,23 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_domain; } +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (iommu->secure) + domain->domain->secure = true; +#endif + ret = iommu_attach_group(domain->domain, group->iommu_group); if (ret) goto out_domain; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (iommu->secure) { + ret = virtcca_attach_secure_dev(domain->domain, group->iommu_group); + if (ret) + goto out_domain; + } +#endif + /* Get aperture info */ geo = &domain->domain->geometry; if (vfio_iommu_aper_conflict(iommu, geo->aperture_start, @@ -2807,6 +2823,12 @@ static void *vfio_iommu_type1_open(unsigned long arg) case VFIO_TYPE1v2_IOMMU: iommu->v2 = true; break; +#ifdef CONFIG_HISI_VIRTCCA_HOST + case VFIO_TYPE1v2_S_IOMMU: + iommu->v2 = true; + iommu->secure = true; + break; +#endif default: kfree(iommu); return ERR_PTR(-EINVAL); @@ -2898,6 +2920,9 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, switch (arg) { case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: +#ifdef CONFIG_HISI_VIRTCCA_HOST + case VFIO_TYPE1v2_S_IOMMU: +#endif case VFIO_TYPE1_NESTING_IOMMU: case VFIO_UNMAP_ALL: return 1; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8fb93470a134..01cdee938990 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1762,4 +1762,9 @@ static inline void iopf_group_response(struct iopf_group *group, { } #endif /* CONFIG_IOMMU_IOPF */ + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); +int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); +#endif #endif /* __LINUX_IOMMU_H */ diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 2b78d03c0c15..03167374f348 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -25,6 +25,7 @@ #define VFIO_TYPE1_IOMMU 1 #define VFIO_SPAPR_TCE_IOMMU 2 #define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_TYPE1v2_S_IOMMU 12 /* * IOMMU enforces DMA cache coherence (ex. PCIe NoSnoop stripping). This * capability is subject to change as groups are added or removed. -- Gitee From a8078c65197c6f2034431a431780538c7c324f79 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Tue, 10 Sep 2024 16:45:19 +0800 Subject: [PATCH 02/12] virtcca feature: bind the kvm handle of cvm with smmu domain virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Bind the kvm handle of cvm with smmu domain Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/kvm/virtcca_cvm.c | 38 ++++++++++ drivers/iommu/iommu.c | 10 +++ drivers/vfio/group.c | 31 ++++++++ include/linux/iommu.h | 1 + include/linux/vfio.h | 3 + virt/kvm/vfio.c | 135 +++++++++++++++++++++++++++++++++++ virt/kvm/vfio.h | 7 ++ 7 files changed, 225 insertions(+) diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 93dd2392a1db..70588b9340fd 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include #include @@ -13,6 +14,8 @@ #include #include +#include "../virt/kvm/vfio.h" + /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); static unsigned long *cvm_vmid_bitmap; @@ -852,3 +855,38 @@ int kvm_init_cvm_vm(struct kvm *kvm) return 0; } + +/* + * Coda (Confidential device assignment) feature + * enable devices to pass directly to confidential virtual machines + */ + +/** + * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CVM + * @group: Iommu group + * + * Returns: + * %0 if smmu_domain has been associate cvm or associate cvm successfully + * %-ENXIO if the iommu group does not have smmu domain + */ +int cvm_arm_smmu_domain_set_kvm(void *group) +{ + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct iommu_domain *domain; + struct kvm *kvm; + + domain = virtcca_iommu_group_get_domain((struct iommu_group *)group); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain->kvm) + return 0; + + kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (kvm && kvm_is_virtcca_cvm(kvm)) + arm_smmu_domain->kvm = kvm; + + return 0; +} + diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 80bfc4b0e815..5d6fc4641aa7 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2498,6 +2498,16 @@ int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *g return ret; } EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); + +/* Obtain domain information through iommu group */ +struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group) +{ + if (iommu_group) + return iommu_group->domain; + + return NULL; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_group_get_domain); #endif int iommu_map(struct iommu_domain *domain, unsigned long iova, diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index 610a429c6191..33a66b6c5759 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -957,3 +957,34 @@ void vfio_group_cleanup(void) vfio.class = NULL; vfio_container_cleanup(); } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_vfio_file_iommu_group - Return the struct iommu_group for the vfio group file + * @file: VFIO group file + * + * The returned iommu_group is valid as long as a ref is held on the file. This + * returns a reference on the group. This function is deprecated, only the SPAPR + * path in kvm should call it. + */ +struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file) +{ + struct vfio_group *group = vfio_group_from_file(file); + struct iommu_group *iommu_group = NULL; + + if (!IS_ENABLED(CONFIG_HISI_VIRTCCA_HOST)) + return NULL; + + if (!group) + return NULL; + + mutex_lock(&group->group_lock); + if (group->iommu_group) { + iommu_group = group->iommu_group; + iommu_group_ref_get(iommu_group); + } + mutex_unlock(&group->group_lock); + return iommu_group; +} +EXPORT_SYMBOL_GPL(virtcca_vfio_file_iommu_group); +#endif diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 01cdee938990..92ac0a0cc87f 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1766,5 +1766,6 @@ static inline void iopf_group_response(struct iopf_group *group, #ifdef CONFIG_HISI_VIRTCCA_HOST int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); +struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group); #endif #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 5ac5f182ce0b..b2ad93893612 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -290,6 +290,9 @@ void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, * External user API */ struct iommu_group *vfio_file_iommu_group(struct file *file); +#ifdef CONFIG_HISI_VIRTCCA_HOST +struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file); +#endif #if IS_ENABLED(CONFIG_VFIO_GROUP) bool vfio_file_is_group(struct file *file); diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index ca24ce120906..40d9441aba50 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -21,6 +21,10 @@ #include #endif +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#endif + struct kvm_vfio_file { struct list_head node; struct file *file; @@ -140,6 +144,67 @@ static void kvm_vfio_update_coherency(struct kvm_device *dev) } } +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * cvm_vfio_file_iommu_group - Get iommu group from vfio file + * @file: Vfio file + * + * Returns: + * %NULL if the virtcca_vfio_file_iommu_group func is not defined + * or CONFIG_HISI_VIRTCCA_HOST is not enable, group is null + * %iommu_group if get the iommu group from file success + */ +static struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) +{ + struct iommu_group *(*fn)(struct file *file); + struct iommu_group *ret; + + fn = symbol_get(virtcca_vfio_file_iommu_group); + if (!fn) + return NULL; + + ret = fn(file); + + symbol_put(virtcca_vfio_file_iommu_group); + + return ret; +} + +/** + * cvm_vfio_add_kvm_to_smmu_domain - Bind the confidential + * virtual machine to smmu domain + * @filp: The handle of file + * @kvm: The kvm belone to confidential virtual machine + * + * Returns: + * %-ENXIO if set kvm failed or iommu group is null + * %0 if set kvm success + */ +static int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, struct kvm_vfio *kv) +{ + struct iommu_group *iommu_group; + int ret = 0; + + if (!is_virtcca_cvm_enable()) + return ret; + + mutex_unlock(&kv->lock); + iommu_group = cvm_vfio_file_iommu_group(filp); + if (!iommu_group) { + ret = -ENXIO; + goto out_lock; + } + if (cvm_arm_smmu_domain_set_kvm((void *)iommu_group)) { + ret = -ENXIO; + goto out_lock; + } + +out_lock: + mutex_lock(&kv->lock); + return ret; +} +#endif + static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { struct kvm_vfio *kv = dev->private; @@ -178,6 +243,9 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvm_arch_start_assignment(dev->kvm); kvm_vfio_file_set_kvm(kvf->file, dev->kvm); kvm_vfio_update_coherency(dev); +#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = cvm_vfio_add_kvm_to_smmu_domain(filp, kv); +#endif out_unlock: mutex_unlock(&kv->lock); @@ -392,3 +460,70 @@ void kvm_vfio_ops_exit(void) { kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); } + + +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_arm_smmu_get_kvm - Find the kvm + * with vfio devices through SMMU domain + * @domain: Smmu domain + * + * Returns: + * %kvm if find the kvm with vfio devices + * %NULL if kvm is null + */ +struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) +{ + int ret = -1; + struct kvm *kvm; + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + struct iommu_group *iommu_group; + + unsigned long flags; + struct arm_smmu_master *master; + + spin_lock_irqsave(&domain->devices_lock, flags); + list_for_each_entry(master, &domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + ret = 0; + break; + } + } + spin_unlock_irqrestore(&domain->devices_lock, flags); + if (ret) + return NULL; + + ret = -1; + iommu_group = master->dev->iommu_group; + mutex_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + mutex_lock(&kvm->lock); + list_for_each_entry(dev, &kvm->devices, vm_node) { + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + if (cvm_vfio_file_iommu_group(kvf->file) == iommu_group) { + ret = 0; + break; + } + } + mutex_unlock(&kv->lock); + if (!ret) + break; + } + } + mutex_unlock(&kvm->lock); + if (!ret) + break; + } + mutex_unlock(&kvm_lock); + + if (ret) + return NULL; + return kvm; +} +EXPORT_SYMBOL_GPL(virtcca_arm_smmu_get_kvm); +#endif diff --git a/virt/kvm/vfio.h b/virt/kvm/vfio.h index e130a4a03530..38b2caad9d0d 100644 --- a/virt/kvm/vfio.h +++ b/virt/kvm/vfio.h @@ -2,6 +2,10 @@ #ifndef __KVM_VFIO_H #define __KVM_VFIO_H +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include "../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" +#endif + #ifdef CONFIG_KVM_VFIO int kvm_vfio_ops_init(void); void kvm_vfio_ops_exit(void); @@ -15,4 +19,7 @@ static inline void kvm_vfio_ops_exit(void) } #endif +#ifdef CONFIG_HISI_VIRTCCA_HOST +struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); +#endif #endif -- Gitee From 6ae1290a135c545470e99e6b117d9be75836d0ac Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Tue, 10 Sep 2024 20:58:57 +0800 Subject: [PATCH 03/12] virtcca feature: vfio driver dma map virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Vfio driver want to map or unmap ram memory and mmio space, which need to use tmi interface, complete the secure memory map and secure world stage2 map. Msi space map also need to use tmi interface complete stage2 map Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/kvm_tmi.h | 3 + arch/arm64/include/asm/kvm_tmm.h | 5 + arch/arm64/kvm/tmi.c | 9 + arch/arm64/kvm/virtcca_cvm.c | 149 ++++++++++- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 56 ++++ drivers/iommu/dma-iommu.c | 49 ++++ drivers/iommu/io-pgtable-arm.c | 112 ++++++++ drivers/iommu/iommu.c | 150 +++++++++++ drivers/vfio/vfio_iommu_type1.c | 244 ++++++++++++++++++ include/linux/iommu.h | 14 + 10 files changed, 788 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 36fceec57490..5d9a3ba90fbf 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -247,6 +247,7 @@ struct tmi_tec_run { #define TMI_FNUM_SMMU_WRITE U(0x282) #define TMI_FNUM_SMMU_READ U(0x283) #define TMI_FNUM_SMMU_PCIE_CORE_CHECK U(0x284) +#define TMI_FNUM_DEV_TTT_CREATE U(0x285) /* TMI SMC64 PIDs handled by the SPMD */ #define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) @@ -280,6 +281,7 @@ struct tmi_tec_run { #define TMI_TMM_SMMU_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_WRITE) #define TMI_TMM_SMMU_READ TMI_FID(SMC_64, TMI_FNUM_SMMU_READ) #define TMI_TMM_SMMU_PCIE_CORE_CHECK TMI_FID(SMC_64, TMI_FNUM_SMMU_PCIE_CORE_CHECK) +#define TMI_TMM_DEV_TTT_CREATE TMI_FID(SMC_64, TMI_FNUM_DEV_TTT_CREATE) #define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) @@ -381,6 +383,7 @@ u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_n u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id); u64 tmi_mem_info_show(u64 mem_info_addr); +u64 tmi_dev_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level); u64 tmi_smmu_queue_create(u64 params_ptr); u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id); u64 tmi_smmu_ste_create(u64 params_ptr); diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index ac1cb415919a..651a0a508fe7 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -100,9 +100,14 @@ int kvm_cvm_map_range(struct kvm *kvm); int cvm_arm_smmu_domain_set_kvm(void *group); int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map); +int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, + unsigned long map_size, uint32_t is_map); int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size); +bool check_virtcca_cvm_ram_range(struct kvm *kvm, uint64_t iova); +bool check_virtcca_cvm_vfio_map_dma(struct kvm *kvm, uint64_t iova); + #define CVM_TTT_BLOCK_LEVEL 2 #define CVM_TTT_MAX_LEVEL 3 diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 769c00961222..62edac67b06f 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -309,3 +309,12 @@ u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits) } EXPORT_SYMBOL(tmi_smmu_read); +/* Create device ttt */ +u64 tmi_dev_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_TTT_CREATE, numa_set, rd, map_addr, level, &res); + return res.a1; +} + diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 70588b9340fd..ea394adead5f 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -196,6 +196,7 @@ void kvm_destroy_cvm(struct kvm *kvm) if (!tmi_cvm_destroy(cvm->rd)) kvm_info("KVM has destroyed cVM: %d\n", cvm->cvm_vmid); + cvm->is_mapped = false; kfree(cvm); kvm->arch.virtcca_cvm = NULL; } @@ -517,7 +518,11 @@ int kvm_cvm_map_range(struct kvm *kvm) } } } - + /* Vfio driver will pin memory in advance, + * if the ram already mapped, activate cvm + * does not need to map twice + */ + cvm->is_mapped = true; return ret; } @@ -528,7 +533,7 @@ static int kvm_activate_cvm(struct kvm *kvm) if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; - if (kvm_cvm_map_range(kvm)) + if (!cvm->is_mapped && kvm_cvm_map_range(kvm)) return -EFAULT; if (tmi_cvm_activate(cvm->rd)) { @@ -862,7 +867,53 @@ int kvm_init_cvm_vm(struct kvm *kvm) */ /** - * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CVM + * check_virtcca_cvm_ram_range - Check if the iova belongs + * to the cvm ram range + * @kvm: The handle of kvm + * @iova: Ipa address + * + * Returns: + * %true if the iova belongs to cvm ram + * %false if the iova is not within the scope of cvm ram + */ +bool check_virtcca_cvm_ram_range(struct kvm *kvm, uint64_t iova) +{ + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + if (iova >= virtcca_cvm->loader_start && + iova < virtcca_cvm->loader_start + virtcca_cvm->ram_size) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(check_virtcca_cvm_ram_range); + +/** + * check_virtcca_cvm_vfio_map_dma - Whether the vfio need + * to map the dma address + * @kvm: The handle of kvm + * @iova: Ipa address + * + * Returns: + * %true if virtcca cvm ram is nort mapped or + * virtcca_cvm_ram is mapped and the iova does not + * belong to cvm ram range + * %false if virtcca_cvm_ram is mapped and the iova belong + * to cvm ram range + */ +bool check_virtcca_cvm_vfio_map_dma(struct kvm *kvm, uint64_t iova) +{ + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + if (!virtcca_cvm->is_mapped) + return true; + + return !check_virtcca_cvm_ram_range(kvm, iova); +} +EXPORT_SYMBOL_GPL(check_virtcca_cvm_vfio_map_dma); + +/** + * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CV * @group: Iommu group * * Returns: @@ -890,3 +941,95 @@ int cvm_arm_smmu_domain_set_kvm(void *group) return 0; } +static int kvm_cvm_dev_ttt_create(struct virtcca_cvm *cvm, + unsigned long addr, + int level, + u64 numa_set) +{ + addr = ALIGN_DOWN(addr, cvm_ttt_level_mapsize(level - 1)); + return tmi_dev_ttt_create(numa_set, cvm->rd, addr, level); +} + +/* CVM create ttt level information about device */ +int kvm_cvm_create_dev_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, + unsigned long ipa, + int level, + int max_level, + struct kvm_mmu_memory_cache *mc) +{ + int ret = 0; + + if (WARN_ON(level == max_level)) + return 0; + + while (level++ < max_level) { + u64 numa_set = kvm_get_first_binded_numa_set(kvm); + + ret = kvm_cvm_dev_ttt_create(cvm, ipa, level, numa_set); + if (ret) + return -ENXIO; + } + + return 0; +} + +/** + * cvm_map_unmap_ipa_range - Vfio driver map or + * unmap cvm ipa + * @kvm: The handle of kvm + * @ipa_base: Ipa address + * @pa: Physical address + * @map_size: Map range + * @is_map: Map type + * + * Returns: + * %0 if cvm map/unmap address successfully + * %-ENXIO if map/unmap failed + */ +int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size, uint32_t is_map) +{ + unsigned long size; + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long ipa = ipa_base; + unsigned long phys = pa; + int ret = 0; + + for (size = 0; size < map_size; size += PAGE_SIZE) { + if (is_map) + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + else + ret = tmi_mmio_unmap(rd, ipa, CVM_TTT_MAX_LEVEL); + + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + if (is_map) { + ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, ipa, + level_fault, CVM_TTT_MAX_LEVEL, NULL); + if (ret) + goto err; + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + } else { + ret = tmi_mmio_unmap(rd, ipa, level_fault); + } + } + + if (ret) + goto err; + + if (size + PAGE_SIZE >= map_size) + break; + ipa += PAGE_SIZE; + phys += PAGE_SIZE; + } + + return 0; + +err: + if (!tmi_cvm_destroy(rd)) + kvm_info("Vfio map failed, kvm has destroyed cVM: %d\n", virtcca_cvm->cvm_vmid); + return -ENXIO; +} diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c index e824adbaeb65..18f49c6525b7 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -1114,6 +1114,62 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) return true; } +/** + * virtcca_smmu_map_pages - Iommu driver calls to this point, + * and then calls the map function in the + * io-pgtable to perform mapping + * @domain: Iommu domain + * @iova: Ipa address + * @paddr: Physical address + * @pgsize: Page size + * @pgcount: Page count + * @prot: Iommu attribute + * @mapped: Maped size + * + * Returns: + * %0 if map success + * %-ENODEV if ops is null + */ +int virtcca_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, size_t *mapped) +{ + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + + if (!ops) + return -ENODEV; + + return virtcca_map_pages(ops, iova, paddr, pgsize, pgcount, prot, mapped); +} +EXPORT_SYMBOL_GPL(virtcca_smmu_map_pages); + +/** + * virtcca_smmu_unmap_pages - Iommu driver calls to this point, + * and then calls the unmap function in the + * io-pgtable to perform unmapping + * @domain: Iommu domain + * @iova: Ipa address + * @pgsize: Page size + * @pgcount: Page count + * + * Returns: + * %0 if map success + * %-ENODEV if ops is null + */ + +size_t virtcca_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + + if (!ops) + return 0; + + return virtcca_unmap_pages(ops, iova, pgsize, pgcount); +} +EXPORT_SYMBOL_GPL(virtcca_smmu_unmap_pages); + /** * virtcca_smmu_secure_dev_operator - Implement security settings for corresponding devices * targeting the secure smmu domain diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index b733b1fb443e..76b21c101ba2 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -33,6 +33,10 @@ #include "dma-iommu.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#endif + struct iommu_dma_msi_page { struct list_head list; dma_addr_t iova; @@ -1811,6 +1815,47 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) } EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* Virtcca map msi address */ +static struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, + phys_addr_t msi_addr, struct iommu_domain *domain) +{ + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iommu_dma_msi_page *msi_page; + dma_addr_t iova; + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + size_t size = cookie_msi_granule(cookie); + + msi_addr &= ~(phys_addr_t)(size - 1); + list_for_each_entry(msi_page, &cookie->msi_page_list, list) + if (msi_page->phys == msi_addr) + return msi_page; + + msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); + if (!msi_page) + return NULL; + + iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); + if (!iova) + goto out_free_page; + + if (virtcca_iommu_map(domain, iova, msi_addr, size, prot)) + goto out_free_iova; + + INIT_LIST_HEAD(&msi_page->list); + msi_page->phys = msi_addr; + msi_page->iova = iova; + list_add(&msi_page->list, &cookie->msi_page_list); + return msi_page; + +out_free_iova: + iommu_dma_free_iova(cookie, iova, size, NULL); +out_free_page: + kfree(msi_page); + return NULL; +} +#endif + static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, phys_addr_t msi_addr, struct iommu_domain *domain) { @@ -1820,6 +1865,10 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(cookie); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && domain->secure) + return virtcca_iommu_dma_get_msi_page(dev, msi_addr, domain); +#endif msi_addr &= ~(phys_addr_t)(size - 1); list_for_each_entry(msi_page, &cookie->msi_page_list, list) if (msi_page->phys == msi_addr) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index fb54baed3f49..9b6a581cfa30 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -22,6 +22,11 @@ #include "io-pgtable-arm.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#include "../virt/kvm/vfio.h" +#endif + #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -498,6 +503,113 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, return pte; } +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* Obtain kvm from smmu domain */ +static struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data) +{ + struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data->iop.cookie; + + if (!smmu_domain) + return NULL; + + return smmu_domain->kvm; +} + +/** + * virtcca_map_pages - Virtcca need map the secure + * memory with paddr + * @ops: the handle of io_pgtable_ops + * @iova: Ipa address + * @paddr: Physical address + * @pgsize: Page size + * @pgcount: Page count + * @iommu_prot: iommu attribute + * @mapped: mapped size + * + * Returns: + * %0 if map pages success + */ +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped) +{ + struct kvm *kvm; + u64 loader_start; + u64 ram_size; + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + long iaext = (s64)iova >> cfg->ias; + int ret = 0; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) + return -EINVAL; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + kvm = virtcca_smmu_domain_get_kvm(data); + if (kvm) { + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + + loader_start = virtcca_cvm->loader_start; + ram_size = virtcca_cvm->ram_size; + if (iova >= loader_start && + iova < loader_start + ram_size && + !virtcca_cvm->is_mapped) { + ret = kvm_cvm_map_range(kvm); + } else if (iova < loader_start) { + if (iova == CVM_MSI_ORIG_IOVA) + iova += CVM_MSI_IOVA_OFFSET; + ret = cvm_map_unmap_ipa_range(kvm, iova, paddr, pgsize * pgcount, true); + } + if (mapped) + *mapped += pgsize * pgcount; + } + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_map_pages); + +/** + * virtcca_unmap_pages - Virtcca unmap the iova + * @ops: the handle of io_pgtable_ops + * @iova: Ipa address + * @pgsize: Page size + * @pgcount: Page count + * + * Returns: + * %0 if map pages success or parameter is invalid + */ +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount) +{ + struct kvm *kvm; + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + long iaext = (s64)iova >> cfg->ias; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext)) + return 0; + + kvm = virtcca_smmu_domain_get_kvm(data); + if (!kvm) + return 0; + + return cvm_map_unmap_ipa_range(kvm, iova, 0, pgsize * pgcount, false); +} +EXPORT_SYMBOL_GPL(virtcca_unmap_pages); +#endif + static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int iommu_prot, gfp_t gfp, size_t *mapped) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 5d6fc4641aa7..13451f29fe13 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2474,6 +2474,156 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, } #ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_iommu_map - Iommu driver map pages, and then + * calls the map function in the + * smmu to perform mapping + * @domain: Iommu domain + * @iova: Ipa address + * @paddr: Physical address + * @size: Map size + * @prot: Iommu attribute + * + * Returns: + * %0 if map success + * %-EINVAL if domain type is not paging + * %-ENODEV if the domain pgsize_bitmap is zero or parameter is invalid + */ +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int min_pagesz; + int ret = 0; + unsigned long orig_iova = iova; + size_t orig_size = size; + const struct iommu_domain_ops *ops = domain->ops; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return -EINVAL; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return -ENODEV; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * both the virtual address and the physical one, as well as + * the size of the mapping, must be aligned (at least) to the + * size of the smallest page supported by the hardware + */ + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); + return -EINVAL; + } + + pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); + + while (size) { + size_t pgsize, count, mapped = 0; + + pgsize = iommu_pgsize(domain, iova, paddr, size, &count); + + pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n", + iova, &paddr, pgsize, count); + ret = virtcca_smmu_map_pages(domain, iova, paddr, pgsize, + count, prot, &mapped); + /* + * Some pages may have been mapped, even if an error occurred, + * so we should account for those so they can be unmapped. + */ + size -= mapped; + + if (ret) + break; + + iova += mapped; + paddr += mapped; + } + + /* unroll mapping in case something went wrong */ + if (ret) + virtcca_iommu_unmap(domain, orig_iova, orig_size - size); + + if (ret == 0 && ops->iotlb_sync_map) { + ret = ops->iotlb_sync_map(domain, iova, size); + if (ret) + goto out_err; + } + + return ret; + +out_err: + /* undo mappings already done */ + virtcca_iommu_unmap(domain, iova, size); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_map); + +/** + * virtcca_iommu_unmap - Iommu driver unmap pages, and then + * calls the map function in the + * smmu to perform unmapping + * @domain: Iommu domain + * @iova: Ipa address + * @size: Map size + * + * Returns: + * %0 if map success or domain type and parameter is invalid + */ + +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + size_t unmapped_page, unmapped = 0; + unsigned int min_pagesz; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return 0; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return 0; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * The virtual address, as well as the size of the mapping, must be + * aligned (at least) to the size of the smallest page supported + * by the hardware + */ + if (!IS_ALIGNED(iova | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + iova, size, min_pagesz); + return 0; + } + + pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); + + /* + * Keep iterating until we either unmap 'size' bytes (or more) + * or we hit an area that isn't mapped. + */ + while (unmapped < size) { + size_t pgsize, count; + + pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count); + unmapped_page = virtcca_smmu_unmap_pages(domain, iova, pgsize, count); + if (!unmapped_page) + break; + + pr_debug("unmapped: iova 0x%lx size 0x%zx\n", + iova, unmapped_page); + + iova += unmapped_page; + unmapped += unmapped_page; + } + return unmapped; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_unmap); + /** * virtcca_attach_secure_dev - Attach the device of iommu * group to confidential virtual machine diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 29cb28863195..6ba9e54202f1 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -38,6 +38,10 @@ #include #include #include "vfio.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#include "../virt/kvm/vfio.h" +#endif #define DRIVER_VERSION "0.2" #define DRIVER_AUTHOR "Alex Williamson " @@ -1040,6 +1044,231 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain, return unmapped; } +#ifdef CONFIG_HISI_VIRTCCA_HOST +static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma); + +/* Whether the kvm is cvm */ +static bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm) +{ + struct arm_smmu_domain *arm_smmu_domain; + + arm_smmu_domain = to_smmu_domain(domain); + *kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (*kvm) + return (*kvm)->arch.is_virtcca_cvm; + + return false; +} + +static bool virtcca_check_kvm_is_cvm(struct vfio_iommu *iommu, struct kvm **kvm) +{ + struct vfio_domain *domain; + bool is_virtcca_cvm = false; + + if (!iommu || !kvm) + return false; + + list_for_each_entry(domain, &iommu->domain_list, next) { + if (domain && domain->domain && virtcca_iommu_domain_get_kvm(domain->domain, kvm)) { + is_virtcca_cvm = true; + break; + } + } + + return is_virtcca_cvm; +} + +/* Traverse all domains and perform mapping operations */ +static int virtcca_vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, + unsigned long pfn, long npage, int prot) +{ + struct vfio_domain *d; + int ret; + + list_for_each_entry(d, &iommu->domain_list, next) { + ret = virtcca_iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, + npage << PAGE_SHIFT, prot | IOMMU_CACHE); + if (ret) + goto unwind; + + cond_resched(); + } + + return 0; + +unwind: + list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) { + virtcca_iommu_unmap(d->domain, iova, npage << PAGE_SHIFT); + cond_resched(); + } + + return ret; +} + +/** + * virtcca_vfio_pin_map_dma - Vfio need to map iova + * @iommu: The handle of iommu + * @dma: Dma information + * @map_size: Map size + * + * Returns: + * %0 if map success + */ +static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, + size_t map_size) +{ + struct kvm *kvm; + long npage; + dma_addr_t iova = dma->iova; + unsigned long vaddr = dma->vaddr; + struct vfio_batch batch; + size_t size = map_size; + unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + int ret = 0; + bool is_virtcca_cvm = virtcca_check_kvm_is_cvm(iommu, &kvm); + + vfio_batch_init(&batch); + + while (size) { + + if (is_virtcca_cvm && !check_virtcca_cvm_vfio_map_dma(kvm, dma->iova)) + break; + /* Pin a contiguous chunk of memory */ + npage = vfio_pin_pages_remote(dma, vaddr + dma->size, + size >> PAGE_SHIFT, &pfn, limit, + &batch); + if (npage <= 0) { + WARN_ON(!npage); + ret = (int)npage; + break; + } + + /* Map it! */ + ret = virtcca_vfio_iommu_map(iommu, iova + dma->size, pfn, npage, + dma->prot); + if (ret) { + vfio_unpin_pages_remote(dma, iova + dma->size, pfn, + npage, true); + vfio_batch_unpin(&batch, dma); + break; + } + + if (is_virtcca_cvm && check_virtcca_cvm_ram_range(kvm, iova)) { + vfio_unpin_pages_remote(dma, iova + dma->size, pfn, + npage, true); + vfio_batch_unpin(&batch, dma); + } + size -= npage << PAGE_SHIFT; + dma->size += npage << PAGE_SHIFT; + } + + vfio_batch_fini(&batch); + dma->iommu_mapped = true; + + if (ret) + vfio_remove_dma(iommu, dma); + + return ret; +} + +/** + * virtcca_vfio_unmap_unpin - Vfio need to unmap iova + * @iommu: The handle of iommu + * @dma: Dma information + * @do_accounting: Need to account or not + * + * Returns: + * %0 if unmap success or the cvm is destroyed + */ +static long virtcca_vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, + bool do_accounting) +{ + struct kvm *kvm; + struct vfio_domain *domain, *d; + LIST_HEAD(unmapped_region_list); + struct iommu_iotlb_gather iotlb_gather; + int unmapped_region_cnt = 0; + long unlocked = 0; + dma_addr_t iova = dma->iova, end = dma->iova + dma->size; + + if (!virtcca_check_kvm_is_cvm(iommu, &kvm)) + return 0; + + if (!dma->size) + return 0; + + if (list_empty(&iommu->domain_list)) + return 0; + + /* + * We use the IOMMU to track the physical addresses, otherwise we'd + * need a much more complicated tracking system. Unfortunately that + * means we need to use one of the iommu domains to figure out the + * pfns to unpin. The rest need to be unmapped in advance so we have + * no iommu translations remaining when the pages are unpinned. + */ + domain = d = list_first_entry(&iommu->domain_list, + struct vfio_domain, next); + + list_for_each_entry_continue(d, &iommu->domain_list, next) { + virtcca_iommu_unmap(d->domain, dma->iova, dma->size); + cond_resched(); + } + + iommu_iotlb_gather_init(&iotlb_gather); + while (iova < end) { + size_t unmapped, len; + phys_addr_t phys, next; + + phys = iommu_iova_to_phys(domain->domain, iova); + if (WARN_ON(!phys)) { + iova += PAGE_SIZE; + continue; + } + + /* + * To optimize for fewer iommu_unmap() calls, each of which + * may require hardware cache flushing, try to find the + * largest contiguous physical memory chunk to unmap. + */ + for (len = PAGE_SIZE; + !domain->fgsp && iova + len < end; len += PAGE_SIZE) { + next = iommu_iova_to_phys(domain->domain, iova + len); + if (next != phys + len) + break; + } + + /* + * First, try to use fast unmap/unpin. In case of failure, + * switch to slow unmap/unpin path. + */ + unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys, + &unlocked, &unmapped_region_list, + &unmapped_region_cnt, + &iotlb_gather); + if (!unmapped) { + unmapped = unmap_unpin_slow(domain, dma, &iova, len, + phys, &unlocked); + if (WARN_ON(!unmapped)) + break; + } + } + + dma->iommu_mapped = false; + + if (unmapped_region_cnt) { + unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list, + &iotlb_gather); + } + + if (do_accounting) { + vfio_lock_acct(dma, -unlocked, true); + return 0; + } + return unlocked; +} +#endif + static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, bool do_accounting) { @@ -1050,6 +1279,11 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, int unmapped_region_cnt = 0; long unlocked = 0; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && iommu->secure) + return virtcca_vfio_unmap_unpin(iommu, dma, do_accounting); +#endif + if (!dma->size) return 0; @@ -1633,6 +1867,11 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && iommu->secure) + return virtcca_vfio_pin_map_dma(iommu, dma, map_size); +#endif + vfio_batch_init(&batch); while (size) { @@ -1856,6 +2095,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && iommu->secure) + return 0; +#endif + /* Arbitrarily pick the first domain in the list for lookups */ if (!list_empty(&iommu->domain_list)) d = list_first_entry(&iommu->domain_list, diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 92ac0a0cc87f..0b8feb6de953 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1767,5 +1767,19 @@ static inline void iopf_group_response(struct iopf_group *group, int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group); +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size); +int virtcca_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, size_t *mapped); +size_t virtcca_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount); +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped); +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount); #endif #endif /* __LINUX_IOMMU_H */ -- Gitee From 02347918ebff9d14561f04904ff2f1c542e50315 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 08:54:50 +0800 Subject: [PATCH 04/12] virtcca feature: secure device page fault virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: User memory abort, if secure world triggers a page fault due to accessing device space, it is necessary to call tmi interface again for stage2 map Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/kvm_tmi.h | 2 + arch/arm64/kvm/mmu.c | 8 ++- arch/arm64/kvm/virtcca_cvm.c | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 5d9a3ba90fbf..b1af7f230b17 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -407,6 +407,8 @@ int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, bool serror_pending, bool ext_dabt_pending); int kvm_init_cvm_vm(struct kvm *kvm); int kvm_enable_virtcca_cvm(struct kvm *kvm); +int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, + unsigned long map_size, enum kvm_pgtable_prot prot, int ret); #endif #endif diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f3ab2c39f764..ca2abf7d98eb 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -19,7 +19,9 @@ #include #include #include - +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#endif #include "trace.h" static struct kvm_pgtable *hyp_pgtable; @@ -1606,6 +1608,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED); +#ifdef CONFIG_HISI_VIRTCCA_HOST + ret = kvm_cvm_map_ipa(kvm, fault_ipa, pfn, vma_pagesize, prot, ret); +#endif + /* Mark the page dirty only if the fault is handled successfully */ if (writable && !ret) { kvm_set_pfn_dirty(pfn); diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index ea394adead5f..662fc3eb3d0c 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1033,3 +1034,89 @@ int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, kvm_info("Vfio map failed, kvm has destroyed cVM: %d\n", virtcca_cvm->cvm_vmid); return -ENXIO; } + +/** + * kvm_cvm_map_ipa_mmio - Map the mmio address when page fault + * @kvm: The handle of kvm + * @ipa_base: Ipa address + * @pa: Physical address + * @map_size: Map range + * + * Returns: + * %0 if cvm map address successfully + * %-ENXIO if map failed + */ +int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size) +{ + unsigned long size; + gfn_t gfn; + kvm_pfn_t pfn; + struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long ipa = ipa_base; + unsigned long phys = pa; + int ret = 0; + + if (WARN_ON(!IS_ALIGNED(ipa, map_size))) + return -EINVAL; + + for (size = 0; size < map_size; size += PAGE_SIZE) { + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + if (ret == TMI_ERROR_TTT_CREATED) { + ret = 0; + goto label; + } + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, ipa, level_fault, + CVM_TTT_MAX_LEVEL, NULL); + + if (ret) + goto err; + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + } + + if (ret) + goto err; +label: + if (size + PAGE_SIZE >= map_size) + break; + + ipa += PAGE_SIZE; + gfn = gpa_to_gfn(ipa); + pfn = gfn_to_pfn(kvm, gfn); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + phys = (uint64_t)__pfn_to_phys(pfn); + + } + + return 0; + +err: + if (!tmi_cvm_destroy(rd)) + kvm_info("MMIO map failed, kvm has destroyed cVM: %d\n", virtcca_cvm->cvm_vmid); + return -ENXIO; +} + +/* Page fault map ipa */ +int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, + unsigned long map_size, enum kvm_pgtable_prot prot, int ret) +{ + if (!is_virtcca_cvm_enable() || !kvm_is_virtcca_cvm(kvm)) + return ret; + + struct page *dst_page = pfn_to_page(pfn); + phys_addr_t dst_phys = page_to_phys(dst_page); + + if (WARN_ON(!(prot & KVM_PGTABLE_PROT_W))) + return -EFAULT; + + if (prot & KVM_PGTABLE_PROT_DEVICE) + return kvm_cvm_map_ipa_mmio(kvm, ipa, dst_phys, map_size); + + return 0; +} -- Gitee From ea28bb7b68e8e853bc21d00cf6afb89d3ef6c4f5 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 09:08:44 +0800 Subject: [PATCH 05/12] virtcca feature: set device security properties virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Set device security properties Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/kvm_tmi.h | 2 +- arch/arm64/kvm/virtcca_cvm.c | 15 +++++++++++++++ drivers/vfio/pci/vfio_pci_core.c | 7 +++++++ include/linux/pci.h | 4 ++++ include/uapi/linux/vfio.h | 1 + 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index b1af7f230b17..d0e2cda5841c 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -409,6 +409,6 @@ int kvm_init_cvm_vm(struct kvm *kvm); int kvm_enable_virtcca_cvm(struct kvm *kvm); int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, unsigned long map_size, enum kvm_pgtable_prot prot, int ret); - +void virtcca_cvm_set_secure_flag(void *vdev, void *info); #endif #endif diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 662fc3eb3d0c..966d568d925e 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -1120,3 +1121,17 @@ int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, return 0; } + +/* Set device secure flag */ +void virtcca_cvm_set_secure_flag(void *vdev, void *info) +{ + if (!is_virtcca_cvm_enable()) + return; + + if (!is_cc_dev(pci_dev_id(((struct vfio_pci_core_device *)vdev)->pdev))) + return; + + ((struct vfio_device_info *)info)->flags |= VFIO_DEVICE_FLAGS_SECURE; +} +EXPORT_SYMBOL_GPL(virtcca_cvm_set_secure_flag); + diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 0524c90cedea..2bff38b56f97 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -31,6 +31,9 @@ #if IS_ENABLED(CONFIG_EEH) #include #endif +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#endif #include "vfio_pci_priv.h" @@ -973,6 +976,10 @@ static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET; +#ifdef CONFIG_HISI_VIRTCCA_HOST + virtcca_cvm_set_secure_flag((void *)vdev, (void *)&info); +#endif + info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; diff --git a/include/linux/pci.h b/include/linux/pci.h index 82f3571a36d9..54e340c81972 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2710,6 +2710,10 @@ static inline bool pci_is_thunderbolt_attached(struct pci_dev *pdev) void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type); #endif +#ifdef CONFIG_HISI_VIRTCCA_HOST +bool is_cc_dev(u32 sid); +#endif + #include #define pci_printk(level, pdev, fmt, arg...) \ diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 03167374f348..df6cf5661d40 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -225,6 +225,7 @@ struct vfio_device_info { #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6) /* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7) /* Info supports caps */ #define VFIO_DEVICE_FLAGS_CDX (1 << 8) /* vfio-cdx device */ +#define VFIO_DEVICE_FLAGS_SECURE (1 << 9) /* vfio-secure device */ __u32 num_regions; /* Max region index + 1 */ __u32 num_irqs; /* Max IRQ index + 1 */ __u32 cap_offset; /* Offset within info struct of first cap */ -- Gitee From c57a5a1f767336eff9721cb833c067657ade31de Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 10:12:38 +0800 Subject: [PATCH 06/12] virtcca feature: activate confidential vm to set security device s_s2ttbr virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Set the security device stage2 map when activate confidential virtual machine Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/kvm/virtcca_cvm.c | 21 +++++++++++++++ virt/kvm/vfio.c | 50 ++++++++++++++++++++++++++++++++++++ virt/kvm/vfio.h | 2 ++ 3 files changed, 73 insertions(+) diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 966d568d925e..d66ddc8b8145 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -180,10 +180,19 @@ void kvm_destroy_cvm(struct kvm *kvm) { struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; uint32_t cvm_vmid; + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; if (!cvm) return; + /* Unmap the cvm with arm smmu domain */ + kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain->kvm && arm_smmu_domain->kvm == kvm) + arm_smmu_domain->kvm = NULL; + } + cvm_vmid = cvm->cvm_vmid; kfree(cvm->params); cvm->params = NULL; @@ -530,6 +539,9 @@ int kvm_cvm_map_range(struct kvm *kvm) static int kvm_activate_cvm(struct kvm *kvm) { + int ret; + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) @@ -538,6 +550,15 @@ static int kvm_activate_cvm(struct kvm *kvm) if (!cvm->is_mapped && kvm_cvm_map_range(kvm)) return -EFAULT; + kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain) { + ret = virtcca_smmu_tmi_dev_attach(arm_smmu_domain, kvm); + if (ret) + return ret; + } + } + if (tmi_cvm_activate(cvm->rd)) { kvm_err("tmi_cvm_activate failed!\n"); return -ENXIO; diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index 40d9441aba50..d15ac5d25ceb 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -526,4 +526,54 @@ struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) return kvm; } EXPORT_SYMBOL_GPL(virtcca_arm_smmu_get_kvm); + +/** + * find_arm_smmu_domain - Find smmu domain list from kvm vfio file + * @kvf: Kvm vfio file + * @smmu_domain_group_list: List of smmu domain group + */ +void find_arm_smmu_domain(struct kvm_vfio_file *kvf, struct list_head *smmu_domain_group_list) +{ + struct iommu_group *iommu_group; + int ret = 0; + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct arm_smmu_domain *arm_smmu_domain_node = NULL; + + iommu_group = cvm_vfio_file_iommu_group(kvf->file); + arm_smmu_domain = to_smmu_domain(virtcca_iommu_group_get_domain(iommu_group)); + list_for_each_entry(arm_smmu_domain_node, + smmu_domain_group_list, node) { + if (arm_smmu_domain_node == arm_smmu_domain) { + ret = -1; + break; + } + } + if (!ret) + list_add_tail(&arm_smmu_domain->node, smmu_domain_group_list); +} + +/** + * kvm_get_arm_smmu_domain - Find kvm vfio file from kvm + * @kvm: Kvm handle + * @smmu_domain_group_list: List of smmu domain group + */ +void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list) +{ + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + + INIT_LIST_HEAD(smmu_domain_group_list); + + list_for_each_entry(dev, &kvm->devices, vm_node) { + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + find_arm_smmu_domain(kvf, smmu_domain_group_list); + } + mutex_unlock(&kv->lock); + } + } +} #endif diff --git a/virt/kvm/vfio.h b/virt/kvm/vfio.h index 38b2caad9d0d..4f263bf43f18 100644 --- a/virt/kvm/vfio.h +++ b/virt/kvm/vfio.h @@ -4,6 +4,7 @@ #ifdef CONFIG_HISI_VIRTCCA_HOST #include "../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" +#include "../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" #endif #ifdef CONFIG_KVM_VFIO @@ -21,5 +22,6 @@ static inline void kvm_vfio_ops_exit(void) #ifdef CONFIG_HISI_VIRTCCA_HOST struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); +void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); #endif #endif -- Gitee From 95542c9cac98d1fe33e1c1a4616cc9e638e73973 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 10:39:07 +0800 Subject: [PATCH 07/12] virtcca feature: read or write config space forwarding to secure world virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: If the bdf number is belong to security device, then read or write generic config space need use tmi interface transfer to secure world Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/kvm_tmi.h | 1 + arch/arm64/include/asm/kvm_tmm.h | 7 +++++ arch/arm64/kvm/tmi.c | 25 +++++++++++++++ drivers/pci/access.c | 54 ++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+) diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index d0e2cda5841c..3f8c445e14b8 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -399,6 +399,7 @@ u64 tmi_smmu_pcie_core_check(u64 smmu_base); u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits); u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits); +u64 iova_to_pa(void *addr); void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); int kvm_load_user_data(struct kvm *kvm, unsigned long arg); unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index 651a0a508fe7..6de037e0e7da 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -14,6 +14,13 @@ #define CVM_MSI_ORIG_IOVA 0x8000000 #define CVM_MSI_IOVA_OFFSET (-0x1000000) +#define CVM_RW_8_BIT 0x8 +#define CVM_RW_16_BIT 0x10 +#define CVM_RW_32_BIT 0x20 +#define CVM_RW_64_BIT 0x40 + +#define BUS_NUM_SHIFT 0x8 + enum virtcca_cvm_state { CVM_STATE_NONE = 1, CVM_STATE_NEW, diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 62edac67b06f..0ac851bb09e2 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -6,6 +6,31 @@ #include #include +/* Supported io_va transfer to pa */ +u64 iova_to_pa(void *addr) +{ + uint64_t pa, par_el1; + + asm volatile( + "AT S1E1W, %0\n" + ::"r"((uint64_t)(addr)) + ); + isb(); + asm volatile( + "mrs %0, par_el1\n" + : "=r"(par_el1) + ); + + pa = ((uint64_t)(addr) & (PAGE_SIZE - 1)) | + (par_el1 & ULL(0x000ffffffffff000)); + + if (par_el1 & UL(1 << 0)) + return (uint64_t)(addr); + else + return pa; +} +EXPORT_SYMBOL(iova_to_pa); + u64 tmi_version(void) { struct arm_smccc_res res; diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 6554a2e89d36..1433daed0601 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -5,6 +5,14 @@ #include #include +#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifndef __GENKSYMS__ +#include +#include +#include +#endif +#endif + #include "pci.h" /* @@ -77,6 +85,42 @@ EXPORT_SYMBOL(pci_bus_write_config_byte); EXPORT_SYMBOL(pci_bus_write_config_word); EXPORT_SYMBOL(pci_bus_write_config_dword); +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* If device is secure dev, read config need transfer to tmm module */ +static int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val) +{ + if (size == 1) + *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_8_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + else if (size == 2) + *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_16_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + else + *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_32_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + + return PCIBIOS_SUCCESSFUL; +} + +/* If device is secure dev, write config need transfer to tmm module */ +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val) +{ + if (size == 1) + WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, + CVM_RW_8_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + else if (size == 2) + WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, + CVM_RW_16_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + else + WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, + CVM_RW_32_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + + return PCIBIOS_SUCCESSFUL; +} +#endif + int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { @@ -86,6 +130,11 @@ int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && is_cc_dev((bus->number << BUS_NUM_SHIFT) | devfn)) + return virtcca_pci_generic_config_read(addr, bus->number, devfn, size, val); +#endif + if (size == 1) *val = readb(addr); else if (size == 2) @@ -106,6 +155,11 @@ int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && is_cc_dev((bus->number << BUS_NUM_SHIFT) | devfn)) + return virtcca_pci_generic_config_write(addr, bus->number, devfn, size, val); +#endif + if (size == 1) writeb(val, addr); else if (size == 2) -- Gitee From ffeea40f823d07c69a9bd807f7477c0ac02240a4 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 10:54:52 +0800 Subject: [PATCH 08/12] virtcca feature: read or write bar space forwarding to the secure world virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: If the bdf number is belong to security device, then read or write bar space need use tmi interface transfer to secure world Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- drivers/vfio/pci/vfio_pci_rdwr.c | 92 ++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index e27de61ac9fe..aeb050ab8422 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -19,6 +19,13 @@ #include "vfio_pci_priv.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifndef __GENKSYMS__ +#include +#include +#endif +#endif + #ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 #define vfio_iowrite64 iowrite64 @@ -37,6 +44,64 @@ #define vfio_ioread8 ioread8 #define vfio_iowrite8 iowrite8 +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* Judge startup virtcca_cvm_host is enable and device is secure or not */ +static bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + + if (cc_dev) + return true; + + return false; +} + +/* Transfer to tmm write io value */ +static void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + WARN_ON(tmi_mmio_write(iova_to_pa(io), val, size, pci_dev_id(pdev))); +} + +/* Transfer to tmm read io value */ +static u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + return tmi_mmio_read(iova_to_pa(io), size, pci_dev_id(pdev)); +} + +#define VFIO_IOWRITE(size) \ +static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (is_virtcca_pci_io_rw(vdev)) { \ + virtcca_pci_io_write(vdev, val, size, io); \ + } else { \ + vfio_iowrite##size(val, io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else #define VFIO_IOWRITE(size) \ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io) \ @@ -56,6 +121,7 @@ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif VFIO_IOWRITE(8) VFIO_IOWRITE(16) @@ -64,6 +130,31 @@ VFIO_IOWRITE(32) VFIO_IOWRITE(64) #endif +#ifdef CONFIG_HISI_VIRTCCA_HOST +#define VFIO_IOREAD(size) \ +static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size * val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (is_virtcca_pci_io_rw(vdev)) { \ + *val = virtcca_pci_io_read(vdev, size, io); \ + } else { \ + *val = vfio_ioread##size(io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else #define VFIO_IOREAD(size) \ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size *val, void __iomem *io) \ @@ -83,6 +174,7 @@ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif VFIO_IOREAD(8) VFIO_IOREAD(16) -- Gitee From 08f1a0fd8d56393d785b5ef168ebfaefd9914c41 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 11 Sep 2024 13:14:18 +0800 Subject: [PATCH 09/12] virtcca feature: read or write msi forwarding to the secure world virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: If the bdf number is belong to security device, then read or write msi msg need use tmi interface transfer to secure world Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- drivers/pci/msi/msi.c | 117 ++++++++++++++++++++++++++++++++++++++++++ drivers/pci/msi/msi.h | 61 ++++++++++++++++++++++ 2 files changed, 178 insertions(+) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 161c3ac171a3..73579c22654e 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -147,6 +147,73 @@ void pci_msi_unmask_irq(struct irq_data *data) } EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_pci_read_msi_msg - secure dev read msi msg + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @msg: Msg information + * @base: Msi base address + * + **/ +static inline void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base) +{ + u64 pbase = iova_to_pa(base); + + msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/** + * virtcca_pci_write_msi_msg - secure dev write msi msg + * @desc: MSI-X description + * @msg: Msg information + * + **/ +static inline int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + void __iomem *base = pci_msix_desc_addr(desc); + u32 ctrl = desc->pci.msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + u64 pbase = iova_to_pa(base); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (!is_cc_dev(pci_dev_id(pdev))) + return 0; + + u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); + + addr += CVM_MSI_IOVA_OFFSET; + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, + msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); + + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl); + tmi_mmio_read(iova_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), + CVM_RW_32_BIT, pci_dev_id(pdev)); + + return 1; +} + +static inline void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr) +{ + desc->pci.msix_ctrl = tmi_mmio_read(iova_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + CVM_RW_32_BIT, pci_dev_id(dev)); +} +#endif + void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); @@ -159,6 +226,10 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return; +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && dev != NULL && is_cc_dev(pci_dev_id(dev))) + return virtcca_pci_read_msi_msg(dev, msg, base); +#endif msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); msg->data = readl(base + PCI_MSIX_ENTRY_DATA); @@ -221,6 +292,10 @@ static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg if (unmasked) pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (virtcca_pci_write_msg_msi(desc, msg)) + return; +#endif writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); writel(msg->data, base + PCI_MSIX_ENTRY_DATA); @@ -639,6 +714,10 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) if (desc->pci.msi_attrib.can_mask) { void __iomem *addr = pci_msix_desc_addr(desc); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) + return virtcca_msix_prepare_msi_desc(dev, desc, addr); +#endif desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } } @@ -724,6 +803,40 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries return ret; } + +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * msix_mask_all_cc - mask all secure dev msix c + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @base: Io address + * @tsize: Number of entry + * @dev_num: Dev number + * + * Returns: + * %0 if msix mask all cc device success + **/ +static int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) +{ + int i; + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + u64 pbase = iova_to_pa(base); + + if (pci_msi_ignore_mask) + goto out; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, + ctrl, CVM_RW_32_BIT, dev_num); + } + +out: + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); + + pcibios_free_irq(dev); + return 0; +} +#endif + /** * msix_capability_init - configure device's MSI-X capability * @dev: pointer to the pci_dev data structure of MSI-X device function @@ -776,6 +889,10 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, * which takes the MSI-X mask bits into account even * when MSI-X is disabled, which prevents MSI delivery. */ +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) + return msix_mask_all_cc(dev, dev->msix_base, tsize, pci_dev_id(dev)); +#endif msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h index ee53cf079f4e..afc9371985c5 100644 --- a/drivers/pci/msi/msi.h +++ b/drivers/pci/msi/msi.h @@ -3,6 +3,13 @@ #include #include +#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifndef __GENKSYMS__ +#include +#include +#endif +#endif + #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); @@ -26,6 +33,50 @@ static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) return desc->pci.mask_base + desc->msi_index * PCI_MSIX_ENTRY_SIZE; } +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* + * If it is a safety device, write vector ctrl need + * use tmi interface + */ +static inline int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + void __iomem *desc_addr = pci_msix_desc_addr(desc); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return 0; + + if (desc->pci.msi_attrib.can_mask) + tmi_mmio_write(iova_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); + return 1; +} + +/* + * If it is a safety device, read msix need + * use tmi interface + */ +static inline int virtcca_pci_msix_mask(struct msi_desc *desc) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return 0; + + /* Flush write to device */ + tmi_mmio_read(iova_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); + return 1; +} +#endif + /* * This internal function does not flush PCI writes to the device. All * users must ensure that they read from the device before either assuming @@ -36,6 +87,11 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { void __iomem *desc_addr = pci_msix_desc_addr(desc); +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (virtcca_pci_msix_write_vector_ctrl(desc, ctrl)) + return; +#endif + if (desc->pci.msi_attrib.can_mask) writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } @@ -44,6 +100,11 @@ static inline void pci_msix_mask(struct msi_desc *desc) { desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); + +#ifdef CONFIG_HISI_VIRTCCA_HOST + if (virtcca_pci_msix_mask(desc)) + return; +#endif /* Flush write to device */ readl(desc->pci.mask_base); } -- Gitee From 4038e2b62385825eed0154d2cfb96b42f9a2f5dd Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 18 Sep 2024 15:40:42 +0800 Subject: [PATCH 10/12] virtcca feature: extract the function to the virtcca_coda.c file virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Extract the function to the virtcca_coda.c file Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/kvm_tmi.h | 2 +- arch/arm64/include/asm/kvm_tmm.h | 4 +- arch/arm64/include/asm/virtcca_coda.h | 61 +++ arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/virtcca_coda.c | 510 ++++++++++++++++++ arch/arm64/kvm/tmi.c | 12 +- arch/arm64/kvm/virtcca_cvm.c | 32 +- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 58 +- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h | 1 - drivers/iommu/dma-iommu.c | 1 + drivers/iommu/io-pgtable-arm.c | 139 +---- drivers/iommu/iommu.c | 230 ++------ drivers/pci/access.c | 39 +- drivers/pci/msi/msi.c | 105 +--- drivers/pci/msi/msi.h | 47 +- drivers/vfio/pci/vfio_pci_rdwr.c | 35 +- drivers/vfio/vfio_iommu_type1.c | 39 +- include/linux/iommu.h | 20 - virt/kvm/vfio.c | 105 ++-- virt/kvm/vfio.h | 9 - 20 files changed, 738 insertions(+), 713 deletions(-) create mode 100644 arch/arm64/include/asm/virtcca_coda.h create mode 100644 arch/arm64/kernel/virtcca_coda.c diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 3f8c445e14b8..eb2a351a3edb 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -399,7 +399,7 @@ u64 tmi_smmu_pcie_core_check(u64 smmu_base); u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits); u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits); -u64 iova_to_pa(void *addr); +u64 mmio_va_to_pa(void *addr); void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); int kvm_load_user_data(struct kvm *kvm, unsigned long arg); unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index 6de037e0e7da..faf1e00e4232 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -112,8 +112,8 @@ int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t p int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size); -bool check_virtcca_cvm_ram_range(struct kvm *kvm, uint64_t iova); -bool check_virtcca_cvm_vfio_map_dma(struct kvm *kvm, uint64_t iova); +bool is_in_virtcca_ram_range(struct kvm *kvm, uint64_t iova); +bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova); #define CVM_TTT_BLOCK_LEVEL 2 #define CVM_TTT_MAX_LEVEL 3 diff --git a/arch/arm64/include/asm/virtcca_coda.h b/arch/arm64/include/asm/virtcca_coda.h new file mode 100644 index 000000000000..e7fcba7bb255 --- /dev/null +++ b/arch/arm64/include/asm/virtcca_coda.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __VIRTCCA_CODA_H +#define __VIRTCCA_CODA_H + +#include +#include + +#include "../../../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" +#include "../../../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" + +#ifdef CONFIG_HISI_VIRTCCA_HOST +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); +int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); + +struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group); +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size); +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped); +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount); + +void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base); +int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg); +void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr); +int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl); +int virtcca_pci_msix_mask(struct msi_desc *desc); +int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num); + +int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val); +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val); + +bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev); +void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io); +u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io); + +bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm); +bool virtcca_check_kvm_is_cvm(void *iommu, struct kvm **kvm); +int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, + unsigned long pfn, long npage, int prot); +int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv); +struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); +void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); +struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops); +struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *data); +struct iommu_group *cvm_vfio_file_iommu_group(struct file *file); +struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data); +#endif +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 4ce58887302a..fe6334bbe93e 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -82,7 +82,7 @@ obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o -obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o +obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o virtcca_coda.o CFLAGS_patch-scs.o += -mbranch-protection=none # Force dependency (vdso*-wrap.S includes vdso.so through incbin) diff --git a/arch/arm64/kernel/virtcca_coda.c b/arch/arm64/kernel/virtcca_coda.c new file mode 100644 index 000000000000..1f3ea50ff77b --- /dev/null +++ b/arch/arm64/kernel/virtcca_coda.c @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../drivers/pci/msi/msi.h" +#include "../../drivers/vfio/vfio.h" + +/** + * virtcca_map_pages - Virtcca need map the secure + * memory with paddr + * @ops: the handle of io_pgtable_ops + * @iova: Ipa address + * @paddr: Physical address + * @pgsize: Page size + * @pgcount: Page count + * @iommu_prot: iommu attribute + * @mapped: mapped size + * + * Returns: + * %0 if map pages success + */ +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped) +{ + struct kvm *kvm; + u64 loader_start; + u64 ram_size; + struct arm_lpae_io_pgtable *data = virtcca_io_pgtable_get_data(ops); + struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); + long iaext = (s64)iova >> cfg->ias; + int ret = 0; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) + return -EINVAL; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + kvm = virtcca_smmu_domain_get_kvm(data); + if (kvm) { + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + loader_start = virtcca_cvm->loader_start; + ram_size = virtcca_cvm->ram_size; + if (iova >= loader_start && + iova < loader_start + ram_size && + !virtcca_cvm->is_mapped) { + ret = kvm_cvm_map_range(kvm); + } else if (iova < loader_start) { + if (iova == CVM_MSI_ORIG_IOVA) + iova += CVM_MSI_IOVA_OFFSET; + ret = cvm_map_unmap_ipa_range(kvm, iova, paddr, pgsize * pgcount, true); + } + if (mapped) + *mapped += pgsize * pgcount; + } + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_map_pages); + +/** + * virtcca_unmap_pages - Virtcca unmap the iova + * @ops: the handle of io_pgtable_ops + * @iova: Ipa address + * @pgsize: Page size + * @pgcount: Page count + * + * Returns: + * %0 if map pages success or parameter is invalid + */ +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount) +{ + struct kvm *kvm; + struct arm_lpae_io_pgtable *data = virtcca_io_pgtable_get_data(ops); + struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); + long iaext = (s64)iova >> cfg->ias; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext)) + return 0; + + kvm = virtcca_smmu_domain_get_kvm(data); + if (!kvm) + return 0; + + return cvm_map_unmap_ipa_range(kvm, iova, 0, pgsize * pgcount, false); +} +EXPORT_SYMBOL_GPL(virtcca_unmap_pages); + +/** + * virtcca_iommu_map - Iommu driver map pages, and then + * calls the map function in the + * smmu to perform mapping + * @domain: Iommu domain + * @iova: Ipa address + * @paddr: Physical address + * @size: Map size + * @prot: Iommu attribute + * + * Returns: + * %0 if map success + * %-EINVAL if domain type is not paging + * %-ENODEV if the domain pgsize_bitmap is zero or parameter is invalid + */ +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int min_pagesz; + int ret = 0; + unsigned long orig_iova = iova; + size_t orig_size = size; + const struct iommu_domain_ops *ops = domain->ops; + struct io_pgtable_ops *io_ops = to_smmu_domain(domain)->pgtbl_ops; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return -EINVAL; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return -ENODEV; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * both the virtual address and the physical one, as well as + * the size of the mapping, must be aligned (at least) to the + * size of the smallest page supported by the hardware + */ + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); + return -EINVAL; + } + + if (!io_ops) + return -ENODEV; + + while (size) { + size_t pgsize, count, mapped = 0; + + pgsize = iommu_pgsize(domain, iova, paddr, size, &count); + + ret = virtcca_map_pages(io_ops, iova, paddr, pgsize, + count, prot, &mapped); + /* + * Some pages may have been mapped, even if an error occurred, + * so we should account for those so they can be unmapped. + */ + size -= mapped; + + if (ret) + break; + + iova += mapped; + paddr += mapped; + } + + /* unroll mapping in case something went wrong */ + if (ret) + virtcca_iommu_unmap(domain, orig_iova, orig_size - size); + + if (ret == 0 && ops->iotlb_sync_map) { + ret = ops->iotlb_sync_map(domain, iova, size); + if (ret) + goto out_err; + } + + return ret; + +out_err: + /* undo mappings already done */ + virtcca_iommu_unmap(domain, iova, size); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_map); + +/** + * virtcca_iommu_unmap - Iommu driver unmap pages, and then + * calls the map function in the + * smmu to perform unmapping + * @domain: Iommu domain + * @iova: Ipa address + * @size: Map size + * + * Returns: + * %0 if map success or domain type and parameter is invalid + */ + +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + size_t unmapped_page, unmapped = 0; + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + unsigned int min_pagesz; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return 0; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return 0; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * The virtual address, as well as the size of the mapping, must be + * aligned (at least) to the size of the smallest page supported + * by the hardware + */ + if (!IS_ALIGNED(iova | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + iova, size, min_pagesz); + return 0; + } + + if (!ops) + return 0; + + /* + * Keep iterating until we either unmap 'size' bytes (or more) + * or we hit an area that isn't mapped. + */ + while (unmapped < size) { + size_t pgsize, count; + + pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count); + unmapped_page = virtcca_unmap_pages(ops, iova, pgsize, count); + if (!unmapped_page) + break; + + iova += unmapped_page; + unmapped += unmapped_page; + } + return unmapped; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_unmap); + +/** + * virtcca_pci_read_msi_msg - secure dev read msi msg + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @msg: Msg information + * @base: Msi base address + * + **/ +void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base) +{ + u64 pbase = mmio_va_to_pa(base); + + msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/** + * virtcca_pci_write_msi_msg - secure dev write msi msg + * @desc: MSI-X description + * @msg: Msg information + * + **/ +int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + void __iomem *base = pci_msix_desc_addr(desc); + u32 ctrl = desc->pci.msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + u64 pbase = mmio_va_to_pa(base); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (!is_cc_dev(pci_dev_id(pdev))) + return 0; + + u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); + + addr += CVM_MSI_IOVA_OFFSET; + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, + msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); + + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl); + tmi_mmio_read(mmio_va_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), + CVM_RW_32_BIT, pci_dev_id(pdev)); + + return 1; +} + +void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr) +{ + desc->pci.msix_ctrl = tmi_mmio_read(mmio_va_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/* + * If it is a safety device, write vector ctrl need + * use tmi interface + */ +int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + void __iomem *desc_addr = pci_msix_desc_addr(desc); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return 0; + + if (desc->pci.msi_attrib.can_mask) + tmi_mmio_write(mmio_va_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); + return 1; +} + +/* + * If it is a safety device, read msix need + * use tmi interface + */ +int virtcca_pci_msix_mask(struct msi_desc *desc) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return 0; + + /* Flush write to device */ + tmi_mmio_read(mmio_va_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); + return 1; +} + +/** + * msix_mask_all_cc - mask all secure dev msix c + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @base: Io address + * @tsize: Number of entry + * @dev_num: Dev number + * + * Returns: + * %0 if msix mask all cc device success + **/ +int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) +{ + int i; + u16 rw_ctrl; + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + u64 pbase = mmio_va_to_pa(base); + + if (pci_msi_ignore_mask) + goto out; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, + ctrl, CVM_RW_32_BIT, dev_num); + } + +out: + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &rw_ctrl); + rw_ctrl &= ~PCI_MSIX_FLAGS_MASKALL; + rw_ctrl |= 0; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, rw_ctrl); + + pcibios_free_irq(dev); + return 0; +} + +/* If device is secure dev, read config need transfer to tmm module */ +int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val) +{ + if (size == 1) + *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_8_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + else if (size == 2) + *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_16_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + else + *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_32_BIT, + ((bus_num << BUS_NUM_SHIFT) | devfn)); + + return 0; +} + +/* If device is secure dev, write config need transfer to tmm module */ +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val) +{ + if (size == 1) + WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, + CVM_RW_8_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + else if (size == 2) + WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, + CVM_RW_16_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + else + WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, + CVM_RW_32_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); + + return 0; +} + +/* Judge startup virtcca_cvm_host is enable and device is secure or not */ +bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + + if (cc_dev) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(is_virtcca_pci_io_rw); + +/* Transfer to tmm write io value */ +void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + WARN_ON(tmi_mmio_write(mmio_va_to_pa(io), val, size, pci_dev_id(pdev))); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_write); + +/* Transfer to tmm read io value */ +u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + return tmi_mmio_read(mmio_va_to_pa(io), size, pci_dev_id(pdev)); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_read); + +/* Whether the kvm is cvm */ +bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm) +{ + struct arm_smmu_domain *arm_smmu_domain; + + arm_smmu_domain = to_smmu_domain(domain); + *kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (*kvm) + return (*kvm)->arch.is_virtcca_cvm; + + return false; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_domain_get_kvm); + +/** + * cvm_vfio_file_iommu_group - Get iommu group from vfio file + * @file: Vfio file + * + * Returns: + * %NULL if the virtcca_vfio_file_iommu_group func is not defined + * or CONFIG_HISI_VIRTCCA_HOST is not enable, group is null + * %iommu_group if get the iommu group from file success + */ +struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) +{ + struct iommu_group *(*fn)(struct file *file); + struct iommu_group *ret; + + fn = symbol_get(virtcca_vfio_file_iommu_group); + if (!fn) + return NULL; + + ret = fn(file); + + symbol_put(virtcca_vfio_file_iommu_group); + + return ret; +} diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 0ac851bb09e2..cd38be6f2fba 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -6,8 +6,14 @@ #include #include -/* Supported io_va transfer to pa */ -u64 iova_to_pa(void *addr) +/** + * mmio_va_to_pa - To convert the virtual address of the mmio space + * to a physical address, it is necessary to implement this interface + * because the kernel insterface __pa has an error when converting the + * physical address of the virtual address of the mmio space + * @addr: MMIO virtual address + */ +u64 mmio_va_to_pa(void *addr) { uint64_t pa, par_el1; @@ -29,7 +35,7 @@ u64 iova_to_pa(void *addr) else return pa; } -EXPORT_SYMBOL(iova_to_pa); +EXPORT_SYMBOL(mmio_va_to_pa); u64 tmi_version(void) { diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index d66ddc8b8145..6b29f65018ba 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -11,13 +11,12 @@ #include #include #include +#include #include #include #include #include -#include "../virt/kvm/vfio.h" - /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); static unsigned long *cvm_vmid_bitmap; @@ -885,12 +884,12 @@ int kvm_init_cvm_vm(struct kvm *kvm) } /* - * Coda (Confidential device assignment) feature + * Coda (Confidential Device Assignment) feature * enable devices to pass directly to confidential virtual machines */ /** - * check_virtcca_cvm_ram_range - Check if the iova belongs + * is_in_virtcca_ram_range - Check if the iova belongs * to the cvm ram range * @kvm: The handle of kvm * @iova: Ipa address @@ -899,8 +898,11 @@ int kvm_init_cvm_vm(struct kvm *kvm) * %true if the iova belongs to cvm ram * %false if the iova is not within the scope of cvm ram */ -bool check_virtcca_cvm_ram_range(struct kvm *kvm, uint64_t iova) +bool is_in_virtcca_ram_range(struct kvm *kvm, uint64_t iova) { + if (!is_virtcca_cvm_enable()) + return false; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; if (iova >= virtcca_cvm->loader_start && @@ -909,10 +911,10 @@ bool check_virtcca_cvm_ram_range(struct kvm *kvm, uint64_t iova) return false; } -EXPORT_SYMBOL_GPL(check_virtcca_cvm_ram_range); +EXPORT_SYMBOL_GPL(is_in_virtcca_ram_range); /** - * check_virtcca_cvm_vfio_map_dma - Whether the vfio need + * is_virtcca_iova_need_vfio_dma - Whether the vfio need * to map the dma address * @kvm: The handle of kvm * @iova: Ipa address @@ -924,16 +926,19 @@ EXPORT_SYMBOL_GPL(check_virtcca_cvm_ram_range); * %false if virtcca_cvm_ram is mapped and the iova belong * to cvm ram range */ -bool check_virtcca_cvm_vfio_map_dma(struct kvm *kvm, uint64_t iova) +bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova) { + if (!is_virtcca_cvm_enable()) + return false; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; if (!virtcca_cvm->is_mapped) return true; - return !check_virtcca_cvm_ram_range(kvm, iova); + return !is_in_virtcca_ram_range(kvm, iova); } -EXPORT_SYMBOL_GPL(check_virtcca_cvm_vfio_map_dma); +EXPORT_SYMBOL_GPL(is_virtcca_iova_need_vfio_dma); /** * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CV @@ -982,9 +987,6 @@ int kvm_cvm_create_dev_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, { int ret = 0; - if (WARN_ON(level == max_level)) - return 0; - while (level++ < max_level) { u64 numa_set = kvm_get_first_binded_numa_set(kvm); @@ -1013,7 +1015,7 @@ int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map) { unsigned long size; - struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; phys_addr_t rd = virtcca_cvm->rd; unsigned long ipa = ipa_base; unsigned long phys = pa; @@ -1074,7 +1076,7 @@ int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, unsigned long size; gfn_t gfn; kvm_pfn_t pfn; - struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; phys_addr_t rd = virtcca_cvm->rd; unsigned long ipa = ipa_base; unsigned long phys = pa; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c index 18f49c6525b7..b2227bc969bd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -650,7 +650,7 @@ u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct struct arm_smmu_master *master; int ret = 0; u64 cmd[CMDQ_ENT_DWORDS] = {0}; - struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); /* @@ -1114,62 +1114,6 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) return true; } -/** - * virtcca_smmu_map_pages - Iommu driver calls to this point, - * and then calls the map function in the - * io-pgtable to perform mapping - * @domain: Iommu domain - * @iova: Ipa address - * @paddr: Physical address - * @pgsize: Page size - * @pgcount: Page count - * @prot: Iommu attribute - * @mapped: Maped size - * - * Returns: - * %0 if map success - * %-ENODEV if ops is null - */ -int virtcca_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int prot, size_t *mapped) -{ - struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; - - if (!ops) - return -ENODEV; - - return virtcca_map_pages(ops, iova, paddr, pgsize, pgcount, prot, mapped); -} -EXPORT_SYMBOL_GPL(virtcca_smmu_map_pages); - -/** - * virtcca_smmu_unmap_pages - Iommu driver calls to this point, - * and then calls the unmap function in the - * io-pgtable to perform unmapping - * @domain: Iommu domain - * @iova: Ipa address - * @pgsize: Page size - * @pgcount: Page count - * - * Returns: - * %0 if map success - * %-ENODEV if ops is null - */ - -size_t virtcca_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; - - if (!ops) - return 0; - - return virtcca_unmap_pages(ops, iova, pgsize, pgcount); -} -EXPORT_SYMBOL_GPL(virtcca_smmu_unmap_pages); - /** * virtcca_smmu_secure_dev_operator - Implement security settings for corresponding devices * targeting the secure smmu domain diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h index f5013354e841..7ff5e59a4117 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h @@ -178,7 +178,6 @@ bool virtcca_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm); void _arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); void virtcca_smmu_device_init(struct platform_device *pdev, struct arm_smmu_device *smmu, resource_size_t ioaddr, bool resume, bool disable_bypass); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 76b21c101ba2..1d6af11ef04a 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -34,6 +34,7 @@ #include "dma-iommu.h" #ifdef CONFIG_HISI_VIRTCCA_HOST +#include #include #endif diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 9b6a581cfa30..921e68e19753 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -20,13 +20,12 @@ #include -#include "io-pgtable-arm.h" - #ifdef CONFIG_HISI_VIRTCCA_HOST -#include -#include "../virt/kvm/vfio.h" +#include #endif +#include "io-pgtable-arm.h" + #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -503,113 +502,6 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, return pte; } -#ifdef CONFIG_HISI_VIRTCCA_HOST -/* Obtain kvm from smmu domain */ -static struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data) -{ - struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data->iop.cookie; - - if (!smmu_domain) - return NULL; - - return smmu_domain->kvm; -} - -/** - * virtcca_map_pages - Virtcca need map the secure - * memory with paddr - * @ops: the handle of io_pgtable_ops - * @iova: Ipa address - * @paddr: Physical address - * @pgsize: Page size - * @pgcount: Page count - * @iommu_prot: iommu attribute - * @mapped: mapped size - * - * Returns: - * %0 if map pages success - */ -int virtcca_map_pages(void *ops, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int iommu_prot, size_t *mapped) -{ - struct kvm *kvm; - u64 loader_start; - u64 ram_size; - struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); - struct io_pgtable_cfg *cfg = &data->iop.cfg; - long iaext = (s64)iova >> cfg->ias; - int ret = 0; - - if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) - return -EINVAL; - - if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) - iaext = ~iaext; - if (WARN_ON(iaext || paddr >> cfg->oas)) - return -ERANGE; - - /* If no access, then nothing to do */ - if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) - return 0; - - kvm = virtcca_smmu_domain_get_kvm(data); - if (kvm) { - struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; - - loader_start = virtcca_cvm->loader_start; - ram_size = virtcca_cvm->ram_size; - if (iova >= loader_start && - iova < loader_start + ram_size && - !virtcca_cvm->is_mapped) { - ret = kvm_cvm_map_range(kvm); - } else if (iova < loader_start) { - if (iova == CVM_MSI_ORIG_IOVA) - iova += CVM_MSI_IOVA_OFFSET; - ret = cvm_map_unmap_ipa_range(kvm, iova, paddr, pgsize * pgcount, true); - } - if (mapped) - *mapped += pgsize * pgcount; - } - return ret; -} -EXPORT_SYMBOL_GPL(virtcca_map_pages); - -/** - * virtcca_unmap_pages - Virtcca unmap the iova - * @ops: the handle of io_pgtable_ops - * @iova: Ipa address - * @pgsize: Page size - * @pgcount: Page count - * - * Returns: - * %0 if map pages success or parameter is invalid - */ -size_t virtcca_unmap_pages(void *ops, unsigned long iova, - size_t pgsize, size_t pgcount) -{ - struct kvm *kvm; - struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); - struct io_pgtable_cfg *cfg = &data->iop.cfg; - long iaext = (s64)iova >> cfg->ias; - - if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) - return 0; - - if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) - iaext = ~iaext; - if (WARN_ON(iaext)) - return 0; - - kvm = virtcca_smmu_domain_get_kvm(data); - if (!kvm) - return 0; - - return cvm_map_unmap_ipa_range(kvm, iova, 0, pgsize * pgcount, false); -} -EXPORT_SYMBOL_GPL(virtcca_unmap_pages); -#endif - static int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int iommu_prot, gfp_t gfp, size_t *mapped) @@ -1835,3 +1727,28 @@ static int __init arm_lpae_do_selftests(void) } subsys_initcall(arm_lpae_do_selftests); #endif + +#ifdef CONFIG_HISI_VIRTCCA_HOST +/* Obtain kvm from smmu domain */ +struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data) +{ + struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data->iop.cookie; + + if (!smmu_domain) + return NULL; + + return smmu_domain->kvm; +} + +/* Obtain io pgtable data from io pgtable ops */ +struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops) +{ + return io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); +} + +/* Obtain io pgtable cfg from io pgtable data */ +struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *data) +{ + return &data->iop.cfg; +} +#endif diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 13451f29fe13..e843ae332bad 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -33,6 +33,12 @@ #include #include +#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifndef __GENKSYMS__ +#include +#endif +#endif + #include "dma-iommu.h" #include "iommu-priv.h" @@ -2473,193 +2479,6 @@ static int __iommu_map(struct iommu_domain *domain, unsigned long iova, return ret; } -#ifdef CONFIG_HISI_VIRTCCA_HOST -/** - * virtcca_iommu_map - Iommu driver map pages, and then - * calls the map function in the - * smmu to perform mapping - * @domain: Iommu domain - * @iova: Ipa address - * @paddr: Physical address - * @size: Map size - * @prot: Iommu attribute - * - * Returns: - * %0 if map success - * %-EINVAL if domain type is not paging - * %-ENODEV if the domain pgsize_bitmap is zero or parameter is invalid - */ -int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot) -{ - unsigned int min_pagesz; - int ret = 0; - unsigned long orig_iova = iova; - size_t orig_size = size; - const struct iommu_domain_ops *ops = domain->ops; - - if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) - return -EINVAL; - - if (WARN_ON(domain->pgsize_bitmap == 0UL)) - return -ENODEV; - - /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->pgsize_bitmap); - - /* - * both the virtual address and the physical one, as well as - * the size of the mapping, must be aligned (at least) to the - * size of the smallest page supported by the hardware - */ - if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", - iova, &paddr, size, min_pagesz); - return -EINVAL; - } - - pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size); - - while (size) { - size_t pgsize, count, mapped = 0; - - pgsize = iommu_pgsize(domain, iova, paddr, size, &count); - - pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx count %zu\n", - iova, &paddr, pgsize, count); - ret = virtcca_smmu_map_pages(domain, iova, paddr, pgsize, - count, prot, &mapped); - /* - * Some pages may have been mapped, even if an error occurred, - * so we should account for those so they can be unmapped. - */ - size -= mapped; - - if (ret) - break; - - iova += mapped; - paddr += mapped; - } - - /* unroll mapping in case something went wrong */ - if (ret) - virtcca_iommu_unmap(domain, orig_iova, orig_size - size); - - if (ret == 0 && ops->iotlb_sync_map) { - ret = ops->iotlb_sync_map(domain, iova, size); - if (ret) - goto out_err; - } - - return ret; - -out_err: - /* undo mappings already done */ - virtcca_iommu_unmap(domain, iova, size); - - return ret; -} -EXPORT_SYMBOL_GPL(virtcca_iommu_map); - -/** - * virtcca_iommu_unmap - Iommu driver unmap pages, and then - * calls the map function in the - * smmu to perform unmapping - * @domain: Iommu domain - * @iova: Ipa address - * @size: Map size - * - * Returns: - * %0 if map success or domain type and parameter is invalid - */ - -size_t virtcca_iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size) -{ - size_t unmapped_page, unmapped = 0; - unsigned int min_pagesz; - - if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) - return 0; - - if (WARN_ON(domain->pgsize_bitmap == 0UL)) - return 0; - - /* find out the minimum page size supported */ - min_pagesz = 1 << __ffs(domain->pgsize_bitmap); - - /* - * The virtual address, as well as the size of the mapping, must be - * aligned (at least) to the size of the smallest page supported - * by the hardware - */ - if (!IS_ALIGNED(iova | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", - iova, size, min_pagesz); - return 0; - } - - pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); - - /* - * Keep iterating until we either unmap 'size' bytes (or more) - * or we hit an area that isn't mapped. - */ - while (unmapped < size) { - size_t pgsize, count; - - pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count); - unmapped_page = virtcca_smmu_unmap_pages(domain, iova, pgsize, count); - if (!unmapped_page) - break; - - pr_debug("unmapped: iova 0x%lx size 0x%zx\n", - iova, unmapped_page); - - iova += unmapped_page; - unmapped += unmapped_page; - } - return unmapped; -} -EXPORT_SYMBOL_GPL(virtcca_iommu_unmap); - -/** - * virtcca_attach_secure_dev - Attach the device of iommu - * group to confidential virtual machine - * @domain: The handle of iommu domain - * @group: Iommu group - * - * Returns: - * %0 if attach the all devices success - * %-EINVAL if the smmu does not initialize secure state - * %-ENOMEM if the device create secure ste failed - * %-ENOENT if the device does not have fwspec - */ -int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) -{ - struct group_device *gdev; - int ret = 0; - - mutex_lock(&group->mutex); - for_each_group_device(group, gdev) - ret = virtcca_smmu_secure_dev_operator(domain, gdev->dev); - mutex_unlock(&group->mutex); - return ret; -} -EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); - -/* Obtain domain information through iommu group */ -struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group) -{ - if (iommu_group) - return iommu_group->domain; - - return NULL; -} -EXPORT_SYMBOL_GPL(virtcca_iommu_group_get_domain); -#endif - int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { @@ -3841,3 +3660,40 @@ void iommu_free_global_pasid(ioasid_t pasid) ida_free(&iommu_global_pasid_ida, pasid); } EXPORT_SYMBOL_GPL(iommu_free_global_pasid); + +#ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * virtcca_attach_secure_dev - Attach the device of iommu + * group to confidential virtual machine + * @domain: The handle of iommu domain + * @group: Iommu group + * + * Returns: + * %0 if attach the all devices success + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) +{ + struct group_device *gdev; + int ret = 0; + + mutex_lock(&group->mutex); + for_each_group_device(group, gdev) + ret = virtcca_smmu_secure_dev_operator(domain, gdev->dev); + mutex_unlock(&group->mutex); + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); + +/* Obtain domain information through iommu group */ +struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group) +{ + if (iommu_group) + return iommu_group->domain; + + return NULL; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_group_get_domain); +#endif diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 1433daed0601..2f88c9dcc2ce 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -7,8 +7,9 @@ #ifdef CONFIG_HISI_VIRTCCA_HOST #ifndef __GENKSYMS__ -#include +#include #include +#include #include #endif #endif @@ -85,42 +86,6 @@ EXPORT_SYMBOL(pci_bus_write_config_byte); EXPORT_SYMBOL(pci_bus_write_config_word); EXPORT_SYMBOL(pci_bus_write_config_dword); -#ifdef CONFIG_HISI_VIRTCCA_HOST -/* If device is secure dev, read config need transfer to tmm module */ -static int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, - unsigned int devfn, int size, u32 *val) -{ - if (size == 1) - *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_8_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - else if (size == 2) - *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_16_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - else - *val = tmi_mmio_read(iova_to_pa(addr), CVM_RW_32_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - - return PCIBIOS_SUCCESSFUL; -} - -/* If device is secure dev, write config need transfer to tmm module */ -int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, - unsigned int devfn, int size, u32 val) -{ - if (size == 1) - WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, - CVM_RW_8_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - else if (size == 2) - WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, - CVM_RW_16_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - else - WARN_ON(tmi_mmio_write(iova_to_pa(addr), val, - CVM_RW_32_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - - return PCIBIOS_SUCCESSFUL; -} -#endif - int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 73579c22654e..d2c9863d0368 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -13,6 +13,10 @@ #include "../pci.h" #include "msi.h" +#ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#endif + int pci_msi_enable = 1; int pci_msi_ignore_mask; @@ -147,73 +151,6 @@ void pci_msi_unmask_irq(struct irq_data *data) } EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); -#ifdef CONFIG_HISI_VIRTCCA_HOST -/** - * virtcca_pci_read_msi_msg - secure dev read msi msg - * @dev: Pointer to the pci_dev data structure of MSI-X device function - * @msg: Msg information - * @base: Msi base address - * - **/ -static inline void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, - void __iomem *base) -{ - u64 pbase = iova_to_pa(base); - - msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, - CVM_RW_32_BIT, pci_dev_id(dev)); - msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, - CVM_RW_32_BIT, pci_dev_id(dev)); - msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); -} - -/** - * virtcca_pci_write_msi_msg - secure dev write msi msg - * @desc: MSI-X description - * @msg: Msg information - * - **/ -static inline int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - void __iomem *base = pci_msix_desc_addr(desc); - u32 ctrl = desc->pci.msix_ctrl; - bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - u64 pbase = iova_to_pa(base); - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (!is_cc_dev(pci_dev_id(pdev))) - return 0; - - u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); - - addr += CVM_MSI_IOVA_OFFSET; - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, - lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, - upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, - msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); - - if (unmasked) - pci_msix_write_vector_ctrl(desc, ctrl); - tmi_mmio_read(iova_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), - CVM_RW_32_BIT, pci_dev_id(pdev)); - - return 1; -} - -static inline void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, - struct msi_desc *desc, void __iomem *addr) -{ - desc->pci.msix_ctrl = tmi_mmio_read(iova_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), - CVM_RW_32_BIT, pci_dev_id(dev)); -} -#endif - void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); @@ -803,40 +740,6 @@ static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries return ret; } - -#ifdef CONFIG_HISI_VIRTCCA_HOST -/** - * msix_mask_all_cc - mask all secure dev msix c - * @dev: Pointer to the pci_dev data structure of MSI-X device function - * @base: Io address - * @tsize: Number of entry - * @dev_num: Dev number - * - * Returns: - * %0 if msix mask all cc device success - **/ -static int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) -{ - int i; - u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; - u64 pbase = iova_to_pa(base); - - if (pci_msi_ignore_mask) - goto out; - - for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, - ctrl, CVM_RW_32_BIT, dev_num); - } - -out: - pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); - - pcibios_free_irq(dev); - return 0; -} -#endif - /** * msix_capability_init - configure device's MSI-X capability * @dev: pointer to the pci_dev data structure of MSI-X device function diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h index afc9371985c5..154e2c00f94c 100644 --- a/drivers/pci/msi/msi.h +++ b/drivers/pci/msi/msi.h @@ -5,8 +5,7 @@ #ifdef CONFIG_HISI_VIRTCCA_HOST #ifndef __GENKSYMS__ -#include -#include +#include #endif #endif @@ -33,50 +32,6 @@ static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) return desc->pci.mask_base + desc->msi_index * PCI_MSIX_ENTRY_SIZE; } -#ifdef CONFIG_HISI_VIRTCCA_HOST -/* - * If it is a safety device, write vector ctrl need - * use tmi interface - */ -static inline int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - void __iomem *desc_addr = pci_msix_desc_addr(desc); - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) - return 0; - - if (desc->pci.msi_attrib.can_mask) - tmi_mmio_write(iova_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), - ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); - return 1; -} - -/* - * If it is a safety device, read msix need - * use tmi interface - */ -static inline int virtcca_pci_msix_mask(struct msi_desc *desc) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) - return 0; - - /* Flush write to device */ - tmi_mmio_read(iova_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); - return 1; -} -#endif - /* * This internal function does not flush PCI writes to the device. All * users must ensure that they read from the device before either assuming diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index aeb050ab8422..d7eda4ca1de9 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -21,7 +21,7 @@ #ifdef CONFIG_HISI_VIRTCCA_HOST #ifndef __GENKSYMS__ -#include +#include #include #endif #endif @@ -45,39 +45,6 @@ #define vfio_iowrite8 iowrite8 #ifdef CONFIG_HISI_VIRTCCA_HOST -/* Judge startup virtcca_cvm_host is enable and device is secure or not */ -static bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) -{ - if (!is_virtcca_cvm_enable()) - return false; - - struct pci_dev *pdev = vdev->pdev; - bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); - - if (cc_dev) - return true; - - return false; -} - -/* Transfer to tmm write io value */ -static void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, - u64 size, void __iomem *io) -{ - struct pci_dev *pdev = vdev->pdev; - - WARN_ON(tmi_mmio_write(iova_to_pa(io), val, size, pci_dev_id(pdev))); -} - -/* Transfer to tmm read io value */ -static u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, - u64 size, void __iomem *io) -{ - struct pci_dev *pdev = vdev->pdev; - - return tmi_mmio_read(iova_to_pa(io), size, pci_dev_id(pdev)); -} - #define VFIO_IOWRITE(size) \ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io) \ diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 6ba9e54202f1..b33c145520ad 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -39,8 +39,10 @@ #include #include "vfio.h" #ifdef CONFIG_HISI_VIRTCCA_HOST +#include +#include +#include #include -#include "../virt/kvm/vfio.h" #endif #define DRIVER_VERSION "0.2" @@ -1047,28 +1049,16 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain, #ifdef CONFIG_HISI_VIRTCCA_HOST static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma); -/* Whether the kvm is cvm */ -static bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm) -{ - struct arm_smmu_domain *arm_smmu_domain; - - arm_smmu_domain = to_smmu_domain(domain); - *kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); - if (*kvm) - return (*kvm)->arch.is_virtcca_cvm; - - return false; -} - -static bool virtcca_check_kvm_is_cvm(struct vfio_iommu *iommu, struct kvm **kvm) +bool virtcca_check_kvm_is_cvm(void *iommu, struct kvm **kvm) { struct vfio_domain *domain; bool is_virtcca_cvm = false; + struct vfio_iommu *vfio_iommu = (struct vfio_iommu *)iommu; - if (!iommu || !kvm) + if (!vfio_iommu || !kvm) return false; - list_for_each_entry(domain, &iommu->domain_list, next) { + list_for_each_entry(domain, &vfio_iommu->domain_list, next) { if (domain && domain->domain && virtcca_iommu_domain_get_kvm(domain->domain, kvm)) { is_virtcca_cvm = true; break; @@ -1079,13 +1069,14 @@ static bool virtcca_check_kvm_is_cvm(struct vfio_iommu *iommu, struct kvm **kvm) } /* Traverse all domains and perform mapping operations */ -static int virtcca_vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, +int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot) { struct vfio_domain *d; int ret; + struct vfio_iommu *vfio_iommu = (struct vfio_iommu *)iommu; - list_for_each_entry(d, &iommu->domain_list, next) { + list_for_each_entry(d, &vfio_iommu->domain_list, next) { ret = virtcca_iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, npage << PAGE_SHIFT, prot | IOMMU_CACHE); if (ret) @@ -1097,7 +1088,7 @@ static int virtcca_vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, return 0; unwind: - list_for_each_entry_continue_reverse(d, &iommu->domain_list, next) { + list_for_each_entry_continue_reverse(d, &vfio_iommu->domain_list, next) { virtcca_iommu_unmap(d->domain, iova, npage << PAGE_SHIFT); cond_resched(); } @@ -1125,13 +1116,13 @@ static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *d size_t size = map_size; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; - bool is_virtcca_cvm = virtcca_check_kvm_is_cvm(iommu, &kvm); + bool is_virtcca_cvm = virtcca_check_kvm_is_cvm((void *)iommu, &kvm); vfio_batch_init(&batch); while (size) { - if (is_virtcca_cvm && !check_virtcca_cvm_vfio_map_dma(kvm, dma->iova)) + if (is_virtcca_cvm && !is_virtcca_iova_need_vfio_dma(kvm, dma->iova)) break; /* Pin a contiguous chunk of memory */ npage = vfio_pin_pages_remote(dma, vaddr + dma->size, @@ -1153,7 +1144,7 @@ static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *d break; } - if (is_virtcca_cvm && check_virtcca_cvm_ram_range(kvm, iova)) { + if (is_virtcca_cvm && is_in_virtcca_ram_range(kvm, iova)) { vfio_unpin_pages_remote(dma, iova + dma->size, pfn, npage, true); vfio_batch_unpin(&batch, dma); @@ -1191,7 +1182,7 @@ static long virtcca_vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma * long unlocked = 0; dma_addr_t iova = dma->iova, end = dma->iova + dma->size; - if (!virtcca_check_kvm_is_cvm(iommu, &kvm)) + if (!virtcca_check_kvm_is_cvm((void *)iommu, &kvm)) return 0; if (!dma->size) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0b8feb6de953..8fb93470a134 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1762,24 +1762,4 @@ static inline void iopf_group_response(struct iopf_group *group, { } #endif /* CONFIG_IOMMU_IOPF */ - -#ifdef CONFIG_HISI_VIRTCCA_HOST -int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); -struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group); -int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot); -size_t virtcca_iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size); -int virtcca_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int prot, size_t *mapped); -size_t virtcca_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, - size_t pgsize, size_t pgcount); -int virtcca_map_pages(void *ops, unsigned long iova, - phys_addr_t paddr, size_t pgsize, size_t pgcount, - int iommu_prot, size_t *mapped); -size_t virtcca_unmap_pages(void *ops, unsigned long iova, - size_t pgsize, size_t pgcount); -#endif #endif /* __LINUX_IOMMU_H */ diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index d15ac5d25ceb..1dc81283e456 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -22,6 +22,7 @@ #endif #ifdef CONFIG_HISI_VIRTCCA_HOST +#include #include #endif @@ -144,67 +145,6 @@ static void kvm_vfio_update_coherency(struct kvm_device *dev) } } -#ifdef CONFIG_HISI_VIRTCCA_HOST -/** - * cvm_vfio_file_iommu_group - Get iommu group from vfio file - * @file: Vfio file - * - * Returns: - * %NULL if the virtcca_vfio_file_iommu_group func is not defined - * or CONFIG_HISI_VIRTCCA_HOST is not enable, group is null - * %iommu_group if get the iommu group from file success - */ -static struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) -{ - struct iommu_group *(*fn)(struct file *file); - struct iommu_group *ret; - - fn = symbol_get(virtcca_vfio_file_iommu_group); - if (!fn) - return NULL; - - ret = fn(file); - - symbol_put(virtcca_vfio_file_iommu_group); - - return ret; -} - -/** - * cvm_vfio_add_kvm_to_smmu_domain - Bind the confidential - * virtual machine to smmu domain - * @filp: The handle of file - * @kvm: The kvm belone to confidential virtual machine - * - * Returns: - * %-ENXIO if set kvm failed or iommu group is null - * %0 if set kvm success - */ -static int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, struct kvm_vfio *kv) -{ - struct iommu_group *iommu_group; - int ret = 0; - - if (!is_virtcca_cvm_enable()) - return ret; - - mutex_unlock(&kv->lock); - iommu_group = cvm_vfio_file_iommu_group(filp); - if (!iommu_group) { - ret = -ENXIO; - goto out_lock; - } - if (cvm_arm_smmu_domain_set_kvm((void *)iommu_group)) { - ret = -ENXIO; - goto out_lock; - } - -out_lock: - mutex_lock(&kv->lock); - return ret; -} -#endif - static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { struct kvm_vfio *kv = dev->private; @@ -244,7 +184,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvm_vfio_file_set_kvm(kvf->file, dev->kvm); kvm_vfio_update_coherency(dev); #ifdef CONFIG_HISI_VIRTCCA_HOST - ret = cvm_vfio_add_kvm_to_smmu_domain(filp, kv); + ret = cvm_vfio_add_kvm_to_smmu_domain(filp, (void *)kv); #endif out_unlock: @@ -461,8 +401,46 @@ void kvm_vfio_ops_exit(void) kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); } - #ifdef CONFIG_HISI_VIRTCCA_HOST +/** + * cvm_vfio_add_kvm_to_smmu_domain - Bind the confidential + * virtual machine to smmu domain + * @filp: The handle of file + * @kvm: The kvm belone to confidential virtual machine + * + * Returns: + * %-ENXIO if set kvm failed or iommu group is null + * %0 if set kvm success + */ +int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv) +{ + struct iommu_group *iommu_group; + int ret = 0; + + if (!is_virtcca_cvm_enable()) + return ret; + + /* Upper-level calling interface has added a kv lock, but the + * cvm_arm_smmu_domain_set_kvm interface also need add this lock. + * Therefore, it is necessary to unlock here and completing the + * acquisition, then add the kv lock before return. + */ + mutex_unlock(&((struct kvm_vfio *)kv)->lock); + iommu_group = cvm_vfio_file_iommu_group(filp); + if (!iommu_group) { + ret = -ENXIO; + goto out_lock; + } + if (cvm_arm_smmu_domain_set_kvm((void *)iommu_group)) { + ret = -ENXIO; + goto out_lock; + } + +out_lock: + mutex_lock(&((struct kvm_vfio *)kv)->lock); + return ret; +} + /** * virtcca_arm_smmu_get_kvm - Find the kvm * with vfio devices through SMMU domain @@ -480,7 +458,6 @@ struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) struct kvm_vfio *kv; struct kvm_vfio_file *kvf; struct iommu_group *iommu_group; - unsigned long flags; struct arm_smmu_master *master; diff --git a/virt/kvm/vfio.h b/virt/kvm/vfio.h index 4f263bf43f18..e130a4a03530 100644 --- a/virt/kvm/vfio.h +++ b/virt/kvm/vfio.h @@ -2,11 +2,6 @@ #ifndef __KVM_VFIO_H #define __KVM_VFIO_H -#ifdef CONFIG_HISI_VIRTCCA_HOST -#include "../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" -#include "../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" -#endif - #ifdef CONFIG_KVM_VFIO int kvm_vfio_ops_init(void); void kvm_vfio_ops_exit(void); @@ -20,8 +15,4 @@ static inline void kvm_vfio_ops_exit(void) } #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST -struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); -void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); -#endif #endif -- Gitee From ad540a13eb8292fc92651b3bf281088a502a85f9 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Fri, 20 Sep 2024 10:55:55 +0800 Subject: [PATCH 11/12] virtcca feature: fix msi iova map virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Fix msi iova map Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/configs/openeuler_defconfig | 1 + arch/arm64/include/asm/kvm_tmi.h | 1 - arch/arm64/include/asm/kvm_tmm.h | 21 +- arch/arm64/include/asm/virtcca_coda.h | 50 +- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kvm/mmu.c | 8 +- arch/arm64/kvm/virtcca_cvm.c | 148 +++-- drivers/Kconfig | 2 + drivers/Makefile | 2 + drivers/base/platform-msi.c | 2 +- drivers/coda/Kconfig | 8 + drivers/coda/Makefile | 4 + drivers/coda/coda.c | 546 ++++++++++++++++++ drivers/coda/coda_pci.c | 214 +++++++ .../coda/coda_vfio.c | 301 +++------- drivers/iommu/arm/arm-smmu-v3/Makefile | 2 +- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c | 523 +---------------- drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h | 16 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 10 +- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 6 +- drivers/iommu/dma-iommu.c | 88 +-- drivers/iommu/io-pgtable-arm.c | 21 +- drivers/iommu/iommu.c | 43 -- drivers/pci/access.c | 12 +- drivers/pci/msi/msi.c | 14 +- drivers/pci/msi/msi.h | 10 +- drivers/vfio/group.c | 4 +- drivers/vfio/pci/vfio_pci_core.c | 4 +- drivers/vfio/pci/vfio_pci_rdwr.c | 8 +- drivers/vfio/vfio_iommu_type1.c | 44 +- include/linux/iommu.h | 2 +- include/linux/msi.h | 2 +- include/linux/pci.h | 4 - include/linux/vfio.h | 3 - include/uapi/linux/vfio.h | 2 +- virt/kvm/vfio.c | 163 +++++- 36 files changed, 1281 insertions(+), 1010 deletions(-) create mode 100644 drivers/coda/Kconfig create mode 100644 drivers/coda/Makefile create mode 100644 drivers/coda/coda.c create mode 100644 drivers/coda/coda_pci.c rename arch/arm64/kernel/virtcca_coda.c => drivers/coda/coda_vfio.c (50%) diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 45d9443367e1..f091fa794a19 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -760,6 +760,7 @@ CONFIG_KVM_HISI_VIRT=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_HISI_VIRTCCA_HOST=y +CONFIG_HISI_VIRTCCA_CODA=y # CONFIG_NVHE_EL2_DEBUG is not set CONFIG_KVM_ARM_MULTI_LPI_TRANSLATE_CACHE=y CONFIG_ARCH_VCPU_STAT=y diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index eb2a351a3edb..cab2b4cdd0f2 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -83,7 +83,6 @@ struct tmi_smmu_cfg_params { #define TMI_SMMU_CMD_QUEUE 1 #define TMI_SMMU_EVT_QUEUE 2 struct tmi_smmu_queue_params { - uint64_t ns_src; /* non-secure queue source address */ uint64_t smmu_base_addr; /* smmu base address */ uint64_t size; /* queue size */ uint64_t smmu_id; /* smmu id */ diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index faf1e00e4232..a26cade21586 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -10,17 +10,20 @@ /* * There is a conflict with the internal iova of CVM, * so it is necessary to offset the msi iova. + * According to qemu file(hw/arm/virt.c), 0x0a001000 - 0x0b000000 + * iova is not being used, so it is used as the iova range for msi + * mapping. */ -#define CVM_MSI_ORIG_IOVA 0x8000000 -#define CVM_MSI_IOVA_OFFSET (-0x1000000) +#define CVM_MSI_ORIG_IOVA 0x8000000 +#define CVM_MSI_MIN_IOVA 0x0a001000 +#define CVM_MSI_MAX_IOVA 0x0b000000 +#define CVM_MSI_IOVA_OFFSET 0x1000 #define CVM_RW_8_BIT 0x8 #define CVM_RW_16_BIT 0x10 #define CVM_RW_32_BIT 0x20 #define CVM_RW_64_BIT 0x40 -#define BUS_NUM_SHIFT 0x8 - enum virtcca_cvm_state { CVM_STATE_NONE = 1, CVM_STATE_NEW, @@ -102,11 +105,8 @@ void kvm_free_rd(struct kvm *kvm); int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target); void kvm_cvm_unmap_destroy_range(struct kvm *kvm); - int kvm_cvm_map_range(struct kvm *kvm); -int cvm_arm_smmu_domain_set_kvm(void *group); -int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, - unsigned long map_size, uint32_t is_map); +int virtcca_cvm_arm_smmu_domain_set_kvm(void *group); int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map); int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, @@ -128,6 +128,11 @@ bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova); ((CVM_PAGE_SHIFT - 3) * (4 - (l)) + 3) #define CVM_L2_BLOCK_SIZE BIT(CVM_TTT_LEVEL_SHIFT(2)) +#define TMM_GRANULE_SIZE2 12 +#define TMM_TTT_WIDTH 19 +#define TMM_GRANULE_SIZE (1UL << TMM_GRANULE_SIZE2) +#define tmm_granule_size(level) (TMM_GRANULE_SIZE << ((3 - level)) * TMM_TTT_WIDTH) + static inline unsigned long cvm_ttt_level_mapsize(int level) { if (WARN_ON(level > CVM_TTT_BLOCK_LEVEL)) diff --git a/arch/arm64/include/asm/virtcca_coda.h b/arch/arm64/include/asm/virtcca_coda.h index e7fcba7bb255..7e9469789e47 100644 --- a/arch/arm64/include/asm/virtcca_coda.h +++ b/arch/arm64/include/asm/virtcca_coda.h @@ -11,11 +11,26 @@ #include "../../../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" #include "../../../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA + +#define MAX_CC_DEV_NUM_ORDER 8 +#define MASK_DEV_FUNCTION 0xfff8 +#define MASK_DEV_BUS 0xff + +#define DEV_BUS_NUM 0x8 +#define DEV_FUNCTION_NUM 0x3 + +#define STE_ENTRY_SIZE 0x40 + +#define SMMU_DOMAIN_IS_SAME 0x2 + int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); -struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group); +u64 virtcca_get_iommu_device_msi_addr(struct iommu_group *iommu_group); +int virtcca_iommu_group_set_dev_msi_addr(struct iommu_group *iommu_group, unsigned long *iova); +int virtcca_map_msi_address(struct kvm *kvm, struct arm_smmu_domain *smmu_domain, phys_addr_t pa, + unsigned long map_size); + int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot); size_t virtcca_iommu_unmap(struct iommu_domain *domain, @@ -28,12 +43,12 @@ size_t virtcca_unmap_pages(void *ops, unsigned long iova, void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, void __iomem *base); -int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg); +bool virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg); void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc, void __iomem *addr); -int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl); -int virtcca_pci_msix_mask(struct msi_desc *desc); -int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num); +bool virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl); +bool virtcca_pci_msix_mask(struct msi_desc *desc); +int virtcca_msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num); int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, unsigned int devfn, int size, u32 *val); @@ -47,7 +62,7 @@ u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, u64 size, void __iomem *io); bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm); -bool virtcca_check_kvm_is_cvm(void *iommu, struct kvm **kvm); +bool virtcca_check_is_cvm_or_not(void *iommu, struct kvm **kvm); int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot); int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv); @@ -56,6 +71,23 @@ void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_grou struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops); struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *data); struct iommu_group *cvm_vfio_file_iommu_group(struct file *file); -struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data); +void *virtcca_io_pgtable_get_smmu_domain(struct arm_lpae_io_pgtable *data); + +struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file); + +bool is_cc_vmid(u32 vmid); +/* Has the root bus device number switched to secure */ +bool is_cc_dev(u32 sid); + +u64 get_g_cc_dev_msi_addr(u32 sid); + +void set_g_cc_dev_msi_addr(u32 sid, u64 msi_addr); + +void g_cc_dev_table_init(void); + +u32 virtcca_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm); + +struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, + phys_addr_t msi_addr, struct iommu_domain *domain); #endif #endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fe6334bbe93e..4ce58887302a 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -82,7 +82,7 @@ obj-$(CONFIG_ARM64_ILP32) += vdso-ilp32/ obj-$(CONFIG_UNWIND_PATCH_PAC_INTO_SCS) += patch-scs.o obj-$(CONFIG_IPI_AS_NMI) += ipi_nmi.o obj-$(CONFIG_HISI_VIRTCCA_GUEST) += virtcca_cvm_guest.o virtcca_cvm_tsi.o -obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o virtcca_coda.o +obj-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_host.o CFLAGS_patch-scs.o += -mbranch-protection=none # Force dependency (vdso*-wrap.S includes vdso.so through incbin) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index ca2abf7d98eb..68efca6ef24e 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -19,7 +19,7 @@ #include #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif #include "trace.h" @@ -873,7 +873,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t u64 mmfr0, mmfr1; u32 phys_shift; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if ((type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) && (!kvm_is_virtcca_cvm(kvm))) #else if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) @@ -1420,7 +1420,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); write_fault = kvm_is_write_fault(vcpu); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (vcpu_is_tec(vcpu)) { write_fault = true; prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; @@ -1608,7 +1608,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA ret = kvm_cvm_map_ipa(kvm, fault_ipa, pfn, vma_pagesize, prot, ret); #endif diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 6b29f65018ba..e1fd6fed66cd 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -11,12 +11,13 @@ #include #include #include -#include #include #include #include #include +#include + /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); static unsigned long *cvm_vmid_bitmap; @@ -179,18 +180,22 @@ void kvm_destroy_cvm(struct kvm *kvm) { struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; uint32_t cvm_vmid; +#ifdef CONFIG_HISI_VIRTCCA_CODA struct arm_smmu_domain *arm_smmu_domain; struct list_head smmu_domain_group_list; +#endif if (!cvm) return; +#ifdef CONFIG_HISI_VIRTCCA_CODA /* Unmap the cvm with arm smmu domain */ kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { if (arm_smmu_domain->kvm && arm_smmu_domain->kvm == kvm) arm_smmu_domain->kvm = NULL; } +#endif cvm_vmid = cvm->cvm_vmid; kfree(cvm->params); @@ -538,9 +543,11 @@ int kvm_cvm_map_range(struct kvm *kvm) static int kvm_activate_cvm(struct kvm *kvm) { +#ifdef CONFIG_HISI_VIRTCCA_CODA int ret; struct arm_smmu_domain *arm_smmu_domain; struct list_head smmu_domain_group_list; +#endif struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) @@ -549,14 +556,16 @@ static int kvm_activate_cvm(struct kvm *kvm) if (!cvm->is_mapped && kvm_cvm_map_range(kvm)) return -EFAULT; +#ifdef CONFIG_HISI_VIRTCCA_CODA kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { if (arm_smmu_domain) { - ret = virtcca_smmu_tmi_dev_attach(arm_smmu_domain, kvm); + ret = virtcca_tmi_dev_attach(arm_smmu_domain, kvm); if (ret) return ret; } } +#endif if (tmi_cvm_activate(cvm->rd)) { kvm_err("tmi_cvm_activate failed!\n"); @@ -883,6 +892,7 @@ int kvm_init_cvm_vm(struct kvm *kvm) return 0; } +#ifdef CONFIG_HISI_VIRTCCA_CODA /* * Coda (Confidential Device Assignment) feature * enable devices to pass directly to confidential virtual machines @@ -940,35 +950,6 @@ bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova) } EXPORT_SYMBOL_GPL(is_virtcca_iova_need_vfio_dma); -/** - * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CV - * @group: Iommu group - * - * Returns: - * %0 if smmu_domain has been associate cvm or associate cvm successfully - * %-ENXIO if the iommu group does not have smmu domain - */ -int cvm_arm_smmu_domain_set_kvm(void *group) -{ - struct arm_smmu_domain *arm_smmu_domain = NULL; - struct iommu_domain *domain; - struct kvm *kvm; - - domain = virtcca_iommu_group_get_domain((struct iommu_group *)group); - if (!domain) - return -ENXIO; - - arm_smmu_domain = to_smmu_domain(domain); - if (arm_smmu_domain->kvm) - return 0; - - kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); - if (kvm && kvm_is_virtcca_cvm(kvm)) - arm_smmu_domain->kvm = kvm; - - return 0; -} - static int kvm_cvm_dev_ttt_create(struct virtcca_cvm *cvm, unsigned long addr, int level, @@ -998,6 +979,42 @@ int kvm_cvm_create_dev_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, return 0; } +/** + * cvm_map_max_level_size - MMIO Map according to largest possible granularity + * @map_start: The start of map address + * @map_end: The end of map address + * @map_size: Map range + * + * Returns: + * %level the map level + * %-ENXIO if no suitable mapping level was found + */ +static int cvm_map_max_level_size(unsigned long map_start, unsigned long map_end, + unsigned long *map_size) +{ + int level = 1; + + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + level++; + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + level++; + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + pr_err("level not allow to map size\n"); + return -ENXIO; +} + /** * cvm_map_unmap_ipa_range - Vfio driver map or * unmap cvm ipa @@ -1014,41 +1031,48 @@ int kvm_cvm_create_dev_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map) { - unsigned long size; + unsigned long map_start; + unsigned long map_end; + int level; struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; phys_addr_t rd = virtcca_cvm->rd; - unsigned long ipa = ipa_base; unsigned long phys = pa; int ret = 0; - for (size = 0; size < map_size; size += PAGE_SIZE) { + map_start = ipa_base; + map_end = map_start + map_size; + while (map_start < map_end) { + level = cvm_map_max_level_size(map_start, map_end, &map_size); + if (level < 0) { + ret = -ENXIO; + goto err; + } if (is_map) - ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + ret = tmi_mmio_map(rd, map_start, level, phys); else - ret = tmi_mmio_unmap(rd, ipa, CVM_TTT_MAX_LEVEL); + ret = tmi_mmio_unmap(rd, map_start, level); if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { /* Create missing TTTs and retry */ int level_fault = TMI_RETURN_INDEX(ret); if (is_map) { - ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, ipa, + ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, map_start, level_fault, CVM_TTT_MAX_LEVEL, NULL); if (ret) goto err; - ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + ret = tmi_mmio_map(rd, map_start, level, phys); } else { - ret = tmi_mmio_unmap(rd, ipa, level_fault); + ret = tmi_mmio_unmap(rd, map_start, level_fault); + map_size = tmm_granule_size(level_fault); } } if (ret) goto err; - if (size + PAGE_SIZE >= map_size) - break; - ipa += PAGE_SIZE; - phys += PAGE_SIZE; + map_start += map_size; + phys += map_size; } return 0; @@ -1158,3 +1182,41 @@ void virtcca_cvm_set_secure_flag(void *vdev, void *info) } EXPORT_SYMBOL_GPL(virtcca_cvm_set_secure_flag); +/** + * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CVM + * @dev: The Device under the iommu group + * + * Returns: + * %0 if smmu_domain has been associate cvm or associate cvm successfully + * %-ENXIO if the iommu group does not have smmu domain + */ +int cvm_arm_smmu_domain_set_kvm(struct device *dev, void *data) +{ + struct kvm *kvm; + struct iommu_domain *domain; + struct arm_smmu_domain *arm_smmu_domain = NULL; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain->kvm) + return 1; + + kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (kvm && kvm_is_virtcca_cvm(kvm)) + arm_smmu_domain->kvm = kvm; + + return 1; +} + +int virtcca_cvm_arm_smmu_domain_set_kvm(void *group) +{ + int ret; + + ret = iommu_group_for_each_dev((struct iommu_group *)group, + (void *)NULL, cvm_arm_smmu_domain_set_kvm); + return ret; +} +#endif diff --git a/drivers/Kconfig b/drivers/Kconfig index f391050ac29f..3be1197d872c 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -249,4 +249,6 @@ source "drivers/cpuinspect/Kconfig" source "drivers/roh/Kconfig" +source "drivers/coda/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index f884f6f1f80d..3955e605df14 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -203,3 +203,5 @@ obj-$(CONFIG_CDX_BUS) += cdx/ obj-$(CONFIG_S390) += s390/ obj-$(CONFIG_ROH) += roh/ + +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda/ diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 3133593238e2..3348d4db5f1b 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -221,7 +221,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, } EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA /** * platform_msi_domain_alloc_range_irqs - Allocate specific scope MSI interrupts for @dev * @dev: The device for which to allocate interrupts diff --git a/drivers/coda/Kconfig b/drivers/coda/Kconfig new file mode 100644 index 000000000000..3f41ef22b44c --- /dev/null +++ b/drivers/coda/Kconfig @@ -0,0 +1,8 @@ +config HISI_VIRTCCA_CODA + bool "Support for virtCCA Confidential Device Assignment" + default y + depends on HISI_VIRTCCA_HOST + help + Support VIRTCCA Device Assignment based on S-EL2 + + If unsure, say Y. diff --git a/drivers/coda/Makefile b/drivers/coda/Makefile new file mode 100644 index 000000000000..62ca3132ac4d --- /dev/null +++ b/drivers/coda/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda.o +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda_vfio.o +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda_pci.o diff --git a/drivers/coda/coda.c b/drivers/coda/coda.c new file mode 100644 index 000000000000..acc32dd730db --- /dev/null +++ b/drivers/coda/coda.c @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include +#include + +struct cc_dev_config { + u32 sid; /* BDF number of the device */ + u32 vmid; /* virtual machine id */ + u32 root_bd; /* root bus and device number. */ + bool secure; /* device secure attribute */ + /* MSI addr for confidential device with iommu group granularity */ + u64 msi_addr; + struct hlist_node node; /* device hash table */ +}; + +static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); + +/** + * get_root_bd - Traverse pcie topology to find the root number + * @dev: The device for which to get root bd + * + * Returns: + * %-1 if error or not pci device + */ +static int get_root_bd(struct device *dev) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return -1; + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return -1; + + while (!pci_is_root_bus(pdev->bus)) + pdev = pci_upstream_bridge(pdev); + + return pci_dev_id(pdev) & MASK_DEV_FUNCTION; +} + +/** + * get_child_devices_rec - Traverse pcie topology to find child devices + * If dev is a bridge, get it's children + * If dev is a regular device, get itself + * @dev: Device for which to get child devices + * @devs: All child devices under input dev + * @max_devs: Max num of devs + * @ndev: Num of child devices + */ +static void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, + int max_devs, int *ndev) +{ + struct pci_bus *bus = dev->subordinate; + + if (bus) { /* dev is a bridge */ + struct pci_dev *child; + + list_for_each_entry(child, &bus->devices, bus_list) { + get_child_devices_rec(child, devs, max_devs, ndev); + } + } else { /* dev is a regular device */ + uint16_t bdf = pci_dev_id(dev); + int i; + /* check if bdf is already in devs */ + for (i = 0; i < *ndev; i++) { + if (devs[i] == bdf) + return; + } + /* check overflow */ + if (*ndev >= max_devs) { + pr_warn("S_SMMU: devices num over max devs\n"); + return; + } + devs[*ndev] = bdf; + *ndev = *ndev + 1; + } +} + +/** + * get_sibling_devices - Get all devices which share the same root_bd as dev + * @dev: Device for which to get child devices + * @devs: All child devices under input dev + * @max_devs: Max num of devs + * + * Returns: + * %0 if get child devices failure + */ +static int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) +{ + struct pci_dev *pdev; + int ndev = 0; + + if (!dev_is_pci(dev)) + return ndev; + + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return ndev; + + while (!pci_is_root_bus(pdev->bus)) + pdev = pci_upstream_bridge(pdev); + + get_child_devices_rec(pdev, devs, max_devs, &ndev); + return ndev; +} + +/** + * add_cc_dev_obj - Add device obj to hash tablse + * @sid: Stream id of device + * @vmid: Virtual machine id + * @root_bd: Root port bus device num + * @secure: Whether the device is secure or not + * + * Returns: + * %0 if add obj success + * %-ENOMEM if alloc obj failed + */ +static int add_cc_dev_obj(u32 sid, u32 vmid, u32 root_bd, bool secure) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj->sid == sid) { + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + return 0; + } + } + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) + return -ENOMEM; + + obj->sid = sid; + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + + hash_add(g_cc_dev_htable, &obj->node, sid); + return 0; +} + +/** + * is_cc_root_bd - Whether the root port is secure or not + * @root_bd: Root port bus device num + * + * Returns: + * %true if the root bd is secure + * %false if the root bd is non-secure + */ +static bool is_cc_root_bd(u32 root_bd) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (obj->root_bd == root_bd && obj->secure) + return true; + } + + return false; +} + +/** + * is_cc_vmid - Whether the vm is confidential vm + * @vmid: Virtual machine id + * + * Returns: + * %true if the vm is confidential + * %false if the vm is not confidential + */ +bool is_cc_vmid(u32 vmid) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (vmid > 0 && obj->vmid == vmid) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(is_cc_vmid); + +/** + * is_cc_dev - Whether the stream id of dev is confidential + * @sid: Stream id of dev + * + * Returns: + * %true if the dev is confidential + * %false if the dev is not confidential + */ +bool is_cc_dev(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->secure; + } + + return false; +} +EXPORT_SYMBOL(is_cc_dev); + +/** + * get_g_cc_dev_msi_addr - Obtain the msi address of confidential device + * @sid: Stream id of dev + * + * Returns: + * %0 if does not find the confidential device that matches the stream id + * %msi_addr return the msi address of confidential device that matches the stream id + */ +u64 get_g_cc_dev_msi_addr(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->msi_addr; + } + return 0; +} + +/** + * set_g_cc_dev_msi_addr - Set the msi address of confidential device + * @sid: Stream id of dev + * @msi_addr: Msi address + */ +void set_g_cc_dev_msi_addr(u32 sid, u64 msi_addr) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid && !obj->msi_addr) { + obj->msi_addr = msi_addr; + return; + } + } +} + +/* Secure device hash table init */ +void g_cc_dev_table_init(void) +{ + hash_init(g_cc_dev_htable); +} +EXPORT_SYMBOL(g_cc_dev_table_init); + +/** + * virtcca_tmi_dev_attach - Complete the stage2 page table establishment + * for the security device + * @arm_smmu_domain: The handle of smmu domain + * @kvm: The handle of virtual machine + * + * Returns: + * %0 if attach dev success + * %-ENXIO if the root port of device does not have pcipc capability + */ +u32 virtcca_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm) +{ + unsigned long flags; + int i, j; + struct arm_smmu_master *master; + int ret = 0; + u64 cmd[CMDQ_ENT_DWORDS] = {0}; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); + /* + * Traverse all devices under the secure smmu domain and + * set the correspnding address translation table for each device + */ + list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + ret = tmi_dev_attach(sid, virtcca_cvm->rd, + arm_smmu_domain->smmu->s_smmu_id); + if (ret) { + dev_err(arm_smmu_domain->smmu->dev, "CoDA: dev protected failed!\n"); + ret = -ENXIO; + goto out; + } + /* Need to config ste */ + cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); + tmi_smmu_queue_write(cmd[0], cmd[1], + arm_smmu_domain->smmu->s_smmu_id); + } + } + } + +out: + spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); + return ret; +} + +/** + * virtcca_secure_dev_ste_create - Setting up the STE config content + * for the security device + * @smmu: An SMMUv3 instance + * @master: SMMU private data for each master + * @sid: Stream id of device + * + * Returns: + * %0 if create ste success + * %-ENOMEM alloc ste params failed + * %-EINVAL set ste config content failed + */ +static int virtcca_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid) +{ + struct tmi_smmu_ste_params *params_ptr; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + params_ptr = kzalloc(sizeof(*params_ptr), GFP_KERNEL); + if (!params_ptr) + return -ENOMEM; + + /* Sync Level 2 STE to TMM */ + params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); + params_ptr->sid = sid; + params_ptr->smmu_id = smmu->s_smmu_id; + + if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) { + kfree(params_ptr); + dev_err(smmu->dev, "CoDA: failed to create ste level 2\n"); + return -EINVAL; + } + + kfree(params_ptr); + + return 0; +} + +/** + * virtcca_delegate_secure_dev - Delegate device to secure state + * @smmu: An SMMUv3 instance + * @root_bd: The port where the secure device is located + * @dev: Secure device + * + * Returns: + * %0 if delegate success + * %-ENOMEM if alloc params failed + * %-EINVAL if the dev is invalid + */ +static inline int virtcca_delegate_secure_dev(uint16_t root_bd, struct arm_smmu_device *smmu, + struct device *dev) +{ + int i; + u64 ret = 0; + struct tmi_dev_delegate_params *params = NULL; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + params->root_bd = root_bd; + params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); + if (params->num_dev >= MAX_DEV_PER_PORT) { + ret = -EINVAL; + goto out; + } + + dev_info(smmu->dev, "CoDA: Delegate %d devices as %02x:%02x to secure\n", + params->num_dev, root_bd >> DEV_BUS_NUM, + (root_bd & MASK_DEV_BUS) >> DEV_FUNCTION_NUM); + ret = tmi_dev_delegate(__pa(params)); + if (ret) { + dev_err(smmu->dev, "CoDA: failed to delegate device to secure\n"); + goto out; + } + + for (i = 0; i < params->num_dev; i++) { + ret = add_cc_dev_obj(params->devs[i], 0, root_bd, true); + if (ret) + break; + } + +out: + kfree(params); + return ret; +} + +/** + * add_secure_dev_to_cc_table - Add secure device to hash table + * @smmu: An SMMUv3 instance + * @smmu_domain: The handle of smmu_domain + * @root_bd: The port where the secure device is located + * @master: SMMU private data for each master + * + * Returns: + * %0 if add to hash table success + * %-ENOMEM if alloc obj failed + * %-EINVAL if stream id is invalid + */ +static inline int add_secure_dev_to_cc_table(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, uint16_t root_bd, struct arm_smmu_master *master) +{ + int i, j; + u64 ret = 0; + + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (!is_cc_dev(sid)) { + dev_err(smmu->dev, "CoDA: sid is not cc dev\n"); + return -EINVAL; + } + ret = add_cc_dev_obj(sid, smmu_domain->s2_cfg.vmid, root_bd, true); + if (ret) + break; + } + return ret; +} + +/** + * virtcca_enable_secure_dev - Enable the PCIe protection controller function + * of the security device + * @smmu_domain: The handle of smmu_domain + * @master: SMMU private data for each master + * @dev: Secure device + * + * Returns: + * %0 if the root port of secure dev successfully set up pcipc capability + * %-ENOMEM alloc ste params failed + * %-EINVAL set ste config content failed + */ +static int virtcca_enable_secure_dev(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master, struct device *dev) +{ + u64 ret = 0; + uint16_t root_bd = get_root_bd(dev); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (!is_cc_root_bd(root_bd)) { + ret = virtcca_delegate_secure_dev(root_bd, smmu, dev); + if (ret) + return ret; + } + + ret = add_secure_dev_to_cc_table(smmu, smmu_domain, root_bd, master); + + return ret; +} + +/** + * virtcca_secure_dev_operator - Implement security settings for corresponding devices + * targeting the secure smmu domain + * @domain: The handle of iommu_domain + * @dev: Secure device + * + * Returns: + * %0 if the domain does not need to enable secure or the domain + * successfully set up security features + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_secure_dev_operator(struct device *dev, void *domain) +{ + int i, j; + int ret; + struct iommu_domain *iommu_domain = (struct iommu_domain *)domain; + struct iommu_fwspec *fwspec = NULL; + struct arm_smmu_device *smmu = NULL; + struct arm_smmu_domain *smmu_domain = NULL; + struct arm_smmu_master *master = NULL; + + if (!is_virtcca_cvm_enable()) + return 0; + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) + return -ENOENT; + + smmu_domain = to_smmu_domain(iommu_domain); + master = dev_iommu_priv_get(dev); + smmu = master->smmu; + + if (!smmu && !virtcca_smmu_enable(smmu)) { + dev_err(smmu->dev, "CoDA: security smmu not initialized for the device\n"); + return -EINVAL; + } + + ret = virtcca_enable_secure_dev(smmu_domain, master, dev); + if (ret) + return ret; + + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + /* Bridged PCI devices may end up with duplicated IDs */ + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (virtcca_secure_dev_ste_create(smmu, master, sid)) + return -ENOMEM; + } + + dev_info(smmu->dev, "CoDA: attach confidential dev: %s", dev_name(dev)); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_secure_dev_operator); + +/** + * virtcca_attach_secure_dev - Attach the device of iommu + * group to confidential virtual machine + * @domain: The handle of iommu domain + * @group: Iommu group + * + * Returns: + * %0 if attach the all devices success + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) +{ + int ret; + + ret = iommu_group_for_each_dev(group, (void *)domain, virtcca_secure_dev_operator); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); diff --git a/drivers/coda/coda_pci.c b/drivers/coda/coda_pci.c new file mode 100644 index 000000000000..07aa58c41bf6 --- /dev/null +++ b/drivers/coda/coda_pci.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include + +#include "../drivers/pci/msi/msi.h" + +/** + * virtcca_pci_read_msi_msg - secure dev read msi msg + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @msg: Msg information + * @base: Msi base address + * + **/ +void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base) +{ + u64 pbase = mmio_va_to_pa(base); + + msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/** + * virtcca_pci_write_msi_msg - secure dev write msi msg + * @desc: MSI-X description + * @msg: Msg information + * + **/ +bool virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) +{ + if (!is_virtcca_cvm_enable()) + return false; + + void __iomem *base = pci_msix_desc_addr(desc); + u32 ctrl = desc->pci.msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + u64 pbase = mmio_va_to_pa(base); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (!is_cc_dev(pci_dev_id(pdev))) + return false; + + u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); + + if (addr) { + /* Get the offset of the its register of a specific device */ + u64 offset = addr - CVM_MSI_ORIG_IOVA; + + addr = get_g_cc_dev_msi_addr(pci_dev_id(pdev)); + addr += offset; + if (!addr) + return true; + } + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, + msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); + + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl); + tmi_mmio_read(mmio_va_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), + CVM_RW_32_BIT, pci_dev_id(pdev)); + + return true; +} + +void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr) +{ + desc->pci.msix_ctrl = tmi_mmio_read(mmio_va_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/* + * If it is a safety device, write vector ctrl need + * use tmi interface + */ +bool virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +{ + if (!is_virtcca_cvm_enable()) + return false; + + void __iomem *desc_addr = pci_msix_desc_addr(desc); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return false; + + if (desc->pci.msi_attrib.can_mask) + tmi_mmio_write(mmio_va_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); + return true; +} + +/* + * If it is a safety device, read msix need + * use tmi interface + */ +bool virtcca_pci_msix_mask(struct msi_desc *desc) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return false; + + /* Flush write to device */ + tmi_mmio_read(mmio_va_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); + return true; +} + +/** + * virtcca_msix_mask_all_cc - mask all secure dev msix c + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @base: Io address + * @tsize: Number of entry + * @dev_num: Dev number + * + * Returns: + * %0 if msix mask all cc device success + **/ +int virtcca_msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) +{ + int i; + u16 rw_ctrl; + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + u64 pbase = mmio_va_to_pa(base); + + if (pci_msi_ignore_mask) + goto out; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, + ctrl, CVM_RW_32_BIT, dev_num); + } + +out: + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &rw_ctrl); + rw_ctrl &= ~PCI_MSIX_FLAGS_MASKALL; + rw_ctrl |= 0; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, rw_ctrl); + + pcibios_free_irq(dev); + return 0; +} + +/* If device is secure dev, read config need transfer to tmm module */ +int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val) +{ + u32 cvm_bit = size == 1 ? CVM_RW_8_BIT : size == 2 ? CVM_RW_16_BIT : CVM_RW_32_BIT; + + *val = tmi_mmio_read(mmio_va_to_pa(addr), cvm_bit, PCI_DEVID(bus_num, devfn)); + return 0; +} + +/* If device is secure dev, write config need transfer to tmm module */ +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val) +{ + u32 cvm_bit = size == 1 ? CVM_RW_8_BIT : size == 2 ? CVM_RW_16_BIT : CVM_RW_32_BIT; + + tmi_mmio_write(mmio_va_to_pa(addr), val, cvm_bit, PCI_DEVID(bus_num, devfn)); + return 0; +} + +/* Judge startup virtcca_cvm_host is enable and device is secure or not */ +bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + + if (cc_dev) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(is_virtcca_pci_io_rw); + +/* Transfer to tmm write io value */ +void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + WARN_ON(tmi_mmio_write(mmio_va_to_pa(io), val, size, pci_dev_id(pdev))); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_write); + +/* Transfer to tmm read io value */ +u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + return tmi_mmio_read(mmio_va_to_pa(io), size, pci_dev_id(pdev)); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_read); diff --git a/arch/arm64/kernel/virtcca_coda.c b/drivers/coda/coda_vfio.c similarity index 50% rename from arch/arm64/kernel/virtcca_coda.c rename to drivers/coda/coda_vfio.c index 1f3ea50ff77b..5e2289453b6f 100644 --- a/arch/arm64/kernel/virtcca_coda.c +++ b/drivers/coda/coda_vfio.c @@ -2,29 +2,21 @@ /* * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. */ -#include -#include -#include #include +#include +#include #include -#include -#include -#include -#include - -#include "../../drivers/pci/msi/msi.h" -#include "../../drivers/vfio/vfio.h" /** * virtcca_map_pages - Virtcca need map the secure * memory with paddr - * @ops: the handle of io_pgtable_ops + * @ops: The handle of io_pgtable_ops * @iova: Ipa address * @paddr: Physical address * @pgsize: Page size * @pgcount: Page count - * @iommu_prot: iommu attribute - * @mapped: mapped size + * @iommu_prot: Iommu attribute + * @mapped: Mapped size * * Returns: * %0 if map pages success @@ -40,6 +32,7 @@ int virtcca_map_pages(void *ops, unsigned long iova, struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); long iaext = (s64)iova >> cfg->ias; int ret = 0; + struct arm_smmu_domain *smmu_domain = NULL; if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) return -EINVAL; @@ -53,20 +46,30 @@ int virtcca_map_pages(void *ops, unsigned long iova, if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) return 0; - kvm = virtcca_smmu_domain_get_kvm(data); + smmu_domain = (struct arm_smmu_domain *)virtcca_io_pgtable_get_smmu_domain(data); + if (!smmu_domain) + return -EINVAL; + + kvm = smmu_domain->kvm; if (kvm) { struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; loader_start = virtcca_cvm->loader_start; ram_size = virtcca_cvm->ram_size; + /* Cvm ram space mapping*/ if (iova >= loader_start && - iova < loader_start + ram_size && - !virtcca_cvm->is_mapped) { + iova < loader_start + ram_size && + !virtcca_cvm->is_mapped) { ret = kvm_cvm_map_range(kvm); - } else if (iova < loader_start) { + } else if (iova < loader_start || iova >= loader_start + ram_size) { if (iova == CVM_MSI_ORIG_IOVA) - iova += CVM_MSI_IOVA_OFFSET; - ret = cvm_map_unmap_ipa_range(kvm, iova, paddr, pgsize * pgcount, true); + /* Cvm msi address mapping */ + ret = virtcca_map_msi_address(kvm, smmu_domain, + paddr, pgsize * pgcount); + else + /* Cvm mmio space mapping */ + ret = cvm_map_unmap_ipa_range(kvm, iova, + paddr, pgsize * pgcount, true); } if (mapped) *mapped += pgsize * pgcount; @@ -77,7 +80,7 @@ EXPORT_SYMBOL_GPL(virtcca_map_pages); /** * virtcca_unmap_pages - Virtcca unmap the iova - * @ops: the handle of io_pgtable_ops + * @ops: The handle of io_pgtable_ops * @iova: Ipa address * @pgsize: Page size * @pgcount: Page count @@ -92,6 +95,7 @@ size_t virtcca_unmap_pages(void *ops, unsigned long iova, struct arm_lpae_io_pgtable *data = virtcca_io_pgtable_get_data(ops); struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); long iaext = (s64)iova >> cfg->ias; + struct arm_smmu_domain *smmu_domain = NULL; if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) return 0; @@ -101,7 +105,11 @@ size_t virtcca_unmap_pages(void *ops, unsigned long iova, if (WARN_ON(iaext)) return 0; - kvm = virtcca_smmu_domain_get_kvm(data); + smmu_domain = (struct arm_smmu_domain *)virtcca_io_pgtable_get_smmu_domain(data); + if (!smmu_domain) + return 0; + + kvm = smmu_domain->kvm; if (!kvm) return 0; @@ -258,218 +266,6 @@ size_t virtcca_iommu_unmap(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(virtcca_iommu_unmap); -/** - * virtcca_pci_read_msi_msg - secure dev read msi msg - * @dev: Pointer to the pci_dev data structure of MSI-X device function - * @msg: Msg information - * @base: Msi base address - * - **/ -void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, - void __iomem *base) -{ - u64 pbase = mmio_va_to_pa(base); - - msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, - CVM_RW_32_BIT, pci_dev_id(dev)); - msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, - CVM_RW_32_BIT, pci_dev_id(dev)); - msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); -} - -/** - * virtcca_pci_write_msi_msg - secure dev write msi msg - * @desc: MSI-X description - * @msg: Msg information - * - **/ -int virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - void __iomem *base = pci_msix_desc_addr(desc); - u32 ctrl = desc->pci.msix_ctrl; - bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - u64 pbase = mmio_va_to_pa(base); - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (!is_cc_dev(pci_dev_id(pdev))) - return 0; - - u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); - - addr += CVM_MSI_IOVA_OFFSET; - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, - lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, - upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, - msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); - - if (unmasked) - pci_msix_write_vector_ctrl(desc, ctrl); - tmi_mmio_read(mmio_va_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), - CVM_RW_32_BIT, pci_dev_id(pdev)); - - return 1; -} - -void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, - struct msi_desc *desc, void __iomem *addr) -{ - desc->pci.msix_ctrl = tmi_mmio_read(mmio_va_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), - CVM_RW_32_BIT, pci_dev_id(dev)); -} - -/* - * If it is a safety device, write vector ctrl need - * use tmi interface - */ -int virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - void __iomem *desc_addr = pci_msix_desc_addr(desc); - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) - return 0; - - if (desc->pci.msi_attrib.can_mask) - tmi_mmio_write(mmio_va_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), - ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); - return 1; -} - -/* - * If it is a safety device, read msix need - * use tmi interface - */ -int virtcca_pci_msix_mask(struct msi_desc *desc) -{ - if (!is_virtcca_cvm_enable()) - return 0; - - struct pci_dev *pdev = (desc->dev != NULL && - dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; - - if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) - return 0; - - /* Flush write to device */ - tmi_mmio_read(mmio_va_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); - return 1; -} - -/** - * msix_mask_all_cc - mask all secure dev msix c - * @dev: Pointer to the pci_dev data structure of MSI-X device function - * @base: Io address - * @tsize: Number of entry - * @dev_num: Dev number - * - * Returns: - * %0 if msix mask all cc device success - **/ -int msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) -{ - int i; - u16 rw_ctrl; - u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; - u64 pbase = mmio_va_to_pa(base); - - if (pci_msi_ignore_mask) - goto out; - - for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { - tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, - ctrl, CVM_RW_32_BIT, dev_num); - } - -out: - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &rw_ctrl); - rw_ctrl &= ~PCI_MSIX_FLAGS_MASKALL; - rw_ctrl |= 0; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, rw_ctrl); - - pcibios_free_irq(dev); - return 0; -} - -/* If device is secure dev, read config need transfer to tmm module */ -int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, - unsigned int devfn, int size, u32 *val) -{ - if (size == 1) - *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_8_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - else if (size == 2) - *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_16_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - else - *val = tmi_mmio_read(mmio_va_to_pa(addr), CVM_RW_32_BIT, - ((bus_num << BUS_NUM_SHIFT) | devfn)); - - return 0; -} - -/* If device is secure dev, write config need transfer to tmm module */ -int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, - unsigned int devfn, int size, u32 val) -{ - if (size == 1) - WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, - CVM_RW_8_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - else if (size == 2) - WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, - CVM_RW_16_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - else - WARN_ON(tmi_mmio_write(mmio_va_to_pa(addr), val, - CVM_RW_32_BIT, ((bus_num << BUS_NUM_SHIFT) | devfn))); - - return 0; -} - -/* Judge startup virtcca_cvm_host is enable and device is secure or not */ -bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) -{ - if (!is_virtcca_cvm_enable()) - return false; - - struct pci_dev *pdev = vdev->pdev; - bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); - - if (cc_dev) - return true; - - return false; -} -EXPORT_SYMBOL_GPL(is_virtcca_pci_io_rw); - -/* Transfer to tmm write io value */ -void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, - u64 size, void __iomem *io) -{ - struct pci_dev *pdev = vdev->pdev; - - WARN_ON(tmi_mmio_write(mmio_va_to_pa(io), val, size, pci_dev_id(pdev))); -} -EXPORT_SYMBOL_GPL(virtcca_pci_io_write); - -/* Transfer to tmm read io value */ -u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, - u64 size, void __iomem *io) -{ - struct pci_dev *pdev = vdev->pdev; - - return tmi_mmio_read(mmio_va_to_pa(io), size, pci_dev_id(pdev)); -} -EXPORT_SYMBOL_GPL(virtcca_pci_io_read); - /* Whether the kvm is cvm */ bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm) { @@ -490,7 +286,7 @@ EXPORT_SYMBOL_GPL(virtcca_iommu_domain_get_kvm); * * Returns: * %NULL if the virtcca_vfio_file_iommu_group func is not defined - * or CONFIG_HISI_VIRTCCA_HOST is not enable, group is null + * or CONFIG_HISI_VIRTCCA_CODA is not enable, group is null * %iommu_group if get the iommu group from file success */ struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) @@ -508,3 +304,40 @@ struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) return ret; } + +/* Obtain msi address through iommu group id */ +u64 virtcca_get_iommu_device_msi_addr(struct iommu_group *iommu_group) +{ + u64 msi_addr = iommu_group_id(iommu_group) * CVM_MSI_IOVA_OFFSET + CVM_MSI_MIN_IOVA; + + if (msi_addr >= CVM_MSI_MAX_IOVA || msi_addr < CVM_MSI_MIN_IOVA) { + pr_err("MSI address 0x%llx overflow.\n", msi_addr); + return 0; + } + + return msi_addr; +} +EXPORT_SYMBOL_GPL(virtcca_get_iommu_device_msi_addr); + +/* Set the corresponding MSI address */ +int virtcca_set_dev_msi_addr(struct device *dev, void *iova) +{ + unsigned long *msi_iova = (unsigned long *)iova; + + if (dev_is_pci(dev)) { + struct pci_dev *pci_dev = to_pci_dev(dev); + u16 pci_id = pci_dev_id(pci_dev); + + set_g_cc_dev_msi_addr(pci_id, *msi_iova); + } + return 0; +} + +/* Traverse the devices in the iommu group and set the corresponding MSI address */ +int virtcca_iommu_group_set_dev_msi_addr(struct iommu_group *iommu_group, unsigned long *iova) +{ + int ret; + + ret = iommu_group_for_each_dev(iommu_group, (void *)iova, virtcca_set_dev_msi_addr); + return ret; +} diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index e4020cafcce9..8781d131961d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o -arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_HOST) += arm-s-smmu-v3.o +arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_CODA) += arm-s-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c index b2227bc969bd..34e0060420af 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -4,21 +4,13 @@ */ #include #include +#include #include "arm-smmu-v3.h" #include "arm-s-smmu-v3.h" -struct cc_dev_config { - u32 sid; /* BDF number of the device */ - u32 vmid; /* virtual machine id */ - u32 root_bd; /* root bus and device number. */ - bool secure; /* device secure attribute */ - struct hlist_node node; /* device hash table */ -}; - static bool g_s_smmu_id_map_init; -static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); static DECLARE_BITMAP(g_s_smmu_id_map, ARM_S_SMMU_MAX_IDS); /* @@ -54,194 +46,6 @@ static inline void virtcca_smmu_set_irq(struct arm_smmu_device *smmu) smmu->s_gerr_irq = msi_get_virq(smmu->dev, S_GERROR_MSI_INDEX); } -/** - * get_root_bd - Traverse pcie topology to find the root number - * @dev: The device for which to get root bd - * - * Returns: - * %-1 if error or not pci device - */ -static int get_root_bd(struct device *dev) -{ - struct pci_dev *pdev; - - if (!dev_is_pci(dev)) - return -1; - pdev = to_pci_dev(dev); - if (pdev->bus == NULL) - return -1; - while (pdev->bus->parent != NULL) - pdev = pdev->bus->self; - - return pci_dev_id(pdev) & MASK_DEV_FUNCTION; -} - -/** - * get_child_devices_rec - Traverse pcie topology to find child devices - * If dev is a bridge, get it's children - * If dev is a regular device, get itself - * @dev: Device for which to get child devices - * @devs: All child devices under input dev - * @max_devs: Max num of devs - * @ndev: Num of child devices - */ -static void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, - int max_devs, int *ndev) -{ - struct pci_bus *bus = dev->subordinate; - - if (bus) { /* dev is a bridge */ - struct pci_dev *child; - - list_for_each_entry(child, &bus->devices, bus_list) { - get_child_devices_rec(child, devs, max_devs, ndev); - } - } else { /* dev is a regular device */ - uint16_t bdf = pci_dev_id(dev); - int i; - /* check if bdf is already in devs */ - for (i = 0; i < *ndev; i++) { - if (devs[i] == bdf) - return; - } - /* check overflow */ - if (*ndev >= max_devs) { - pr_warn("S_SMMU: devices num over max devs\n"); - return; - } - devs[*ndev] = bdf; - *ndev = *ndev + 1; - } -} - -/** - * get_sibling_devices - Get all devices which share the same root_bd as dev - * @dev: Device for which to get child devices - * @devs: All child devices under input dev - * @max_devs: Max num of devs - * - * Returns: - * %0 if get child devices failure - */ -static int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) -{ - struct pci_dev *pdev; - int ndev = 0; - - if (!dev_is_pci(dev)) - return ndev; - - pdev = to_pci_dev(dev); - if (pdev->bus == NULL) - return ndev; - - while (pdev->bus->parent != NULL) - pdev = pdev->bus->self; - - get_child_devices_rec(pdev, devs, max_devs, &ndev); - return ndev; -} - -/** - * add_cc_dev_obj - Add device obj to hash tablse - * @sid: Stream id of device - * @vmid: Virtual machine id - * @root_bd: Root port bus device num - * @secure: Whether the device is secure or not - * - * Returns: - * %0 if add obj success - * %-ENOMEM if alloc obj failed - */ -static int add_cc_dev_obj(u32 sid, u32 vmid, u32 root_bd, bool secure) -{ - struct cc_dev_config *obj; - - hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { - if (obj->sid == sid) { - obj->vmid = vmid; - obj->root_bd = root_bd; - obj->secure = secure; - return 0; - } - } - - obj = kzalloc(sizeof(*obj), GFP_KERNEL); - if (!obj) - return -ENOMEM; - - obj->sid = sid; - obj->vmid = vmid; - obj->root_bd = root_bd; - obj->secure = secure; - - hash_add(g_cc_dev_htable, &obj->node, sid); - return 0; -} - -/** - * is_cc_root_bd - Whether the root port is secure or not - * @root_bd: Root port bus device num - * - * Returns: - * %true if the root bd is secure - * %false if the root bd is non-secure - */ -static bool is_cc_root_bd(u32 root_bd) -{ - int bkt; - struct cc_dev_config *obj; - - hash_for_each(g_cc_dev_htable, bkt, obj, node) { - if (obj->root_bd == root_bd && obj->secure) - return true; - } - - return false; -} - -/** - * is_cc_vmid - Whether the vm is confidential vm - * @vmid: Virtual machine id - * - * Returns: - * %true if the vm is confidential - * %false if the vm is not confidential - */ -static bool is_cc_vmid(u32 vmid) -{ - int bkt; - struct cc_dev_config *obj; - - hash_for_each(g_cc_dev_htable, bkt, obj, node) { - if (vmid > 0 && obj->vmid == vmid) - return true; - } - - return false; -} - -/** - * is_cc_dev - Whether the stream id of dev is confidential - * @sid: Stream id of dev - * - * Returns: - * %true if the dev is confidential - * %false if the dev is not confidential - */ -bool is_cc_dev(u32 sid) -{ - struct cc_dev_config *obj; - - hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { - if (obj != NULL && obj->sid == sid) - return obj->secure; - } - - return false; -} -EXPORT_SYMBOL(is_cc_dev); - /** * virtcca_smmu_cmdq_need_forward - Whether the cmd queue need transfer to secure world * @cmd0: Command consists of 128 bits, cmd0 is the low 64 bits @@ -401,7 +205,6 @@ static void virtcca_smmu_init_one_queue(struct arm_smmu_device *smmu, qsz = ((1 << q->llq.max_n_shift) * dwords) << ARM_S_QUEUE_SHIFT_SIZE; if (!strcmp(name, "cmdq")) { - params_ptr->ns_src = q->base_dma; params_ptr->smmu_base_addr = smmu->ioaddr; params_ptr->size = qsz; params_ptr->smmu_id = smmu->s_smmu_id; @@ -410,7 +213,6 @@ static void virtcca_smmu_init_one_queue(struct arm_smmu_device *smmu, } if (!strcmp(name, "evtq")) { - params_ptr->ns_src = q->base_dma; params_ptr->smmu_base_addr = smmu->ioaddr; params_ptr->size = qsz; params_ptr->smmu_id = smmu->s_smmu_id; @@ -633,217 +435,6 @@ static void platform_get_s_irq_byname_optional(struct platform_device *pdev, smmu->s_gerr_irq = irq; } -/** - * virtcca_smmu_tmi_dev_attach - Complete the stage2 page table establishment - * for the security device - * @arm_smmu_domain: The handle of smmu domain - * @kvm: The handle of virtual machine - * - * Returns: - * %0 if attach dev success - * %-ENXIO if the root port of device does not have pcipc capability - */ -u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm) -{ - unsigned long flags; - int i, j; - struct arm_smmu_master *master; - int ret = 0; - u64 cmd[CMDQ_ENT_DWORDS] = {0}; - struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; - - spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); - /* - * Traverse all devices under the secure smmu domain and - * set the correspnding address translation table for each device - */ - list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { - if (master && master->num_streams >= 0) { - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - ret = tmi_dev_attach(sid, virtcca_cvm->rd, - arm_smmu_domain->smmu->s_smmu_id); - if (ret) { - dev_err(arm_smmu_domain->smmu->dev, "S_SMMU: dev protected failed!\n"); - ret = -ENXIO; - goto out; - } - /* Need to config ste */ - cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); - cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); - cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); - tmi_smmu_queue_write(cmd[0], cmd[1], - arm_smmu_domain->smmu->s_smmu_id); - } - } - } - -out: - spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); - return ret; -} - -/** - * virtcca_smmu_secure_dev_ste_create - Setting up the STE config content - * for the security device - * @smmu: An SMMUv3 instance - * @master: SMMU private data for each master - * @sid: Stream id of device - * - * Returns: - * %0 if create ste success - * %-ENOMEM alloc ste params failed - * %-EINVAL set ste config content failed - */ -static int virtcca_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, - struct arm_smmu_master *master, u32 sid) -{ - struct tmi_smmu_ste_params *params_ptr; - struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; - - params_ptr = kzalloc(sizeof(*params_ptr), GFP_KERNEL); - if (!params_ptr) - return -ENOMEM; - - /* Sync Level 2 STE to TMM */ - params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); - params_ptr->sid = sid; - params_ptr->smmu_id = smmu->s_smmu_id; - - if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) { - kfree(params_ptr); - dev_err(smmu->dev, "S_SMMU: failed to create ste level 2\n"); - return -EINVAL; - } - - kfree(params_ptr); - - return 0; -} - -/** - * add_secure_dev_to_cc_table - Add secure device to hash table - * @smmu: An SMMUv3 instance - * @smmu_domain: The handle of smmu_domain - * @root_bd: The port where the secure device is located - * @master: SMMU private data for each master - * - * Returns: - * %0 if add to hash table success - * %-ENOMEM if alloc obj failed - * %-EINVAL if stream id is invalid - */ -static inline int add_secure_dev_to_cc_table(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, uint16_t root_bd, struct arm_smmu_master *master) -{ - int i, j; - u64 ret = 0; - - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - if (!is_cc_dev(sid)) { - dev_err(smmu->dev, "S_SMMU: sid is not cc dev\n"); - return -EINVAL; - } - ret = add_cc_dev_obj(sid, smmu_domain->s2_cfg.vmid, root_bd, true); - if (ret) - break; - } - return ret; -} - -/** - * virtcca_delegate_secure_dev - Delegate device to secure state - * @smmu: An SMMUv3 instance - * @root_bd: The port where the secure device is located - * @dev: Secure device - * - * Returns: - * %0 if delegate success - * %-ENOMEM if alloc params failed - * %-EINVAL if the dev is invalid - */ -static inline int virtcca_delegate_secure_dev(uint16_t root_bd, struct arm_smmu_device *smmu, - struct device *dev) -{ - int i; - u64 ret = 0; - struct tmi_dev_delegate_params *params = NULL; - - params = kzalloc(sizeof(*params), GFP_KERNEL); - if (!params) - return -ENOMEM; - - params->root_bd = root_bd; - params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); - if (params->num_dev >= MAX_DEV_PER_PORT) { - ret = -EINVAL; - goto out; - } - - dev_info(smmu->dev, "S_SMMU: Delegate %d devices as %02x:%02x to secure\n", - params->num_dev, root_bd >> DEV_BUS_NUM, - (root_bd & MASK_DEV_BUS) >> DEV_FUNCTION_NUM); - ret = tmi_dev_delegate(__pa(params)); - if (ret) { - dev_err(smmu->dev, "S_SMMU: failed to delegate device to secure\n"); - goto out; - } - - for (i = 0; i < params->num_dev; i++) { - ret = add_cc_dev_obj(params->devs[i], 0, root_bd, true); - if (ret) - break; - } - -out: - kfree(params); - return ret; -} - -/** - * virtcca_enable_secure_dev - Enable the PCIe protection controller function - * of the security device - * @smmu_domain: The handle of smmu_domain - * @master: SMMU private data for each master - * @dev: Secure device - * - * Returns: - * %0 if the root port of secure dev successfully set up pcipc capability - * %-ENOMEM alloc ste params failed - * %-EINVAL set ste config content failed - */ -static int virtcca_enable_secure_dev(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_master *master, struct device *dev) -{ - u64 ret = 0; - uint16_t root_bd = get_root_bd(dev); - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (!is_cc_root_bd(root_bd)) { - ret = virtcca_delegate_secure_dev(root_bd, smmu, dev); - if (ret) - return ret; - } - - ret = add_secure_dev_to_cc_table(smmu, smmu_domain, root_bd, master); - - return ret; -} - /** * virtcca_smmu_write_msi_msg - Write secure smmu msi msg * @desc: Descriptor structure for MSI based interrupts @@ -945,26 +536,6 @@ static void virtcca_smmu_setup_unique_irqs(struct arm_smmu_device *smmu, bool re } } -/** - * _arm_smmu_write_reg_sync - Write value to smmu registers and wait for completion - * @smmu: An SMMUv3 instance - * @val: Expected value to be written - * @reg_off: Offset of object register - * @ack_off: Acknowledge offset of object register - * - * Returns: - * %0 if write success - */ -static int _arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, - u32 reg_off, u32 ack_off) -{ - u32 reg; - - writel_relaxed(val, smmu->base + reg_off); - return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, - 1, ARM_SMMU_POLL_TIMEOUT_US); -} - /** * virtcca_smmu_setup_irqs - Initialize the smmu irq * @smmu: An SMMUv3 instance @@ -972,15 +543,9 @@ static int _arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, */ static void virtcca_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) { - int irq, ret; + int irq; u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; - ret = _arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, - ARM_SMMU_IRQ_CTRLACK); - if (ret) { - dev_err(smmu->dev, "S_SMMU: failed to disable irqs\n"); - return; - } /* Disable IRQs first */ virtcca_smmu_disable_irq(smmu); @@ -991,12 +556,6 @@ static void virtcca_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) if (smmu->features & ARM_SMMU_FEAT_PRI) irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; - /* Enable interrupt generation on the SMMU */ - ret = _arm_smmu_write_reg_sync(smmu, irqen_flags, - ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); - if (ret) - dev_warn(smmu->dev, "S_SMMU: failed to enable irqs\n"); - virtcca_smmu_enable_irq(smmu, IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN); } @@ -1026,11 +585,11 @@ static int virtcca_smmu_id_alloc(void) * %true if the smmu need to initialize secure state * %false the smmu does not need to initialize secure state */ -static bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) +bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) { if (!g_s_smmu_id_map_init) { set_bit(0, g_s_smmu_id_map); - hash_init(g_cc_dev_htable); + g_cc_dev_table_init(); g_s_smmu_id_map_init = true; } smmu->ioaddr = ioaddr; @@ -1043,6 +602,7 @@ static bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; return false; } +EXPORT_SYMBOL_GPL(virtcca_smmu_map_init); /** * arm_s_smmu_device_enable - Enable the smmu secure state @@ -1106,7 +666,7 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) } if (!(rv & S_IDR1_SEL2)) { - dev_err(smmu->dev, "S_SMMU: secure stage2 translation not supported!\n"); + dev_err(smmu->dev, "S_SMMU: secure stage2 translation is not supported!\n"); smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; return false; } @@ -1114,66 +674,6 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) return true; } -/** - * virtcca_smmu_secure_dev_operator - Implement security settings for corresponding devices - * targeting the secure smmu domain - * @domain: The handle of iommu_domain - * @dev: Secure device - * - * Returns: - * %0 if the domain does not need to enable secure or the domain - * successfully set up security features - * %-EINVAL if the smmu does not initialize secure state - * %-ENOMEM if the device create secure ste failed - * %-ENOENT if the device does not have fwspec - */ -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev) -{ - int i, j; - int ret; - struct iommu_fwspec *fwspec = NULL; - struct arm_smmu_device *smmu = NULL; - struct arm_smmu_domain *smmu_domain = NULL; - struct arm_smmu_master *master = NULL; - - if (!is_virtcca_cvm_enable()) - return 0; - - fwspec = dev_iommu_fwspec_get(dev); - if (!fwspec) - return -ENOENT; - - smmu_domain = to_smmu_domain(domain); - master = dev_iommu_priv_get(dev); - smmu = master->smmu; - - if (!smmu && !virtcca_smmu_enable(smmu)) { - dev_err(smmu->dev, "S_SMMU: security smmu not initialized for the device\n"); - return -EINVAL; - } - - ret = virtcca_enable_secure_dev(smmu_domain, master, dev); - if (ret) - return ret; - - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - /* Bridged PCI devices may end up with duplicated IDs */ - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - if (virtcca_smmu_secure_dev_ste_create(smmu, master, sid)) - return -ENOMEM; - } - - dev_info(smmu->dev, "S_SMMU: attach confidential dev: %s", dev_name(dev)); - - return ret; -} -EXPORT_SYMBOL_GPL(virtcca_smmu_secure_dev_operator); - /** * virtcca_smmu_device_init - Initialize the smmu security features * @pdev: The handle of iommu_domain @@ -1205,9 +705,18 @@ void virtcca_smmu_device_init(struct platform_device *pdev, struct arm_smmu_devi if (!virtcca_smmu_enable(smmu) || !arm_s_smmu_idr1_support_secure(smmu)) return; + /* + * S_SMMU implementation doesn't support unique irq lines, + * use a single irq line for all the S_SMMU interrupts. + */ irq = platform_get_irq_byname_optional(pdev, "combined"); - if (irq <= 0) + if (irq <= 0) { platform_get_s_irq_byname_optional(pdev, smmu); + } else { + dev_err(smmu->dev, "S_SMMU: virtcca does not support combined irq\n"); + smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; + return; + } /* Create cmd queue */ virtcca_smmu_init_one_queue(smmu, &smmu->cmdq.q, CMDQ_ENT_DWORDS, "cmdq"); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h index 7ff5e59a4117..3d1786db7508 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h @@ -4,21 +4,12 @@ */ #ifndef _ARM_S_SMMU_V3_H #define _ARM_S_SMMU_V3_H -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #include #include -#define MAX_CC_DEV_NUM_ORDER 8 -#define MASK_DEV_FUNCTION 0xfff8 -#define MASK_DEV_BUS 0xff - -#define DEV_BUS_NUM 0x8 -#define DEV_FUNCTION_NUM 0x3 - -#define STE_ENTRY_SIZE 0x40 - #define SMMU_PCIE_CORE_IS_VALID 0x1 #define ARM_S_SMMU_MAX_IDS (1 << 5) @@ -168,15 +159,10 @@ result ? result : ((cond) ? 0 : -ETIMEDOUT); \ }) -/* Has the root bus device number switched to secure */ -bool is_cc_dev(u32 sid); - void virtcca_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, struct arm_smmu_ll_queue *llq, struct arm_smmu_queue *q, int n, bool sync); bool virtcca_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); -u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, - struct kvm *kvm); void _arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); void virtcca_smmu_device_init(struct platform_device *pdev, struct arm_smmu_device *smmu, resource_size_t ioaddr, bool resume, bool disable_bypass); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 61a14c186eca..230368d05892 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -30,7 +30,7 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include "arm-s-smmu-v3.h" #endif @@ -999,7 +999,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, * Dependency ordering from the cmpxchg() loop above. */ arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_cmdq_write_entries(smmu, cmds, &llq, &cmdq->q, n, sync); #endif if (sync) { @@ -2383,7 +2383,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_set_stage(domain, smmu_domain); #endif @@ -3692,7 +3692,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); } -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA void _arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) { if (virtcca_smmu_write_msi_msg(desc, msg)) @@ -4876,7 +4876,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) if (ret) return ret; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_device_init(pdev, smmu, ioaddr, false, disable_bypass); #endif diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 94bfcfb7b660..93bb08bd7404 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -13,7 +13,7 @@ #include #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif @@ -757,7 +757,7 @@ struct arm_smmu_device { bool bypass; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA int s_evtq_irq; int s_gerr_irq; resource_size_t ioaddr; @@ -816,7 +816,7 @@ struct arm_smmu_domain { struct list_head mmu_notifiers; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA struct list_head node; struct kvm *kvm; #endif diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 1d6af11ef04a..e47ff5b7d471 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -33,9 +33,9 @@ #include "dma-iommu.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST -#include +#ifdef CONFIG_HISI_VIRTCCA_CODA #include +#include #endif struct iommu_dma_msi_page { @@ -1816,47 +1816,6 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) } EXPORT_SYMBOL_GPL(iommu_setup_dma_ops); -#ifdef CONFIG_HISI_VIRTCCA_HOST -/* Virtcca map msi address */ -static struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, - phys_addr_t msi_addr, struct iommu_domain *domain) -{ - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iommu_dma_msi_page *msi_page; - dma_addr_t iova; - int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; - size_t size = cookie_msi_granule(cookie); - - msi_addr &= ~(phys_addr_t)(size - 1); - list_for_each_entry(msi_page, &cookie->msi_page_list, list) - if (msi_page->phys == msi_addr) - return msi_page; - - msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); - if (!msi_page) - return NULL; - - iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); - if (!iova) - goto out_free_page; - - if (virtcca_iommu_map(domain, iova, msi_addr, size, prot)) - goto out_free_iova; - - INIT_LIST_HEAD(&msi_page->list); - msi_page->phys = msi_addr; - msi_page->iova = iova; - list_add(&msi_page->list, &cookie->msi_page_list); - return msi_page; - -out_free_iova: - iommu_dma_free_iova(cookie, iova, size, NULL); -out_free_page: - kfree(msi_page); - return NULL; -} -#endif - static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, phys_addr_t msi_addr, struct iommu_domain *domain) { @@ -1866,7 +1825,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(cookie); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && domain->secure) return virtcca_iommu_dma_get_msi_page(dev, msi_addr, domain); #endif @@ -1963,3 +1922,44 @@ static int iommu_dma_init(void) return iova_cache_get(); } arch_initcall(iommu_dma_init); + +#ifdef CONFIG_HISI_VIRTCCA_CODA +/* Virtcca map msi address */ +struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, + phys_addr_t msi_addr, struct iommu_domain *domain) +{ + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iommu_dma_msi_page *msi_page; + dma_addr_t iova; + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + size_t size = cookie_msi_granule(cookie); + + msi_addr &= ~(phys_addr_t)(size - 1); + list_for_each_entry(msi_page, &cookie->msi_page_list, list) + if (msi_page->phys == msi_addr) + return msi_page; + + msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); + if (!msi_page) + return NULL; + + iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); + if (!iova) + goto out_free_page; + + if (virtcca_iommu_map(domain, iova, msi_addr, size, prot)) + goto out_free_iova; + + INIT_LIST_HEAD(&msi_page->list); + msi_page->phys = msi_addr; + msi_page->iova = iova; + list_add(&msi_page->list, &cookie->msi_page_list); + return msi_page; + +out_free_iova: + iommu_dma_free_iova(cookie, iova, size, NULL); +out_free_page: + kfree(msi_page); + return NULL; +} +#endif diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 921e68e19753..c00b940130f8 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -20,7 +20,7 @@ #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif @@ -1728,18 +1728,7 @@ static int __init arm_lpae_do_selftests(void) subsys_initcall(arm_lpae_do_selftests); #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST -/* Obtain kvm from smmu domain */ -struct kvm *virtcca_smmu_domain_get_kvm(struct arm_lpae_io_pgtable *data) -{ - struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data->iop.cookie; - - if (!smmu_domain) - return NULL; - - return smmu_domain->kvm; -} - +#ifdef CONFIG_HISI_VIRTCCA_CODA /* Obtain io pgtable data from io pgtable ops */ struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops) { @@ -1751,4 +1740,10 @@ struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *da { return &data->iop.cfg; } + +/* Obtain smmu domain from io pgtable data */ +void *virtcca_io_pgtable_get_smmu_domain(struct arm_lpae_io_pgtable *data) +{ + return data->iop.cookie; +} #endif diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index e843ae332bad..28f63ad432de 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -33,12 +33,6 @@ #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST -#ifndef __GENKSYMS__ -#include -#endif -#endif - #include "dma-iommu.h" #include "iommu-priv.h" @@ -3660,40 +3654,3 @@ void iommu_free_global_pasid(ioasid_t pasid) ida_free(&iommu_global_pasid_ida, pasid); } EXPORT_SYMBOL_GPL(iommu_free_global_pasid); - -#ifdef CONFIG_HISI_VIRTCCA_HOST -/** - * virtcca_attach_secure_dev - Attach the device of iommu - * group to confidential virtual machine - * @domain: The handle of iommu domain - * @group: Iommu group - * - * Returns: - * %0 if attach the all devices success - * %-EINVAL if the smmu does not initialize secure state - * %-ENOMEM if the device create secure ste failed - * %-ENOENT if the device does not have fwspec - */ -int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) -{ - struct group_device *gdev; - int ret = 0; - - mutex_lock(&group->mutex); - for_each_group_device(group, gdev) - ret = virtcca_smmu_secure_dev_operator(domain, gdev->dev); - mutex_unlock(&group->mutex); - return ret; -} -EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); - -/* Obtain domain information through iommu group */ -struct iommu_domain *virtcca_iommu_group_get_domain(struct iommu_group *iommu_group) -{ - if (iommu_group) - return iommu_group->domain; - - return NULL; -} -EXPORT_SYMBOL_GPL(virtcca_iommu_group_get_domain); -#endif diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 2f88c9dcc2ce..39eb93d804ca 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -5,12 +5,12 @@ #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #ifndef __GENKSYMS__ #include #include -#include #include +#include #endif #endif @@ -95,8 +95,8 @@ int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (is_virtcca_cvm_enable() && is_cc_dev((bus->number << BUS_NUM_SHIFT) | devfn)) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(PCI_DEVID(bus->number, devfn))) return virtcca_pci_generic_config_read(addr, bus->number, devfn, size, val); #endif @@ -120,8 +120,8 @@ int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (is_virtcca_cvm_enable() && is_cc_dev((bus->number << BUS_NUM_SHIFT) | devfn)) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(PCI_DEVID(bus->number, devfn))) return virtcca_pci_generic_config_write(addr, bus->number, devfn, size, val); #endif diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index d2c9863d0368..785bbf637ab5 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -13,7 +13,7 @@ #include "../pci.h" #include "msi.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif @@ -163,7 +163,7 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && dev != NULL && is_cc_dev(pci_dev_id(dev))) return virtcca_pci_read_msi_msg(dev, msg, base); #endif @@ -229,8 +229,8 @@ static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg if (unmasked) pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (virtcca_pci_write_msg_msi(desc, msg)) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_write_msg_msi(desc, msg)) return; #endif writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); @@ -651,7 +651,7 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) if (desc->pci.msi_attrib.can_mask) { void __iomem *addr = pci_msix_desc_addr(desc); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) return virtcca_msix_prepare_msi_desc(dev, desc, addr); #endif @@ -792,9 +792,9 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, * which takes the MSI-X mask bits into account even * when MSI-X is disabled, which prevents MSI delivery. */ -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) - return msix_mask_all_cc(dev, dev->msix_base, tsize, pci_dev_id(dev)); + return virtcca_msix_mask_all_cc(dev, dev->msix_base, tsize, pci_dev_id(dev)); #endif msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h index 154e2c00f94c..20119a585690 100644 --- a/drivers/pci/msi/msi.h +++ b/drivers/pci/msi/msi.h @@ -3,7 +3,7 @@ #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #ifndef __GENKSYMS__ #include #endif @@ -42,8 +42,8 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { void __iomem *desc_addr = pci_msix_desc_addr(desc); -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (virtcca_pci_msix_write_vector_ctrl(desc, ctrl)) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_msix_write_vector_ctrl(desc, ctrl)) return; #endif @@ -56,8 +56,8 @@ static inline void pci_msix_mask(struct msi_desc *desc) desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (virtcca_pci_msix_mask(desc)) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_msix_mask(desc)) return; #endif /* Flush write to device */ diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index 33a66b6c5759..dc9e57641c51 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -958,7 +958,7 @@ void vfio_group_cleanup(void) vfio_container_cleanup(); } -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA /** * virtcca_vfio_file_iommu_group - Return the struct iommu_group for the vfio group file * @file: VFIO group file @@ -972,7 +972,7 @@ struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file) struct vfio_group *group = vfio_group_from_file(file); struct iommu_group *iommu_group = NULL; - if (!IS_ENABLED(CONFIG_HISI_VIRTCCA_HOST)) + if (!IS_ENABLED(CONFIG_HISI_VIRTCCA_CODA)) return NULL; if (!group) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 2bff38b56f97..9c130f83b21b 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -31,7 +31,7 @@ #if IS_ENABLED(CONFIG_EEH) #include #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif @@ -976,7 +976,7 @@ static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_cvm_set_secure_flag((void *)vdev, (void *)&info); #endif diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index d7eda4ca1de9..e340da8a9edc 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -19,10 +19,10 @@ #include "vfio_pci_priv.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #ifndef __GENKSYMS__ -#include #include +#include #endif #endif @@ -44,7 +44,7 @@ #define vfio_ioread8 ioread8 #define vfio_iowrite8 iowrite8 -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #define VFIO_IOWRITE(size) \ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io) \ @@ -97,7 +97,7 @@ VFIO_IOWRITE(32) VFIO_IOWRITE(64) #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #define VFIO_IOREAD(size) \ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size * val, void __iomem *io) \ diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index b33c145520ad..af4af1bb02c3 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -38,11 +38,11 @@ #include #include #include "vfio.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #include -#include #include +#include #endif #define DRIVER_VERSION "0.2" @@ -83,8 +83,8 @@ struct vfio_iommu { bool dirty_page_tracking; struct list_head emulated_iommu_groups; bool dirty_log_get_no_clear; -#ifdef CONFIG_HISI_VIRTCCA_HOST - bool secure; +#ifdef CONFIG_HISI_VIRTCCA_CODA + bool secure; /* Whether the vfio iommu is secure or not */ #endif }; @@ -1046,10 +1046,10 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain, return unmapped; } -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma); -bool virtcca_check_kvm_is_cvm(void *iommu, struct kvm **kvm) +bool virtcca_check_is_cvm_or_not(void *iommu, struct kvm **kvm) { struct vfio_domain *domain; bool is_virtcca_cvm = false; @@ -1116,14 +1116,18 @@ static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *d size_t size = map_size; unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; - bool is_virtcca_cvm = virtcca_check_kvm_is_cvm((void *)iommu, &kvm); + bool is_virtcca_cvm = virtcca_check_is_cvm_or_not((void *)iommu, &kvm); vfio_batch_init(&batch); while (size) { - + /* + * Due to cvm ram full mapping, ram space only needs to be mapped once, + * if the iova is in ram space, there is no need to map it again. + */ if (is_virtcca_cvm && !is_virtcca_iova_need_vfio_dma(kvm, dma->iova)) break; + /* Pin a contiguous chunk of memory */ npage = vfio_pin_pages_remote(dma, vaddr + dma->size, size >> PAGE_SHIFT, &pfn, limit, @@ -1145,6 +1149,10 @@ static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *d } if (is_virtcca_cvm && is_in_virtcca_ram_range(kvm, iova)) { + /* + * The cvm ram mapping uses secure memory, + * so non-secure memory needs to be unpinned. + */ vfio_unpin_pages_remote(dma, iova + dma->size, pfn, npage, true); vfio_batch_unpin(&batch, dma); @@ -1182,7 +1190,7 @@ static long virtcca_vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma * long unlocked = 0; dma_addr_t iova = dma->iova, end = dma->iova + dma->size; - if (!virtcca_check_kvm_is_cvm((void *)iommu, &kvm)) + if (!virtcca_check_is_cvm_or_not((void *)iommu, &kvm)) return 0; if (!dma->size) @@ -1270,7 +1278,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, int unmapped_region_cnt = 0; long unlocked = 0; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && iommu->secure) return virtcca_vfio_unmap_unpin(iommu, dma, do_accounting); #endif @@ -1858,7 +1866,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && iommu->secure) return virtcca_vfio_pin_map_dma(iommu, dma, map_size); #endif @@ -2086,7 +2094,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && iommu->secure) return 0; #endif @@ -2692,8 +2700,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_domain; } -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (iommu->secure) +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) domain->domain->secure = true; #endif @@ -2701,8 +2709,8 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, if (ret) goto out_domain; -#ifdef CONFIG_HISI_VIRTCCA_HOST - if (iommu->secure) { +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) { ret = virtcca_attach_secure_dev(domain->domain, group->iommu_group); if (ret) goto out_domain; @@ -3058,7 +3066,7 @@ static void *vfio_iommu_type1_open(unsigned long arg) case VFIO_TYPE1v2_IOMMU: iommu->v2 = true; break; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA case VFIO_TYPE1v2_S_IOMMU: iommu->v2 = true; iommu->secure = true; @@ -3155,7 +3163,7 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, switch (arg) { case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA case VFIO_TYPE1v2_S_IOMMU: #endif case VFIO_TYPE1_NESTING_IOMMU: diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8fb93470a134..0dad8f803586 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -260,7 +260,7 @@ struct iommu_domain { }; struct mutex switch_log_lock; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA KABI_USE(1, bool secure) #else KABI_RESERVE(1) diff --git a/include/linux/msi.h b/include/linux/msi.h index c21099e1f16d..7354ffb14856 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -640,7 +640,7 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, struct irq_domain *parent); int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, irq_write_msi_msg_t write_msi_msg); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA int platform_msi_domain_alloc_range_irqs(struct device *dev, unsigned int start, unsigned int end, irq_write_msi_msg_t write_msi_msg); #endif diff --git a/include/linux/pci.h b/include/linux/pci.h index 54e340c81972..82f3571a36d9 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -2710,10 +2710,6 @@ static inline bool pci_is_thunderbolt_attached(struct pci_dev *pdev) void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type); #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST -bool is_cc_dev(u32 sid); -#endif - #include #define pci_printk(level, pdev, fmt, arg...) \ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index b2ad93893612..5ac5f182ce0b 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -290,9 +290,6 @@ void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes, * External user API */ struct iommu_group *vfio_file_iommu_group(struct file *file); -#ifdef CONFIG_HISI_VIRTCCA_HOST -struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file); -#endif #if IS_ENABLED(CONFIG_VFIO_GROUP) bool vfio_file_is_group(struct file *file); diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index df6cf5661d40..3337c12b3d59 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -25,7 +25,7 @@ #define VFIO_TYPE1_IOMMU 1 #define VFIO_SPAPR_TCE_IOMMU 2 #define VFIO_TYPE1v2_IOMMU 3 -#define VFIO_TYPE1v2_S_IOMMU 12 +#define VFIO_TYPE1v2_S_IOMMU 12 /* Virtcca feature: secure iommu */ /* * IOMMU enforces DMA cache coherence (ex. PCIe NoSnoop stripping). This * capability is subject to change as groups are added or removed. diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index 1dc81283e456..e3c603aeb66c 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -21,9 +21,9 @@ #include #endif -#ifdef CONFIG_HISI_VIRTCCA_HOST -#include +#ifdef CONFIG_HISI_VIRTCCA_CODA #include +#include #endif struct kvm_vfio_file { @@ -183,7 +183,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvm_arch_start_assignment(dev->kvm); kvm_vfio_file_set_kvm(kvf->file, dev->kvm); kvm_vfio_update_coherency(dev); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA ret = cvm_vfio_add_kvm_to_smmu_domain(filp, (void *)kv); #endif @@ -401,7 +401,7 @@ void kvm_vfio_ops_exit(void) kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); } -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA /** * cvm_vfio_add_kvm_to_smmu_domain - Bind the confidential * virtual machine to smmu domain @@ -416,28 +416,29 @@ int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv) { struct iommu_group *iommu_group; int ret = 0; + struct kvm_vfio *_kv = (struct kvm_vfio *)kv; if (!is_virtcca_cvm_enable()) return ret; /* Upper-level calling interface has added a kv lock, but the - * cvm_arm_smmu_domain_set_kvm interface also need add this lock. + * virtcca_cvm_arm_smmu_domain_set_kvm interface also need add this lock. * Therefore, it is necessary to unlock here and completing the * acquisition, then add the kv lock before return. */ - mutex_unlock(&((struct kvm_vfio *)kv)->lock); + mutex_unlock(&_kv->lock); iommu_group = cvm_vfio_file_iommu_group(filp); if (!iommu_group) { ret = -ENXIO; goto out_lock; } - if (cvm_arm_smmu_domain_set_kvm((void *)iommu_group)) { + if (virtcca_cvm_arm_smmu_domain_set_kvm((void *)iommu_group) != 1) { ret = -ENXIO; goto out_lock; } out_lock: - mutex_lock(&((struct kvm_vfio *)kv)->lock); + mutex_lock(&_kv->lock); return ret; } @@ -462,6 +463,7 @@ struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) struct arm_smmu_master *master; spin_lock_irqsave(&domain->devices_lock, flags); + /* Get smmu master from smmu domain */ list_for_each_entry(master, &domain->devices, domain_head) { if (master && master->num_streams >= 0) { ret = 0; @@ -477,20 +479,24 @@ struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) mutex_lock(&kvm_lock); list_for_each_entry(kvm, &vm_list, vm_list) { mutex_lock(&kvm->lock); + /* Get kvm vfio device list from kvm */ list_for_each_entry(dev, &kvm->devices, vm_node) { - if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { - kv = (struct kvm_vfio *)dev->private; - mutex_lock(&kv->lock); - list_for_each_entry(kvf, &kv->file_list, node) { - if (cvm_vfio_file_iommu_group(kvf->file) == iommu_group) { - ret = 0; - break; - } - } - mutex_unlock(&kv->lock); - if (!ret) + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") != 0) + continue; + + /* Find the kvm vfio which device name is kvm-vfio */ + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + /* Get iommu_group from vfio file */ + if (cvm_vfio_file_iommu_group(kvf->file) == iommu_group) { + ret = 0; break; + } } + mutex_unlock(&kv->lock); + if (!ret) + break; } mutex_unlock(&kvm->lock); if (!ret) @@ -509,15 +515,16 @@ EXPORT_SYMBOL_GPL(virtcca_arm_smmu_get_kvm); * @kvf: Kvm vfio file * @smmu_domain_group_list: List of smmu domain group */ -void find_arm_smmu_domain(struct kvm_vfio_file *kvf, struct list_head *smmu_domain_group_list) +int find_arm_smmu_domain(struct device *dev, void *data) { - struct iommu_group *iommu_group; int ret = 0; + struct iommu_domain *domain = NULL; struct arm_smmu_domain *arm_smmu_domain = NULL; struct arm_smmu_domain *arm_smmu_domain_node = NULL; + struct list_head *smmu_domain_group_list = (struct list_head *)data; - iommu_group = cvm_vfio_file_iommu_group(kvf->file); - arm_smmu_domain = to_smmu_domain(virtcca_iommu_group_get_domain(iommu_group)); + domain = iommu_get_domain_for_dev(dev); + arm_smmu_domain = to_smmu_domain(domain); list_for_each_entry(arm_smmu_domain_node, smmu_domain_group_list, node) { if (arm_smmu_domain_node == arm_smmu_domain) { @@ -527,6 +534,8 @@ void find_arm_smmu_domain(struct kvm_vfio_file *kvf, struct list_head *smmu_doma } if (!ret) list_add_tail(&arm_smmu_domain->node, smmu_domain_group_list); + + return 1; } /** @@ -539,18 +548,124 @@ void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_grou struct kvm_device *dev; struct kvm_vfio *kv; struct kvm_vfio_file *kvf; + struct iommu_group *iommu_group = NULL; INIT_LIST_HEAD(smmu_domain_group_list); list_for_each_entry(dev, &kvm->devices, vm_node) { + /* The device name passed through the vfio driver is called kvm-vfio */ + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") != 0) + continue; + + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + iommu_group = cvm_vfio_file_iommu_group(kvf->file); + iommu_group_for_each_dev(iommu_group, + (void *)smmu_domain_group_list, find_arm_smmu_domain); + } + mutex_unlock(&kv->lock); + } +} + +/** + * cvm_smmu_domain_judge - Find the iommu group corresponding to the smmu domain + * @dev: The handle of device + * @data: Smmu domain + * + * Returns: + * %-ENXIO if domain is null + * %SMMU_DOMAIN_IS_SAME if find the iommu group corresponding to the smmu domain + * %1 if does not find the iommu group + */ +int cvm_smmu_domain_judge(struct device *dev, void *data) +{ + struct iommu_domain *domain = NULL; + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain == smmu_domain) + return SMMU_DOMAIN_IS_SAME; + + return 1; +} + +/* Judge the iommu group correspond to the smmu domain */ +int virtcca_cvm_smmu_domain_judge(struct iommu_group *group, struct arm_smmu_domain *smmu_domain) +{ + int ret; + + ret = iommu_group_for_each_dev((struct iommu_group *)group, + (void *)smmu_domain, cvm_smmu_domain_judge); + return ret; +} + +/** + * virtcca_iommu_group_map_msi_address - Find iommu group from kvm vfio file, map it + * @kvm: The handle of kvm + * @kvf: Kvm vfio file + * @smmu_domain: Smmu domain + * @pa: Physical address + * @map_size: Mapped size + */ +int virtcca_iommu_group_map_msi_address(struct kvm *kvm, struct kvm_vfio_file *kvf, + struct arm_smmu_domain *smmu_domain, phys_addr_t pa, unsigned long map_size) +{ + unsigned long iova; + int ret = 0; + struct iommu_group *iommu_group = NULL; + + iommu_group = cvm_vfio_file_iommu_group(kvf->file); + if (iommu_group) { + if (virtcca_cvm_smmu_domain_judge(iommu_group, smmu_domain) == + SMMU_DOMAIN_IS_SAME) { + iova = virtcca_get_iommu_device_msi_addr(iommu_group); + if (!iova) + return -ENXIO; + + ret = virtcca_iommu_group_set_dev_msi_addr(iommu_group, &iova); + if (ret) + return ret; + + ret = cvm_map_unmap_ipa_range(kvm, iova, pa, map_size, true); + if (ret) + return ret; + } + } + return ret; +} + +/* Get iommu group from specific smmu domain, map it */ +int virtcca_map_msi_address(struct kvm *kvm, struct arm_smmu_domain *smmu_domain, + phys_addr_t pa, unsigned long map_size) +{ + int ret = 0; + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + + mutex_lock(&kvm->lock); + list_for_each_entry(dev, &kvm->devices, vm_node) { + /* Get kvm vfio device list */ if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { kv = (struct kvm_vfio *)dev->private; mutex_lock(&kv->lock); list_for_each_entry(kvf, &kv->file_list, node) { - find_arm_smmu_domain(kvf, smmu_domain_group_list); + /* Get iommu_group from vfio file, map it */ + ret = virtcca_iommu_group_map_msi_address(kvm, kvf, smmu_domain, + pa, map_size); } mutex_unlock(&kv->lock); + if (ret) + break; } } + mutex_unlock(&kvm->lock); + return ret; } #endif -- Gitee From 5063d32882709a1e97e7212838cc41e6232bb3a9 Mon Sep 17 00:00:00 2001 From: yangxiangkai Date: Wed, 16 Oct 2024 15:59:23 +0800 Subject: [PATCH 12/12] virtcca feature: msi place modify virtcca inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAQON6 -------------------------------- VirtCCA Coda Feature: Msi place modify Signed-off-by: Xiangkai Yang Signed-off-by: Junbin Li --- --- arch/arm64/include/asm/virtcca_coda.h | 8 ++-- drivers/coda/coda.c | 1 + drivers/coda/coda_vfio.c | 52 ++++++++++++++++++++--- drivers/iommu/dma-iommu.c | 59 +++++++-------------------- drivers/iommu/io-pgtable-arm.c | 24 ----------- drivers/vfio/pci/vfio_pci_intrs.c | 10 +++++ include/uapi/linux/vfio.h | 2 +- 7 files changed, 76 insertions(+), 80 deletions(-) diff --git a/arch/arm64/include/asm/virtcca_coda.h b/arch/arm64/include/asm/virtcca_coda.h index 7e9469789e47..93ea972cbb98 100644 --- a/arch/arm64/include/asm/virtcca_coda.h +++ b/arch/arm64/include/asm/virtcca_coda.h @@ -68,10 +68,7 @@ int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv); struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); -struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops); -struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *data); struct iommu_group *cvm_vfio_file_iommu_group(struct file *file); -void *virtcca_io_pgtable_get_smmu_domain(struct arm_lpae_io_pgtable *data); struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file); @@ -87,7 +84,8 @@ void g_cc_dev_table_init(void); u32 virtcca_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm); -struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, - phys_addr_t msi_addr, struct iommu_domain *domain); +void virtcca_iommu_dma_get_msi_page(void *cookie, dma_addr_t *iova, phys_addr_t *phys); + +int virtcca_msi_map(struct vfio_pci_core_device *vdev); #endif #endif diff --git a/drivers/coda/coda.c b/drivers/coda/coda.c index acc32dd730db..bfbb4607557c 100644 --- a/drivers/coda/coda.c +++ b/drivers/coda/coda.c @@ -226,6 +226,7 @@ u64 get_g_cc_dev_msi_addr(u32 sid) } return 0; } +EXPORT_SYMBOL_GPL(get_g_cc_dev_msi_addr); /** * set_g_cc_dev_msi_addr - Set the msi address of confidential device diff --git a/drivers/coda/coda_vfio.c b/drivers/coda/coda_vfio.c index 5e2289453b6f..9de5d50a6bff 100644 --- a/drivers/coda/coda_vfio.c +++ b/drivers/coda/coda_vfio.c @@ -28,8 +28,8 @@ int virtcca_map_pages(void *ops, unsigned long iova, struct kvm *kvm; u64 loader_start; u64 ram_size; - struct arm_lpae_io_pgtable *data = virtcca_io_pgtable_get_data(ops); - struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); + struct io_pgtable *data = io_pgtable_ops_to_pgtable(ops); + struct io_pgtable_cfg *cfg = &data->cfg; long iaext = (s64)iova >> cfg->ias; int ret = 0; struct arm_smmu_domain *smmu_domain = NULL; @@ -46,7 +46,7 @@ int virtcca_map_pages(void *ops, unsigned long iova, if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) return 0; - smmu_domain = (struct arm_smmu_domain *)virtcca_io_pgtable_get_smmu_domain(data); + smmu_domain = (struct arm_smmu_domain *)(data->cookie); if (!smmu_domain) return -EINVAL; @@ -92,8 +92,8 @@ size_t virtcca_unmap_pages(void *ops, unsigned long iova, size_t pgsize, size_t pgcount) { struct kvm *kvm; - struct arm_lpae_io_pgtable *data = virtcca_io_pgtable_get_data(ops); - struct io_pgtable_cfg *cfg = virtcca_io_pgtable_get_cfg(data); + struct io_pgtable *data = io_pgtable_ops_to_pgtable(ops); + struct io_pgtable_cfg *cfg = &data->cfg; long iaext = (s64)iova >> cfg->ias; struct arm_smmu_domain *smmu_domain = NULL; @@ -105,7 +105,7 @@ size_t virtcca_unmap_pages(void *ops, unsigned long iova, if (WARN_ON(iaext)) return 0; - smmu_domain = (struct arm_smmu_domain *)virtcca_io_pgtable_get_smmu_domain(data); + smmu_domain = (struct arm_smmu_domain *)(data->cookie); if (!smmu_domain) return 0; @@ -341,3 +341,43 @@ int virtcca_iommu_group_set_dev_msi_addr(struct iommu_group *iommu_group, unsign ret = iommu_group_for_each_dev(iommu_group, (void *)iova, virtcca_set_dev_msi_addr); return ret; } + +/** + * virtcca_msi_map - Vfio driver mapping device side msi address + * @vdev: Vfio pci core device + * + * Returns: + * %0 if map success + */ +int virtcca_msi_map(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + int ret; + dma_addr_t iova; + phys_addr_t msi_addr; + struct iommu_domain *domain = NULL; + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + + /* + * If the device is secure and has not done MSI address mapping, + * Mapping is required. + */ + if (cc_dev && !get_g_cc_dev_msi_addr(pci_dev_id(pdev))) { + domain = iommu_get_domain_for_dev(&(pdev->dev)); + /* Get the MSI address of the device */ + virtcca_iommu_dma_get_msi_page((void *)domain->iova_cookie, &iova, &msi_addr); + /* Release non-secure side device MSI address mapping */ + iommu_unmap(domain, iova, PAGE_SIZE); + /* Mapping secure side MSI address */ + ret = virtcca_iommu_map(domain, iova, msi_addr, PAGE_SIZE, prot); + if (ret) + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtcca_msi_map); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index e47ff5b7d471..9a3dc42a3309 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -33,11 +33,6 @@ #include "dma-iommu.h" -#ifdef CONFIG_HISI_VIRTCCA_CODA -#include -#include -#endif - struct iommu_dma_msi_page { struct list_head list; dma_addr_t iova; @@ -1825,10 +1820,6 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(cookie); -#ifdef CONFIG_HISI_VIRTCCA_CODA - if (is_virtcca_cvm_enable() && domain->secure) - return virtcca_iommu_dma_get_msi_page(dev, msi_addr, domain); -#endif msi_addr &= ~(phys_addr_t)(size - 1); list_for_each_entry(msi_page, &cookie->msi_page_list, list) if (msi_page->phys == msi_addr) @@ -1924,42 +1915,22 @@ static int iommu_dma_init(void) arch_initcall(iommu_dma_init); #ifdef CONFIG_HISI_VIRTCCA_CODA -/* Virtcca map msi address */ -struct iommu_dma_msi_page *virtcca_iommu_dma_get_msi_page(struct device *dev, - phys_addr_t msi_addr, struct iommu_domain *domain) +/** + * virtcca_iommu_dma_get_msi_page() - Traverse the MSI page list to obtain the last + * MSI page + * @cookie: Iommu dma cookie + * @iova: Iova address + * @phys: Physical address + * + */ +void virtcca_iommu_dma_get_msi_page(void *cookie, dma_addr_t *iova, phys_addr_t *phys) { - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iommu_dma_msi_page *msi_page; - dma_addr_t iova; - int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; - size_t size = cookie_msi_granule(cookie); - - msi_addr &= ~(phys_addr_t)(size - 1); - list_for_each_entry(msi_page, &cookie->msi_page_list, list) - if (msi_page->phys == msi_addr) - return msi_page; + struct iommu_dma_msi_page *msi_page = NULL; + struct iommu_dma_cookie *dma_cookie = (struct iommu_dma_cookie *)cookie; - msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL); - if (!msi_page) - return NULL; - - iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev); - if (!iova) - goto out_free_page; - - if (virtcca_iommu_map(domain, iova, msi_addr, size, prot)) - goto out_free_iova; - - INIT_LIST_HEAD(&msi_page->list); - msi_page->phys = msi_addr; - msi_page->iova = iova; - list_add(&msi_page->list, &cookie->msi_page_list); - return msi_page; - -out_free_iova: - iommu_dma_free_iova(cookie, iova, size, NULL); -out_free_page: - kfree(msi_page); - return NULL; + msi_page = list_last_entry(&dma_cookie->msi_page_list, struct iommu_dma_msi_page, list); + *iova = msi_page->iova; + *phys = msi_page->phys; } +EXPORT_SYMBOL_GPL(virtcca_iommu_dma_get_msi_page); #endif diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index c00b940130f8..fb54baed3f49 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -20,10 +20,6 @@ #include -#ifdef CONFIG_HISI_VIRTCCA_CODA -#include -#endif - #include "io-pgtable-arm.h" #define ARM_LPAE_MAX_ADDR_BITS 52 @@ -1727,23 +1723,3 @@ static int __init arm_lpae_do_selftests(void) } subsys_initcall(arm_lpae_do_selftests); #endif - -#ifdef CONFIG_HISI_VIRTCCA_CODA -/* Obtain io pgtable data from io pgtable ops */ -struct arm_lpae_io_pgtable *virtcca_io_pgtable_get_data(void *ops) -{ - return io_pgtable_ops_to_data((struct io_pgtable_ops *)ops); -} - -/* Obtain io pgtable cfg from io pgtable data */ -struct io_pgtable_cfg *virtcca_io_pgtable_get_cfg(struct arm_lpae_io_pgtable *data) -{ - return &data->iop.cfg; -} - -/* Obtain smmu domain from io pgtable data */ -void *virtcca_io_pgtable_get_smmu_domain(struct arm_lpae_io_pgtable *data) -{ - return data->iop.cookie; -} -#endif diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 620134041b48..a80a80aa20ed 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -22,6 +22,10 @@ #include "vfio_pci_priv.h" +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#endif + struct vfio_pci_irq_ctx { struct eventfd_ctx *trigger; struct virqfd *unmask; @@ -712,6 +716,12 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, if (ret) return ret; +#ifdef CONFIG_HISI_VIRTCCA_CODA + ret = virtcca_msi_map(vdev); + if (ret) + return ret; +#endif + ret = vfio_msi_set_block(vdev, start, count, fds, msix); if (ret) vfio_msi_disable(vdev, msix); diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 3337c12b3d59..df0790085d3d 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -225,7 +225,7 @@ struct vfio_device_info { #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6) /* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7) /* Info supports caps */ #define VFIO_DEVICE_FLAGS_CDX (1 << 8) /* vfio-cdx device */ -#define VFIO_DEVICE_FLAGS_SECURE (1 << 9) /* vfio-secure device */ +#define VFIO_DEVICE_FLAGS_SECURE (1 << 15) /* vfio-secure device */ __u32 num_regions; /* Max region index + 1 */ __u32 num_irqs; /* Max IRQ index + 1 */ __u32 cap_offset; /* Offset within info struct of first cap */ -- Gitee