diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 45d9443367e19136ef8cf59848cda7b3353e3ee2..f091fa794a19f0465e7ef48f53b2bf79fe963ef8 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -760,6 +760,7 @@ CONFIG_KVM_HISI_VIRT=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y CONFIG_HISI_VIRTCCA_HOST=y +CONFIG_HISI_VIRTCCA_CODA=y # CONFIG_NVHE_EL2_DEBUG is not set CONFIG_KVM_ARM_MULTI_LPI_TRANSLATE_CACHE=y CONFIG_ARCH_VCPU_STAT=y diff --git a/arch/arm64/include/asm/kvm_tmi.h b/arch/arm64/include/asm/kvm_tmi.h index 36fceec5749072b461057f3f43039fc4249759c0..cab2b4cdd0f2bd9e12776d10510e9f19f5abccfc 100644 --- a/arch/arm64/include/asm/kvm_tmi.h +++ b/arch/arm64/include/asm/kvm_tmi.h @@ -83,7 +83,6 @@ struct tmi_smmu_cfg_params { #define TMI_SMMU_CMD_QUEUE 1 #define TMI_SMMU_EVT_QUEUE 2 struct tmi_smmu_queue_params { - uint64_t ns_src; /* non-secure queue source address */ uint64_t smmu_base_addr; /* smmu base address */ uint64_t size; /* queue size */ uint64_t smmu_id; /* smmu id */ @@ -247,6 +246,7 @@ struct tmi_tec_run { #define TMI_FNUM_SMMU_WRITE U(0x282) #define TMI_FNUM_SMMU_READ U(0x283) #define TMI_FNUM_SMMU_PCIE_CORE_CHECK U(0x284) +#define TMI_FNUM_DEV_TTT_CREATE U(0x285) /* TMI SMC64 PIDs handled by the SPMD */ #define TMI_TMM_VERSION_REQ TMI_FID(SMC_64, TMI_FNUM_VERSION_REQ) @@ -280,6 +280,7 @@ struct tmi_tec_run { #define TMI_TMM_SMMU_WRITE TMI_FID(SMC_64, TMI_FNUM_SMMU_WRITE) #define TMI_TMM_SMMU_READ TMI_FID(SMC_64, TMI_FNUM_SMMU_READ) #define TMI_TMM_SMMU_PCIE_CORE_CHECK TMI_FID(SMC_64, TMI_FNUM_SMMU_PCIE_CORE_CHECK) +#define TMI_TMM_DEV_TTT_CREATE TMI_FID(SMC_64, TMI_FNUM_DEV_TTT_CREATE) #define TMI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) #define TMI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) @@ -381,6 +382,7 @@ u64 tmi_ttt_map_range(u64 rd, u64 map_addr, u64 size, u64 cur_node, u64 target_n u64 tmi_ttt_unmap_range(u64 rd, u64 map_addr, u64 size, u64 node_id); u64 tmi_mem_info_show(u64 mem_info_addr); +u64 tmi_dev_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level); u64 tmi_smmu_queue_create(u64 params_ptr); u64 tmi_smmu_queue_write(uint64_t cmd0, uint64_t cmd1, u64 smmu_id); u64 tmi_smmu_ste_create(u64 params_ptr); @@ -396,6 +398,7 @@ u64 tmi_smmu_pcie_core_check(u64 smmu_base); u64 tmi_smmu_write(u64 smmu_base, u64 reg_offset, u64 val, u64 bits); u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits); +u64 mmio_va_to_pa(void *addr); void kvm_cvm_vcpu_put(struct kvm_vcpu *vcpu); int kvm_load_user_data(struct kvm *kvm, unsigned long arg); unsigned long cvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu, @@ -404,6 +407,8 @@ int kvm_cvm_vcpu_set_events(struct kvm_vcpu *vcpu, bool serror_pending, bool ext_dabt_pending); int kvm_init_cvm_vm(struct kvm *kvm); int kvm_enable_virtcca_cvm(struct kvm *kvm); - +int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, + unsigned long map_size, enum kvm_pgtable_prot prot, int ret); +void virtcca_cvm_set_secure_flag(void *vdev, void *info); #endif #endif diff --git a/arch/arm64/include/asm/kvm_tmm.h b/arch/arm64/include/asm/kvm_tmm.h index ac1cb415919a7a7086c97e6f70efd2c59a1ad1e3..a26cade215865391ceaec67da84406a95a3c88ee 100644 --- a/arch/arm64/include/asm/kvm_tmm.h +++ b/arch/arm64/include/asm/kvm_tmm.h @@ -10,9 +10,19 @@ /* * There is a conflict with the internal iova of CVM, * so it is necessary to offset the msi iova. + * According to qemu file(hw/arm/virt.c), 0x0a001000 - 0x0b000000 + * iova is not being used, so it is used as the iova range for msi + * mapping. */ -#define CVM_MSI_ORIG_IOVA 0x8000000 -#define CVM_MSI_IOVA_OFFSET (-0x1000000) +#define CVM_MSI_ORIG_IOVA 0x8000000 +#define CVM_MSI_MIN_IOVA 0x0a001000 +#define CVM_MSI_MAX_IOVA 0x0b000000 +#define CVM_MSI_IOVA_OFFSET 0x1000 + +#define CVM_RW_8_BIT 0x8 +#define CVM_RW_16_BIT 0x10 +#define CVM_RW_32_BIT 0x20 +#define CVM_RW_64_BIT 0x40 enum virtcca_cvm_state { CVM_STATE_NONE = 1, @@ -95,14 +105,16 @@ void kvm_free_rd(struct kvm *kvm); int cvm_psci_complete(struct kvm_vcpu *calling, struct kvm_vcpu *target); void kvm_cvm_unmap_destroy_range(struct kvm *kvm); - int kvm_cvm_map_range(struct kvm *kvm); -int cvm_arm_smmu_domain_set_kvm(void *group); -int kvm_cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, +int virtcca_cvm_arm_smmu_domain_set_kvm(void *group); +int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size, uint32_t is_map); int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, phys_addr_t pa, unsigned long map_size); +bool is_in_virtcca_ram_range(struct kvm *kvm, uint64_t iova); +bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova); + #define CVM_TTT_BLOCK_LEVEL 2 #define CVM_TTT_MAX_LEVEL 3 @@ -116,6 +128,11 @@ int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, ((CVM_PAGE_SHIFT - 3) * (4 - (l)) + 3) #define CVM_L2_BLOCK_SIZE BIT(CVM_TTT_LEVEL_SHIFT(2)) +#define TMM_GRANULE_SIZE2 12 +#define TMM_TTT_WIDTH 19 +#define TMM_GRANULE_SIZE (1UL << TMM_GRANULE_SIZE2) +#define tmm_granule_size(level) (TMM_GRANULE_SIZE << ((3 - level)) * TMM_TTT_WIDTH) + static inline unsigned long cvm_ttt_level_mapsize(int level) { if (WARN_ON(level > CVM_TTT_BLOCK_LEVEL)) diff --git a/arch/arm64/include/asm/virtcca_coda.h b/arch/arm64/include/asm/virtcca_coda.h new file mode 100644 index 0000000000000000000000000000000000000000..93ea972cbb98b3180592c5c79464cd59c5e7c0b2 --- /dev/null +++ b/arch/arm64/include/asm/virtcca_coda.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __VIRTCCA_CODA_H +#define __VIRTCCA_CODA_H + +#include +#include + +#include "../../../drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h" +#include "../../../drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h" + +#ifdef CONFIG_HISI_VIRTCCA_CODA + +#define MAX_CC_DEV_NUM_ORDER 8 +#define MASK_DEV_FUNCTION 0xfff8 +#define MASK_DEV_BUS 0xff + +#define DEV_BUS_NUM 0x8 +#define DEV_FUNCTION_NUM 0x3 + +#define STE_ENTRY_SIZE 0x40 + +#define SMMU_DOMAIN_IS_SAME 0x2 + +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group); + +u64 virtcca_get_iommu_device_msi_addr(struct iommu_group *iommu_group); +int virtcca_iommu_group_set_dev_msi_addr(struct iommu_group *iommu_group, unsigned long *iova); +int virtcca_map_msi_address(struct kvm *kvm, struct arm_smmu_domain *smmu_domain, phys_addr_t pa, + unsigned long map_size); + +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size); +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped); +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount); + +void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base); +bool virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg); +void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr); +bool virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl); +bool virtcca_pci_msix_mask(struct msi_desc *desc); +int virtcca_msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num); + +int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val); +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val); + +bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev); +void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io); +u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io); + +bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm); +bool virtcca_check_is_cvm_or_not(void *iommu, struct kvm **kvm); +int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, + unsigned long pfn, long npage, int prot); +int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv); +struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain); +void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list); +struct iommu_group *cvm_vfio_file_iommu_group(struct file *file); + +struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file); + +bool is_cc_vmid(u32 vmid); +/* Has the root bus device number switched to secure */ +bool is_cc_dev(u32 sid); + +u64 get_g_cc_dev_msi_addr(u32 sid); + +void set_g_cc_dev_msi_addr(u32 sid, u64 msi_addr); + +void g_cc_dev_table_init(void); + +u32 virtcca_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm); + +void virtcca_iommu_dma_get_msi_page(void *cookie, dma_addr_t *iova, phys_addr_t *phys); + +int virtcca_msi_map(struct vfio_pci_core_device *vdev); +#endif +#endif diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index f3ab2c39f7647afa3c11a6bf9ba36ec3a93ed467..68efca6ef24e7a4faf0c96ebfe421b51ec16bd97 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -19,7 +19,9 @@ #include #include #include - +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#endif #include "trace.h" static struct kvm_pgtable *hyp_pgtable; @@ -871,7 +873,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t u64 mmfr0, mmfr1; u32 phys_shift; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if ((type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) && (!kvm_is_virtcca_cvm(kvm))) #else if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) @@ -1418,7 +1420,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, fault_granule = 1UL << ARM64_HW_PGTABLE_LEVEL_SHIFT(fault_level); write_fault = kvm_is_write_fault(vcpu); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA if (vcpu_is_tec(vcpu)) { write_fault = true; prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; @@ -1606,6 +1608,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED); +#ifdef CONFIG_HISI_VIRTCCA_CODA + ret = kvm_cvm_map_ipa(kvm, fault_ipa, pfn, vma_pagesize, prot, ret); +#endif + /* Mark the page dirty only if the fault is handled successfully */ if (writable && !ret) { kvm_set_pfn_dirty(pfn); diff --git a/arch/arm64/kvm/tmi.c b/arch/arm64/kvm/tmi.c index 769c0096122277f20f29aa7bb71e6597212e9f46..cd38be6f2fbaf49830818c7d564d1d4a27bccd6d 100644 --- a/arch/arm64/kvm/tmi.c +++ b/arch/arm64/kvm/tmi.c @@ -6,6 +6,37 @@ #include #include +/** + * mmio_va_to_pa - To convert the virtual address of the mmio space + * to a physical address, it is necessary to implement this interface + * because the kernel insterface __pa has an error when converting the + * physical address of the virtual address of the mmio space + * @addr: MMIO virtual address + */ +u64 mmio_va_to_pa(void *addr) +{ + uint64_t pa, par_el1; + + asm volatile( + "AT S1E1W, %0\n" + ::"r"((uint64_t)(addr)) + ); + isb(); + asm volatile( + "mrs %0, par_el1\n" + : "=r"(par_el1) + ); + + pa = ((uint64_t)(addr) & (PAGE_SIZE - 1)) | + (par_el1 & ULL(0x000ffffffffff000)); + + if (par_el1 & UL(1 << 0)) + return (uint64_t)(addr); + else + return pa; +} +EXPORT_SYMBOL(mmio_va_to_pa); + u64 tmi_version(void) { struct arm_smccc_res res; @@ -309,3 +340,12 @@ u64 tmi_smmu_read(u64 smmu_base, u64 reg_offset, u64 bits) } EXPORT_SYMBOL(tmi_smmu_read); +/* Create device ttt */ +u64 tmi_dev_ttt_create(u64 numa_set, u64 rd, u64 map_addr, u64 level) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_smc(TMI_TMM_DEV_TTT_CREATE, numa_set, rd, map_addr, level, &res); + return res.a1; +} + diff --git a/arch/arm64/kvm/virtcca_cvm.c b/arch/arm64/kvm/virtcca_cvm.c index 93dd2392a1db166b87ebda0ab3515532fa2842ef..e1fd6fed66cd13bb67b1ba36cbf4ded6fc323486 100644 --- a/arch/arm64/kvm/virtcca_cvm.c +++ b/arch/arm64/kvm/virtcca_cvm.c @@ -4,15 +4,20 @@ */ #include #include +#include +#include #include #include #include #include #include +#include #include #include #include +#include + /* Protects access to cvm_vmid_bitmap */ static DEFINE_SPINLOCK(cvm_vmid_lock); static unsigned long *cvm_vmid_bitmap; @@ -175,10 +180,23 @@ void kvm_destroy_cvm(struct kvm *kvm) { struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; uint32_t cvm_vmid; +#ifdef CONFIG_HISI_VIRTCCA_CODA + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; +#endif if (!cvm) return; +#ifdef CONFIG_HISI_VIRTCCA_CODA + /* Unmap the cvm with arm smmu domain */ + kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain->kvm && arm_smmu_domain->kvm == kvm) + arm_smmu_domain->kvm = NULL; + } +#endif + cvm_vmid = cvm->cvm_vmid; kfree(cvm->params); cvm->params = NULL; @@ -193,6 +211,7 @@ void kvm_destroy_cvm(struct kvm *kvm) if (!tmi_cvm_destroy(cvm->rd)) kvm_info("KVM has destroyed cVM: %d\n", cvm->cvm_vmid); + cvm->is_mapped = false; kfree(cvm); kvm->arch.virtcca_cvm = NULL; } @@ -514,20 +533,40 @@ int kvm_cvm_map_range(struct kvm *kvm) } } } - + /* Vfio driver will pin memory in advance, + * if the ram already mapped, activate cvm + * does not need to map twice + */ + cvm->is_mapped = true; return ret; } static int kvm_activate_cvm(struct kvm *kvm) { +#ifdef CONFIG_HISI_VIRTCCA_CODA + int ret; + struct arm_smmu_domain *arm_smmu_domain; + struct list_head smmu_domain_group_list; +#endif struct virtcca_cvm *cvm = kvm->arch.virtcca_cvm; if (virtcca_cvm_state(kvm) != CVM_STATE_NEW) return -EINVAL; - if (kvm_cvm_map_range(kvm)) + if (!cvm->is_mapped && kvm_cvm_map_range(kvm)) return -EFAULT; +#ifdef CONFIG_HISI_VIRTCCA_CODA + kvm_get_arm_smmu_domain(kvm, &smmu_domain_group_list); + list_for_each_entry(arm_smmu_domain, &smmu_domain_group_list, node) { + if (arm_smmu_domain) { + ret = virtcca_tmi_dev_attach(arm_smmu_domain, kvm); + if (ret) + return ret; + } + } +#endif + if (tmi_cvm_activate(cvm->rd)) { kvm_err("tmi_cvm_activate failed!\n"); return -ENXIO; @@ -852,3 +891,332 @@ int kvm_init_cvm_vm(struct kvm *kvm) return 0; } + +#ifdef CONFIG_HISI_VIRTCCA_CODA +/* + * Coda (Confidential Device Assignment) feature + * enable devices to pass directly to confidential virtual machines + */ + +/** + * is_in_virtcca_ram_range - Check if the iova belongs + * to the cvm ram range + * @kvm: The handle of kvm + * @iova: Ipa address + * + * Returns: + * %true if the iova belongs to cvm ram + * %false if the iova is not within the scope of cvm ram + */ +bool is_in_virtcca_ram_range(struct kvm *kvm, uint64_t iova) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + if (iova >= virtcca_cvm->loader_start && + iova < virtcca_cvm->loader_start + virtcca_cvm->ram_size) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(is_in_virtcca_ram_range); + +/** + * is_virtcca_iova_need_vfio_dma - Whether the vfio need + * to map the dma address + * @kvm: The handle of kvm + * @iova: Ipa address + * + * Returns: + * %true if virtcca cvm ram is nort mapped or + * virtcca_cvm_ram is mapped and the iova does not + * belong to cvm ram range + * %false if virtcca_cvm_ram is mapped and the iova belong + * to cvm ram range + */ +bool is_virtcca_iova_need_vfio_dma(struct kvm *kvm, uint64_t iova) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + if (!virtcca_cvm->is_mapped) + return true; + + return !is_in_virtcca_ram_range(kvm, iova); +} +EXPORT_SYMBOL_GPL(is_virtcca_iova_need_vfio_dma); + +static int kvm_cvm_dev_ttt_create(struct virtcca_cvm *cvm, + unsigned long addr, + int level, + u64 numa_set) +{ + addr = ALIGN_DOWN(addr, cvm_ttt_level_mapsize(level - 1)); + return tmi_dev_ttt_create(numa_set, cvm->rd, addr, level); +} + +/* CVM create ttt level information about device */ +int kvm_cvm_create_dev_ttt_levels(struct kvm *kvm, struct virtcca_cvm *cvm, + unsigned long ipa, + int level, + int max_level, + struct kvm_mmu_memory_cache *mc) +{ + int ret = 0; + + while (level++ < max_level) { + u64 numa_set = kvm_get_first_binded_numa_set(kvm); + + ret = kvm_cvm_dev_ttt_create(cvm, ipa, level, numa_set); + if (ret) + return -ENXIO; + } + + return 0; +} + +/** + * cvm_map_max_level_size - MMIO Map according to largest possible granularity + * @map_start: The start of map address + * @map_end: The end of map address + * @map_size: Map range + * + * Returns: + * %level the map level + * %-ENXIO if no suitable mapping level was found + */ +static int cvm_map_max_level_size(unsigned long map_start, unsigned long map_end, + unsigned long *map_size) +{ + int level = 1; + + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + level++; + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + level++; + *map_size = tmm_granule_size(level); + if (IS_ALIGNED(map_start, *map_size) && + (map_start + *map_size <= map_end)) + return level; + + pr_err("level not allow to map size\n"); + return -ENXIO; +} + +/** + * cvm_map_unmap_ipa_range - Vfio driver map or + * unmap cvm ipa + * @kvm: The handle of kvm + * @ipa_base: Ipa address + * @pa: Physical address + * @map_size: Map range + * @is_map: Map type + * + * Returns: + * %0 if cvm map/unmap address successfully + * %-ENXIO if map/unmap failed + */ +int cvm_map_unmap_ipa_range(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size, uint32_t is_map) +{ + unsigned long map_start; + unsigned long map_end; + int level; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long phys = pa; + int ret = 0; + + map_start = ipa_base; + map_end = map_start + map_size; + while (map_start < map_end) { + level = cvm_map_max_level_size(map_start, map_end, &map_size); + if (level < 0) { + ret = -ENXIO; + goto err; + } + if (is_map) + ret = tmi_mmio_map(rd, map_start, level, phys); + else + ret = tmi_mmio_unmap(rd, map_start, level); + + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + if (is_map) { + ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, map_start, + level_fault, CVM_TTT_MAX_LEVEL, NULL); + if (ret) + goto err; + ret = tmi_mmio_map(rd, map_start, level, phys); + } else { + ret = tmi_mmio_unmap(rd, map_start, level_fault); + map_size = tmm_granule_size(level_fault); + } + } + + if (ret) + goto err; + + map_start += map_size; + phys += map_size; + } + + return 0; + +err: + if (!tmi_cvm_destroy(rd)) + kvm_info("Vfio map failed, kvm has destroyed cVM: %d\n", virtcca_cvm->cvm_vmid); + return -ENXIO; +} + +/** + * kvm_cvm_map_ipa_mmio - Map the mmio address when page fault + * @kvm: The handle of kvm + * @ipa_base: Ipa address + * @pa: Physical address + * @map_size: Map range + * + * Returns: + * %0 if cvm map address successfully + * %-ENXIO if map failed + */ +int kvm_cvm_map_ipa_mmio(struct kvm *kvm, phys_addr_t ipa_base, + phys_addr_t pa, unsigned long map_size) +{ + unsigned long size; + gfn_t gfn; + kvm_pfn_t pfn; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + phys_addr_t rd = virtcca_cvm->rd; + unsigned long ipa = ipa_base; + unsigned long phys = pa; + int ret = 0; + + if (WARN_ON(!IS_ALIGNED(ipa, map_size))) + return -EINVAL; + + for (size = 0; size < map_size; size += PAGE_SIZE) { + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + if (ret == TMI_ERROR_TTT_CREATED) { + ret = 0; + goto label; + } + if (TMI_RETURN_STATUS(ret) == TMI_ERROR_TTT_WALK) { + /* Create missing TTTs and retry */ + int level_fault = TMI_RETURN_INDEX(ret); + + ret = kvm_cvm_create_dev_ttt_levels(kvm, virtcca_cvm, ipa, level_fault, + CVM_TTT_MAX_LEVEL, NULL); + + if (ret) + goto err; + ret = tmi_mmio_map(rd, ipa, CVM_TTT_MAX_LEVEL, phys); + } + + if (ret) + goto err; +label: + if (size + PAGE_SIZE >= map_size) + break; + + ipa += PAGE_SIZE; + gfn = gpa_to_gfn(ipa); + pfn = gfn_to_pfn(kvm, gfn); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + phys = (uint64_t)__pfn_to_phys(pfn); + + } + + return 0; + +err: + if (!tmi_cvm_destroy(rd)) + kvm_info("MMIO map failed, kvm has destroyed cVM: %d\n", virtcca_cvm->cvm_vmid); + return -ENXIO; +} + +/* Page fault map ipa */ +int kvm_cvm_map_ipa(struct kvm *kvm, phys_addr_t ipa, kvm_pfn_t pfn, + unsigned long map_size, enum kvm_pgtable_prot prot, int ret) +{ + if (!is_virtcca_cvm_enable() || !kvm_is_virtcca_cvm(kvm)) + return ret; + + struct page *dst_page = pfn_to_page(pfn); + phys_addr_t dst_phys = page_to_phys(dst_page); + + if (WARN_ON(!(prot & KVM_PGTABLE_PROT_W))) + return -EFAULT; + + if (prot & KVM_PGTABLE_PROT_DEVICE) + return kvm_cvm_map_ipa_mmio(kvm, ipa, dst_phys, map_size); + + return 0; +} + +/* Set device secure flag */ +void virtcca_cvm_set_secure_flag(void *vdev, void *info) +{ + if (!is_virtcca_cvm_enable()) + return; + + if (!is_cc_dev(pci_dev_id(((struct vfio_pci_core_device *)vdev)->pdev))) + return; + + ((struct vfio_device_info *)info)->flags |= VFIO_DEVICE_FLAGS_SECURE; +} +EXPORT_SYMBOL_GPL(virtcca_cvm_set_secure_flag); + +/** + * cvm_arm_smmu_domain_set_kvm - Associate SMMU domain with CVM + * @dev: The Device under the iommu group + * + * Returns: + * %0 if smmu_domain has been associate cvm or associate cvm successfully + * %-ENXIO if the iommu group does not have smmu domain + */ +int cvm_arm_smmu_domain_set_kvm(struct device *dev, void *data) +{ + struct kvm *kvm; + struct iommu_domain *domain; + struct arm_smmu_domain *arm_smmu_domain = NULL; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain->kvm) + return 1; + + kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (kvm && kvm_is_virtcca_cvm(kvm)) + arm_smmu_domain->kvm = kvm; + + return 1; +} + +int virtcca_cvm_arm_smmu_domain_set_kvm(void *group) +{ + int ret; + + ret = iommu_group_for_each_dev((struct iommu_group *)group, + (void *)NULL, cvm_arm_smmu_domain_set_kvm); + return ret; +} +#endif diff --git a/drivers/Kconfig b/drivers/Kconfig index f391050ac29f95ae4b13b23e16480582181d697a..3be1197d872c1d64796f3d454e768b9f0e01f899 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -249,4 +249,6 @@ source "drivers/cpuinspect/Kconfig" source "drivers/roh/Kconfig" +source "drivers/coda/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index f884f6f1f80df349ddfd76a9ba956eee8084fdfc..3955e605df14ffb72cce5203c1d1b60df19d75ee 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -203,3 +203,5 @@ obj-$(CONFIG_CDX_BUS) += cdx/ obj-$(CONFIG_S390) += s390/ obj-$(CONFIG_ROH) += roh/ + +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda/ diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 3133593238e2725b29a8b895f15c20d56c25d267..3348d4db5f1b3cdb24fd62d2421eb7a87ca50c94 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -221,7 +221,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, } EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA /** * platform_msi_domain_alloc_range_irqs - Allocate specific scope MSI interrupts for @dev * @dev: The device for which to allocate interrupts diff --git a/drivers/coda/Kconfig b/drivers/coda/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..3f41ef22b44cf63f20faf48152f792752094eab2 --- /dev/null +++ b/drivers/coda/Kconfig @@ -0,0 +1,8 @@ +config HISI_VIRTCCA_CODA + bool "Support for virtCCA Confidential Device Assignment" + default y + depends on HISI_VIRTCCA_HOST + help + Support VIRTCCA Device Assignment based on S-EL2 + + If unsure, say Y. diff --git a/drivers/coda/Makefile b/drivers/coda/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..62ca3132ac4d10843ea8b51bba0806a3643e88b9 --- /dev/null +++ b/drivers/coda/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda.o +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda_vfio.o +obj-$(CONFIG_HISI_VIRTCCA_CODA) += coda_pci.o diff --git a/drivers/coda/coda.c b/drivers/coda/coda.c new file mode 100644 index 0000000000000000000000000000000000000000..bfbb4607557ccf0b1ebadfea8a0615de044d09c6 --- /dev/null +++ b/drivers/coda/coda.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include +#include + +struct cc_dev_config { + u32 sid; /* BDF number of the device */ + u32 vmid; /* virtual machine id */ + u32 root_bd; /* root bus and device number. */ + bool secure; /* device secure attribute */ + /* MSI addr for confidential device with iommu group granularity */ + u64 msi_addr; + struct hlist_node node; /* device hash table */ +}; + +static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); + +/** + * get_root_bd - Traverse pcie topology to find the root number + * @dev: The device for which to get root bd + * + * Returns: + * %-1 if error or not pci device + */ +static int get_root_bd(struct device *dev) +{ + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) + return -1; + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return -1; + + while (!pci_is_root_bus(pdev->bus)) + pdev = pci_upstream_bridge(pdev); + + return pci_dev_id(pdev) & MASK_DEV_FUNCTION; +} + +/** + * get_child_devices_rec - Traverse pcie topology to find child devices + * If dev is a bridge, get it's children + * If dev is a regular device, get itself + * @dev: Device for which to get child devices + * @devs: All child devices under input dev + * @max_devs: Max num of devs + * @ndev: Num of child devices + */ +static void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, + int max_devs, int *ndev) +{ + struct pci_bus *bus = dev->subordinate; + + if (bus) { /* dev is a bridge */ + struct pci_dev *child; + + list_for_each_entry(child, &bus->devices, bus_list) { + get_child_devices_rec(child, devs, max_devs, ndev); + } + } else { /* dev is a regular device */ + uint16_t bdf = pci_dev_id(dev); + int i; + /* check if bdf is already in devs */ + for (i = 0; i < *ndev; i++) { + if (devs[i] == bdf) + return; + } + /* check overflow */ + if (*ndev >= max_devs) { + pr_warn("S_SMMU: devices num over max devs\n"); + return; + } + devs[*ndev] = bdf; + *ndev = *ndev + 1; + } +} + +/** + * get_sibling_devices - Get all devices which share the same root_bd as dev + * @dev: Device for which to get child devices + * @devs: All child devices under input dev + * @max_devs: Max num of devs + * + * Returns: + * %0 if get child devices failure + */ +static int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) +{ + struct pci_dev *pdev; + int ndev = 0; + + if (!dev_is_pci(dev)) + return ndev; + + pdev = to_pci_dev(dev); + if (pdev->bus == NULL) + return ndev; + + while (!pci_is_root_bus(pdev->bus)) + pdev = pci_upstream_bridge(pdev); + + get_child_devices_rec(pdev, devs, max_devs, &ndev); + return ndev; +} + +/** + * add_cc_dev_obj - Add device obj to hash tablse + * @sid: Stream id of device + * @vmid: Virtual machine id + * @root_bd: Root port bus device num + * @secure: Whether the device is secure or not + * + * Returns: + * %0 if add obj success + * %-ENOMEM if alloc obj failed + */ +static int add_cc_dev_obj(u32 sid, u32 vmid, u32 root_bd, bool secure) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj->sid == sid) { + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + return 0; + } + } + + obj = kzalloc(sizeof(*obj), GFP_KERNEL); + if (!obj) + return -ENOMEM; + + obj->sid = sid; + obj->vmid = vmid; + obj->root_bd = root_bd; + obj->secure = secure; + + hash_add(g_cc_dev_htable, &obj->node, sid); + return 0; +} + +/** + * is_cc_root_bd - Whether the root port is secure or not + * @root_bd: Root port bus device num + * + * Returns: + * %true if the root bd is secure + * %false if the root bd is non-secure + */ +static bool is_cc_root_bd(u32 root_bd) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (obj->root_bd == root_bd && obj->secure) + return true; + } + + return false; +} + +/** + * is_cc_vmid - Whether the vm is confidential vm + * @vmid: Virtual machine id + * + * Returns: + * %true if the vm is confidential + * %false if the vm is not confidential + */ +bool is_cc_vmid(u32 vmid) +{ + int bkt; + struct cc_dev_config *obj; + + hash_for_each(g_cc_dev_htable, bkt, obj, node) { + if (vmid > 0 && obj->vmid == vmid) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(is_cc_vmid); + +/** + * is_cc_dev - Whether the stream id of dev is confidential + * @sid: Stream id of dev + * + * Returns: + * %true if the dev is confidential + * %false if the dev is not confidential + */ +bool is_cc_dev(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->secure; + } + + return false; +} +EXPORT_SYMBOL(is_cc_dev); + +/** + * get_g_cc_dev_msi_addr - Obtain the msi address of confidential device + * @sid: Stream id of dev + * + * Returns: + * %0 if does not find the confidential device that matches the stream id + * %msi_addr return the msi address of confidential device that matches the stream id + */ +u64 get_g_cc_dev_msi_addr(u32 sid) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid) + return obj->msi_addr; + } + return 0; +} +EXPORT_SYMBOL_GPL(get_g_cc_dev_msi_addr); + +/** + * set_g_cc_dev_msi_addr - Set the msi address of confidential device + * @sid: Stream id of dev + * @msi_addr: Msi address + */ +void set_g_cc_dev_msi_addr(u32 sid, u64 msi_addr) +{ + struct cc_dev_config *obj; + + hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { + if (obj != NULL && obj->sid == sid && !obj->msi_addr) { + obj->msi_addr = msi_addr; + return; + } + } +} + +/* Secure device hash table init */ +void g_cc_dev_table_init(void) +{ + hash_init(g_cc_dev_htable); +} +EXPORT_SYMBOL(g_cc_dev_table_init); + +/** + * virtcca_tmi_dev_attach - Complete the stage2 page table establishment + * for the security device + * @arm_smmu_domain: The handle of smmu domain + * @kvm: The handle of virtual machine + * + * Returns: + * %0 if attach dev success + * %-ENXIO if the root port of device does not have pcipc capability + */ +u32 virtcca_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm) +{ + unsigned long flags; + int i, j; + struct arm_smmu_master *master; + int ret = 0; + u64 cmd[CMDQ_ENT_DWORDS] = {0}; + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); + /* + * Traverse all devices under the secure smmu domain and + * set the correspnding address translation table for each device + */ + list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + ret = tmi_dev_attach(sid, virtcca_cvm->rd, + arm_smmu_domain->smmu->s_smmu_id); + if (ret) { + dev_err(arm_smmu_domain->smmu->dev, "CoDA: dev protected failed!\n"); + ret = -ENXIO; + goto out; + } + /* Need to config ste */ + cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); + cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); + cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); + tmi_smmu_queue_write(cmd[0], cmd[1], + arm_smmu_domain->smmu->s_smmu_id); + } + } + } + +out: + spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); + return ret; +} + +/** + * virtcca_secure_dev_ste_create - Setting up the STE config content + * for the security device + * @smmu: An SMMUv3 instance + * @master: SMMU private data for each master + * @sid: Stream id of device + * + * Returns: + * %0 if create ste success + * %-ENOMEM alloc ste params failed + * %-EINVAL set ste config content failed + */ +static int virtcca_secure_dev_ste_create(struct arm_smmu_device *smmu, + struct arm_smmu_master *master, u32 sid) +{ + struct tmi_smmu_ste_params *params_ptr; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + params_ptr = kzalloc(sizeof(*params_ptr), GFP_KERNEL); + if (!params_ptr) + return -ENOMEM; + + /* Sync Level 2 STE to TMM */ + params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); + params_ptr->sid = sid; + params_ptr->smmu_id = smmu->s_smmu_id; + + if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) { + kfree(params_ptr); + dev_err(smmu->dev, "CoDA: failed to create ste level 2\n"); + return -EINVAL; + } + + kfree(params_ptr); + + return 0; +} + +/** + * virtcca_delegate_secure_dev - Delegate device to secure state + * @smmu: An SMMUv3 instance + * @root_bd: The port where the secure device is located + * @dev: Secure device + * + * Returns: + * %0 if delegate success + * %-ENOMEM if alloc params failed + * %-EINVAL if the dev is invalid + */ +static inline int virtcca_delegate_secure_dev(uint16_t root_bd, struct arm_smmu_device *smmu, + struct device *dev) +{ + int i; + u64 ret = 0; + struct tmi_dev_delegate_params *params = NULL; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + params->root_bd = root_bd; + params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); + if (params->num_dev >= MAX_DEV_PER_PORT) { + ret = -EINVAL; + goto out; + } + + dev_info(smmu->dev, "CoDA: Delegate %d devices as %02x:%02x to secure\n", + params->num_dev, root_bd >> DEV_BUS_NUM, + (root_bd & MASK_DEV_BUS) >> DEV_FUNCTION_NUM); + ret = tmi_dev_delegate(__pa(params)); + if (ret) { + dev_err(smmu->dev, "CoDA: failed to delegate device to secure\n"); + goto out; + } + + for (i = 0; i < params->num_dev; i++) { + ret = add_cc_dev_obj(params->devs[i], 0, root_bd, true); + if (ret) + break; + } + +out: + kfree(params); + return ret; +} + +/** + * add_secure_dev_to_cc_table - Add secure device to hash table + * @smmu: An SMMUv3 instance + * @smmu_domain: The handle of smmu_domain + * @root_bd: The port where the secure device is located + * @master: SMMU private data for each master + * + * Returns: + * %0 if add to hash table success + * %-ENOMEM if alloc obj failed + * %-EINVAL if stream id is invalid + */ +static inline int add_secure_dev_to_cc_table(struct arm_smmu_device *smmu, + struct arm_smmu_domain *smmu_domain, uint16_t root_bd, struct arm_smmu_master *master) +{ + int i, j; + u64 ret = 0; + + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (!is_cc_dev(sid)) { + dev_err(smmu->dev, "CoDA: sid is not cc dev\n"); + return -EINVAL; + } + ret = add_cc_dev_obj(sid, smmu_domain->s2_cfg.vmid, root_bd, true); + if (ret) + break; + } + return ret; +} + +/** + * virtcca_enable_secure_dev - Enable the PCIe protection controller function + * of the security device + * @smmu_domain: The handle of smmu_domain + * @master: SMMU private data for each master + * @dev: Secure device + * + * Returns: + * %0 if the root port of secure dev successfully set up pcipc capability + * %-ENOMEM alloc ste params failed + * %-EINVAL set ste config content failed + */ +static int virtcca_enable_secure_dev(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master, struct device *dev) +{ + u64 ret = 0; + uint16_t root_bd = get_root_bd(dev); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (!is_cc_root_bd(root_bd)) { + ret = virtcca_delegate_secure_dev(root_bd, smmu, dev); + if (ret) + return ret; + } + + ret = add_secure_dev_to_cc_table(smmu, smmu_domain, root_bd, master); + + return ret; +} + +/** + * virtcca_secure_dev_operator - Implement security settings for corresponding devices + * targeting the secure smmu domain + * @domain: The handle of iommu_domain + * @dev: Secure device + * + * Returns: + * %0 if the domain does not need to enable secure or the domain + * successfully set up security features + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_secure_dev_operator(struct device *dev, void *domain) +{ + int i, j; + int ret; + struct iommu_domain *iommu_domain = (struct iommu_domain *)domain; + struct iommu_fwspec *fwspec = NULL; + struct arm_smmu_device *smmu = NULL; + struct arm_smmu_domain *smmu_domain = NULL; + struct arm_smmu_master *master = NULL; + + if (!is_virtcca_cvm_enable()) + return 0; + + fwspec = dev_iommu_fwspec_get(dev); + if (!fwspec) + return -ENOENT; + + smmu_domain = to_smmu_domain(iommu_domain); + master = dev_iommu_priv_get(dev); + smmu = master->smmu; + + if (!smmu && !virtcca_smmu_enable(smmu)) { + dev_err(smmu->dev, "CoDA: security smmu not initialized for the device\n"); + return -EINVAL; + } + + ret = virtcca_enable_secure_dev(smmu_domain, master, dev); + if (ret) + return ret; + + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + /* Bridged PCI devices may end up with duplicated IDs */ + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + if (virtcca_secure_dev_ste_create(smmu, master, sid)) + return -ENOMEM; + } + + dev_info(smmu->dev, "CoDA: attach confidential dev: %s", dev_name(dev)); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_secure_dev_operator); + +/** + * virtcca_attach_secure_dev - Attach the device of iommu + * group to confidential virtual machine + * @domain: The handle of iommu domain + * @group: Iommu group + * + * Returns: + * %0 if attach the all devices success + * %-EINVAL if the smmu does not initialize secure state + * %-ENOMEM if the device create secure ste failed + * %-ENOENT if the device does not have fwspec + */ +int virtcca_attach_secure_dev(struct iommu_domain *domain, struct iommu_group *group) +{ + int ret; + + ret = iommu_group_for_each_dev(group, (void *)domain, virtcca_secure_dev_operator); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_attach_secure_dev); diff --git a/drivers/coda/coda_pci.c b/drivers/coda/coda_pci.c new file mode 100644 index 0000000000000000000000000000000000000000..07aa58c41bf6fc99e7816c25af506b644f47fa6e --- /dev/null +++ b/drivers/coda/coda_pci.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include + +#include "../drivers/pci/msi/msi.h" + +/** + * virtcca_pci_read_msi_msg - secure dev read msi msg + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @msg: Msg information + * @base: Msi base address + * + **/ +void virtcca_pci_read_msi_msg(struct pci_dev *dev, struct msi_msg *msg, + void __iomem *base) +{ + u64 pbase = mmio_va_to_pa(base); + + msg->address_lo = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->address_hi = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + CVM_RW_32_BIT, pci_dev_id(dev)); + msg->data = tmi_mmio_read(pbase + PCI_MSIX_ENTRY_DATA, CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/** + * virtcca_pci_write_msi_msg - secure dev write msi msg + * @desc: MSI-X description + * @msg: Msg information + * + **/ +bool virtcca_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) +{ + if (!is_virtcca_cvm_enable()) + return false; + + void __iomem *base = pci_msix_desc_addr(desc); + u32 ctrl = desc->pci.msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + u64 pbase = mmio_va_to_pa(base); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (!is_cc_dev(pci_dev_id(pdev))) + return false; + + u64 addr = (u64)msg->address_lo | ((u64)msg->address_hi << 32); + + if (addr) { + /* Get the offset of the its register of a specific device */ + u64 offset = addr - CVM_MSI_ORIG_IOVA; + + addr = get_g_cc_dev_msi_addr(pci_dev_id(pdev)); + addr += offset; + if (!addr) + return true; + } + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, + lower_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, + upper_32_bits(addr), CVM_RW_32_BIT, pci_dev_id(pdev)); + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_DATA, + msg->data, CVM_RW_32_BIT, pci_dev_id(pdev)); + + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl); + tmi_mmio_read(mmio_va_to_pa((void *)pbase + PCI_MSIX_ENTRY_DATA), + CVM_RW_32_BIT, pci_dev_id(pdev)); + + return true; +} + +void virtcca_msix_prepare_msi_desc(struct pci_dev *dev, + struct msi_desc *desc, void __iomem *addr) +{ + desc->pci.msix_ctrl = tmi_mmio_read(mmio_va_to_pa(addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + CVM_RW_32_BIT, pci_dev_id(dev)); +} + +/* + * If it is a safety device, write vector ctrl need + * use tmi interface + */ +bool virtcca_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +{ + if (!is_virtcca_cvm_enable()) + return false; + + void __iomem *desc_addr = pci_msix_desc_addr(desc); + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return false; + + if (desc->pci.msi_attrib.can_mask) + tmi_mmio_write(mmio_va_to_pa(desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL), + ctrl, CVM_RW_32_BIT, pci_dev_id(pdev)); + return true; +} + +/* + * If it is a safety device, read msix need + * use tmi interface + */ +bool virtcca_pci_msix_mask(struct msi_desc *desc) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = (desc->dev != NULL && + dev_is_pci(desc->dev)) ? to_pci_dev(desc->dev) : NULL; + + if (pdev == NULL || !is_cc_dev(pci_dev_id(pdev))) + return false; + + /* Flush write to device */ + tmi_mmio_read(mmio_va_to_pa(desc->pci.mask_base), CVM_RW_32_BIT, pci_dev_id(pdev)); + return true; +} + +/** + * virtcca_msix_mask_all_cc - mask all secure dev msix c + * @dev: Pointer to the pci_dev data structure of MSI-X device function + * @base: Io address + * @tsize: Number of entry + * @dev_num: Dev number + * + * Returns: + * %0 if msix mask all cc device success + **/ +int virtcca_msix_mask_all_cc(struct pci_dev *dev, void __iomem *base, int tsize, u64 dev_num) +{ + int i; + u16 rw_ctrl; + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + u64 pbase = mmio_va_to_pa(base); + + if (pci_msi_ignore_mask) + goto out; + + for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) { + tmi_mmio_write(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, + ctrl, CVM_RW_32_BIT, dev_num); + } + +out: + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &rw_ctrl); + rw_ctrl &= ~PCI_MSIX_FLAGS_MASKALL; + rw_ctrl |= 0; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, rw_ctrl); + + pcibios_free_irq(dev); + return 0; +} + +/* If device is secure dev, read config need transfer to tmm module */ +int virtcca_pci_generic_config_read(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 *val) +{ + u32 cvm_bit = size == 1 ? CVM_RW_8_BIT : size == 2 ? CVM_RW_16_BIT : CVM_RW_32_BIT; + + *val = tmi_mmio_read(mmio_va_to_pa(addr), cvm_bit, PCI_DEVID(bus_num, devfn)); + return 0; +} + +/* If device is secure dev, write config need transfer to tmm module */ +int virtcca_pci_generic_config_write(void __iomem *addr, unsigned char bus_num, + unsigned int devfn, int size, u32 val) +{ + u32 cvm_bit = size == 1 ? CVM_RW_8_BIT : size == 2 ? CVM_RW_16_BIT : CVM_RW_32_BIT; + + tmi_mmio_write(mmio_va_to_pa(addr), val, cvm_bit, PCI_DEVID(bus_num, devfn)); + return 0; +} + +/* Judge startup virtcca_cvm_host is enable and device is secure or not */ +bool is_virtcca_pci_io_rw(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return false; + + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + + if (cc_dev) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(is_virtcca_pci_io_rw); + +/* Transfer to tmm write io value */ +void virtcca_pci_io_write(struct vfio_pci_core_device *vdev, u64 val, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + WARN_ON(tmi_mmio_write(mmio_va_to_pa(io), val, size, pci_dev_id(pdev))); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_write); + +/* Transfer to tmm read io value */ +u64 virtcca_pci_io_read(struct vfio_pci_core_device *vdev, + u64 size, void __iomem *io) +{ + struct pci_dev *pdev = vdev->pdev; + + return tmi_mmio_read(mmio_va_to_pa(io), size, pci_dev_id(pdev)); +} +EXPORT_SYMBOL_GPL(virtcca_pci_io_read); diff --git a/drivers/coda/coda_vfio.c b/drivers/coda/coda_vfio.c new file mode 100644 index 0000000000000000000000000000000000000000..9de5d50a6bffbe9ea8388a23064d078b3540d894 --- /dev/null +++ b/drivers/coda/coda_vfio.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include +#include +#include + +/** + * virtcca_map_pages - Virtcca need map the secure + * memory with paddr + * @ops: The handle of io_pgtable_ops + * @iova: Ipa address + * @paddr: Physical address + * @pgsize: Page size + * @pgcount: Page count + * @iommu_prot: Iommu attribute + * @mapped: Mapped size + * + * Returns: + * %0 if map pages success + */ +int virtcca_map_pages(void *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, size_t *mapped) +{ + struct kvm *kvm; + u64 loader_start; + u64 ram_size; + struct io_pgtable *data = io_pgtable_ops_to_pgtable(ops); + struct io_pgtable_cfg *cfg = &data->cfg; + long iaext = (s64)iova >> cfg->ias; + int ret = 0; + struct arm_smmu_domain *smmu_domain = NULL; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) + return -EINVAL; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext || paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + smmu_domain = (struct arm_smmu_domain *)(data->cookie); + if (!smmu_domain) + return -EINVAL; + + kvm = smmu_domain->kvm; + if (kvm) { + struct virtcca_cvm *virtcca_cvm = kvm->arch.virtcca_cvm; + + loader_start = virtcca_cvm->loader_start; + ram_size = virtcca_cvm->ram_size; + /* Cvm ram space mapping*/ + if (iova >= loader_start && + iova < loader_start + ram_size && + !virtcca_cvm->is_mapped) { + ret = kvm_cvm_map_range(kvm); + } else if (iova < loader_start || iova >= loader_start + ram_size) { + if (iova == CVM_MSI_ORIG_IOVA) + /* Cvm msi address mapping */ + ret = virtcca_map_msi_address(kvm, smmu_domain, + paddr, pgsize * pgcount); + else + /* Cvm mmio space mapping */ + ret = cvm_map_unmap_ipa_range(kvm, iova, + paddr, pgsize * pgcount, true); + } + if (mapped) + *mapped += pgsize * pgcount; + } + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_map_pages); + +/** + * virtcca_unmap_pages - Virtcca unmap the iova + * @ops: The handle of io_pgtable_ops + * @iova: Ipa address + * @pgsize: Page size + * @pgcount: Page count + * + * Returns: + * %0 if map pages success or parameter is invalid + */ +size_t virtcca_unmap_pages(void *ops, unsigned long iova, + size_t pgsize, size_t pgcount) +{ + struct kvm *kvm; + struct io_pgtable *data = io_pgtable_ops_to_pgtable(ops); + struct io_pgtable_cfg *cfg = &data->cfg; + long iaext = (s64)iova >> cfg->ias; + struct arm_smmu_domain *smmu_domain = NULL; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) + iaext = ~iaext; + if (WARN_ON(iaext)) + return 0; + + smmu_domain = (struct arm_smmu_domain *)(data->cookie); + if (!smmu_domain) + return 0; + + kvm = smmu_domain->kvm; + if (!kvm) + return 0; + + return cvm_map_unmap_ipa_range(kvm, iova, 0, pgsize * pgcount, false); +} +EXPORT_SYMBOL_GPL(virtcca_unmap_pages); + +/** + * virtcca_iommu_map - Iommu driver map pages, and then + * calls the map function in the + * smmu to perform mapping + * @domain: Iommu domain + * @iova: Ipa address + * @paddr: Physical address + * @size: Map size + * @prot: Iommu attribute + * + * Returns: + * %0 if map success + * %-EINVAL if domain type is not paging + * %-ENODEV if the domain pgsize_bitmap is zero or parameter is invalid + */ +int virtcca_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int min_pagesz; + int ret = 0; + unsigned long orig_iova = iova; + size_t orig_size = size; + const struct iommu_domain_ops *ops = domain->ops; + struct io_pgtable_ops *io_ops = to_smmu_domain(domain)->pgtbl_ops; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return -EINVAL; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return -ENODEV; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * both the virtual address and the physical one, as well as + * the size of the mapping, must be aligned (at least) to the + * size of the smallest page supported by the hardware + */ + if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); + return -EINVAL; + } + + if (!io_ops) + return -ENODEV; + + while (size) { + size_t pgsize, count, mapped = 0; + + pgsize = iommu_pgsize(domain, iova, paddr, size, &count); + + ret = virtcca_map_pages(io_ops, iova, paddr, pgsize, + count, prot, &mapped); + /* + * Some pages may have been mapped, even if an error occurred, + * so we should account for those so they can be unmapped. + */ + size -= mapped; + + if (ret) + break; + + iova += mapped; + paddr += mapped; + } + + /* unroll mapping in case something went wrong */ + if (ret) + virtcca_iommu_unmap(domain, orig_iova, orig_size - size); + + if (ret == 0 && ops->iotlb_sync_map) { + ret = ops->iotlb_sync_map(domain, iova, size); + if (ret) + goto out_err; + } + + return ret; + +out_err: + /* undo mappings already done */ + virtcca_iommu_unmap(domain, iova, size); + + return ret; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_map); + +/** + * virtcca_iommu_unmap - Iommu driver unmap pages, and then + * calls the map function in the + * smmu to perform unmapping + * @domain: Iommu domain + * @iova: Ipa address + * @size: Map size + * + * Returns: + * %0 if map success or domain type and parameter is invalid + */ + +size_t virtcca_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + size_t unmapped_page, unmapped = 0; + struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + unsigned int min_pagesz; + + if (unlikely(!(domain->type & __IOMMU_DOMAIN_PAGING))) + return 0; + + if (WARN_ON(domain->pgsize_bitmap == 0UL)) + return 0; + + /* find out the minimum page size supported */ + min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + + /* + * The virtual address, as well as the size of the mapping, must be + * aligned (at least) to the size of the smallest page supported + * by the hardware + */ + if (!IS_ALIGNED(iova | size, min_pagesz)) { + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + iova, size, min_pagesz); + return 0; + } + + if (!ops) + return 0; + + /* + * Keep iterating until we either unmap 'size' bytes (or more) + * or we hit an area that isn't mapped. + */ + while (unmapped < size) { + size_t pgsize, count; + + pgsize = iommu_pgsize(domain, iova, iova, size - unmapped, &count); + unmapped_page = virtcca_unmap_pages(ops, iova, pgsize, count); + if (!unmapped_page) + break; + + iova += unmapped_page; + unmapped += unmapped_page; + } + return unmapped; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_unmap); + +/* Whether the kvm is cvm */ +bool virtcca_iommu_domain_get_kvm(struct iommu_domain *domain, struct kvm **kvm) +{ + struct arm_smmu_domain *arm_smmu_domain; + + arm_smmu_domain = to_smmu_domain(domain); + *kvm = virtcca_arm_smmu_get_kvm(arm_smmu_domain); + if (*kvm) + return (*kvm)->arch.is_virtcca_cvm; + + return false; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_domain_get_kvm); + +/** + * cvm_vfio_file_iommu_group - Get iommu group from vfio file + * @file: Vfio file + * + * Returns: + * %NULL if the virtcca_vfio_file_iommu_group func is not defined + * or CONFIG_HISI_VIRTCCA_CODA is not enable, group is null + * %iommu_group if get the iommu group from file success + */ +struct iommu_group *cvm_vfio_file_iommu_group(struct file *file) +{ + struct iommu_group *(*fn)(struct file *file); + struct iommu_group *ret; + + fn = symbol_get(virtcca_vfio_file_iommu_group); + if (!fn) + return NULL; + + ret = fn(file); + + symbol_put(virtcca_vfio_file_iommu_group); + + return ret; +} + +/* Obtain msi address through iommu group id */ +u64 virtcca_get_iommu_device_msi_addr(struct iommu_group *iommu_group) +{ + u64 msi_addr = iommu_group_id(iommu_group) * CVM_MSI_IOVA_OFFSET + CVM_MSI_MIN_IOVA; + + if (msi_addr >= CVM_MSI_MAX_IOVA || msi_addr < CVM_MSI_MIN_IOVA) { + pr_err("MSI address 0x%llx overflow.\n", msi_addr); + return 0; + } + + return msi_addr; +} +EXPORT_SYMBOL_GPL(virtcca_get_iommu_device_msi_addr); + +/* Set the corresponding MSI address */ +int virtcca_set_dev_msi_addr(struct device *dev, void *iova) +{ + unsigned long *msi_iova = (unsigned long *)iova; + + if (dev_is_pci(dev)) { + struct pci_dev *pci_dev = to_pci_dev(dev); + u16 pci_id = pci_dev_id(pci_dev); + + set_g_cc_dev_msi_addr(pci_id, *msi_iova); + } + return 0; +} + +/* Traverse the devices in the iommu group and set the corresponding MSI address */ +int virtcca_iommu_group_set_dev_msi_addr(struct iommu_group *iommu_group, unsigned long *iova) +{ + int ret; + + ret = iommu_group_for_each_dev(iommu_group, (void *)iova, virtcca_set_dev_msi_addr); + return ret; +} + +/** + * virtcca_msi_map - Vfio driver mapping device side msi address + * @vdev: Vfio pci core device + * + * Returns: + * %0 if map success + */ +int virtcca_msi_map(struct vfio_pci_core_device *vdev) +{ + if (!is_virtcca_cvm_enable()) + return 0; + + int ret; + dma_addr_t iova; + phys_addr_t msi_addr; + struct iommu_domain *domain = NULL; + struct pci_dev *pdev = vdev->pdev; + bool cc_dev = pdev == NULL ? false : is_cc_dev(pci_dev_id(pdev)); + int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; + + /* + * If the device is secure and has not done MSI address mapping, + * Mapping is required. + */ + if (cc_dev && !get_g_cc_dev_msi_addr(pci_dev_id(pdev))) { + domain = iommu_get_domain_for_dev(&(pdev->dev)); + /* Get the MSI address of the device */ + virtcca_iommu_dma_get_msi_page((void *)domain->iova_cookie, &iova, &msi_addr); + /* Release non-secure side device MSI address mapping */ + iommu_unmap(domain, iova, PAGE_SIZE); + /* Mapping secure side MSI address */ + ret = virtcca_iommu_map(domain, iova, msi_addr, PAGE_SIZE, prot); + if (ret) + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(virtcca_msi_map); diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index e4020cafcce9fa305d11b8dd202f357bc98f35aa..8781d131961d849fbbecfe665bf0e2b7e89d0a55 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-objs-y += arm-smmu-v3.o -arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_HOST) += arm-s-smmu-v3.o +arm_smmu_v3-objs-$(CONFIG_HISI_VIRTCCA_CODA) += arm-s-smmu-v3.o arm_smmu_v3-objs-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o arm_smmu_v3-objs := $(arm_smmu_v3-objs-y) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c index d247168c68f773e79f256275b8ab99370f174ce6..34e0060420af72eb5039d6531b86722d8795bde4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.c @@ -4,21 +4,13 @@ */ #include #include +#include #include "arm-smmu-v3.h" #include "arm-s-smmu-v3.h" -struct cc_dev_config { - u32 sid; /* BDF number of the device */ - u32 vmid; /* virtual machine id */ - u32 root_bd; /* root bus and device number. */ - bool secure; /* device secure attribute */ - struct hlist_node node; /* device hash table */ -}; - static bool g_s_smmu_id_map_init; -static DEFINE_HASHTABLE(g_cc_dev_htable, MAX_CC_DEV_NUM_ORDER); static DECLARE_BITMAP(g_s_smmu_id_map, ARM_S_SMMU_MAX_IDS); /* @@ -54,194 +46,6 @@ static inline void virtcca_smmu_set_irq(struct arm_smmu_device *smmu) smmu->s_gerr_irq = msi_get_virq(smmu->dev, S_GERROR_MSI_INDEX); } -/** - * get_root_bd - Traverse pcie topology to find the root number - * @dev: The device for which to get root bd - * - * Returns: - * %-1 if error or not pci device - */ -static int get_root_bd(struct device *dev) -{ - struct pci_dev *pdev; - - if (!dev_is_pci(dev)) - return -1; - pdev = to_pci_dev(dev); - if (pdev->bus == NULL) - return -1; - while (pdev->bus->parent != NULL) - pdev = pdev->bus->self; - - return pci_dev_id(pdev) & MASK_DEV_FUNCTION; -} - -/** - * get_child_devices_rec - Traverse pcie topology to find child devices - * If dev is a bridge, get it's children - * If dev is a regular device, get itself - * @dev: Device for which to get child devices - * @devs: All child devices under input dev - * @max_devs: Max num of devs - * @ndev: Num of child devices - */ -static void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, - int max_devs, int *ndev) -{ - struct pci_bus *bus = dev->subordinate; - - if (bus) { /* dev is a bridge */ - struct pci_dev *child; - - list_for_each_entry(child, &bus->devices, bus_list) { - get_child_devices_rec(child, devs, max_devs, ndev); - } - } else { /* dev is a regular device */ - uint16_t bdf = pci_dev_id(dev); - int i; - /* check if bdf is already in devs */ - for (i = 0; i < *ndev; i++) { - if (devs[i] == bdf) - return; - } - /* check overflow */ - if (*ndev >= max_devs) { - pr_warn("S_SMMU: devices num over max devs\n"); - return; - } - devs[*ndev] = bdf; - *ndev = *ndev + 1; - } -} - -/** - * get_sibling_devices - Get all devices which share the same root_bd as dev - * @dev: Device for which to get child devices - * @devs: All child devices under input dev - * @max_devs: Max num of devs - * - * Returns: - * %0 if get child devices failure - */ -static int get_sibling_devices(struct device *dev, uint16_t *devs, int max_devs) -{ - struct pci_dev *pdev; - int ndev = 0; - - if (!dev_is_pci(dev)) - return ndev; - - pdev = to_pci_dev(dev); - if (pdev->bus == NULL) - return ndev; - - while (pdev->bus->parent != NULL) - pdev = pdev->bus->self; - - get_child_devices_rec(pdev, devs, max_devs, &ndev); - return ndev; -} - -/** - * add_cc_dev_obj - Add device obj to hash tablse - * @sid: Stream id of device - * @vmid: Virtual machine id - * @root_bd: Root port bus device num - * @secure: Whether the device is secure or not - * - * Returns: - * %0 if add obj success - * %-ENOMEM if alloc obj failed - */ -static int add_cc_dev_obj(u32 sid, u32 vmid, u32 root_bd, bool secure) -{ - struct cc_dev_config *obj; - - hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { - if (obj->sid == sid) { - obj->vmid = vmid; - obj->root_bd = root_bd; - obj->secure = secure; - return 0; - } - } - - obj = kzalloc(sizeof(*obj), GFP_KERNEL); - if (!obj) - return -ENOMEM; - - obj->sid = sid; - obj->vmid = vmid; - obj->root_bd = root_bd; - obj->secure = secure; - - hash_add(g_cc_dev_htable, &obj->node, sid); - return 0; -} - -/** - * is_cc_root_bd - Whether the root port is secure or not - * @root_bd: Root port bus device num - * - * Returns: - * %true if the root bd is secure - * %false if the root bd is non-secure - */ -static bool is_cc_root_bd(u32 root_bd) -{ - int bkt; - struct cc_dev_config *obj; - - hash_for_each(g_cc_dev_htable, bkt, obj, node) { - if (obj->root_bd == root_bd && obj->secure) - return true; - } - - return false; -} - -/** - * is_cc_vmid - Whether the vm is confidential vm - * @vmid: Virtual machine id - * - * Returns: - * %true if the vm is confidential - * %false if the vm is not confidential - */ -static bool is_cc_vmid(u32 vmid) -{ - int bkt; - struct cc_dev_config *obj; - - hash_for_each(g_cc_dev_htable, bkt, obj, node) { - if (vmid > 0 && obj->vmid == vmid) - return true; - } - - return false; -} - -/** - * is_cc_dev - Whether the stream id of dev is confidential - * @sid: Stream id of dev - * - * Returns: - * %true if the dev is confidential - * %false if the dev is not confidential - */ -bool is_cc_dev(u32 sid) -{ - struct cc_dev_config *obj; - - hash_for_each_possible(g_cc_dev_htable, obj, node, sid) { - if (obj != NULL && obj->sid == sid) - return obj->secure; - } - - return false; -} -EXPORT_SYMBOL(is_cc_dev); - /** * virtcca_smmu_cmdq_need_forward - Whether the cmd queue need transfer to secure world * @cmd0: Command consists of 128 bits, cmd0 is the low 64 bits @@ -401,7 +205,6 @@ static void virtcca_smmu_init_one_queue(struct arm_smmu_device *smmu, qsz = ((1 << q->llq.max_n_shift) * dwords) << ARM_S_QUEUE_SHIFT_SIZE; if (!strcmp(name, "cmdq")) { - params_ptr->ns_src = q->base_dma; params_ptr->smmu_base_addr = smmu->ioaddr; params_ptr->size = qsz; params_ptr->smmu_id = smmu->s_smmu_id; @@ -410,7 +213,6 @@ static void virtcca_smmu_init_one_queue(struct arm_smmu_device *smmu, } if (!strcmp(name, "evtq")) { - params_ptr->ns_src = q->base_dma; params_ptr->smmu_base_addr = smmu->ioaddr; params_ptr->size = qsz; params_ptr->smmu_id = smmu->s_smmu_id; @@ -633,217 +435,6 @@ static void platform_get_s_irq_byname_optional(struct platform_device *pdev, smmu->s_gerr_irq = irq; } -/** - * virtcca_smmu_tmi_dev_attach - Complete the stage2 page table establishment - * for the security device - * @arm_smmu_domain: The handle of smmu domain - * @kvm: The handle of virtual machine - * - * Returns: - * %0 if attach dev success - * %-ENXIO if the root port of device does not have pcipc capability - */ -u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, struct kvm *kvm) -{ - unsigned long flags; - int i, j; - struct arm_smmu_master *master; - int ret = 0; - u64 cmd[CMDQ_ENT_DWORDS] = {0}; - struct virtcca_cvm *virtcca_cvm = (struct virtcca_cvm *)kvm->arch.virtcca_cvm; - - spin_lock_irqsave(&arm_smmu_domain->devices_lock, flags); - /* - * Traverse all devices under the secure smmu domain and - * set the correspnding address translation table for each device - */ - list_for_each_entry(master, &arm_smmu_domain->devices, domain_head) { - if (master && master->num_streams >= 0) { - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - ret = tmi_dev_attach(sid, virtcca_cvm->rd, - arm_smmu_domain->smmu->s_smmu_id); - if (ret) { - dev_err(arm_smmu_domain->smmu->dev, "S_SMMU: dev protected failed!\n"); - ret = -ENXIO; - goto out; - } - /* Need to config ste */ - cmd[0] |= FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CFGI_STE); - cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, sid); - cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, true); - tmi_smmu_queue_write(cmd[0], cmd[1], - arm_smmu_domain->smmu->s_smmu_id); - } - } - } - -out: - spin_unlock_irqrestore(&arm_smmu_domain->devices_lock, flags); - return ret; -} - -/** - * virtcca_smmu_secure_dev_ste_create - Setting up the STE config content - * for the security device - * @smmu: An SMMUv3 instance - * @master: SMMU private data for each master - * @sid: Stream id of device - * - * Returns: - * %0 if create ste success - * %-ENOMEM alloc ste params failed - * %-EINVAL set ste config content failed - */ -static int virtcca_smmu_secure_dev_ste_create(struct arm_smmu_device *smmu, - struct arm_smmu_master *master, u32 sid) -{ - struct tmi_smmu_ste_params *params_ptr; - struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; - - params_ptr = kzalloc(sizeof(*params_ptr), GFP_KERNEL); - if (!params_ptr) - return -ENOMEM; - - /* Sync Level 2 STE to TMM */ - params_ptr->ns_src = desc->l2ptr_dma + ((sid & ((1 << STRTAB_SPLIT) - 1)) * STE_ENTRY_SIZE); - params_ptr->sid = sid; - params_ptr->smmu_id = smmu->s_smmu_id; - - if (tmi_smmu_ste_create(__pa(params_ptr)) != 0) { - kfree(params_ptr); - dev_err(smmu->dev, "S_SMMU: failed to create ste level 2\n"); - return -EINVAL; - } - - kfree(params_ptr); - - return 0; -} - -/** - * add_secure_dev_to_cc_table - Add secure device to hash table - * @smmu: An SMMUv3 instance - * @smmu_domain: The handle of smmu_domain - * @root_bd: The port where the secure device is located - * @master: SMMU private data for each master - * - * Returns: - * %0 if add to hash table success - * %-ENOMEM if alloc obj failed - * %-EINVAL if stream id is invalid - */ -static inline int add_secure_dev_to_cc_table(struct arm_smmu_device *smmu, - struct arm_smmu_domain *smmu_domain, uint16_t root_bd, struct arm_smmu_master *master) -{ - int i, j; - u64 ret = 0; - - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - if (!is_cc_dev(sid)) { - dev_err(smmu->dev, "S_SMMU: sid is not cc dev\n"); - return -EINVAL; - } - ret = add_cc_dev_obj(sid, smmu_domain->s2_cfg.vmid, root_bd, true); - if (ret) - break; - } - return ret; -} - -/** - * virtcca_delegate_secure_dev - Delegate device to secure state - * @smmu: An SMMUv3 instance - * @root_bd: The port where the secure device is located - * @dev: Secure device - * - * Returns: - * %0 if delegate success - * %-ENOMEM if alloc params failed - * %-EINVAL if the dev is invalid - */ -static inline int virtcca_delegate_secure_dev(uint16_t root_bd, struct arm_smmu_device *smmu, - struct device *dev) -{ - int i; - u64 ret = 0; - struct tmi_dev_delegate_params *params = NULL; - - params = kzalloc(sizeof(*params), GFP_KERNEL); - if (!params) - return -ENOMEM; - - params->root_bd = root_bd; - params->num_dev = get_sibling_devices(dev, params->devs, MAX_DEV_PER_PORT); - if (params->num_dev >= MAX_DEV_PER_PORT) { - ret = -EINVAL; - goto out; - } - - dev_info(smmu->dev, "S_SMMU: Delegate %d devices as %02x:%02x to secure\n", - params->num_dev, root_bd >> DEV_BUS_NUM, - (root_bd & MASK_DEV_BUS) >> DEV_FUNCTION_NUM); - ret = tmi_dev_delegate(__pa(params)); - if (ret) { - dev_err(smmu->dev, "S_SMMU: failed to delegate device to secure\n"); - goto out; - } - - for (i = 0; i < params->num_dev; i++) { - ret = add_cc_dev_obj(params->devs[i], 0, root_bd, true); - if (ret) - break; - } - -out: - kfree(params); - return ret; -} - -/** - * virtcca_enable_secure_dev - Enable the PCIe protection controller function - * of the security device - * @smmu_domain: The handle of smmu_domain - * @master: SMMU private data for each master - * @dev: Secure device - * - * Returns: - * %0 if the root port of secure dev successfully set up pcipc capability - * %-ENOMEM alloc ste params failed - * %-EINVAL set ste config content failed - */ -static int virtcca_enable_secure_dev(struct arm_smmu_domain *smmu_domain, - struct arm_smmu_master *master, struct device *dev) -{ - u64 ret = 0; - uint16_t root_bd = get_root_bd(dev); - struct arm_smmu_device *smmu = smmu_domain->smmu; - - if (!is_cc_root_bd(root_bd)) { - ret = virtcca_delegate_secure_dev(root_bd, smmu, dev); - if (ret) - return ret; - } - - ret = add_secure_dev_to_cc_table(smmu, smmu_domain, root_bd, master); - - return ret; -} - /** * virtcca_smmu_write_msi_msg - Write secure smmu msi msg * @desc: Descriptor structure for MSI based interrupts @@ -945,26 +536,6 @@ static void virtcca_smmu_setup_unique_irqs(struct arm_smmu_device *smmu, bool re } } -/** - * _arm_smmu_write_reg_sync - Write value to smmu registers and wait for completion - * @smmu: An SMMUv3 instance - * @val: Expected value to be written - * @reg_off: Offset of object register - * @ack_off: Acknowledge offset of object register - * - * Returns: - * %0 if write success - */ -static int _arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, - u32 reg_off, u32 ack_off) -{ - u32 reg; - - writel_relaxed(val, smmu->base + reg_off); - return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, - 1, ARM_SMMU_POLL_TIMEOUT_US); -} - /** * virtcca_smmu_setup_irqs - Initialize the smmu irq * @smmu: An SMMUv3 instance @@ -972,15 +543,9 @@ static int _arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, */ static void virtcca_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) { - int irq, ret; + int irq; u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; - ret = _arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, - ARM_SMMU_IRQ_CTRLACK); - if (ret) { - dev_err(smmu->dev, "S_SMMU: failed to disable irqs\n"); - return; - } /* Disable IRQs first */ virtcca_smmu_disable_irq(smmu); @@ -991,12 +556,6 @@ static void virtcca_smmu_setup_irqs(struct arm_smmu_device *smmu, bool resume) if (smmu->features & ARM_SMMU_FEAT_PRI) irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; - /* Enable interrupt generation on the SMMU */ - ret = _arm_smmu_write_reg_sync(smmu, irqen_flags, - ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); - if (ret) - dev_warn(smmu->dev, "S_SMMU: failed to enable irqs\n"); - virtcca_smmu_enable_irq(smmu, IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN); } @@ -1026,11 +585,11 @@ static int virtcca_smmu_id_alloc(void) * %true if the smmu need to initialize secure state * %false the smmu does not need to initialize secure state */ -static bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) +bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) { if (!g_s_smmu_id_map_init) { set_bit(0, g_s_smmu_id_map); - hash_init(g_cc_dev_htable); + g_cc_dev_table_init(); g_s_smmu_id_map_init = true; } smmu->ioaddr = ioaddr; @@ -1043,6 +602,7 @@ static bool virtcca_smmu_map_init(struct arm_smmu_device *smmu, resource_size_t smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; return false; } +EXPORT_SYMBOL_GPL(virtcca_smmu_map_init); /** * arm_s_smmu_device_enable - Enable the smmu secure state @@ -1106,7 +666,7 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) } if (!(rv & S_IDR1_SEL2)) { - dev_err(smmu->dev, "S_SMMU: secure stage2 translation not supported!\n"); + dev_err(smmu->dev, "S_SMMU: secure stage2 translation is not supported!\n"); smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; return false; } @@ -1114,65 +674,6 @@ static bool arm_s_smmu_idr1_support_secure(struct arm_smmu_device *smmu) return true; } -/** - * virtcca_smmu_secure_dev_operator - Implement security settings for corresponding devices - * targeting the secure smmu domain - * @domain: The handle of iommu_domain - * @dev: Secure device - * - * Returns: - * %0 if the domain does not need to enable secure or the domain - * successfully set up security features - * %-EINVAL if the smmu does not initialize secure state - * %-ENOMEM if the device create secure ste failed - * %-ENOENT if the device does not have fwspec - */ -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev) -{ - int i, j; - int ret; - struct iommu_fwspec *fwspec = NULL; - struct arm_smmu_device *smmu = NULL; - struct arm_smmu_domain *smmu_domain = NULL; - struct arm_smmu_master *master = NULL; - - if (!is_virtcca_cvm_enable()) - return 0; - - fwspec = dev_iommu_fwspec_get(dev); - if (!fwspec) - return -ENOENT; - - smmu_domain = to_smmu_domain(domain); - master = dev_iommu_priv_get(dev); - smmu = master->smmu; - - if (!smmu && !virtcca_smmu_enable(smmu)) { - dev_err(smmu->dev, "S_SMMU: security smmu not initialized for the device\n"); - return -EINVAL; - } - - ret = virtcca_enable_secure_dev(smmu_domain, master, dev); - if (ret) - return ret; - - for (i = 0; i < master->num_streams; i++) { - u32 sid = master->streams[i].id; - /* Bridged PCI devices may end up with duplicated IDs */ - for (j = 0; j < i; j++) - if (master->streams[j].id == sid) - break; - if (j < i) - continue; - if (virtcca_smmu_secure_dev_ste_create(smmu, master, sid)) - return -ENOMEM; - } - - dev_info(smmu->dev, "S_SMMU: attach confidential dev: %s", dev_name(dev)); - - return ret; -} - /** * virtcca_smmu_device_init - Initialize the smmu security features * @pdev: The handle of iommu_domain @@ -1204,9 +705,18 @@ void virtcca_smmu_device_init(struct platform_device *pdev, struct arm_smmu_devi if (!virtcca_smmu_enable(smmu) || !arm_s_smmu_idr1_support_secure(smmu)) return; + /* + * S_SMMU implementation doesn't support unique irq lines, + * use a single irq line for all the S_SMMU interrupts. + */ irq = platform_get_irq_byname_optional(pdev, "combined"); - if (irq <= 0) + if (irq <= 0) { platform_get_s_irq_byname_optional(pdev, smmu); + } else { + dev_err(smmu->dev, "S_SMMU: virtcca does not support combined irq\n"); + smmu->s_smmu_id = ARM_S_SMMU_INVALID_ID; + return; + } /* Create cmd queue */ virtcca_smmu_init_one_queue(smmu, &smmu->cmdq.q, CMDQ_ENT_DWORDS, "cmdq"); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h index f5013354e84104874a71bfb61dc9b3aa031e8c86..3d1786db75081f125d45c89c01924bab5a1bdda5 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-s-smmu-v3.h @@ -4,21 +4,12 @@ */ #ifndef _ARM_S_SMMU_V3_H #define _ARM_S_SMMU_V3_H -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #include #include -#define MAX_CC_DEV_NUM_ORDER 8 -#define MASK_DEV_FUNCTION 0xfff8 -#define MASK_DEV_BUS 0xff - -#define DEV_BUS_NUM 0x8 -#define DEV_FUNCTION_NUM 0x3 - -#define STE_ENTRY_SIZE 0x40 - #define SMMU_PCIE_CORE_IS_VALID 0x1 #define ARM_S_SMMU_MAX_IDS (1 << 5) @@ -168,17 +159,11 @@ result ? result : ((cond) ? 0 : -ETIMEDOUT); \ }) -/* Has the root bus device number switched to secure */ -bool is_cc_dev(u32 sid); - void virtcca_smmu_cmdq_write_entries(struct arm_smmu_device *smmu, u64 *cmds, struct arm_smmu_ll_queue *llq, struct arm_smmu_queue *q, int n, bool sync); bool virtcca_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); -u32 virtcca_smmu_tmi_dev_attach(struct arm_smmu_domain *arm_smmu_domain, - struct kvm *kvm); void _arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg); -int virtcca_smmu_secure_dev_operator(struct iommu_domain *domain, struct device *dev); void virtcca_smmu_device_init(struct platform_device *pdev, struct arm_smmu_device *smmu, resource_size_t ioaddr, bool resume, bool disable_bypass); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 61a14c186ecae35a48b2403d14d1af8c28bd88e2..230368d05892af0c47ecae7c2806788b8d9edcfa 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -30,7 +30,7 @@ #include "arm-smmu-v3.h" #include "../../dma-iommu.h" -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include "arm-s-smmu-v3.h" #endif @@ -999,7 +999,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, * Dependency ordering from the cmpxchg() loop above. */ arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_cmdq_write_entries(smmu, cmds, &llq, &cmdq->q, n, sync); #endif if (sync) { @@ -2383,7 +2383,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_set_stage(domain, smmu_domain); #endif @@ -3692,7 +3692,7 @@ static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); } -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA void _arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) { if (virtcca_smmu_write_msi_msg(desc, msg)) @@ -4876,7 +4876,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) if (ret) return ret; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA virtcca_smmu_device_init(pdev, smmu, ioaddr, false, disable_bypass); #endif diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 94bfcfb7b6600f4cc24cb2b7b6170b628887eb10..93bb08bd74046ea089b740874b774716cb0e5d58 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -13,7 +13,7 @@ #include #include #include -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif @@ -757,7 +757,7 @@ struct arm_smmu_device { bool bypass; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA int s_evtq_irq; int s_gerr_irq; resource_size_t ioaddr; @@ -816,7 +816,7 @@ struct arm_smmu_domain { struct list_head mmu_notifiers; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA struct list_head node; struct kvm *kvm; #endif diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index b733b1fb443e2518bad2806aab841916c2937e05..9a3dc42a3309f401d8dcd739a1ee4e155b7062bb 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1913,3 +1913,24 @@ static int iommu_dma_init(void) return iova_cache_get(); } arch_initcall(iommu_dma_init); + +#ifdef CONFIG_HISI_VIRTCCA_CODA +/** + * virtcca_iommu_dma_get_msi_page() - Traverse the MSI page list to obtain the last + * MSI page + * @cookie: Iommu dma cookie + * @iova: Iova address + * @phys: Physical address + * + */ +void virtcca_iommu_dma_get_msi_page(void *cookie, dma_addr_t *iova, phys_addr_t *phys) +{ + struct iommu_dma_msi_page *msi_page = NULL; + struct iommu_dma_cookie *dma_cookie = (struct iommu_dma_cookie *)cookie; + + msi_page = list_last_entry(&dma_cookie->msi_page_list, struct iommu_dma_msi_page, list); + *iova = msi_page->iova; + *phys = msi_page->phys; +} +EXPORT_SYMBOL_GPL(virtcca_iommu_dma_get_msi_page); +#endif diff --git a/drivers/pci/access.c b/drivers/pci/access.c index 6554a2e89d3612dcb8e232802269a34c42d28e83..39eb93d804cae24b8a316ca28600390e037443d9 100644 --- a/drivers/pci/access.c +++ b/drivers/pci/access.c @@ -5,6 +5,15 @@ #include #include +#ifdef CONFIG_HISI_VIRTCCA_CODA +#ifndef __GENKSYMS__ +#include +#include +#include +#include +#endif +#endif + #include "pci.h" /* @@ -86,6 +95,11 @@ int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(PCI_DEVID(bus->number, devfn))) + return virtcca_pci_generic_config_read(addr, bus->number, devfn, size, val); +#endif + if (size == 1) *val = readb(addr); else if (size == 2) @@ -106,6 +120,11 @@ int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, if (!addr) return PCIBIOS_DEVICE_NOT_FOUND; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(PCI_DEVID(bus->number, devfn))) + return virtcca_pci_generic_config_write(addr, bus->number, devfn, size, val); +#endif + if (size == 1) writeb(val, addr); else if (size == 2) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 161c3ac171a3b7a05197a922e2aeb4c1c4bcaa50..785bbf637ab5d997fde9acc68c96e7085f5e4f5e 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -13,6 +13,10 @@ #include "../pci.h" #include "msi.h" +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#endif + int pci_msi_enable = 1; int pci_msi_ignore_mask; @@ -159,6 +163,10 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) return; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && dev != NULL && is_cc_dev(pci_dev_id(dev))) + return virtcca_pci_read_msi_msg(dev, msg, base); +#endif msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); msg->data = readl(base + PCI_MSIX_ENTRY_DATA); @@ -221,6 +229,10 @@ static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg if (unmasked) pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_write_msg_msi(desc, msg)) + return; +#endif writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); writel(msg->data, base + PCI_MSIX_ENTRY_DATA); @@ -639,6 +651,10 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) if (desc->pci.msi_attrib.can_mask) { void __iomem *addr = pci_msix_desc_addr(desc); +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) + return virtcca_msix_prepare_msi_desc(dev, desc, addr); +#endif desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } } @@ -776,6 +792,10 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, * which takes the MSI-X mask bits into account even * when MSI-X is disabled, which prevents MSI delivery. */ +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) + return virtcca_msix_mask_all_cc(dev, dev->msix_base, tsize, pci_dev_id(dev)); +#endif msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h index ee53cf079f4e18233efacc8e748abe77acfd6f75..20119a585690f57bfd6344d946f7c3d8fff03bf8 100644 --- a/drivers/pci/msi/msi.h +++ b/drivers/pci/msi/msi.h @@ -3,6 +3,12 @@ #include #include +#ifdef CONFIG_HISI_VIRTCCA_CODA +#ifndef __GENKSYMS__ +#include +#endif +#endif + #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); @@ -36,6 +42,11 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) { void __iomem *desc_addr = pci_msix_desc_addr(desc); +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_msix_write_vector_ctrl(desc, ctrl)) + return; +#endif + if (desc->pci.msi_attrib.can_mask) writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } @@ -44,6 +55,11 @@ static inline void pci_msix_mask(struct msi_desc *desc) { desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); + +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && virtcca_pci_msix_mask(desc)) + return; +#endif /* Flush write to device */ readl(desc->pci.mask_base); } diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index 610a429c61912568e35dba0431923f393afe718f..dc9e57641c5121552a529c3cb0203a563f24ed54 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -957,3 +957,34 @@ void vfio_group_cleanup(void) vfio.class = NULL; vfio_container_cleanup(); } + +#ifdef CONFIG_HISI_VIRTCCA_CODA +/** + * virtcca_vfio_file_iommu_group - Return the struct iommu_group for the vfio group file + * @file: VFIO group file + * + * The returned iommu_group is valid as long as a ref is held on the file. This + * returns a reference on the group. This function is deprecated, only the SPAPR + * path in kvm should call it. + */ +struct iommu_group *virtcca_vfio_file_iommu_group(struct file *file) +{ + struct vfio_group *group = vfio_group_from_file(file); + struct iommu_group *iommu_group = NULL; + + if (!IS_ENABLED(CONFIG_HISI_VIRTCCA_CODA)) + return NULL; + + if (!group) + return NULL; + + mutex_lock(&group->group_lock); + if (group->iommu_group) { + iommu_group = group->iommu_group; + iommu_group_ref_get(iommu_group); + } + mutex_unlock(&group->group_lock); + return iommu_group; +} +EXPORT_SYMBOL_GPL(virtcca_vfio_file_iommu_group); +#endif diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 0524c90cedea6c70ad8ae31fc1ab506ff062acaa..9c130f83b21bb452925da006d566280691b8d353 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -31,6 +31,9 @@ #if IS_ENABLED(CONFIG_EEH) #include #endif +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#endif #include "vfio_pci_priv.h" @@ -973,6 +976,10 @@ static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, if (vdev->reset_works) info.flags |= VFIO_DEVICE_FLAGS_RESET; +#ifdef CONFIG_HISI_VIRTCCA_CODA + virtcca_cvm_set_secure_flag((void *)vdev, (void *)&info); +#endif + info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; info.num_irqs = VFIO_PCI_NUM_IRQS; diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 620134041b48810b6d21617ba4f2cfa4babf8679..a80a80aa20eda0c1b04f38ea862e2026c0f2c891 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -22,6 +22,10 @@ #include "vfio_pci_priv.h" +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#endif + struct vfio_pci_irq_ctx { struct eventfd_ctx *trigger; struct virqfd *unmask; @@ -712,6 +716,12 @@ static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, if (ret) return ret; +#ifdef CONFIG_HISI_VIRTCCA_CODA + ret = virtcca_msi_map(vdev); + if (ret) + return ret; +#endif + ret = vfio_msi_set_block(vdev, start, count, fds, msix); if (ret) vfio_msi_disable(vdev, msix); diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index e27de61ac9fe75f5818dc8d7386270c592c05a07..e340da8a9edc11d9226954ca2876009f962a7d2f 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -19,6 +19,13 @@ #include "vfio_pci_priv.h" +#ifdef CONFIG_HISI_VIRTCCA_CODA +#ifndef __GENKSYMS__ +#include +#include +#endif +#endif + #ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 #define vfio_iowrite64 iowrite64 @@ -37,6 +44,31 @@ #define vfio_ioread8 ioread8 #define vfio_iowrite8 iowrite8 +#ifdef CONFIG_HISI_VIRTCCA_CODA +#define VFIO_IOWRITE(size) \ +static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (is_virtcca_pci_io_rw(vdev)) { \ + virtcca_pci_io_write(vdev, val, size, io); \ + } else { \ + vfio_iowrite##size(val, io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else #define VFIO_IOWRITE(size) \ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size val, void __iomem *io) \ @@ -56,6 +88,7 @@ static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif VFIO_IOWRITE(8) VFIO_IOWRITE(16) @@ -64,6 +97,31 @@ VFIO_IOWRITE(32) VFIO_IOWRITE(64) #endif +#ifdef CONFIG_HISI_VIRTCCA_CODA +#define VFIO_IOREAD(size) \ +static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size * val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + if (is_virtcca_pci_io_rw(vdev)) { \ + *val = virtcca_pci_io_read(vdev, size, io); \ + } else { \ + *val = vfio_ioread##size(io); \ + } \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} +#else #define VFIO_IOREAD(size) \ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ bool test_mem, u##size *val, void __iomem *io) \ @@ -83,6 +141,7 @@ static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \ \ return 0; \ } +#endif VFIO_IOREAD(8) VFIO_IOREAD(16) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 9c4adf11dbbe357c14ffc926fe496a40b0f4ba22..af4af1bb02c324881eaf9a6a3ba57b147d20095e 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -38,6 +38,12 @@ #include #include #include "vfio.h" +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#include +#include +#include +#endif #define DRIVER_VERSION "0.2" #define DRIVER_AUTHOR "Alex Williamson " @@ -77,6 +83,9 @@ struct vfio_iommu { bool dirty_page_tracking; struct list_head emulated_iommu_groups; bool dirty_log_get_no_clear; +#ifdef CONFIG_HISI_VIRTCCA_CODA + bool secure; /* Whether the vfio iommu is secure or not */ +#endif }; struct vfio_domain { @@ -1037,6 +1046,228 @@ static size_t unmap_unpin_slow(struct vfio_domain *domain, return unmapped; } +#ifdef CONFIG_HISI_VIRTCCA_CODA +static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma); + +bool virtcca_check_is_cvm_or_not(void *iommu, struct kvm **kvm) +{ + struct vfio_domain *domain; + bool is_virtcca_cvm = false; + struct vfio_iommu *vfio_iommu = (struct vfio_iommu *)iommu; + + if (!vfio_iommu || !kvm) + return false; + + list_for_each_entry(domain, &vfio_iommu->domain_list, next) { + if (domain && domain->domain && virtcca_iommu_domain_get_kvm(domain->domain, kvm)) { + is_virtcca_cvm = true; + break; + } + } + + return is_virtcca_cvm; +} + +/* Traverse all domains and perform mapping operations */ +int virtcca_vfio_iommu_map(void *iommu, dma_addr_t iova, + unsigned long pfn, long npage, int prot) +{ + struct vfio_domain *d; + int ret; + struct vfio_iommu *vfio_iommu = (struct vfio_iommu *)iommu; + + list_for_each_entry(d, &vfio_iommu->domain_list, next) { + ret = virtcca_iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, + npage << PAGE_SHIFT, prot | IOMMU_CACHE); + if (ret) + goto unwind; + + cond_resched(); + } + + return 0; + +unwind: + list_for_each_entry_continue_reverse(d, &vfio_iommu->domain_list, next) { + virtcca_iommu_unmap(d->domain, iova, npage << PAGE_SHIFT); + cond_resched(); + } + + return ret; +} + +/** + * virtcca_vfio_pin_map_dma - Vfio need to map iova + * @iommu: The handle of iommu + * @dma: Dma information + * @map_size: Map size + * + * Returns: + * %0 if map success + */ +static int virtcca_vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, + size_t map_size) +{ + struct kvm *kvm; + long npage; + dma_addr_t iova = dma->iova; + unsigned long vaddr = dma->vaddr; + struct vfio_batch batch; + size_t size = map_size; + unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + int ret = 0; + bool is_virtcca_cvm = virtcca_check_is_cvm_or_not((void *)iommu, &kvm); + + vfio_batch_init(&batch); + + while (size) { + /* + * Due to cvm ram full mapping, ram space only needs to be mapped once, + * if the iova is in ram space, there is no need to map it again. + */ + if (is_virtcca_cvm && !is_virtcca_iova_need_vfio_dma(kvm, dma->iova)) + break; + + /* Pin a contiguous chunk of memory */ + npage = vfio_pin_pages_remote(dma, vaddr + dma->size, + size >> PAGE_SHIFT, &pfn, limit, + &batch); + if (npage <= 0) { + WARN_ON(!npage); + ret = (int)npage; + break; + } + + /* Map it! */ + ret = virtcca_vfio_iommu_map(iommu, iova + dma->size, pfn, npage, + dma->prot); + if (ret) { + vfio_unpin_pages_remote(dma, iova + dma->size, pfn, + npage, true); + vfio_batch_unpin(&batch, dma); + break; + } + + if (is_virtcca_cvm && is_in_virtcca_ram_range(kvm, iova)) { + /* + * The cvm ram mapping uses secure memory, + * so non-secure memory needs to be unpinned. + */ + vfio_unpin_pages_remote(dma, iova + dma->size, pfn, + npage, true); + vfio_batch_unpin(&batch, dma); + } + size -= npage << PAGE_SHIFT; + dma->size += npage << PAGE_SHIFT; + } + + vfio_batch_fini(&batch); + dma->iommu_mapped = true; + + if (ret) + vfio_remove_dma(iommu, dma); + + return ret; +} + +/** + * virtcca_vfio_unmap_unpin - Vfio need to unmap iova + * @iommu: The handle of iommu + * @dma: Dma information + * @do_accounting: Need to account or not + * + * Returns: + * %0 if unmap success or the cvm is destroyed + */ +static long virtcca_vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, + bool do_accounting) +{ + struct kvm *kvm; + struct vfio_domain *domain, *d; + LIST_HEAD(unmapped_region_list); + struct iommu_iotlb_gather iotlb_gather; + int unmapped_region_cnt = 0; + long unlocked = 0; + dma_addr_t iova = dma->iova, end = dma->iova + dma->size; + + if (!virtcca_check_is_cvm_or_not((void *)iommu, &kvm)) + return 0; + + if (!dma->size) + return 0; + + if (list_empty(&iommu->domain_list)) + return 0; + + /* + * We use the IOMMU to track the physical addresses, otherwise we'd + * need a much more complicated tracking system. Unfortunately that + * means we need to use one of the iommu domains to figure out the + * pfns to unpin. The rest need to be unmapped in advance so we have + * no iommu translations remaining when the pages are unpinned. + */ + domain = d = list_first_entry(&iommu->domain_list, + struct vfio_domain, next); + + list_for_each_entry_continue(d, &iommu->domain_list, next) { + virtcca_iommu_unmap(d->domain, dma->iova, dma->size); + cond_resched(); + } + + iommu_iotlb_gather_init(&iotlb_gather); + while (iova < end) { + size_t unmapped, len; + phys_addr_t phys, next; + + phys = iommu_iova_to_phys(domain->domain, iova); + if (WARN_ON(!phys)) { + iova += PAGE_SIZE; + continue; + } + + /* + * To optimize for fewer iommu_unmap() calls, each of which + * may require hardware cache flushing, try to find the + * largest contiguous physical memory chunk to unmap. + */ + for (len = PAGE_SIZE; + !domain->fgsp && iova + len < end; len += PAGE_SIZE) { + next = iommu_iova_to_phys(domain->domain, iova + len); + if (next != phys + len) + break; + } + + /* + * First, try to use fast unmap/unpin. In case of failure, + * switch to slow unmap/unpin path. + */ + unmapped = unmap_unpin_fast(domain, dma, &iova, len, phys, + &unlocked, &unmapped_region_list, + &unmapped_region_cnt, + &iotlb_gather); + if (!unmapped) { + unmapped = unmap_unpin_slow(domain, dma, &iova, len, + phys, &unlocked); + if (WARN_ON(!unmapped)) + break; + } + } + + dma->iommu_mapped = false; + + if (unmapped_region_cnt) { + unlocked += vfio_sync_unpin(dma, domain, &unmapped_region_list, + &iotlb_gather); + } + + if (do_accounting) { + vfio_lock_acct(dma, -unlocked, true); + return 0; + } + return unlocked; +} +#endif + static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, bool do_accounting) { @@ -1047,6 +1278,11 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, int unmapped_region_cnt = 0; long unlocked = 0; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) + return virtcca_vfio_unmap_unpin(iommu, dma, do_accounting); +#endif + if (!dma->size) return 0; @@ -1630,6 +1866,11 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) + return virtcca_vfio_pin_map_dma(iommu, dma, map_size); +#endif + vfio_batch_init(&batch); while (size) { @@ -1853,6 +2094,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) + return 0; +#endif + /* Arbitrarily pick the first domain in the list for lookups */ if (!list_empty(&iommu->domain_list)) d = list_first_entry(&iommu->domain_list, @@ -2454,10 +2700,23 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, goto out_domain; } +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) + domain->domain->secure = true; +#endif + ret = iommu_attach_group(domain->domain, group->iommu_group); if (ret) goto out_domain; +#ifdef CONFIG_HISI_VIRTCCA_CODA + if (is_virtcca_cvm_enable() && iommu->secure) { + ret = virtcca_attach_secure_dev(domain->domain, group->iommu_group); + if (ret) + goto out_domain; + } +#endif + /* Get aperture info */ geo = &domain->domain->geometry; if (vfio_iommu_aper_conflict(iommu, geo->aperture_start, @@ -2807,6 +3066,12 @@ static void *vfio_iommu_type1_open(unsigned long arg) case VFIO_TYPE1v2_IOMMU: iommu->v2 = true; break; +#ifdef CONFIG_HISI_VIRTCCA_CODA + case VFIO_TYPE1v2_S_IOMMU: + iommu->v2 = true; + iommu->secure = true; + break; +#endif default: kfree(iommu); return ERR_PTR(-EINVAL); @@ -2898,6 +3163,9 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, switch (arg) { case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: +#ifdef CONFIG_HISI_VIRTCCA_CODA + case VFIO_TYPE1v2_S_IOMMU: +#endif case VFIO_TYPE1_NESTING_IOMMU: case VFIO_UNMAP_ALL: return 1; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8fb93470a1340c36166879466474033e258e1b89..0dad8f8035863093b3f89eff4f56439f28904f76 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -260,7 +260,7 @@ struct iommu_domain { }; struct mutex switch_log_lock; -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA KABI_USE(1, bool secure) #else KABI_RESERVE(1) diff --git a/include/linux/msi.h b/include/linux/msi.h index c21099e1f16d9a2df5aa3f90166387cf311da185..7354ffb148568503273a07969c49c6361b0af61f 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -640,7 +640,7 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, struct irq_domain *parent); int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, irq_write_msi_msg_t write_msi_msg); -#ifdef CONFIG_HISI_VIRTCCA_HOST +#ifdef CONFIG_HISI_VIRTCCA_CODA int platform_msi_domain_alloc_range_irqs(struct device *dev, unsigned int start, unsigned int end, irq_write_msi_msg_t write_msi_msg); #endif diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 2b78d03c0c15c0c1ce42774510b2ce27e4a49711..df0790085d3dfdb7474738843aa8466ab98df611 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -25,6 +25,7 @@ #define VFIO_TYPE1_IOMMU 1 #define VFIO_SPAPR_TCE_IOMMU 2 #define VFIO_TYPE1v2_IOMMU 3 +#define VFIO_TYPE1v2_S_IOMMU 12 /* Virtcca feature: secure iommu */ /* * IOMMU enforces DMA cache coherence (ex. PCIe NoSnoop stripping). This * capability is subject to change as groups are added or removed. @@ -224,6 +225,7 @@ struct vfio_device_info { #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6) /* vfio-fsl-mc device */ #define VFIO_DEVICE_FLAGS_CAPS (1 << 7) /* Info supports caps */ #define VFIO_DEVICE_FLAGS_CDX (1 << 8) /* vfio-cdx device */ +#define VFIO_DEVICE_FLAGS_SECURE (1 << 15) /* vfio-secure device */ __u32 num_regions; /* Max region index + 1 */ __u32 num_irqs; /* Max IRQ index + 1 */ __u32 cap_offset; /* Offset within info struct of first cap */ diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index ca24ce1209068e0f50577318b84c67a59eaf1ee5..e3c603aeb66c9808fd39d60fd1afd3328ba18333 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -21,6 +21,11 @@ #include #endif +#ifdef CONFIG_HISI_VIRTCCA_CODA +#include +#include +#endif + struct kvm_vfio_file { struct list_head node; struct file *file; @@ -178,6 +183,9 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvm_arch_start_assignment(dev->kvm); kvm_vfio_file_set_kvm(kvf->file, dev->kvm); kvm_vfio_update_coherency(dev); +#ifdef CONFIG_HISI_VIRTCCA_CODA + ret = cvm_vfio_add_kvm_to_smmu_domain(filp, (void *)kv); +#endif out_unlock: mutex_unlock(&kv->lock); @@ -392,3 +400,272 @@ void kvm_vfio_ops_exit(void) { kvm_unregister_device_ops(KVM_DEV_TYPE_VFIO); } + +#ifdef CONFIG_HISI_VIRTCCA_CODA +/** + * cvm_vfio_add_kvm_to_smmu_domain - Bind the confidential + * virtual machine to smmu domain + * @filp: The handle of file + * @kvm: The kvm belone to confidential virtual machine + * + * Returns: + * %-ENXIO if set kvm failed or iommu group is null + * %0 if set kvm success + */ +int cvm_vfio_add_kvm_to_smmu_domain(struct file *filp, void *kv) +{ + struct iommu_group *iommu_group; + int ret = 0; + struct kvm_vfio *_kv = (struct kvm_vfio *)kv; + + if (!is_virtcca_cvm_enable()) + return ret; + + /* Upper-level calling interface has added a kv lock, but the + * virtcca_cvm_arm_smmu_domain_set_kvm interface also need add this lock. + * Therefore, it is necessary to unlock here and completing the + * acquisition, then add the kv lock before return. + */ + mutex_unlock(&_kv->lock); + iommu_group = cvm_vfio_file_iommu_group(filp); + if (!iommu_group) { + ret = -ENXIO; + goto out_lock; + } + if (virtcca_cvm_arm_smmu_domain_set_kvm((void *)iommu_group) != 1) { + ret = -ENXIO; + goto out_lock; + } + +out_lock: + mutex_lock(&_kv->lock); + return ret; +} + +/** + * virtcca_arm_smmu_get_kvm - Find the kvm + * with vfio devices through SMMU domain + * @domain: Smmu domain + * + * Returns: + * %kvm if find the kvm with vfio devices + * %NULL if kvm is null + */ +struct kvm *virtcca_arm_smmu_get_kvm(struct arm_smmu_domain *domain) +{ + int ret = -1; + struct kvm *kvm; + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + struct iommu_group *iommu_group; + unsigned long flags; + struct arm_smmu_master *master; + + spin_lock_irqsave(&domain->devices_lock, flags); + /* Get smmu master from smmu domain */ + list_for_each_entry(master, &domain->devices, domain_head) { + if (master && master->num_streams >= 0) { + ret = 0; + break; + } + } + spin_unlock_irqrestore(&domain->devices_lock, flags); + if (ret) + return NULL; + + ret = -1; + iommu_group = master->dev->iommu_group; + mutex_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) { + mutex_lock(&kvm->lock); + /* Get kvm vfio device list from kvm */ + list_for_each_entry(dev, &kvm->devices, vm_node) { + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") != 0) + continue; + + /* Find the kvm vfio which device name is kvm-vfio */ + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + /* Get iommu_group from vfio file */ + if (cvm_vfio_file_iommu_group(kvf->file) == iommu_group) { + ret = 0; + break; + } + } + mutex_unlock(&kv->lock); + if (!ret) + break; + } + mutex_unlock(&kvm->lock); + if (!ret) + break; + } + mutex_unlock(&kvm_lock); + + if (ret) + return NULL; + return kvm; +} +EXPORT_SYMBOL_GPL(virtcca_arm_smmu_get_kvm); + +/** + * find_arm_smmu_domain - Find smmu domain list from kvm vfio file + * @kvf: Kvm vfio file + * @smmu_domain_group_list: List of smmu domain group + */ +int find_arm_smmu_domain(struct device *dev, void *data) +{ + int ret = 0; + struct iommu_domain *domain = NULL; + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct arm_smmu_domain *arm_smmu_domain_node = NULL; + struct list_head *smmu_domain_group_list = (struct list_head *)data; + + domain = iommu_get_domain_for_dev(dev); + arm_smmu_domain = to_smmu_domain(domain); + list_for_each_entry(arm_smmu_domain_node, + smmu_domain_group_list, node) { + if (arm_smmu_domain_node == arm_smmu_domain) { + ret = -1; + break; + } + } + if (!ret) + list_add_tail(&arm_smmu_domain->node, smmu_domain_group_list); + + return 1; +} + +/** + * kvm_get_arm_smmu_domain - Find kvm vfio file from kvm + * @kvm: Kvm handle + * @smmu_domain_group_list: List of smmu domain group + */ +void kvm_get_arm_smmu_domain(struct kvm *kvm, struct list_head *smmu_domain_group_list) +{ + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + struct iommu_group *iommu_group = NULL; + + INIT_LIST_HEAD(smmu_domain_group_list); + + list_for_each_entry(dev, &kvm->devices, vm_node) { + /* The device name passed through the vfio driver is called kvm-vfio */ + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") != 0) + continue; + + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + iommu_group = cvm_vfio_file_iommu_group(kvf->file); + iommu_group_for_each_dev(iommu_group, + (void *)smmu_domain_group_list, find_arm_smmu_domain); + } + mutex_unlock(&kv->lock); + } +} + +/** + * cvm_smmu_domain_judge - Find the iommu group corresponding to the smmu domain + * @dev: The handle of device + * @data: Smmu domain + * + * Returns: + * %-ENXIO if domain is null + * %SMMU_DOMAIN_IS_SAME if find the iommu group corresponding to the smmu domain + * %1 if does not find the iommu group + */ +int cvm_smmu_domain_judge(struct device *dev, void *data) +{ + struct iommu_domain *domain = NULL; + struct arm_smmu_domain *arm_smmu_domain = NULL; + struct arm_smmu_domain *smmu_domain = (struct arm_smmu_domain *)data; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -ENXIO; + + arm_smmu_domain = to_smmu_domain(domain); + if (arm_smmu_domain == smmu_domain) + return SMMU_DOMAIN_IS_SAME; + + return 1; +} + +/* Judge the iommu group correspond to the smmu domain */ +int virtcca_cvm_smmu_domain_judge(struct iommu_group *group, struct arm_smmu_domain *smmu_domain) +{ + int ret; + + ret = iommu_group_for_each_dev((struct iommu_group *)group, + (void *)smmu_domain, cvm_smmu_domain_judge); + return ret; +} + +/** + * virtcca_iommu_group_map_msi_address - Find iommu group from kvm vfio file, map it + * @kvm: The handle of kvm + * @kvf: Kvm vfio file + * @smmu_domain: Smmu domain + * @pa: Physical address + * @map_size: Mapped size + */ +int virtcca_iommu_group_map_msi_address(struct kvm *kvm, struct kvm_vfio_file *kvf, + struct arm_smmu_domain *smmu_domain, phys_addr_t pa, unsigned long map_size) +{ + unsigned long iova; + int ret = 0; + struct iommu_group *iommu_group = NULL; + + iommu_group = cvm_vfio_file_iommu_group(kvf->file); + if (iommu_group) { + if (virtcca_cvm_smmu_domain_judge(iommu_group, smmu_domain) == + SMMU_DOMAIN_IS_SAME) { + iova = virtcca_get_iommu_device_msi_addr(iommu_group); + if (!iova) + return -ENXIO; + + ret = virtcca_iommu_group_set_dev_msi_addr(iommu_group, &iova); + if (ret) + return ret; + + ret = cvm_map_unmap_ipa_range(kvm, iova, pa, map_size, true); + if (ret) + return ret; + } + } + return ret; +} + +/* Get iommu group from specific smmu domain, map it */ +int virtcca_map_msi_address(struct kvm *kvm, struct arm_smmu_domain *smmu_domain, + phys_addr_t pa, unsigned long map_size) +{ + int ret = 0; + struct kvm_device *dev; + struct kvm_vfio *kv; + struct kvm_vfio_file *kvf; + + mutex_lock(&kvm->lock); + list_for_each_entry(dev, &kvm->devices, vm_node) { + /* Get kvm vfio device list */ + if (dev->ops && strcmp(dev->ops->name, "kvm-vfio") == 0) { + kv = (struct kvm_vfio *)dev->private; + mutex_lock(&kv->lock); + list_for_each_entry(kvf, &kv->file_list, node) { + /* Get iommu_group from vfio file, map it */ + ret = virtcca_iommu_group_map_msi_address(kvm, kvf, smmu_domain, + pa, map_size); + } + mutex_unlock(&kv->lock); + if (ret) + break; + } + } + mutex_unlock(&kvm->lock); + return ret; +} +#endif