diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 3d0d3e5305b375e674e5936242b0c443728b2ed6..1e06d0795bcb805d16ff0918970ba43a8df1772d 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5408,3 +5408,7 @@ A hex value specifying bitmask with supplemental xhci host controller quirks. Meaning of each bit can be consulted in header drivers/usb/host/xhci.h. + + zhaoxin_patch_bitmask= + [X86] Bitmask for Zhaoxin Platform's patch. + bit 0: enable KH-40000 dma patch's node check function diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h index ce4d176b3d139d9636f5483103ad1ac9b20d9be9..a2cf2eaa261540710c938c03a5108bc546a8d560 100644 --- a/arch/x86/include/asm/dma-mapping.h +++ b/arch/x86/include/asm/dma-mapping.h @@ -30,6 +30,27 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) return dma_ops; } +extern bool is_zhaoxin_kh40000(void); + +#if IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) +phys_addr_t kh40000_iommu_iova_to_phys(struct device *dev, dma_addr_t paddr); +void kh40000_sync_single_dma_for_cpu(struct device *dev, dma_addr_t paddr, + enum dma_data_direction dir, bool is_iommu); +void *kh40000_dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs); +void kh40000_set_direct_dma_ops(void); +void kh40000_set_swiotlb_dma_ops(void); +#else +static inline void kh40000_set_direct_dma_ops(void) +{ + +} +static inline void kh40000_set_swiotlb_dma_ops(void) +{ + +} +#endif /* CONFIG_INTEL_IOMMU && CONFIG_X86_64 */ + bool arch_dma_alloc_attrs(struct device **dev); #define arch_dma_alloc_attrs arch_dma_alloc_attrs diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c index 50d5848bf22efb5ffed1292bd8aa354689d2f7dd..f508b078db2e2759655b27bdd4b9de725baaa038 100644 --- a/arch/x86/kernel/early-quirks.c +++ b/arch/x86/kernel/early-quirks.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -666,6 +667,22 @@ static void __init apple_airport_reset(int bus, int slot, int func) early_iounmap(mmio, BCM4331_MMIO_SIZE); } +bool __ro_after_init zhaoxin_kh40000; + +bool is_zhaoxin_kh40000(void) +{ + return zhaoxin_kh40000; +} + +static void __init zhaoxin_platform_check(int num, int slot, int func) +{ + if (read_pci_config_byte(num, slot, func, PCI_REVISION_ID) == 0x10) { + zhaoxin_kh40000 = true; + kh40000_set_direct_dma_ops(); + pr_info("KH-40000 platform detected!\n"); + } +} + #define QFLAG_APPLY_ONCE 0x1 #define QFLAG_APPLIED 0x2 #define QFLAG_DONE (QFLAG_APPLY_ONCE|QFLAG_APPLIED) @@ -683,6 +700,10 @@ static struct chipset early_qrk[] __initdata = { PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, nvidia_bugs }, { PCI_VENDOR_ID_VIA, PCI_ANY_ID, PCI_CLASS_BRIDGE_PCI, PCI_ANY_ID, QFLAG_APPLY_ONCE, via_bugs }, + { PCI_VENDOR_ID_ZHAOXIN, 0x1001, PCI_CLASS_BRIDGE_ISA, + PCI_ANY_ID, QFLAG_APPLY_ONCE, zhaoxin_platform_check }, + { PCI_VENDOR_ID_ZHAOXIN, 0x345B, PCI_CLASS_BRIDGE_ISA, + PCI_ANY_ID, QFLAG_APPLY_ONCE, zhaoxin_platform_check }, { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_K8_NB, PCI_CLASS_BRIDGE_HOST, PCI_ANY_ID, 0, fix_hypertransport_config }, { PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_IXP400_SMBUS, diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 7ba73fe0d917de810abf81b3f344f2b68fa2c491..46794612f225a21a27340c54d232d7ac56577736 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -175,6 +175,87 @@ static int __init pci_iommu_init(void) /* Must execute after PCI subsystem */ rootfs_initcall(pci_iommu_init); +#if IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) +/*** + * usage: + * set "zhaoxin_patch_bitmask=" in cmdline + * value description: + * bit 0: enable(1) node check or not(0). default 1 + */ +enum { + ZHAOXIN_P2CW_NODE_CHECK = BIT(0), + ZHAOXIN_PATCH_CODE_MAX = ZHAOXIN_P2CW_NODE_CHECK, +}; + +#define ZHAOXIN_PATCH_CODE_DEFAULT ZHAOXIN_P2CW_NODE_CHECK + +unsigned long zhaoxin_patch_code = ZHAOXIN_PATCH_CODE_DEFAULT; + +static int __init zhaoxin_patch_code_setup(char *str) +{ + int err = kstrtoul(str, 0, &zhaoxin_patch_code); + + if (err || (zhaoxin_patch_code > ZHAOXIN_PATCH_CODE_MAX)) { + pr_err("cmdline 'zhaoxin_patch_bitmask=%s' inappropriate\n", str); + zhaoxin_patch_code = ZHAOXIN_PATCH_CODE_DEFAULT; + return err; + } + + if (ZHAOXIN_P2CW_NODE_CHECK & zhaoxin_patch_code) + pr_info("zhaoxin dma patch node check is enabled\n"); + + return 0; +} +__setup("zhaoxin_patch_bitmask=", zhaoxin_patch_code_setup); + +static struct pci_dev *kh40000_get_pci_dev(struct device *dev) +{ + if (dev_is_pci(dev)) + return to_pci_dev(dev); + + if (dev->parent) + return kh40000_get_pci_dev(dev->parent); + + return NULL; +} + +void kh40000_sync_single_dma_for_cpu(struct device *dev, dma_addr_t paddr, + enum dma_data_direction dir, bool is_iommu) +{ + u8 vid; + struct pci_dev *pci; + u64 dma_mask = *dev->dma_mask; + + /* check direction */ + if ((dir != DMA_FROM_DEVICE) && (dir != DMA_BIDIRECTIONAL)) + return; + + /* check dma capability */ + if (dma_mask <= DMA_BIT_MASK(32)) + return; + + /* check device type */ + pci = kh40000_get_pci_dev(dev); + if (pci == NULL) + return; + + /* get real physical address */ + if (is_iommu) + paddr = kh40000_iommu_iova_to_phys(dev, paddr); + + /* check node or not */ + if ((zhaoxin_patch_code & ZHAOXIN_P2CW_NODE_CHECK)) { + unsigned long pfn = PFN_DOWN(paddr); + + if (pfn_to_nid(pfn) == dev_to_node(dev)) + return; + } + + pci_read_config_byte(pci, PCI_VENDOR_ID, &vid); +} + +#endif /* CONFIG_INTEL_IOMMU && IS_BUILTIN(CONFIG_X86_64) */ + #ifdef CONFIG_PCI /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index 71c0b01d93b1b3d1befa524c31ecc88ae65f0b82..a2eaa2afa1d8014a7d3efe5f6c8a06967dbb2972 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -65,6 +65,9 @@ void __init pci_swiotlb_init(void) if (swiotlb) { swiotlb_init(0); dma_ops = &swiotlb_dma_ops; + + if (is_zhaoxin_kh40000()) + kh40000_set_swiotlb_dma_ops(); } } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 3a08b0998cb4c9cf86653d84baa715769ad0f220..aa826684b08180540a055c0ddab45ffe870ed154 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4757,6 +4757,7 @@ const struct attribute_group *intel_iommu_groups[] = { NULL, }; +const struct dma_map_ops kh40000_iommu_dma_ops; int __init intel_iommu_init(void) { int ret = -ENODEV; @@ -4850,6 +4851,9 @@ int __init intel_iommu_init(void) #endif dma_ops = &intel_dma_ops; + if (is_zhaoxin_kh40000()) + dma_ops = &kh40000_iommu_dma_ops; + init_iommu_pm_ops(); for_each_active_iommu(iommu, drhd) { @@ -5499,3 +5503,111 @@ static void __init check_tylersburg_isoch(void) pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", vtisochctrl); } +#if IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) +static void *kh40000_iommu_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t flags, unsigned long attrs) +{ + struct page *page = NULL; + int order; + nodemask_t nodemask; + int node = dev_to_node(dev); + + nodes_clear(nodemask); + size = PAGE_ALIGN(size); + order = get_order(size); + + if (!iommu_no_mapping(dev)) + flags &= ~(GFP_DMA | GFP_DMA32); + else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) { + if (dev->coherent_dma_mask < DMA_BIT_MASK(32)) + flags |= GFP_DMA; + else + flags |= GFP_DMA32; + } + + if (node == NUMA_NO_NODE) { + page = __alloc_pages_nodemask(flags, order, numa_mem_id(), NULL); + } else { + if (!(flags & (GFP_DMA | GFP_DMA32))) { + node_set(node, nodemask); + page = __alloc_pages_nodemask(flags | __GFP_HIGH, order, node, &nodemask); + } else { + page = __alloc_pages_nodemask(flags | __GFP_HIGH, order, node, NULL); + } + } + + if (!page) + return NULL; + memset(page_address(page), 0, size); + + *dma_handle = __intel_map_single(dev, page_to_phys(page), size, DMA_BIDIRECTIONAL, + dev->coherent_dma_mask); + if (*dma_handle) + return page_address(page); + + return NULL; +} + +phys_addr_t kh40000_iommu_iova_to_phys(struct device *dev, dma_addr_t paddr) +{ + /* only patch remote DMA access */ + if (!iommu_no_mapping(dev)) { + struct dmar_domain *domain = find_domain(dev); + + paddr = intel_iommu_iova_to_phys(&(domain->domain), paddr); + } + return paddr; +} + +static void kh40000_iommu_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 1); +} + +static void kh40000_iommu_unmap_page(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 1); + intel_unmap_page(dev, addr, size, dir, attrs); +} + +static void kh40000_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, + enum dma_data_direction dir, unsigned long attrs) +{ + dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK; + unsigned long nrpages = 0; + struct scatterlist *sg; + int i; + + for_each_sg(sglist, sg, nelems, i) { + nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg)); + kh40000_iommu_sync_single_for_cpu(dev, sg->dma_address, sg_dma_len(sg), dir); + } + + intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT); +} + +static void kh40000_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) + kh40000_iommu_sync_single_for_cpu(dev, sg->dma_address, sg_dma_len(sg), dir); +} + +const struct dma_map_ops kh40000_iommu_dma_ops = { + .alloc = kh40000_iommu_alloc_coherent, + .free = intel_free_coherent, + .map_sg = intel_map_sg, + .unmap_sg = kh40000_iommu_unmap_sg, + .map_page = intel_map_page, + .unmap_page = kh40000_iommu_unmap_page, + .sync_single_for_cpu = kh40000_iommu_sync_single_for_cpu, + .sync_sg_for_cpu = kh40000_iommu_sync_sg_for_cpu, + .mapping_error = intel_mapping_error, + .dma_supported = dma_direct_supported, +}; +#endif /* IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) */ diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index a608b3713384c04f837bd9c7447938476670eaa0..601cd95dd6f8de9aa757668eb2538ed169c11643 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -223,3 +223,113 @@ const struct dma_map_ops dma_direct_ops = { .mapping_error = dma_direct_mapping_error, }; EXPORT_SYMBOL(dma_direct_ops); + +#if IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) +void *kh40000_dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, + unsigned long attrs) +{ + int page_order = get_order(size); + struct page *page = NULL; + void *ret; + int node = dev_to_node(dev); + nodemask_t nodemask; + + nodes_clear(nodemask); + + gfp &= ~__GFP_ZERO; + + /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ + if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) + gfp |= GFP_DMA; + if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) + gfp |= GFP_DMA32; + +again: + if (node == NUMA_NO_NODE) { + page = __alloc_pages_nodemask(gfp, page_order, numa_mem_id(), NULL); + } else { + if (!(gfp & (GFP_DMA | GFP_DMA32))) { + node_set(node, nodemask); + page = __alloc_pages_nodemask(gfp | __GFP_HIGH, page_order, node, + &nodemask); + } else { + page = __alloc_pages_nodemask(gfp | __GFP_HIGH, page_order, node, NULL); + } + } + + if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { + __free_pages(page, page_order); + page = NULL; + + if (IS_ENABLED(CONFIG_ZONE_DMA32) && dev->coherent_dma_mask < DMA_BIT_MASK(64) && + !(gfp & (GFP_DMA32 | GFP_DMA))) { + gfp |= GFP_DMA32; + goto again; + } + + if (IS_ENABLED(CONFIG_ZONE_DMA) && dev->coherent_dma_mask < DMA_BIT_MASK(32) && + !(gfp & GFP_DMA)) { + gfp = (gfp & ~GFP_DMA32) | GFP_DMA; + goto again; + } + } + + if (!page) + return NULL; + ret = page_address(page); + if (force_dma_unencrypted()) { + set_memory_decrypted((unsigned long)ret, 1 << page_order); + *dma_handle = __phys_to_dma(dev, page_to_phys(page)); + } else { + *dma_handle = phys_to_dma(dev, page_to_phys(page)); + } + memset(ret, 0, size); + return ret; +} + +static void kh40000_direct_unmap_page(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 0); +} + +static void kh40000_direct_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 0); +} + +static void kh40000_direct_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) + kh40000_sync_single_dma_for_cpu(dev, sg_dma_address(sg), dir, 0); +} + +static void kh40000_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, int nelems, + enum dma_data_direction dir, unsigned long attrs) +{ + kh40000_direct_sync_sg_for_cpu(dev, sgl, nelems, dir); +} + +const struct dma_map_ops kh40000_direct_dma_ops = { + .alloc = kh40000_dma_direct_alloc, + .free = dma_direct_free, + .map_page = dma_direct_map_page, + .map_sg = dma_direct_map_sg, + .unmap_sg = kh40000_direct_unmap_sg, + .unmap_page = kh40000_direct_unmap_page, + .sync_sg_for_cpu = kh40000_direct_sync_sg_for_cpu, + .sync_single_for_cpu = kh40000_direct_sync_single_for_cpu, + .dma_supported = dma_direct_supported, + .mapping_error = dma_direct_mapping_error, +}; + +void kh40000_set_direct_dma_ops(void) +{ + dma_ops = &kh40000_direct_dma_ops; +} +#endif /* IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) */ diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index ffefd8a58345e2121efe5e8a3b56e25341219a23..570f8bdce2909394fc6374112d57b3cf2ac5b28a 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1085,3 +1085,120 @@ const struct dma_map_ops swiotlb_dma_ops = { .dma_supported = dma_direct_supported, }; EXPORT_SYMBOL(swiotlb_dma_ops); + +#if IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) +static void *kh40000_swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, + unsigned long attrs) +{ + phys_addr_t phys_addr; + u64 dma_mask = DMA_BIT_MASK(32); + + if (dev && dev->coherent_dma_mask) + dma_mask = dev->coherent_dma_mask; + + if (swiotlb_force == SWIOTLB_NO_FORCE) + goto out_warn; + + if (dma_mask > DMA_BIT_MASK(32)) + return NULL; + + phys_addr = swiotlb_tbl_map_single(dev, __phys_to_dma(dev, io_tlb_start), 0, size, + DMA_FROM_DEVICE, attrs); + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; + + *dma_handle = __phys_to_dma(dev, phys_addr); + if (!dma_coherent_ok(dev, *dma_handle, size)) + goto out_unmap; + + memset(phys_to_virt(phys_addr), 0, size); + return phys_to_virt(phys_addr); + +out_unmap: + dev_warn(dev, "hwdev DMA mask = 0x%016llx, dev_addr = 0x%016llx\n", + (unsigned long long)dev->coherent_dma_mask, (unsigned long long)*dma_handle); + + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); +out_warn: + if (!(attrs & DMA_ATTR_NO_WARN)) { + dev_warn_ratelimited(dev, "swiotlb: coherent allocation failed, size=%zu\n", size); + dump_stack(); + } + return NULL; +} + +static void *kh40000_swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs) +{ + void *vaddr; + + if (gfp & __GFP_NOWARN) + attrs |= DMA_ATTR_NO_WARN; + + gfp |= __GFP_NOWARN; + + vaddr = kh40000_dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!vaddr) + vaddr = kh40000_swiotlb_alloc_buffer(dev, size, dma_handle, attrs); + + return vaddr; +} + +static void kh40000_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 0); + swiotlb_sync_single_for_cpu(dev, addr, size, dir); +} + +static void kh40000_swiotlb_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, int nelems, + enum dma_data_direction dir) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) + kh40000_sync_single_dma_for_cpu(dev, sg_dma_address(sg), dir, 0); + + swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir); +} + +static void kh40000_swiotlb_unmap_sg(struct device *dev, struct scatterlist *sgl, int nelems, + enum dma_data_direction dir, unsigned long attrs) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) + kh40000_sync_single_dma_for_cpu(dev, sg_dma_address(sg), dir, 0); + + swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs); +} + +static void kh40000_swiotlb_unmap_page(struct device *dev, dma_addr_t addr, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + kh40000_sync_single_dma_for_cpu(dev, addr, dir, 0); + swiotlb_unmap_page(dev, addr, size, dir, attrs); +} + +const struct dma_map_ops kh40000_swiotlb_dma_ops = { + .mapping_error = swiotlb_dma_mapping_error, + .alloc = kh40000_swiotlb_alloc, + .free = swiotlb_free, + .sync_single_for_cpu = kh40000_swiotlb_sync_single_for_cpu, + .sync_single_for_device = swiotlb_sync_single_for_device, + .sync_sg_for_cpu = kh40000_swiotlb_sync_sg_for_cpu, + .sync_sg_for_device = swiotlb_sync_sg_for_device, + .map_sg = swiotlb_map_sg_attrs, + .unmap_sg = kh40000_swiotlb_unmap_sg, + .map_page = swiotlb_map_page, + .unmap_page = kh40000_swiotlb_unmap_page, + .dma_supported = dma_direct_supported, +}; + +void kh40000_set_swiotlb_dma_ops(void) +{ + dma_ops = &kh40000_swiotlb_dma_ops; +} +#endif /* IS_BUILTIN(CONFIG_INTEL_IOMMU) && IS_BUILTIN(CONFIG_X86_64) */