From e59d4d8b5ba1d21f7a3e1ed33c7cc92bf64e4d4b Mon Sep 17 00:00:00 2001 From: He Sheng Date: Mon, 15 May 2023 10:38:52 +0800 Subject: [PATCH 001/150] sw64: clean up useless cpu_idle wrap Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- In commit 77ba50381cbc ("sw64: split out the idle loop in idle.c"), cpu_idle() was added and wrapped into arch_cpu_idle(). That is unnecessary, so remove it. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/kernel/idle.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/sw_64/kernel/idle.c b/arch/sw_64/kernel/idle.c index 8193a7093b57..eb4738dedac5 100644 --- a/arch/sw_64/kernel/idle.c +++ b/arch/sw_64/kernel/idle.c @@ -9,7 +9,7 @@ #include #include -void cpu_idle(void) +void arch_cpu_idle(void) { local_irq_enable(); cpu_relax(); @@ -32,8 +32,3 @@ void cpu_idle(void) : "$1"); } } - -void arch_cpu_idle(void) -{ - cpu_idle(); -} -- Gitee From 0be196cbacba1358aade46f046466c8b7eda591a Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Sat, 6 May 2023 09:24:09 +0800 Subject: [PATCH 002/150] sw64: send a fake signal to all blocking tasks Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- This is porting of commit 43347d56c8d9 ("livepatch: send a fake signal to all blocking tasks"). Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/thread_info.h | 4 +++- arch/sw_64/kernel/entry.S | 2 +- arch/sw_64/kernel/signal.c | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index a8341d64ad43..c111dc88bb22 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -92,6 +92,7 @@ static __always_inline u64 rtid(void) #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) +#define _TIF_PATCH_PENDING (1 << TIF_PATCH_PENDING) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_SECCOMP (1 << TIF_SECCOMP) @@ -100,7 +101,8 @@ static __always_inline u64 rtid(void) /* Work to do on interrupt/exception return. */ #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_RESUME | _TIF_UPROBE) + _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ + _TIF_PATCH_PENDING) #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index 013656e78d08..77eab978b90d 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -292,7 +292,7 @@ $ret_success: .align 4 .ent work_pending work_pending: - and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_UPROBE, $2 + and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_UPROBE | _TIF_PATCH_PENDING, $2 bne $2, $work_notifysig $work_resched: diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 6414654a0f59..e1b41587a541 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -423,6 +424,9 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags, if (thread_flags & _TIF_UPROBE) uprobe_notify_resume(regs); + if (thread_flags & _TIF_PATCH_PENDING) + klp_update_patch_state(current); + if (thread_flags & _TIF_SIGPENDING) { do_signal(regs, r0, r19); r0 = 0; -- Gitee From 77467acaade5796a23f4a18ffc745fe9b796fa72 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 11 May 2023 09:54:03 +0800 Subject: [PATCH 003/150] sw64: fix KBUILD_DEFCONFIG Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- There is no "defconfig" in configs. Make it openeuler_defconfig. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 7d86e80362f6..3175b0fa2a19 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -31,7 +31,7 @@ cflags-y += $(call cc-option, -fno-jump-tables) cflags-y += $(cpuflags-y) KBUILD_CFLAGS += $(cflags-y) -KBUILD_DEFCONFIG = defconfig +KBUILD_DEFCONFIG = openeuler_defconfig head-y := arch/sw_64/kernel/head.o -- Gitee From e30634a64c38d7efc00264f0e4f4c434d414175a Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Mon, 15 May 2023 14:07:39 +0800 Subject: [PATCH 004/150] sw64: selftests/vm: fix virtual address range mapping test Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CACS -------------------------------- The address space on SW64 is 4PB and no high mappings are supported, so fix the test to make it pass on sw64 as well. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/testing/selftests/vm/virtual_address_range.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/testing/selftests/vm/virtual_address_range.c b/tools/testing/selftests/vm/virtual_address_range.c index c0592646ed93..d9eeaaee58eb 100644 --- a/tools/testing/selftests/vm/virtual_address_range.c +++ b/tools/testing/selftests/vm/virtual_address_range.c @@ -48,6 +48,11 @@ #define HIGH_ADDR_SHIFT 49 #define NR_CHUNKS_LOW NR_CHUNKS_256TB #define NR_CHUNKS_HIGH 0 +#elif defined __sw_64__ +#define HIGH_ADDR_MARK ADDR_MARK_128TB * 32UL +#define HIGH_ADDR_SHIFT 53 +#define NR_CHUNKS_LOW NR_CHUNKS_128TB * 32UL +#define NR_CHUNKS_HIGH 0 #else #define HIGH_ADDR_MARK ADDR_MARK_128TB #define HIGH_ADDR_SHIFT 48 -- Gitee From 36040f558d099c9500e7500c378df6a3b0cad8ff Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Wed, 17 May 2023 09:26:04 +0800 Subject: [PATCH 005/150] sw64: kvm: fix an error with non-8MB aligned guest RAM size Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CADA -------------------------------- For non-8MB aligned guest RAM size, it will cause "vm_insert_page failed: -14" because of the inconsistent size of user vma to map and pages number. To fix it, this patch: - Use the memory size passed by hypervisor instead of the rounded up one during physical page remmaping to ensure correctness. - Alloc and free physical memory for guest with the rounded up size to ensure the values stored in CSR:VPCR are not lost due to rounding down. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm-sw64.c | 2 +- arch/sw_64/kvm/vmem.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/kvm-sw64.c index 122fb02957ce..ec1294a24377 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/kvm-sw64.c @@ -481,7 +481,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, #endif info->start = addr; - info->size = size; + info->size = mem->memory_size; vma->vm_private_data = (void *) info; vma->vm_ops = &vmem_vm_ops; diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index c6f9d6cdf03b..efac24ce5ede 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -72,7 +72,7 @@ static void vmem_vm_close(struct vm_area_struct *vma) info = vma->vm_private_data; addr = info->start; - size = info->size; + size = round_up(info->size, 8 << 20); if (atomic_dec_and_test(&info->refcnt)) { if (sw64_kvm_pool && addr_in_pool(sw64_kvm_pool, addr, size)) { @@ -123,7 +123,7 @@ static int vmem_mmap(struct file *flip, struct vm_area_struct *vma) return -ENOMEM; if (flip->private_data == NULL) { - addr = gen_pool_alloc(sw64_kvm_pool, size); + addr = gen_pool_alloc(sw64_kvm_pool, round_up(size, 8 << 20)); if (!addr) return -ENOMEM; -- Gitee From b1a5ab3150672b4ce817e8cd92c71d214c82da9f Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Fri, 26 May 2023 12:29:45 +0800 Subject: [PATCH 006/150] sw64: fix shared_cpu_map of tcache's cacheinfo Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAEK -------------------------------- The shared_cpu_map of tcache's cacheinfo was set as cpu_online_mask, but it should be set as topology_llc_cpumask(cpu). Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cacheinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/cacheinfo.c b/arch/sw_64/kernel/cacheinfo.c index 87d3f4bcd10f..e340c53690a9 100644 --- a/arch/sw_64/kernel/cacheinfo.c +++ b/arch/sw_64/kernel/cacheinfo.c @@ -89,7 +89,7 @@ int populate_cache_leaves(unsigned int cpu) } if (c->tcache.size) { - cpumask_copy(&this_leaf->shared_cpu_map, cpu_online_mask); + cpumask_copy(&this_leaf->shared_cpu_map, topology_llc_cpumask(cpu)); populate_cache(tcache, this_leaf, 3, CACHE_TYPE_UNIFIED, topo->package_id); } -- Gitee From 2b291329115b90c565779d969e46fe4849675367 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Fri, 2 Jun 2023 14:05:25 +0800 Subject: [PATCH 007/150] sw64: fix a compile error for headers_install Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Add missing "WITH Linux-syscall-note" for SPDX-License-Identifier to make headers_install successfully. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/bpf_perf_event.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/uapi/asm/bpf_perf_event.h b/arch/sw_64/include/uapi/asm/bpf_perf_event.h index b551b741653d..5e1e648aeec4 100644 --- a/arch/sw_64/include/uapi/asm/bpf_perf_event.h +++ b/arch/sw_64/include/uapi/asm/bpf_perf_event.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ #define _UAPI__ASM_BPF_PERF_EVENT_H__ -- Gitee From 4257f9fcd196d0f79ab3ad91cb776acd4772f276 Mon Sep 17 00:00:00 2001 From: He Chuyue Date: Fri, 2 Jun 2023 16:06:46 +0800 Subject: [PATCH 008/150] sw64: perf: add libbfd-buildid and disassembler-four-args support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAF9 -------------------------------- This patch specifies the linked dynamic library explicitly to provide libbfd-buildid and disassembler-four-args support for sw64 arch. Signed-off-by: He Chuyue Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/build/feature/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 401f6cf22a66..6a8a35b8c7ea 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -240,10 +240,18 @@ else endif $(OUTPUT)test-libbfd-buildid.bin: +ifeq ($(ARCH),sw_64) + $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl -liberty -lz +else $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl +endif $(OUTPUT)test-disassembler-four-args.bin: +ifeq ($(ARCH),sw_64) + $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes -liberty -lz +else $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes +endif $(OUTPUT)test-disassembler-init-styled.bin: $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes -- Gitee From a1d2098d761792ac8ab3fcfc7ee6fbdfb4515b7d Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Wed, 7 Jun 2023 10:37:06 +0800 Subject: [PATCH 009/150] sw64: kvm: fix some build errors for KVM module Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Commit 7495e22bb165 ("KVM: Move running VCPU from ARM to common code") has moved running VCPU to common code, leading to multiple definition of `__pcpu_unique_kvm_running_vcpu`, so we remove the duplicated one from KVM/SW64 to fix it. This patch also fixes the duplicated definition of VM_STAT, VCPU_STAT and some complile warnings for KVM. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm-sw64.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/kvm-sw64.c index ec1294a24377..e485c47a74a2 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/kvm-sw64.c @@ -39,20 +39,9 @@ extern bool bind_vcpu_enabled; #define HARDWARE_VPN_MASK ((1UL << WIDTH_HARDWARE_VPN) - 1) #define VPN_SHIFT (64 - WIDTH_HARDWARE_VPN) -#define VCPU_STAT(n, x, ...) \ - { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } -#define VM_STAT(n, x, ...) \ - { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } #define DFX_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu_stat, x), DFX_STAT_U64, ## __VA_ARGS__ } -static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); - -static void kvm_set_running_vcpu(struct kvm_vcpu *vcpu) -{ - __this_cpu_write(kvm_running_vcpu, vcpu); -} - int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) { set_bit(number, (vcpu->arch.irqs_pending)); @@ -272,7 +261,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, { /* * At this point memslot has been committed and there is an - * allocated dirty_bitmap[], dirty pages will be be tracked while the + * allocated dirty_bitmap[], dirty pages will be tracked while the * memory slot is write protected. */ @@ -597,7 +586,6 @@ static void update_steal_time(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { vcpu->cpu = cpu; - kvm_set_running_vcpu(vcpu); update_steal_time(vcpu); } @@ -609,7 +597,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) * optimized make_all_cpus_request path. */ vcpu->cpu = -1; - kvm_set_running_vcpu(NULL); } int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, @@ -656,7 +643,7 @@ void _debug_printk_vcpu(struct kvm_vcpu *vcpu) disp16 = insn & 0xffff; if (opc == 0x06 && disp16 == 0x1000) /* RD_F */ - pr_info("vcpu exit: pc = %#lx (%px), insn[%x] : rd_f r%d [%#lx]\n", + pr_info("vcpu exit: pc = %#lx (%p), insn[%x] : rd_f r%d [%#lx]\n", pc, pc_phys, insn, ra, vcpu_get_reg(vcpu, ra)); } -- Gitee From cc77bd14ff9f7a5081474f4d14b8b6bb720bd8d9 Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Wed, 21 Jun 2023 15:34:31 +0800 Subject: [PATCH 010/150] sw64: hugetlbfs: add arch_hugetlb_valid_size Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56U83 -------------------------------- This patch is arch porting of commit ae94da8981339 ("hugetlbfs: add arch_hugetlb_valid_size") and commit 359f25443a8d ("hugetlbfs: move hugepagesz= parsing to arch independent code"). Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/mm/hugetlbpage.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c index 2a40225af4d8..240d791b5c33 100644 --- a/arch/sw_64/mm/hugetlbpage.c +++ b/arch/sw_64/mm/hugetlbpage.c @@ -306,19 +306,13 @@ arch_initcall(sw64_256m_hugetlb_init); #endif #endif /* CONFIG_HUGETLB_PAGE */ -static __init int setup_hugepagesz(char *opt) +bool __init arch_hugetlb_valid_size(unsigned long size) { - unsigned long ps = memparse(opt, &opt); - - if (ps == PMD_SIZE) { - hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); - } else if (ps == (PMD_SIZE << 5)) { - hugetlb_add_hstate(PMD_SHIFT + 5 - PAGE_SHIFT); - } else { - printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n", - ps >> 20); - return 0; + switch (size) { + case PMD_SIZE: + case (PMD_SIZE<<5): + return true; } - return 1; + + return false; } -__setup("hugepagesz=", setup_hugepagesz); -- Gitee From 3da6416f557d4f083465eed9eae18f1b9b915a0a Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Fri, 24 Feb 2023 08:58:39 +0800 Subject: [PATCH 011/150] sw64: kvm: fix memory hotplug support for guest Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAH4 -------------------------------- This patch: - Change the value saved to the segment table used for memory hotplug to deal with non-1GB aligned vmbase. - Round up the entry number of the segment table to deal with non-1GB aligned ram size of guest. But anyway, due to the limitations of the currently supported memory- hotplug granularity, the allocated ram size of guest still need to be 1GB aligned if you are going to hot add or remove memory. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kvm/kvm-sw64.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/kvm-sw64.c index e485c47a74a2..0669fa47dfa1 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/kvm-sw64.c @@ -389,14 +389,17 @@ static void setup_segment_table(struct kvm *kvm, struct kvm_memory_slot *memslot, unsigned long addr, size_t size) { unsigned long *seg_pgd = kvm->arch.seg_pgd; - unsigned int num_of_entry = size >> 30; - unsigned long base_hpa = addr >> 30; - int i; + unsigned long num_of_entry; + unsigned long base_hpa = addr; + unsigned long i; + + num_of_entry = round_up(size, 1 << 30) >> 30; for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + i; + *seg_pgd = base_hpa + (i << 30); seg_pgd++; } + } #endif @@ -928,11 +931,11 @@ void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) unsigned long *seg_pgd; unsigned long num_of_entry = slot->npages >> 17; unsigned long base_hpa = slot->arch.host_phys_addr; - int i; + unsigned long i; seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); for (i = 0; i < num_of_entry; i++) { - *seg_pgd = (base_hpa >> 30) + i; + *seg_pgd = base_hpa + (i << 30); seg_pgd++; } } -- Gitee From ff8eaefde452da6c5b3ac5e0b467f2dac06b9435 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 30 May 2023 10:50:18 +0800 Subject: [PATCH 012/150] sw64: fix CONFIG_GENERIC_SCHED_CLOCK=y Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Add missing header file. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/time.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 8be7a11df51d..872a19802ade 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -133,6 +133,7 @@ void __init setup_sched_clock(void) } #ifdef CONFIG_GENERIC_SCHED_CLOCK +#include static u64 notrace sched_clock_read(void) { return (rdtc() - sc_start) >> sc_shift; -- Gitee From 55ed87dbe25910d2c9f570fe50d63d1ad8a6a242 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Tue, 23 May 2023 16:52:33 +0800 Subject: [PATCH 013/150] sw64: hibernation: update ktp to vcpucb after restoring it Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAHK -------------------------------- It has to update vcpucb.ktp to make it consistent with current one. Otherwise, S4 wake up will fail due to incorrect ktp. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hibernate_asm.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/hibernate_asm.S b/arch/sw_64/kernel/hibernate_asm.S index 1e9abcf77bee..ff997cd76c5a 100644 --- a/arch/sw_64/kernel/hibernate_asm.S +++ b/arch/sw_64/kernel/hibernate_asm.S @@ -114,7 +114,7 @@ $hibernate_setfpec_over: ldl sp, PSTATE_SP($16) ldl $8, PSTATE_KTP($16) - + sys_call HMC_wrktp ldi $0, 0($31) -- Gitee From 0235bb9b867903c289c03a9c34e2a9cf10f70dcc Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Wed, 31 May 2023 14:12:24 +0800 Subject: [PATCH 014/150] sw64: selftests/ftrace: add sw64 support for kprobe args tests Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAI0 -------------------------------- Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- .../selftests/ftrace/test.d/kprobe/kprobe_args_string.tc | 3 +++ .../selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc index 84285a6f60b0..60506d6626a9 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc @@ -22,6 +22,9 @@ ppc64*) ppc*) ARG1=%r3 ;; +sw_64) + ARG1=%r16 +;; *) echo "Please implement other architecture here" exit_untested diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc index 474ca1a9a088..8879f9517b42 100644 --- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc +++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc @@ -32,6 +32,10 @@ ppc*) GOODREG=%r3 BADREG=%msr ;; +sw_64) + GOODREG=%r16 + BADREG=%ps +;; *) echo "Please implement other architecture here" exit_untested -- Gitee From e4a9ceb561913364890dde7c449d76b693dbacb9 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Thu, 1 Jun 2023 17:22:46 +0800 Subject: [PATCH 015/150] sw64: fix a compile error with CONFIG_TRANSPARENT_HUGEPAGE=n Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 6610e62f4123..c9276d588e28 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -521,6 +521,7 @@ static inline int pmd_protnone(pmd_t pmd) } #endif +#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -543,12 +544,6 @@ static inline int has_transparent_hugepage(void) } #ifdef CONFIG_ARCH_HAS_PTE_DEVMAP -#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) -static inline int pte_devmap(pte_t a) -{ - return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; -} - static inline int pmd_devmap(pmd_t pmd) { return !!(pmd_val(pmd) & _PAGE_DEVMAP); @@ -571,13 +566,20 @@ static inline int pgd_devmap(pgd_t pgd) return 0; } #endif -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ static inline pmd_t pmd_mkdevmap(pmd_t pmd) { pmd_val(pmd) |= _PAGE_DEVMAP; return pmd; } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pte_devmap(pte_t a) +{ + return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; +} +#endif #define __HAVE_ARCH_PMDP_GET_AND_CLEAR static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, -- Gitee From 314a1e3e7253ce96e6903e00661a99287b72dc22 Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Fri, 2 Jun 2023 16:32:21 +0800 Subject: [PATCH 016/150] sw64: update cpufreq's info when cpu frequency changed Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56QDM -------------------------------- Update cpufreq's information when cpu frequency changed and optimize cpufreq related code. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 18 ++-- arch/sw_64/chip/chip3/Makefile | 1 + .../{kernel/clock.c => chip/chip3/cpufreq.c} | 16 +--- arch/sw_64/chip/chip3/cpufreq_debugfs.c | 89 +++++-------------- arch/sw_64/chip/chip3/cpufreq_platform.c | 68 ++++++++++++++ arch/sw_64/include/asm/{clock.h => cpufreq.h} | 16 +++- arch/sw_64/include/asm/hw_init.h | 4 +- arch/sw_64/kernel/Makefile | 1 - arch/sw_64/kernel/cpuautoplug.c | 2 +- arch/sw_64/kernel/platform.c | 20 ----- arch/sw_64/kernel/setup.c | 4 +- drivers/cpufreq/sw64_cpufreq.c | 51 +---------- 12 files changed, 123 insertions(+), 167 deletions(-) rename arch/sw_64/{kernel/clock.c => chip/chip3/cpufreq.c} (82%) create mode 100644 arch/sw_64/chip/chip3/cpufreq_platform.c rename arch/sw_64/include/asm/{clock.h => cpufreq.h} (73%) delete mode 100644 arch/sw_64/kernel/platform.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 6067b0e47c9c..308f55fddd23 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -235,14 +235,6 @@ config SW64_CHIP3_ASIC_DEBUG help Used for debug -config CPUFREQ_DEBUGFS - bool "CPU Frequency debugfs interface for Chip3 Asic" - depends on SW64_CHIP3 && DEBUG_FS - help - Turns on the DebugFS interface for CPU Frequency. - - If you don't know what to do here, say N. - choice prompt "Platform Type" @@ -268,6 +260,7 @@ config LOCK_MEMB bool "Insert mem barrier before lock instruction" default y +menu "CPU Power Management" source "drivers/cpufreq/Kconfig" config SW64_CPUAUTOPLUG @@ -277,6 +270,15 @@ config SW64_CPUAUTOPLUG help Turns on the interface for SW64_CPU CPUAUTOPLUG. +config CPUFREQ_DEBUGFS + bool "CPU Frequency debugfs interface for Chip3 Asic" + depends on SW64_CHIP3 && DEBUG_FS && CPU_FREQ + help + Turns on the DebugFS interface for CPU Frequency. + + If you don't know what to do here, say N. + +endmenu # clear all implied options (don't want default values for those): # Most of these machines have ISA slots; not exactly sure which don't, # and this doesn't activate hordes of code, so do it always. diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile index ba0ab3f67f98..bc262e8f12ef 100644 --- a/arch/sw_64/chip/chip3/Makefile +++ b/arch/sw_64/chip/chip3/Makefile @@ -4,4 +4,5 @@ obj-y := chip.o i2c-lib.o obj-$(CONFIG_PCI) += pci-quirks.o obj-$(CONFIG_PCI_MSI) += msi.o vt_msi.o +obj-$(CONFIG_CPU_FREQ) += cpufreq_platform.o cpufreq.o obj-$(CONFIG_CPUFREQ_DEBUGFS) += cpufreq_debugfs.o diff --git a/arch/sw_64/kernel/clock.c b/arch/sw_64/chip/chip3/cpufreq.c similarity index 82% rename from arch/sw_64/kernel/clock.c rename to arch/sw_64/chip/chip3/cpufreq.c index 8f2bec5fd782..e859d822870c 100644 --- a/arch/sw_64/kernel/clock.c +++ b/arch/sw_64/chip/chip3/cpufreq.c @@ -10,20 +10,8 @@ #include #include #include -#include +#include -#define CLK_PRT 0x1UL -#define CORE_CLK0_V (0x1UL << 1) -#define CORE_CLK0_R (0x1UL << 2) -#define CORE_CLK2_V (0x1UL << 15) -#define CORE_CLK2_R (0x1UL << 16) - -#define CLK_LV1_SEL_PRT 0x1UL -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) - -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL2_CFG_SHIFT 18 char curruent_policy[CPUFREQ_NAME_LEN]; @@ -48,7 +36,7 @@ unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy) val = sw64_io_read(0, CLK_CTL) >> CORE_PLL2_CFG_SHIFT; for (i = 0; ft[i].frequency != CPUFREQ_TABLE_END; i++) { - if (val == ft[i].driver_data) + if (val == i) return ft[i].frequency; } return 0; diff --git a/arch/sw_64/chip/chip3/cpufreq_debugfs.c b/arch/sw_64/chip/chip3/cpufreq_debugfs.c index c58f1cee3907..bb4ae26bc22b 100644 --- a/arch/sw_64/chip/chip3/cpufreq_debugfs.c +++ b/arch/sw_64/chip/chip3/cpufreq_debugfs.c @@ -6,40 +6,27 @@ #include #include - -#define CLK_PRT 0x1UL -#define CORE_CLK0_V (0x1UL << 1) -#define CORE_CLK0_R (0x1UL << 2) -#define CORE_CLK2_V (0x1UL << 15) -#define CORE_CLK2_R (0x1UL << 16) - -#define CLK_LV1_SEL_PRT 0x1UL -#define CLK_LV1_SEL_MUXA (0x1UL << 2) -#define CLK_LV1_SEL_MUXB (0x1UL << 3) - -#define CORE_PLL0_CFG_SHIFT 4 -#define CORE_PLL2_CFG_SHIFT 18 - -static int cpu_freq[16] = { - 200, 1200, 1800, 1900, - 1950, 2000, 2050, 2100, - 2150, 2200, 2250, 2300, - 2350, 2400, 2450, 2500 -}; +#include static int cpufreq_show(struct seq_file *m, void *v) { int i; u64 val; + int freq; val = sw64_io_read(0, CLK_CTL); val = val >> CORE_PLL2_CFG_SHIFT; - for (i = 0; i < sizeof(cpu_freq)/sizeof(int); i++) { - if (cpu_freq[val] == cpu_freq[i]) - seq_printf(m, "[%d] ", cpu_freq[i]); + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; else - seq_printf(m, "%d ", cpu_freq[i]); + freq = freq_table[i].driver_data; + + if (val == i) + seq_printf(m, "[%d] ", freq); + else + seq_printf(m, "%d ", freq); } seq_puts(m, "\n"); @@ -56,8 +43,7 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, { char buf[5]; size_t size; - int cf, i, err, index; - u64 val; + int cf, i, err, index, freq; size = min(sizeof(buf) - 1, len); if (copy_from_user(buf, user_buf, size)) @@ -69,8 +55,13 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, return err; index = -1; - for (i = 0; i < sizeof(cpu_freq)/sizeof(int); i++) { - if (cf == cpu_freq[i]) { + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (freq_table[i].frequency != CPUFREQ_ENTRY_INVALID) + freq = freq_table[i].frequency; + else + freq = freq_table[i].driver_data; + + if (cf == freq) { index = i; break; } @@ -79,46 +70,8 @@ static ssize_t cpufreq_set(struct file *file, const char __user *user_buf, if (index < 0) return -EINVAL; - /* Set CLK_CTL PLL2 */ - sw64_io_write(0, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); - sw64_io_write(1, CLK_CTL, CORE_CLK2_R | CORE_CLK2_V | CLK_PRT); - val = sw64_io_read(0, CLK_CTL); - - sw64_io_write(0, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | index << CORE_PLL2_CFG_SHIFT); - - udelay(1); - - sw64_io_write(0, CLK_CTL, CORE_CLK2_V | CLK_PRT - | index << CORE_PLL2_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, CORE_CLK2_V | CLK_PRT - | index << CORE_PLL2_CFG_SHIFT); - val = sw64_io_read(0, CLK_CTL); - - /* LV1 select PLL1/PLL2 */ - sw64_io_write(0, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); - sw64_io_write(1, CLU_LV1_SEL, CLK_LV1_SEL_MUXA | CLK_LV1_SEL_PRT); - - /* Set CLK_CTL PLL0 */ - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V); - - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_R | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - udelay(1); - - sw64_io_write(0, CLK_CTL, val | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - sw64_io_write(1, CLK_CTL, val | CORE_CLK0_V - | index << CORE_PLL0_CFG_SHIFT); - - /* LV1 select PLL0/PLL1 */ - sw64_io_write(0, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); - sw64_io_write(1, CLU_LV1_SEL, CLK_LV1_SEL_MUXB | CLK_LV1_SEL_PRT); - + sw64_set_rate(index); + update_cpu_freq(freq); return len; } diff --git a/arch/sw_64/chip/chip3/cpufreq_platform.c b/arch/sw_64/chip/chip3/cpufreq_platform.c new file mode 100644 index 000000000000..60fdc08a3169 --- /dev/null +++ b/arch/sw_64/chip/chip3/cpufreq_platform.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +#define CRYSTAL_BIT (1UL << 34) + +/* Minimum CLK support */ +enum { + DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, + DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV +}; + +struct cpufreq_frequency_table freq_table[] = { + {0, 200, CPUFREQ_ENTRY_INVALID}, + {0, DC_1, CPUFREQ_ENTRY_INVALID}, + {0, DC_2, 0}, + {0, DC_3, 0}, + {0, DC_4, 0}, + {0, DC_5, 0}, + {0, DC_6, 0}, + {0, DC_7, 0}, + {0, DC_8, 0}, + {0, DC_9, 0}, + {0, DC_10, 0}, + {0, DC_11, 0}, + {0, DC_12, 0}, + {0, DC_13, 0}, + {0, DC_14, 0}, + {0, DC_15, 0}, + {-1, DC_RESV, CPUFREQ_TABLE_END}, +}; + + +static struct platform_device sw64_cpufreq_device = { + .name = "sw64_cpufreq", + .id = -1, +}; + +static int __init sw64_cpufreq_init(void) +{ + int i; + unsigned long max_rate, freq_off; + max_rate = get_cpu_freq() / 1000000; + + if (sw64_io_read(0, INIT_CTL) & CRYSTAL_BIT) + freq_off = 50; + else + freq_off = 60; + + /* clock table init */ + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (i == 1) + freq_table[i].driver_data = freq_off * 24; + if (i == 2) + freq_table[i].frequency = freq_off * 36; + if (i > 2) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); + + if (freq_table[i].frequency == max_rate) + freq_table[i + 1].frequency = CPUFREQ_TABLE_END; + } + + return platform_device_register(&sw64_cpufreq_device); +} +arch_initcall(sw64_cpufreq_init); diff --git a/arch/sw_64/include/asm/clock.h b/arch/sw_64/include/asm/cpufreq.h similarity index 73% rename from arch/sw_64/include/asm/clock.h rename to arch/sw_64/include/asm/cpufreq.h index 8a6548aa0a0d..0741282e8d4c 100644 --- a/arch/sw_64/include/asm/clock.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -40,8 +40,22 @@ struct clk { #define CLK_ALWAYS_ENABLED (1 << 0) #define CLK_RATE_PROPAGATES (1 << 1) -int clk_init(void); +#define CLK_PRT 0x1UL +#define CORE_CLK0_V (0x1UL << 1) +#define CORE_CLK0_R (0x1UL << 2) +#define CORE_CLK2_V (0x1UL << 15) +#define CORE_CLK2_R (0x1UL << 16) + +#define CLK_LV1_SEL_PRT 0x1UL +#define CLK_LV1_SEL_MUXA (0x1UL << 2) +#define CLK_LV1_SEL_MUXB (0x1UL << 3) + +#define CORE_PLL0_CFG_SHIFT 4 +#define CORE_PLL2_CFG_SHIFT 18 +extern struct cpufreq_frequency_table freq_table[]; + +int clk_init(void); void sw64_set_rate(unsigned int index); struct clk *sw64_clk_get(struct device *dev, const char *id); diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 1426892c83b4..65ced5bbdc72 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -85,9 +85,7 @@ static inline unsigned long get_cpu_freq(void) static inline void update_cpu_freq(unsigned long freq) { - freq = freq * 1000000; - if (cpu_desc.frequency != freq) - cpu_desc.frequency = freq; + cpu_desc.frequency = freq * 1000000; } #define EMUL_FLAG (0x1UL << 63) diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 667e06039987..615556ff7b5a 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -46,7 +46,6 @@ obj-y += kvm_cma.o endif # Core logic support -obj-$(CONFIG_SW64_CPUFREQ) += platform.o clock.o obj-$(CONFIG_SW64_CPUAUTOPLUG) += cpuautoplug.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index de6f77086185..033173fd6200 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -12,7 +12,7 @@ #include #include -#include +#include #include #include diff --git a/arch/sw_64/kernel/platform.c b/arch/sw_64/kernel/platform.c deleted file mode 100644 index f4c880acaa40..000000000000 --- a/arch/sw_64/kernel/platform.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/sw/kernel/setup.c - * - * Copyright (C) 1995 Linus Torvalds - */ - -#include - -static struct platform_device sw64_cpufreq_device = { - .name = "sw64_cpufreq", - .id = -1, -}; - -static int __init sw64_cpufreq_init(void) -{ - return platform_device_register(&sw64_cpufreq_device); -} - -arch_initcall(sw64_cpufreq_init); diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 16c13646f3c7..466160cb372d 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -897,7 +897,7 @@ show_cpuinfo(struct seq_file *f, void *slot) int i; unsigned long cpu_freq; - cpu_freq = get_cpu_freq() / 1000 / 1000; + cpu_freq = cpuid(GET_CPU_FREQ, 0); for_each_online_cpu(i) { /* @@ -921,7 +921,7 @@ show_cpuinfo(struct seq_file *f, void *slot) "cache size\t: %u KB\n" "physical id\t: %d\n" "bogomips\t: %lu.%02lu\n", - cpu_freq, cpu_data[i].tcache.size >> 10, + get_cpu_freq() / 1000 / 1000, cpu_data[i].tcache.size >> 10, cpu_topology[i].package_id, loops_per_jiffy / (500000/HZ), (loops_per_jiffy / (5000/HZ)) % 100); diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index 9259753c8f06..6ac7081c3222 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -20,40 +20,13 @@ #include #include -#include +#include #include -#define CRYSTAL_BIT (1UL << 34) - static uint nowait; static struct clk *cpuclk; -/* Minimum CLK support */ -enum { - DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, - DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV -}; - -static struct cpufreq_frequency_table freq_table[] = { - {0, DC_0, CPUFREQ_ENTRY_INVALID}, - {0, DC_1, CPUFREQ_ENTRY_INVALID}, - {0, DC_2, 0}, - {0, DC_3, 0}, - {0, DC_4, 0}, - {0, DC_5, 0}, - {0, DC_6, 0}, - {0, DC_7, 0}, - {0, DC_8, 0}, - {0, DC_9, 0}, - {0, DC_10, 0}, - {0, DC_11, 0}, - {0, DC_12, 0}, - {0, DC_13, 0}, - {0, DC_14, 0}, - {0, DC_15, 0}, - {-1, DC_RESV, CPUFREQ_TABLE_END}, -}; static int sw64_cpu_freq_notifier(struct notifier_block *nb, unsigned long val, void *data); @@ -102,39 +75,19 @@ static int sw64_cpufreq_target(struct cpufreq_policy *policy, /* setting the cpu frequency */ sw64_set_rate(index); + update_cpu_freq(freq_table[index].frequency); return 0; } static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) { - int i; - unsigned long max_rate, freq_off; - cpuclk = sw64_clk_get(NULL, "cpu_clk"); if (IS_ERR(cpuclk)) { pr_err("couldn't get CPU clk\n"); return PTR_ERR(cpuclk); } - max_rate = get_cpu_freq() / 1000000; - - if (sw64_io_read(0, INIT_CTL) & CRYSTAL_BIT) - freq_off = 50; - else - freq_off = 60; - - /* clock table init */ - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (i == 2) - freq_table[i].frequency = freq_off * 36; - if (i > 2) - freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); - - if (freq_table[i].frequency == max_rate) - freq_table[i + 1].frequency = CPUFREQ_TABLE_END; - } - policy->clk = cpuclk; cpufreq_generic_init(policy, freq_table, 0); -- Gitee From f44330f9b418e0e4328e947329df884dc816720d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 2 Jun 2023 13:53:36 +0800 Subject: [PATCH 017/150] sw64: fix some include format Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hw_irq.h | 2 +- arch/sw_64/include/asm/switch_to.h | 2 +- arch/sw_64/kernel/dup_print.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/hw_irq.h b/arch/sw_64/include/asm/hw_irq.h index ad5aed26efb7..3cfc725f7517 100644 --- a/arch/sw_64/include/asm/hw_irq.h +++ b/arch/sw_64/include/asm/hw_irq.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_HW_IRQ_H #define _ASM_SW64_HW_IRQ_H -#include +#include extern volatile unsigned long irq_err_count; DECLARE_PER_CPU(unsigned long, irq_pmi_count); diff --git a/arch/sw_64/include/asm/switch_to.h b/arch/sw_64/include/asm/switch_to.h index e5596a735b2d..c81b004cfdd8 100644 --- a/arch/sw_64/include/asm/switch_to.h +++ b/arch/sw_64/include/asm/switch_to.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_SWITCH_TO_H #define _ASM_SW64_SWITCH_TO_H -#include +#include extern void __fpstate_save(struct task_struct *save_to); extern void __fpstate_restore(struct task_struct *restore_from); diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index e28e0053239c..107363dc0513 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -52,7 +52,7 @@ int sw64_printk(const char *fmt, va_list args) #endif #ifdef CONFIG_SW64_RRU -#include +#include static DEFINE_SPINLOCK(printf_lock); #define USER_PRINT_BUFF_BASE (0x600000UL + __START_KERNEL_map) -- Gitee From fa4926d36688d32b733bdf1fd501cad54eed3f82 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 2 Jun 2023 13:46:04 +0800 Subject: [PATCH 018/150] sw64: fix audit_syscall_exit() Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAKA -------------------------------- SW64 has special syscall return convention, so add is_syscall_success() and modify regs_return_value() to make sure audit_syscall_exit() can get the correct syscall return value. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ptrace.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 2c60bc6730ad..95a26be29667 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -94,8 +94,16 @@ extern int regs_query_register_offset(const char *name); extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); -static inline unsigned long regs_return_value(struct pt_regs *regs) +static inline int is_syscall_success(struct pt_regs *regs) { - return regs->r0; + return !regs->r19; +} + +static inline long regs_return_value(struct pt_regs *regs) +{ + if (is_syscall_success(regs) || !user_mode(regs)) + return regs->r0; + else + return -regs->r0; } #endif /* _ASM_SW64_PTRACE_H */ -- Gitee From a25dfc8fa541271a84c27c527bd584663631d3ef Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 5 Jun 2023 15:19:59 +0800 Subject: [PATCH 019/150] sw64: rename ptrace macros Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Add "PT_" prefix to ptrace macros to avoid naming conflicts. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/ptrace.h | 36 ++++++++++----------- arch/sw_64/kernel/ptrace.c | 48 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/arch/sw_64/include/uapi/asm/ptrace.h b/arch/sw_64/include/uapi/asm/ptrace.h index 5cf3ca1d3dd8..e48dd68c5b4d 100644 --- a/arch/sw_64/include/uapi/asm/ptrace.h +++ b/arch/sw_64/include/uapi/asm/ptrace.h @@ -29,23 +29,23 @@ struct user_fpsimd_state { /* PTRACE_ATTACH is 16 */ /* PTRACE_DETACH is 17 */ -#define REG_BASE 0 -#define REG_END 29 -#define USP 30 -#define FPREG_BASE 32 -#define FPREG_END 62 -#define FPCR 63 -#define PC 64 -#define TP 65 -#define UNIQUE TP -#define VECREG_BASE 67 -#define VECREG_END 161 -#define F31_V1 98 -#define F31_V2 130 -#define DA_MATCH 163 -#define DA_MASK 164 -#define DV_MATCH 165 -#define DV_MASK 166 -#define DC_CTL 167 +#define PT_REG_BASE 0 +#define PT_REG_END 29 +#define PT_USP 30 +#define PT_FPREG_BASE 32 +#define PT_FPREG_END 62 +#define PT_FPCR 63 +#define PT_PC 64 +#define PT_TP 65 +#define PT_UNIQUE PT_TP +#define PT_VECREG_BASE 67 +#define PT_VECREG_END 161 +#define PT_F31_V1 98 +#define PT_F31_V2 130 +#define PT_DA_MATCH 163 +#define PT_DA_MASK 164 +#define PT_DV_MATCH 165 +#define PT_DV_MASK 166 +#define PT_DC_CTL 167 #endif /* _UAPI_ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 2ca8d5804537..b509576257cc 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -61,13 +61,13 @@ short regoffsets[32] = { #define PCB_OFF(var) offsetof(struct pcb_struct, var) static int pcboff[] = { - [USP] = PCB_OFF(usp), - [TP] = PCB_OFF(tp), - [DA_MATCH] = PCB_OFF(da_match), - [DA_MASK] = PCB_OFF(da_mask), - [DV_MATCH] = PCB_OFF(dv_match), - [DV_MASK] = PCB_OFF(dv_mask), - [DC_CTL] = PCB_OFF(dc_ctl) + [PT_USP] = PCB_OFF(usp), + [PT_TP] = PCB_OFF(tp), + [PT_DA_MATCH] = PCB_OFF(da_match), + [PT_DA_MASK] = PCB_OFF(da_mask), + [PT_DV_MATCH] = PCB_OFF(dv_match), + [PT_DV_MASK] = PCB_OFF(dv_mask), + [PT_DC_CTL] = PCB_OFF(dc_ctl) }; static unsigned long zero; @@ -83,39 +83,39 @@ get_reg_addr(struct task_struct *task, unsigned long regno) int fno, vno; switch (regno) { - case USP: - case UNIQUE: - case DA_MATCH: - case DA_MASK: - case DV_MATCH: - case DV_MASK: - case DC_CTL: + case PT_USP: + case PT_UNIQUE: + case PT_DA_MATCH: + case PT_DA_MASK: + case PT_DV_MATCH: + case PT_DV_MASK: + case PT_DC_CTL: addr = (void *)task_thread_info(task) + pcboff[regno]; break; - case REG_BASE ... REG_END: + case PT_REG_BASE ... PT_REG_END: addr = (void *)task_pt_regs(task) + regoffsets[regno]; break; - case FPREG_BASE ... FPREG_END: - fno = regno - FPREG_BASE; + case PT_FPREG_BASE ... PT_FPREG_END: + fno = regno - PT_FPREG_BASE; addr = &task->thread.fpstate.fp[fno].v[0]; break; - case VECREG_BASE ... VECREG_END: + case PT_VECREG_BASE ... PT_VECREG_END: /* * return addr for zero value if we catch vectors of f31 * v0 and v3 of f31 are not in this range so ignore them */ - if (regno == F31_V1 || regno == F31_V2) { + if (regno == PT_F31_V1 || regno == PT_F31_V2) { addr = &zero; break; } - fno = (regno - VECREG_BASE) & 0x1f; - vno = 1 + ((regno - VECREG_BASE) >> 5); + fno = (regno - PT_VECREG_BASE) & 0x1f; + vno = 1 + ((regno - PT_VECREG_BASE) >> 5); addr = &task->thread.fpstate.fp[fno].v[vno]; break; - case FPCR: + case PT_FPCR: addr = &task->thread.fpstate.fpcr; break; - case PC: + case PT_PC: addr = (void *)task_pt_regs(task) + PT_REGS_PC; break; default: @@ -170,7 +170,7 @@ ptrace_set_bpt(struct task_struct *child) unsigned int insn, op_code; unsigned long pc; - pc = get_reg(child, PC); + pc = get_reg(child, PT_PC); res = read_int(child, pc, (int *)&insn); if (res < 0) return res; -- Gitee From 3e9bfa2c4e7d30beb9e86b09aafaca18b3839281 Mon Sep 17 00:00:00 2001 From: Xu Chenjiao Date: Mon, 12 Jun 2023 14:23:02 +0800 Subject: [PATCH 020/150] sw64: remove S3 flag Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CALA -------------------------------- There's no need to set S3 flag in kernel. Remove the redundant code. Signed-off-by: Xu Chenjiao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/chip.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 819b9c3728f6..2f39c6b6206b 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -649,9 +649,6 @@ static void chip3_suspend(bool wakeup) chip3_intpu_restore(); chip3_spbu_restore(); } else { - /* Set S3 flag */ - cpld_write(0x64, 0x34, 0x33); - chip3_spbu_save(); chip3_intpu_save(); chip3_pcie_save(); -- Gitee From 15534321ae27fb8925e1a1392bc2e74c998bc069 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Wed, 14 Jun 2023 13:53:25 +0800 Subject: [PATCH 021/150] sw64: correct the definition of MAX_DMA_ADDRESS Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- On sw64 platform, we don't have any address limitations on DMA zone, so we correct the definition of MAX_DMA_ADDRESS to ~0UL. Besides, we introduce a new macro MAX_DMA32_PFN to represent DMA32 zone which will allow to use the dma-direct method. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/dma.h | 12 +++--------- arch/sw_64/kernel/setup.c | 2 +- arch/sw_64/mm/init.c | 5 +---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/arch/sw_64/include/asm/dma.h b/arch/sw_64/include/asm/dma.h index 1211b71f347e..cf6a9cf75233 100644 --- a/arch/sw_64/include/asm/dma.h +++ b/arch/sw_64/include/asm/dma.h @@ -88,20 +88,14 @@ */ #define MAX_ISA_DMA_ADDRESS 0x100000000UL +#define MAX_DMA32_PFN (1UL << (32 - PAGE_SHIFT)) + /* * If we have the iommu, we don't have any address limitations on DMA. * Otherwise (Nautilus, RX164), we have to have 0-16 Mb DMA zone * like i386. */ -#ifdef CONFIG_IOMMU_SUPPORT -#ifndef CONFIG_SWICH_GPU -#define MAX_DMA_ADDRESS (PAGE_OFFSET + 0x100000000) -#else -#define MAX_DMA_ADDRESS (~0UL) -#endif /* CONFIG_IOMMU_SUPPORT */ -#else -#define MAX_DMA_ADDRESS (PAGE_OFFSET + 0x80000000) -#endif +#define MAX_DMA_ADDRESS ~0UL /* 8237 DMA controllers */ #define IO_DMA1_BASE 0x00 /* 8 bit slave DMA, channels 0..3 */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 466160cb372d..aa90d284de9b 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -361,7 +361,7 @@ void __init process_memmap(void) static int i; // Make it static so we won't start over again every time. int ret; phys_addr_t base, size; - unsigned long dma_end __maybe_unused = virt_to_phys((void *)MAX_DMA_ADDRESS); + unsigned long dma_end __maybe_unused = (MAX_DMA32_PFN << PAGE_SHIFT); if (!memblock_initialized) return; diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 76104c097309..a002e83bae34 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -104,14 +104,11 @@ void __init callback_init(void) void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES]; - unsigned long dma_pfn; memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); - dma_pfn = PFN_DOWN(virt_to_phys((void *)MAX_DMA_ADDRESS)); - #ifdef CONFIG_ZONE_DMA32 - max_zone_pfns[ZONE_DMA32] = min(dma_pfn, max_low_pfn); + max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; -- Gitee From fc54be1be3b9f7bf6f584e4e17bb40b2dfdf5610 Mon Sep 17 00:00:00 2001 From: Zhang Fuxing Date: Thu, 15 Jun 2023 15:13:18 +0000 Subject: [PATCH 022/150] =?UTF-8?q?fbdev:=C2=A0sw64=C2=A0use=C2=A0=5F=5Fra?= =?UTF-8?q?w=C2=A0I/O=C2=A0memory=C2=A0api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAO9 -------------------------------- Since sw64 also has __raw I/O accessors, add __sw_64__ in fb.h. Signed-off-by: Zhang Fuxing Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/fb.h b/include/linux/fb.h index ecfbcc0553a5..916a4b4e5e84 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -545,7 +545,7 @@ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) { #elif defined(__i386__) || defined(__alpha__) || defined(__x86_64__) || \ defined(__hppa__) || defined(__sh__) || defined(__powerpc__) || \ - defined(__arm__) || defined(__aarch64__) + defined(__arm__) || defined(__aarch64__) || defined(__sw_64__) #define fb_readb __raw_readb #define fb_readw __raw_readw -- Gitee From 5a710f25f9d2b9ec7d497fa62679aa9e0b38e935 Mon Sep 17 00:00:00 2001 From: Zhang Yi Date: Thu, 4 May 2023 19:55:17 +0800 Subject: [PATCH 023/150] sw64: kvm: add support for cpu offline and online in guest Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAOV -------------------------------- This patch adds support for it: - Migrate interrupts from offline vCPU to other online vCPUs. - Virtualize NMII in guest for reset vCPU. Signed-off-by: Zhang Yi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/vt_msi.c | 45 +++++++++++++++++++++++++++++++ arch/sw_64/include/asm/kvm_host.h | 2 +- arch/sw_64/kernel/smp.c | 4 +++ arch/sw_64/kvm/handle_exit.c | 2 +- arch/sw_64/kvm/kvm-sw64.c | 15 ++++++++++- 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/chip/chip3/vt_msi.c b/arch/sw_64/chip/chip3/vt_msi.c index 0cdf7f196e8a..3d7f56f3acdf 100644 --- a/arch/sw_64/chip/chip3/vt_msi.c +++ b/arch/sw_64/chip/chip3/vt_msi.c @@ -32,12 +32,57 @@ static void vt_irq_msi_update_msg(struct irq_data *irqd, pci_write_msi_msg(irqd->irq, msg); } +static int +vt_set_affinity(struct irq_data *irqd, const struct cpumask *cpumask, + bool force) +{ + struct sw64_msi_chip_data *cdata; + struct cpumask searchmask; + int cpu, vector; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target coreid is already in the new mask, + * and is online then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = irqd->irq; + spin_lock(&cdata->cdata_lock); + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->prev_vector = cdata->vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + vt_irq_msi_update_msg(irqd, irqd->chip_data); + + return 0; +} + static struct irq_chip vt_pci_msi_controller = { .name = "PCI-MSI", .irq_unmask = pci_msi_unmask_irq, .irq_mask = pci_msi_mask_irq, .irq_ack = sw64_irq_noop, .irq_compose_msi_msg = vt_irq_msi_compose_msg, + .irq_set_affinity = vt_set_affinity, }; int chip_setup_vt_msix_irq(struct pci_dev *dev, struct msi_desc *desc) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 5433a3b21b87..56e7ef56c82b 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -132,7 +132,7 @@ void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr); #endif int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index, struct hcall_args *hargs); -void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid); +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type); static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 003fee7f0ea1..78ef01b54da9 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -268,6 +268,10 @@ int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; + if (smp_booted) { + /* irq must be disabled before reset vCPU */ + reset_cpu(cpu); + } smp_boot_one_cpu(cpu, tidle); return cpu_online(cpu) ? 0 : -ENOSYS; diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index 5d14f2a22f1f..7ff826d0ed09 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -54,7 +54,7 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, return 1; case SW64_KVM_EXIT_IPI: vcpu->stat.ipi_exits++; - vcpu_send_ipi(vcpu, hargs->arg0); + vcpu_send_ipi(vcpu, hargs->arg0, hargs->arg1); return 1; case SW64_KVM_EXIT_DEBUG: vcpu->stat.debug_exits++; diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/kvm-sw64.c index 0669fa47dfa1..c9182f6deadd 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/kvm-sw64.c @@ -16,6 +16,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include "trace.h" @@ -39,6 +40,8 @@ extern bool bind_vcpu_enabled; #define HARDWARE_VPN_MASK ((1UL << WIDTH_HARDWARE_VPN) - 1) #define VPN_SHIFT (64 - WIDTH_HARDWARE_VPN) +#define GUEST_RESET_PC 0xffffffff80011100 + #define DFX_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu_stat, x), DFX_STAT_U64, ## __VA_ARGS__ } @@ -734,6 +737,13 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) memset(&hargs, 0, sizeof(hargs)); clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + irq = interrupt_pending(vcpu, &more); if (irq < SWVM_IRQS) try_deliver_interrupt(vcpu, irq, more); @@ -943,10 +953,13 @@ void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) } #endif -void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid) +void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) { struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); + if (type == II_RESET) + target_vcpu->arch.restart = 1; + if (target_vcpu != NULL) vcpu_interrupt_line(target_vcpu, 1, 1); } -- Gitee From 224f43a92bf72086ecdc40011473309e7f56c48b Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Tue, 6 Jun 2023 17:16:30 +0800 Subject: [PATCH 024/150] sw64: fix some cpu autoplug bugs Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- This patch: - Rewrite get_min_busy_time(), make index start with 0 to avoid compile optimization in decrease_cores(). - Fix argument of find_min_busy_cpu() and decrease_cores(). - Fix the bug that ap_info.mincpus is always 16. - Disable this feature in guest os because of hardware limitation. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cpuautoplug.c | 21 ++++++++++----------- drivers/cpufreq/sw64_cpufreq.c | 5 +++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index 033173fd6200..e867d5f544c0 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -257,18 +257,17 @@ static inline cputime64_t sw64_get_idle_time(cputime64_t *wall) static cputime64_t get_min_busy_time(cputime64_t arr[], int size) { - int loop, min_idx; + int i, min_cpu_idx; cputime64_t min_time = arr[0]; - for (loop = 1; loop < size; loop++) { - if (arr[loop] > 0) { - if (arr[loop] < min_time) { - min_time = arr[loop]; - min_idx = loop; - } + for (i = 0; i < size; i++) { + if (arr[i] > 0 && arr[i] < min_time) { + min_time = arr[i]; + min_cpu_idx = i; } } - return min_idx; + + return min_cpu_idx; } static int find_min_busy_cpu(void) @@ -284,8 +283,6 @@ static int find_min_busy_cpu(void) b_time[cpus] = busy_time; } target_cpu = get_min_busy_time(b_time, nr_all_cpus); - pr_info("The target_cpu is %d, the cpu_num is %d\n", - target_cpu, num_online_cpus() - 1); return target_cpu; } @@ -321,6 +318,8 @@ static void decrease_cores(int cur_cpus) per_cpu(cpu_adjusting, dev->id) = -1; lock_device_hotplug(); cpu_device_down(dev); + pr_info("The target_cpu is %d. After cpu_down, the cpu_num is %d\n", + cur_cpus, num_online_cpus()); get_cpu_device(dev->id)->offline = true; unlock_device_hotplug(); per_cpu(cpu_adjusting, dev->id) = 0; @@ -452,7 +451,7 @@ static int __init cpuautoplug_init(void) ap_info.maxcpus = setup_max_cpus > nr_cpu_ids ? nr_cpu_ids : setup_max_cpus; - ap_info.mincpus = 16; + ap_info.mincpus = ap_info.maxcpus / 4; ap_info.dec_reqs = 0; ap_info.inc_reqs = 0; ap_info.sampling_rate = 720; /* 720ms */ diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index 6ac7081c3222..91bd05f245e9 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -140,6 +140,11 @@ static int __init cpufreq_init(void) { int ret; + if (is_in_guest()) { + pr_warn("Now sw_64 CPUFreq does not support virtual machines\n"); + return -ENODEV; + } + /* Register platform stuff */ ret = platform_driver_register(&platform_driver); if (ret) -- Gitee From a626f3b929d677849cea9e0fd7fa2f4d61dfdafb Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 21 Jun 2023 15:55:16 +0800 Subject: [PATCH 025/150] sw64: fix uretprobe implementation Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAPA -------------------------------- SW64 uses ra to calculate gp after returning from function call, so ra has to be restored after uretprobe. In commit 1a1339853c20 ("sw64: fix incorrect gp after uretprobe triggered"), a new function was added to restore ra, but it was not called in the right place. In addition, pc has been changed in handle_trampoline(), which is called before this function, so we need to save pc before calling handle_trampoline(), and pass it to this function manually. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uprobes.h | 7 ++++++- arch/sw_64/kernel/signal.c | 7 ++++++- arch/sw_64/kernel/traps.c | 4 ++-- arch/sw_64/kernel/uprobes.c | 7 ++----- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/include/asm/uprobes.h b/arch/sw_64/include/asm/uprobes.h index 2a5b268cb88f..fcd2026c3622 100644 --- a/arch/sw_64/include/asm/uprobes.h +++ b/arch/sw_64/include/asm/uprobes.h @@ -35,6 +35,11 @@ struct arch_uprobe_task { unsigned long saved_trap_nr; }; -extern void sw64_fix_uretprobe(struct pt_regs *regs); +#ifdef CONFIG_UPROBES +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc); +#else +static inline void +sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) {} +#endif #endif /* _ASM_SW64_UPROBES_H */ diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index e1b41587a541..0b7db801a7e8 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -421,8 +422,12 @@ do_work_pending(struct pt_regs *regs, unsigned long thread_flags, } else { local_irq_enable(); - if (thread_flags & _TIF_UPROBE) + if (thread_flags & _TIF_UPROBE) { + unsigned long pc = regs->pc; + uprobe_notify_resume(regs); + sw64_fix_uretprobe(regs, pc - 4); + } if (thread_flags & _TIF_PATCH_PENDING) klp_update_patch_state(current); diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index fda9ef61a2e4..cdbd6621e79c 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -300,10 +300,10 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) #ifdef CONFIG_UPROBES case UPROBE_BRK_UPROBE: if (notify_die(DIE_UPROBE, "uprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) - return sw64_fix_uretprobe(regs); + return; case UPROBE_BRK_UPROBE_XOL: if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) - return sw64_fix_uretprobe(regs); + return; #endif } diff --git a/arch/sw_64/kernel/uprobes.c b/arch/sw_64/kernel/uprobes.c index e25793f4a058..0e56134a8251 100644 --- a/arch/sw_64/kernel/uprobes.c +++ b/arch/sw_64/kernel/uprobes.c @@ -172,14 +172,11 @@ static unsigned long get_trampoline_vaddr(void) return trampoline_vaddr; } -void sw64_fix_uretprobe(struct pt_regs *regs) +void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) { - unsigned long bp_vaddr; - - bp_vaddr = uprobe_get_swbp_addr(regs); /* * regs->pc has been changed to orig_ret_vaddr in handle_trampoline(). */ - if (bp_vaddr == get_trampoline_vaddr()) + if (exc_pc == get_trampoline_vaddr()) regs->r26 = regs->pc; } -- Gitee From 4b340d7fabf57e8372d91f1028fc52ec5ad2383a Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Tue, 27 Jun 2023 14:49:35 +0800 Subject: [PATCH 026/150] sw64: determine external clock according to specification Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAPS -------------------------------- IOR:INIT_CTL[34] was used by software to determine the external clock frequency. But it has been deprecated in sunway platform specification v2.0, which specifies that SROM should copy board configuration from flash to the specified memory, and software can access it to determine external clock frequency. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/cpufreq_platform.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/chip/chip3/cpufreq_platform.c b/arch/sw_64/chip/chip3/cpufreq_platform.c index 60fdc08a3169..a37e524f7b14 100644 --- a/arch/sw_64/chip/chip3/cpufreq_platform.c +++ b/arch/sw_64/chip/chip3/cpufreq_platform.c @@ -5,8 +5,6 @@ #include #include -#define CRYSTAL_BIT (1UL << 34) - /* Minimum CLK support */ enum { DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, @@ -42,13 +40,16 @@ static struct platform_device sw64_cpufreq_device = { static int __init sw64_cpufreq_init(void) { int i; + unsigned char external_clk; unsigned long max_rate, freq_off; max_rate = get_cpu_freq() / 1000000; - if (sw64_io_read(0, INIT_CTL) & CRYSTAL_BIT) - freq_off = 50; - else + external_clk = *((unsigned char *)__va(0x908011)); + + if (external_clk == 240) freq_off = 60; + else + freq_off = 50; /* clock table init */ for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { -- Gitee From 95d55bc90851e14088c0aac7425724562f9ac673 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Wed, 28 Jun 2023 16:50:13 +0800 Subject: [PATCH 027/150] sw64: tools: fix selftests/sigaltstack for sw64 Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAQQ -------------------------------- On sw64 the compilation of the file sas.c in directory tools/testing/selftests/sigaltstack fails with invalid register name "sp". However, the stack pointer on sw64 is register $30. So, make it platform dependend to fix the compile error. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- tools/testing/selftests/sigaltstack/current_stack_pointer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/sigaltstack/current_stack_pointer.h b/tools/testing/selftests/sigaltstack/current_stack_pointer.h index ea9bdf3a90b1..47c0dc383430 100644 --- a/tools/testing/selftests/sigaltstack/current_stack_pointer.h +++ b/tools/testing/selftests/sigaltstack/current_stack_pointer.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#if __alpha__ +#if __alpha__ || __sw_64__ register unsigned long sp asm("$30"); #elif __arm__ || __aarch64__ || __csky__ || __m68k__ || __mips__ || __riscv register unsigned long sp asm("sp"); -- Gitee From 42f4f089ff7a63953108233f859b4e5ded06a69e Mon Sep 17 00:00:00 2001 From: Gao Chen Date: Fri, 30 Jun 2023 13:55:19 +0800 Subject: [PATCH 028/150] sw64: add save_stack_trace_regs() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CATS -------------------------------- The save_stack_trace_regs() was not implemented on sw64 yet. This patch implements save_stack_trace_regs(), so that some trace tools that depend on it will work. Signed-off-by: Gao Chen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/stacktrace.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c index 71c0e5b7b3e0..57d4fcc35ec5 100644 --- a/arch/sw_64/kernel/stacktrace.c +++ b/arch/sw_64/kernel/stacktrace.c @@ -174,6 +174,19 @@ int save_trace(unsigned long pc, void *d) return (trace->nr_entries >= trace->max_entries); } +void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) +{ + struct stack_trace_data data; + + data.trace = trace; + data.nosched = 0; + + walk_stackframe(current, regs, save_trace, &data); + + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + static void __save_stack_trace(struct task_struct *tsk, struct stack_trace *trace, unsigned int nosched) { -- Gitee From f8b85cc91092981d7e04415c07d662da2ab2f7d4 Mon Sep 17 00:00:00 2001 From: Zhu Yiming Date: Mon, 10 Jul 2023 09:11:06 +0800 Subject: [PATCH 029/150] sw64: update openeuler_defconfig Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- When CONFIG_NLS_ISO8859_1=m and CONFIG_NLS_CODEPAGE_437=m, the virtual machine without loading uefi-bios-sw cannot boot the image installed via ISO following new firmware specification, so select CONFIG_NLS_ISO8859_1=y and CONFIG_NLS_CODEPAGE_437=y in openeuler_defconfig. Signed-off-by: Zhu Yiming Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/openeuler_defconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/configs/openeuler_defconfig b/arch/sw_64/configs/openeuler_defconfig index 78bf5d4e0523..92fc21b99b60 100644 --- a/arch/sw_64/configs/openeuler_defconfig +++ b/arch/sw_64/configs/openeuler_defconfig @@ -629,7 +629,7 @@ CONFIG_NFSD_V3_ACL=y CONFIG_NFSD_V4=y CONFIG_NFSD_SCSILAYOUT=y CONFIG_NFSD_V4_SECURITY_LABEL=y -CONFIG_NLS_CODEPAGE_437=m +CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_737=m CONFIG_NLS_CODEPAGE_775=m CONFIG_NLS_CODEPAGE_850=m @@ -653,7 +653,7 @@ CONFIG_NLS_ISO8859_8=m CONFIG_NLS_CODEPAGE_1250=m CONFIG_NLS_CODEPAGE_1251=m CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=m CONFIG_NLS_ISO8859_3=m CONFIG_NLS_ISO8859_4=m -- Gitee From 608dd899b7613c6253320416e23ecdc861baff1f Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Tue, 11 Jul 2023 09:59:55 +0800 Subject: [PATCH 030/150] sw64: modify lock instruction Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAX1 -------------------------------- The new compiler requires lstw/lstl and rd_f should use same register in lock instruction sequence. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/bitops.h | 12 ++++++------ arch/sw_64/include/asm/futex.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h index 8c4844e8d067..add1b3c11366 100644 --- a/arch/sw_64/include/asm/bitops.h +++ b/arch/sw_64/include/asm/bitops.h @@ -44,8 +44,8 @@ set_bit(unsigned long nr, volatile void *addr) " memb\n" #endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" @@ -86,8 +86,8 @@ clear_bit(unsigned long nr, volatile void *addr) " memb\n" #endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" @@ -139,8 +139,8 @@ change_bit(unsigned long nr, volatile void *addr) " memb\n" #endif " lstw %0, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " rd_f %0\n" + " beq %0, 2f\n" ".subsection 2\n" "2: br 1b\n" ".previous" diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h index 50324470173f..344ce5e4da28 100644 --- a/arch/sw_64/include/asm/futex.h +++ b/arch/sw_64/include/asm/futex.h @@ -34,8 +34,8 @@ insn \ LOCK_FIXUP \ "2: lstw %1, 0(%3)\n" \ - " rd_f %2\n" \ - " beq %2, 4f\n" \ + " rd_f %1\n" \ + " beq %1, 4f\n" \ " bis $31, $31, %1\n" \ "3: .subsection 2\n" \ "4: br 1b\n" \ -- Gitee From 87ac3f3c530b8844223f5906b59623d944c7dfee Mon Sep 17 00:00:00 2001 From: Zhou Xuemei Date: Tue, 11 Jul 2023 11:00:48 +0800 Subject: [PATCH 031/150] sw64: pci: use pci_has_flag to define pcibios_assign_all_busses Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CAYU -------------------------------- Since some buggy firmwares may configure invalid bridge bus numbers, the kernel re-assigns all PCI bus numbers in sw64_init_pci(). However, users may trigger a pci bus rescan in the userspace by the command below: > echo 1 > /sys/bus/pci/rescan Unexpected errors may occur on the endpoint devices due to the re-assign bus numbers of upstream bridges. To work around this problem, the flag PCI_REASSIGN_ALL_BUS is cleared at the end of sw64_init_pci(). Signed-off-by: Zhou Xuemei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 2 +- arch/sw_64/kernel/pci.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index de175b3c1043..1a931cc29195 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -74,7 +74,7 @@ struct pci_controller { * bus numbers. */ -#define pcibios_assign_all_busses() 1 +#define pcibios_assign_all_busses() (pci_has_flag(PCI_REASSIGN_ALL_BUS)) #define PCIBIOS_MIN_IO 0 #define PCIBIOS_MIN_MEM 0 diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 6cc872ba9ca5..efa1625cf895 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -690,7 +690,9 @@ void __init sw64_init_irq(void) void __init sw64_init_pci(void) { + pci_add_flags(PCI_REASSIGN_ALL_BUS); common_init_pci(); + pci_clear_flags(PCI_REASSIGN_ALL_BUS); } static int setup_bus_dma_cb(struct pci_dev *pdev, void *data) -- Gitee From f74e06d0a38c7eea6036ac6c5d12d802d3bb0a4a Mon Sep 17 00:00:00 2001 From: Yang Qiang Date: Mon, 24 Jul 2023 08:49:35 +0800 Subject: [PATCH 032/150] i2c: designware: add sunway system I2C support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CB2G -------------------------------- Sunway system Controller is equipped with a dedicated I2C Controller which functionality is based on the DW APB I2C IP-core, the only difference in a way it's registers are accessed. Signed-off-by: Yang Qiang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/i2c/busses/i2c-designware-common.c | 8 ++++++++ drivers/i2c/busses/i2c-designware-core.h | 1 + drivers/i2c/busses/i2c-designware-platdrv.c | 1 + 3 files changed, 10 insertions(+) diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index c9f7783ac7cb..87f36f676b4e 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -63,6 +63,9 @@ static int dw_reg_read(void *context, unsigned int reg, unsigned int *val) { struct dw_i2c_dev *dev = context; + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; + *val = readl(dev->base + reg); return 0; @@ -72,6 +75,9 @@ static int dw_reg_write(void *context, unsigned int reg, unsigned int val) { struct dw_i2c_dev *dev = context; + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = reg << 7; + writel(val, dev->base + reg); return 0; @@ -149,6 +155,8 @@ int i2c_dw_init_regmap(struct dw_i2c_dev *dev) return ret; reg = readl(dev->base + DW_IC_COMP_TYPE); + if ((dev->flags & MODEL_MASK) == MODEL_SUNWAY) + reg = readl(dev->base + (DW_IC_COMP_TYPE << 7)); i2c_dw_release_lock(dev); if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 9b51f3300d47..cccddd3b63be 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -297,6 +297,7 @@ struct dw_i2c_dev { #define MODEL_MSCC_OCELOT 0x00000100 #define MODEL_BAIKAL_BT1 0x00000200 +#define MODEL_SUNWAY 0x00000800 #define MODEL_MASK 0x00000f00 struct i2c_dw_semaphore_callbacks { diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index c0d4f2b990c5..38f5c88fb431 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -154,6 +154,7 @@ static const struct of_device_id dw_i2c_of_match[] = { { .compatible = "snps,designware-i2c", }, { .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT }, { .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 }, + { .compatible = "sunway,suntai-i2c", .data = (void *)MODEL_SUNWAY }, {}, }; MODULE_DEVICE_TABLE(of, dw_i2c_of_match); -- Gitee From 9e1581849d8c771191c4ed81bb5f1cea02ffee53 Mon Sep 17 00:00:00 2001 From: Yang Qiang Date: Wed, 26 Jul 2023 14:36:39 +0800 Subject: [PATCH 033/150] i2c: sunway: do some cleanup for old i2c driver Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CB2G -------------------------------- The driver for i2c-sunway-sw6* is no longer in use, so now ready to remove it. Signed-off-by: Yang Qiang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/i2c/busses/Kconfig | 9 - drivers/i2c/busses/Makefile | 4 - drivers/i2c/busses/i2c-sunway-sw6-common.c | 362 ---------- drivers/i2c/busses/i2c-sunway-sw6-core.h | 325 --------- drivers/i2c/busses/i2c-sunway-sw6-master.c | 755 -------------------- drivers/i2c/busses/i2c-sunway-sw6-platdrv.c | 498 ------------- 6 files changed, 1953 deletions(-) delete mode 100644 drivers/i2c/busses/i2c-sunway-sw6-common.c delete mode 100644 drivers/i2c/busses/i2c-sunway-sw6-core.h delete mode 100644 drivers/i2c/busses/i2c-sunway-sw6-master.c delete mode 100644 drivers/i2c/busses/i2c-sunway-sw6-platdrv.c diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index f745cd6d440f..a490ba486474 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -554,15 +554,6 @@ config I2C_DESIGNWARE_PLATFORM This driver can also be built as a module. If so, the module will be called i2c-designware-platform. -config I2C_SUNWAY_SW6 - tristate "SUNWAY SW6 I2C Controller " - depends on SW64_CHIP3 - default y if PLATFORM_XUELANG - help - If you say yes to this option, support will be included for the - Synopsys DesignWare I2C adapter in MCU of CHIP3. Only master mode - is supported. - config I2C_DESIGNWARE_AMDPSP bool "AMD PSP I2C semaphore support" depends on X86_MSR diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 1271a7511750..d274db87ecd1 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -51,10 +51,6 @@ obj-$(CONFIG_I2C_DAVINCI) += i2c-davinci.o obj-$(CONFIG_I2C_DESIGNWARE_CORE) += i2c-designware-core.o i2c-designware-core-y := i2c-designware-common.o i2c-designware-core-y += i2c-designware-master.o -obj-$(CONFIG_I2C_SUNWAY_SW6) += i2c-sunway-sw6.o -i2c-sunway-sw6-y := i2c-sunway-sw6-common.o -i2c-sunway-sw6-y += i2c-sunway-sw6-master.o -i2c-sunway-sw6-y += i2c-sunway-sw6-platdrv.o i2c-designware-core-$(CONFIG_I2C_DESIGNWARE_SLAVE) += i2c-designware-slave.o obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM) += i2c-designware-platform.o i2c-designware-platform-y := i2c-designware-platdrv.o diff --git a/drivers/i2c/busses/i2c-sunway-sw6-common.c b/drivers/i2c/busses/i2c-sunway-sw6-common.c deleted file mode 100644 index 80a59d199f7d..000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-common.c +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static char *abort_sources[] = { - [ABRT_7B_ADDR_NOACK] = - "slave address not acknowledged (7bit mode)", - [ABRT_10ADDR1_NOACK] = - "first address byte not acknowledged (10bit mode)", - [ABRT_10ADDR2_NOACK] = - "second address byte not acknowledged (10bit mode)", - [ABRT_TXDATA_NOACK] = - "data not acknowledged", - [ABRT_GCALL_NOACK] = - "no acknowledgement for a general call", - [ABRT_GCALL_READ] = - "read after general call", - [ABRT_SBYTE_ACKDET] = - "start byte acknowledged", - [ABRT_SBYTE_NORSTRT] = - "trying to send start byte when restart is disabled", - [ABRT_10B_RD_NORSTRT] = - "trying to read when restart is disabled (10bit mode)", - [ABRT_MASTER_DIS] = - "trying to use disabled adapter", - [ARB_LOST] = - "lost arbitration", - [ABRT_SLAVE_FLUSH_TXFIFO] = - "read command so flush old data in the TX FIFO", - [ABRT_SLAVE_ARBLOST] = - "slave lost the bus while transmitting data to a remote master", - [ABRT_SLAVE_RD_INTX] = - "incorrect slave-transmitter mode configuration", -}; - -u32 dw_readl(struct dw_i2c_dev *dev, int offset) -{ - u32 value; - - offset = offset << 7; - if (dev->flags & ACCESS_16BIT) - value = readw_relaxed(dev->base + offset) | - (readw_relaxed(dev->base + offset + 2) << 16); - else - value = readl_relaxed(dev->base + offset); - - if (dev->flags & ACCESS_SWAP) - return swab32(value); - else - return value; -} - -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset) -{ - offset = offset << 7; - if (dev->flags & ACCESS_SWAP) - b = swab32(b); - - if (dev->flags & ACCESS_16BIT) { - writew_relaxed((u16)b, dev->base + offset); - writew_relaxed((u16)(b >> 16), dev->base + offset + 2); - } else { - writel_relaxed(b, dev->base + offset); - } -} - -/** - * i2c_dw_set_reg_access() - Set register access flags - * @dev: device private data - * - * Autodetects needed register access mode and sets access flags accordingly. - * This must be called before doing any other register access. - */ -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev) -{ - u32 reg; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - reg = dw_readl(dev, DW_IC_COMP_TYPE); - i2c_dw_release_lock(dev); - - if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { - /* Configure register endianness access */ - dev->flags |= ACCESS_SWAP; - } else if (reg == (DW_IC_COMP_TYPE_VALUE & 0x0000ffff)) { - /* Configure register access mode 16bit */ - dev->flags |= ACCESS_16BIT; - } else if (reg != DW_IC_COMP_TYPE_VALUE) { - dev_err(dev->dev, - "Unknown Synopsys component type: 0x%08x\n", reg); - return -ENODEV; - } - - return 0; -} - -u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset) -{ - /* - * DesignWare I2C core doesn't seem to have solid strategy to meet - * the tHD;STA timing spec. Configuring _HCNT based on tHIGH spec - * will result in violation of the tHD;STA spec. - */ - if (cond) - /* - * Conditional expression: - * - * IC_[FS]S_SCL_HCNT + (1+4+3) >= IC_CLK * tHIGH - * - * This is based on the DW manuals, and represents an ideal - * configuration. The resulting I2C bus speed will be - * faster than any of the others. - * - * If your hardware is free from tHD;STA issue, try this one. - */ - return (ic_clk * tSYMBOL + 500000) / 1000000 - 8 + offset; - else - /* - * Conditional expression: - * - * IC_[FS]S_SCL_HCNT + 3 >= IC_CLK * (tHD;STA + tf) - * - * This is just experimental rule; the tHD;STA period turned - * out to be proportinal to (_HCNT + 3). With this setting, - * we could meet both tHIGH and tHD;STA timing specs. - * - * If unsure, you'd better to take this alternative. - * - * The reason why we need to take into account "tf" here, - * is the same as described in i2c_dw_scl_lcnt(). - */ - return (ic_clk * (tSYMBOL + tf) + 500000) / 1000000 - - 3 + offset; -} - -u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset) -{ - /* - * Conditional expression: - * - * IC_[FS]S_SCL_LCNT + 1 >= IC_CLK * (tLOW + tf) - * - * DW I2C core starts counting the SCL CNTs for the LOW period - * of the SCL clock (tLOW) as soon as it pulls the SCL line. - * In order to meet the tLOW timing spec, we need to take into - * account the fall time of SCL signal (tf). Default tf value - * should be 0.3 us, for safety. - */ - return ((ic_clk * (tLOW + tf) + 500000) / 1000000) - 1 + offset; -} - -int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev) -{ - u32 reg; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - /* Configure SDA Hold Time if required */ - reg = dw_readl(dev, DW_IC_COMP_VERSION); - if (reg >= DW_IC_SDA_HOLD_MIN_VERS) { - if (!dev->sda_hold_time) { - /* Keep previous hold time setting if no one set it */ - dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD); - } - - /* - * Workaround for avoiding TX arbitration lost in case I2C - * slave pulls SDA down "too quickly" after falling egde of - * SCL by enabling non-zero SDA RX hold. Specification says it - * extends incoming SDA low to high transition while SCL is - * high but it apprears to help also above issue. - */ - if (!(dev->sda_hold_time & DW_IC_SDA_HOLD_RX_MASK)) - dev->sda_hold_time |= 1 << DW_IC_SDA_HOLD_RX_SHIFT; - - dev_dbg(dev->dev, "SDA Hold Time TX:RX = %d:%d\n", - dev->sda_hold_time & ~(u32)DW_IC_SDA_HOLD_RX_MASK, - dev->sda_hold_time >> DW_IC_SDA_HOLD_RX_SHIFT); - } else if (dev->sda_hold_time) { - dev_warn(dev->dev, - "Hardware too old to adjust SDA hold time.\n"); - dev->sda_hold_time = 0; - } - - i2c_dw_release_lock(dev); - - return 0; -} - -void __i2c_dw_disable(struct dw_i2c_dev *dev) -{ - int timeout = 100; - - do { - __i2c_dw_disable_nowait(dev); - /* - * The enable status register may be unimplemented, but - * in that case this test reads zero and exits the loop. - */ - if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == 0) - return; - - /* - * Wait 10 times the signaling period of the highest I2C - * transfer supported by the driver (for 400KHz this is - * 25us) as described in the DesignWare I2C databook. - */ - usleep_range(25, 250); - } while (timeout--); - - dev_warn(dev->dev, "timeout in disabling adapter\n"); -} - -unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev) -{ - /* - * Clock is not necessary if we got LCNT/HCNT values directly from - * the platform code. - */ - if (WARN_ON_ONCE(!dev->get_clk_rate_khz)) - return 0; - return dev->get_clk_rate_khz(dev); -} - -int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare) -{ - if (IS_ERR(dev->clk)) - return PTR_ERR(dev->clk); - - if (prepare) - return clk_prepare_enable(dev->clk); - - clk_disable_unprepare(dev->clk); - return 0; -} -EXPORT_SYMBOL_GPL(i2c_dw_prepare_clk); - -int i2c_dw_acquire_lock(struct dw_i2c_dev *dev) -{ - int ret; - - if (!dev->acquire_lock) - return 0; - - ret = dev->acquire_lock(dev); - if (!ret) - return 0; - - dev_err(dev->dev, "couldn't acquire bus ownership\n"); - - return ret; -} - -void i2c_dw_release_lock(struct dw_i2c_dev *dev) -{ - if (dev->release_lock) - dev->release_lock(dev); -} - -/* - * Waiting for bus not busy - */ -int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev) -{ - int timeout = TIMEOUT; - - while (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) { - if (timeout <= 0) { - dev_warn(dev->dev, "timeout waiting for bus ready\n"); - i2c_recover_bus(&dev->adapter); - - if (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) - return -ETIMEDOUT; - return 0; - } - timeout--; - usleep_range(1000, 1100); - } - - return 0; -} - -int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev) -{ - unsigned long abort_source = dev->abort_source; - int i; - - if (abort_source & DW_IC_TX_ABRT_NOACK) { - for_each_set_bit(i, &abort_source, ARRAY_SIZE(abort_sources)) - dev_dbg(dev->dev, - "%s: %s\n", __func__, abort_sources[i]); - return -EREMOTEIO; - } - - for_each_set_bit(i, &abort_source, ARRAY_SIZE(abort_sources)) - dev_err(dev->dev, "%s: %s\n", __func__, abort_sources[i]); - - if (abort_source & DW_IC_TX_ARB_LOST) - return -EAGAIN; - else if (abort_source & DW_IC_TX_ABRT_GCALL_READ) - return -EINVAL; /* wrong msgs[] data */ - else - return -EIO; -} - -u32 i2c_dw_func(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - return dev->functionality; -} - -void i2c_dw_disable(struct dw_i2c_dev *dev) -{ - /* Disable controller */ - __i2c_dw_disable(dev); - - /* Disable all interrupts */ - dw_writel(dev, 0, DW_IC_INTR_MASK); - dw_readl(dev, DW_IC_CLR_INTR); -} - -void i2c_dw_disable_int(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 0, DW_IC_INTR_MASK); -} - -u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev) -{ - return dw_readl(dev, DW_IC_COMP_PARAM_1); -} -EXPORT_SYMBOL_GPL(i2c_dw_read_comp_param); - -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core"); -MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-sunway-sw6-core.h b/drivers/i2c/busses/i2c-sunway-sw6-core.h deleted file mode 100644 index 8fd0a0b4a5c9..000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-core.h +++ /dev/null @@ -1,325 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ - -#include -#include - -#define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C | \ - I2C_FUNC_SMBUS_BYTE | \ - I2C_FUNC_SMBUS_BYTE_DATA | \ - I2C_FUNC_SMBUS_WORD_DATA | \ - I2C_FUNC_SMBUS_BLOCK_DATA | \ - I2C_FUNC_SMBUS_I2C_BLOCK) - -#define DW_IC_CON_MASTER 0x1 -#define DW_IC_CON_SPEED_STD 0x2 -#define DW_IC_CON_SPEED_FAST 0x4 -#define DW_IC_CON_SPEED_HIGH 0x6 -#define DW_IC_CON_SPEED_MASK 0x6 -#define DW_IC_CON_10BITADDR_SLAVE 0x8 -#define DW_IC_CON_10BITADDR_MASTER 0x10 -#define DW_IC_CON_RESTART_EN 0x20 -#define DW_IC_CON_SLAVE_DISABLE 0x40 -#define DW_IC_CON_STOP_DET_IFADDRESSED 0x80 -#define DW_IC_CON_TX_EMPTY_CTRL 0x100 -#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL 0x200 - -/* - * Registers offset - */ -#define DW_IC_CON 0x0 -#define DW_IC_TAR 0x4 -#define DW_IC_SAR 0x8 -#define DW_IC_DATA_CMD 0x10 -#define DW_IC_SS_SCL_HCNT 0x14 -#define DW_IC_SS_SCL_LCNT 0x18 -#define DW_IC_FS_SCL_HCNT 0x1c -#define DW_IC_FS_SCL_LCNT 0x20 -#define DW_IC_HS_SCL_HCNT 0x24 -#define DW_IC_HS_SCL_LCNT 0x28 -#define DW_IC_INTR_STAT 0x2c -#define DW_IC_INTR_MASK 0x30 -#define DW_IC_RAW_INTR_STAT 0x34 -#define DW_IC_RX_TL 0x38 -#define DW_IC_TX_TL 0x3c -#define DW_IC_CLR_INTR 0x40 -#define DW_IC_CLR_RX_UNDER 0x44 -#define DW_IC_CLR_RX_OVER 0x48 -#define DW_IC_CLR_TX_OVER 0x4c -#define DW_IC_CLR_RD_REQ 0x50 -#define DW_IC_CLR_TX_ABRT 0x54 -#define DW_IC_CLR_RX_DONE 0x58 -#define DW_IC_CLR_ACTIVITY 0x5c -#define DW_IC_CLR_STOP_DET 0x60 -#define DW_IC_CLR_START_DET 0x64 -#define DW_IC_CLR_GEN_CALL 0x68 -#define DW_IC_ENABLE 0x6c -#define DW_IC_STATUS 0x70 -#define DW_IC_TXFLR 0x74 -#define DW_IC_RXFLR 0x78 -#define DW_IC_SDA_HOLD 0x7c -#define DW_IC_TX_ABRT_SOURCE 0x80 -#define DW_IC_ENABLE_STATUS 0x9c -#define DW_IC_CLR_RESTART_DET 0xa8 -#define DW_IC_COMP_PARAM_1 0xf4 -#define DW_IC_COMP_VERSION 0xf8 -#define DW_IC_SDA_HOLD_MIN_VERS 0x3131312A -#define DW_IC_COMP_TYPE 0xfc -#define DW_IC_COMP_TYPE_VALUE 0x44570140 - -#define DW_IC_INTR_RX_UNDER 0x001 -#define DW_IC_INTR_RX_OVER 0x002 -#define DW_IC_INTR_RX_FULL 0x004 -#define DW_IC_INTR_TX_OVER 0x008 -#define DW_IC_INTR_TX_EMPTY 0x010 -#define DW_IC_INTR_RD_REQ 0x020 -#define DW_IC_INTR_TX_ABRT 0x040 -#define DW_IC_INTR_RX_DONE 0x080 -#define DW_IC_INTR_ACTIVITY 0x100 -#define DW_IC_INTR_STOP_DET 0x200 -#define DW_IC_INTR_START_DET 0x400 -#define DW_IC_INTR_GEN_CALL 0x800 -#define DW_IC_INTR_RESTART_DET 0x1000 - -#define DW_IC_INTR_DEFAULT_MASK (DW_IC_INTR_RX_FULL | \ - DW_IC_INTR_TX_ABRT | \ - DW_IC_INTR_STOP_DET) -#define DW_IC_INTR_MASTER_MASK (DW_IC_INTR_DEFAULT_MASK | \ - DW_IC_INTR_TX_EMPTY) -#define DW_IC_INTR_SLAVE_MASK (DW_IC_INTR_DEFAULT_MASK | \ - DW_IC_INTR_RX_DONE | \ - DW_IC_INTR_RX_UNDER | \ - DW_IC_INTR_RD_REQ) - -#define DW_IC_STATUS_ACTIVITY 0x1 -#define DW_IC_STATUS_TFE BIT(2) -#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) -#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) - -#define DW_IC_SDA_HOLD_RX_SHIFT 16 -#define DW_IC_SDA_HOLD_RX_MASK GENMASK(23, DW_IC_SDA_HOLD_RX_SHIFT) - -#define DW_IC_ERR_TX_ABRT 0x1 - -#define DW_IC_TAR_10BITADDR_MASTER BIT(12) - -#define DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH (BIT(2) | BIT(3)) -#define DW_IC_COMP_PARAM_1_SPEED_MODE_MASK GENMASK(3, 2) - -/* - * status codes - */ -#define STATUS_IDLE 0x0 -#define STATUS_WRITE_IN_PROGRESS 0x1 -#define STATUS_READ_IN_PROGRESS 0x2 - -#define TIMEOUT 20 /* ms */ - -/* - * operation modes - */ -#define DW_IC_MASTER 0 -#define DW_IC_SLAVE 1 - -/* - * Hardware abort codes from the DW_IC_TX_ABRT_SOURCE register - * - * Only expected abort codes are listed here - * refer to the datasheet for the full list - */ -#define ABRT_7B_ADDR_NOACK 0 -#define ABRT_10ADDR1_NOACK 1 -#define ABRT_10ADDR2_NOACK 2 -#define ABRT_TXDATA_NOACK 3 -#define ABRT_GCALL_NOACK 4 -#define ABRT_GCALL_READ 5 -#define ABRT_SBYTE_ACKDET 7 -#define ABRT_SBYTE_NORSTRT 9 -#define ABRT_10B_RD_NORSTRT 10 -#define ABRT_MASTER_DIS 11 -#define ARB_LOST 12 -#define ABRT_SLAVE_FLUSH_TXFIFO 13 -#define ABRT_SLAVE_ARBLOST 14 -#define ABRT_SLAVE_RD_INTX 15 - -#define DW_IC_TX_ABRT_7B_ADDR_NOACK (1UL << ABRT_7B_ADDR_NOACK) -#define DW_IC_TX_ABRT_10ADDR1_NOACK (1UL << ABRT_10ADDR1_NOACK) -#define DW_IC_TX_ABRT_10ADDR2_NOACK (1UL << ABRT_10ADDR2_NOACK) -#define DW_IC_TX_ABRT_TXDATA_NOACK (1UL << ABRT_TXDATA_NOACK) -#define DW_IC_TX_ABRT_GCALL_NOACK (1UL << ABRT_GCALL_NOACK) -#define DW_IC_TX_ABRT_GCALL_READ (1UL << ABRT_GCALL_READ) -#define DW_IC_TX_ABRT_SBYTE_ACKDET (1UL << ABRT_SBYTE_ACKDET) -#define DW_IC_TX_ABRT_SBYTE_NORSTRT (1UL << ABRT_SBYTE_NORSTRT) -#define DW_IC_TX_ABRT_10B_RD_NORSTRT (1UL << ABRT_10B_RD_NORSTRT) -#define DW_IC_TX_ABRT_MASTER_DIS (1UL << ABRT_MASTER_DIS) -#define DW_IC_TX_ARB_LOST (1UL << ARB_LOST) -#define DW_IC_RX_ABRT_SLAVE_RD_INTX (1UL << ABRT_SLAVE_RD_INTX) -#define DW_IC_RX_ABRT_SLAVE_ARBLOST (1UL << ABRT_SLAVE_ARBLOST) -#define DW_IC_RX_ABRT_SLAVE_FLUSH_TXFIFO (1UL << ABRT_SLAVE_FLUSH_TXFIFO) - -#define DW_IC_TX_ABRT_NOACK (DW_IC_TX_ABRT_7B_ADDR_NOACK | \ - DW_IC_TX_ABRT_10ADDR1_NOACK | \ - DW_IC_TX_ABRT_10ADDR2_NOACK | \ - DW_IC_TX_ABRT_TXDATA_NOACK | \ - DW_IC_TX_ABRT_GCALL_NOACK) - - -/** - * struct dw_i2c_dev - private i2c-designware data - * @dev: driver model device node - * @base: IO registers pointer - * @cmd_complete: tx completion indicator - * @clk: input reference clock - * @slave: represent an I2C slave device - * @cmd_err: run time hadware error code - * @msgs: points to an array of messages currently being transferred - * @msgs_num: the number of elements in msgs - * @msg_write_idx: the element index of the current tx message in the msgs - * array - * @tx_buf_len: the length of the current tx buffer - * @tx_buf: the current tx buffer - * @msg_read_idx: the element index of the current rx message in the msgs - * array - * @rx_buf_len: the length of the current rx buffer - * @rx_buf: the current rx buffer - * @msg_err: error status of the current transfer - * @status: i2c master status, one of STATUS_* - * @abort_source: copy of the TX_ABRT_SOURCE register - * @irq: interrupt number for the i2c master - * @adapter: i2c subsystem adapter node - * @slave_cfg: configuration for the slave device - * @tx_fifo_depth: depth of the hardware tx fifo - * @rx_fifo_depth: depth of the hardware rx fifo - * @rx_outstanding: current master-rx elements in tx fifo - * @timings: bus clock frequency, SDA hold and other timings - * @sda_hold_time: SDA hold value - * @ss_hcnt: standard speed HCNT value - * @ss_lcnt: standard speed LCNT value - * @fs_hcnt: fast speed HCNT value - * @fs_lcnt: fast speed LCNT value - * @fp_hcnt: fast plus HCNT value - * @fp_lcnt: fast plus LCNT value - * @hs_hcnt: high speed HCNT value - * @hs_lcnt: high speed LCNT value - * @pm_qos: pm_qos_request used while holding a hardware lock on the bus - * @acquire_lock: function to acquire a hardware lock on the bus - * @release_lock: function to release a hardware lock on the bus - * @pm_disabled: true if power-management should be disabled for this i2c-bus - * @disable: function to disable the controller - * @disable_int: function to disable all interrupts - * @init: function to initialize the I2C hardware - * @mode: operation mode - DW_IC_MASTER or DW_IC_SLAVE - * - * HCNT and LCNT parameters can be used if the platform knows more accurate - * values than the one computed based only on the input clock frequency. - * Leave them to be %0 if not used. - */ -struct dw_i2c_dev { - struct device *dev; - void __iomem *base; - struct completion cmd_complete; - struct clk *clk; - struct reset_control *rst; - struct i2c_client *slave; - u32 (*get_clk_rate_khz)(struct dw_i2c_dev *dev); - struct dw_pci_controller *controller; - int cmd_err; - struct i2c_msg *msgs; - int msgs_num; - int msg_write_idx; - u32 tx_buf_len; - u8 *tx_buf; - int msg_read_idx; - u32 rx_buf_len; - u8 *rx_buf; - int msg_err; - unsigned int status; - u32 abort_source; - int irq; - u32 flags; - struct i2c_adapter adapter; - u32 functionality; - u32 master_cfg; - u32 slave_cfg; - unsigned int tx_fifo_depth; - unsigned int rx_fifo_depth; - int rx_outstanding; - struct i2c_timings timings; - u32 sda_hold_time; - u32 intr_mask_value; - u16 ss_hcnt; - u16 ss_lcnt; - u16 fs_hcnt; - u16 fs_lcnt; - u16 fp_hcnt; - u16 fp_lcnt; - u16 hs_hcnt; - u16 hs_lcnt; - struct pm_qos_request pm_qos; - int (*acquire_lock)(struct dw_i2c_dev *dev); - void (*release_lock)(struct dw_i2c_dev *dev); - bool pm_disabled; - void (*disable)(struct dw_i2c_dev *dev); - void (*disable_int)(struct dw_i2c_dev *dev); - int (*init)(struct dw_i2c_dev *dev); - int mode; - struct i2c_bus_recovery_info rinfo; -}; - -#define ACCESS_SWAP 0x00000001 -#define ACCESS_16BIT 0x00000002 -#define ACCESS_INTR_MASK 0x00000004 - -#define MODEL_CHERRYTRAIL 0x00000100 - -u32 dw_readl(struct dw_i2c_dev *dev, int offset); -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset); -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev); -u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset); -u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset); -int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev); -unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev); -int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare); -int i2c_dw_acquire_lock(struct dw_i2c_dev *dev); -void i2c_dw_release_lock(struct dw_i2c_dev *dev); -int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev); -int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev); -u32 i2c_dw_func(struct i2c_adapter *adap); -void i2c_dw_disable(struct dw_i2c_dev *dev); -void i2c_dw_disable_int(struct dw_i2c_dev *dev); - -static inline void __i2c_dw_enable(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 1, DW_IC_ENABLE); -} - -static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) -{ - dw_writel(dev, 0, DW_IC_ENABLE); -} - -void __i2c_dw_disable(struct dw_i2c_dev *dev); - -extern u32 i2c_dw_read_comp_param(struct dw_i2c_dev *dev); -extern int i2c_dw_probe(struct dw_i2c_dev *dev); -#if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE) -extern int i2c_dw_probe_slave(struct dw_i2c_dev *dev); -#else -static inline int i2c_dw_probe_slave(struct dw_i2c_dev *dev) { return -EINVAL; } -#endif - -#if IS_ENABLED(CONFIG_I2C_DESIGNWARE_BAYTRAIL) -extern int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev); -extern void i2c_dw_remove_lock_support(struct dw_i2c_dev *dev); -#else -static inline int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev) { return 0; } -static inline void i2c_dw_remove_lock_support(struct dw_i2c_dev *dev) {} -#endif diff --git a/drivers/i2c/busses/i2c-sunway-sw6-master.c b/drivers/i2c/busses/i2c-sunway-sw6-master.c deleted file mode 100644 index eccfd338ab36..000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-master.c +++ /dev/null @@ -1,755 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver (master only). - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static void i2c_dw_configure_fifo_master(struct dw_i2c_dev *dev) -{ - /* Configure Tx/Rx FIFO threshold levels */ - dw_writel(dev, dev->tx_fifo_depth / 2, DW_IC_TX_TL); - dw_writel(dev, 0, DW_IC_RX_TL); - - /* Configure the I2C master */ - dw_writel(dev, dev->master_cfg, DW_IC_CON); -} - -static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) -{ - const char *mode_str, *fp_str = ""; - u32 comp_param1; - u32 sda_falling_time, scl_falling_time; - struct i2c_timings *t = &dev->timings; - u32 ic_clk; - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1); - i2c_dw_release_lock(dev); - - /* Set standard and fast speed dividers for high/low periods */ - sda_falling_time = t->sda_fall_ns ?: 300; /* ns */ - scl_falling_time = t->scl_fall_ns ?: 300; /* ns */ - - /* Calculate SCL timing parameters for standard mode if not set */ - if (!dev->ss_hcnt || !dev->ss_lcnt) { - ic_clk = i2c_dw_clk_rate(dev); - dev->ss_hcnt = - i2c_dw_scl_hcnt(ic_clk, - 4000, /* tHD;STA = tHIGH = 4.0 us */ - sda_falling_time, - 0, /* 0: DW default, 1: Ideal */ - 0); /* No offset */ - dev->ss_lcnt = - i2c_dw_scl_lcnt(ic_clk, - 4700, /* tLOW = 4.7 us */ - scl_falling_time, - 0); /* No offset */ - } - dev_dbg(dev->dev, "Standard Mode HCNT:LCNT = %d:%d\n", - dev->ss_hcnt, dev->ss_lcnt); - - /* - * Set SCL timing parameters for fast mode or fast mode plus. Only - * difference is the timing parameter values since the registers are - * the same. - */ - if (t->bus_freq_hz == 1000000) { - /* - * Check are fast mode plus parameters available and use - * fast mode if not. - */ - if (dev->fp_hcnt && dev->fp_lcnt) { - dev->fs_hcnt = dev->fp_hcnt; - dev->fs_lcnt = dev->fp_lcnt; - fp_str = " Plus"; - } - } - /* - * Calculate SCL timing parameters for fast mode if not set. They are - * needed also in high speed mode. - */ - if (!dev->fs_hcnt || !dev->fs_lcnt) { - ic_clk = i2c_dw_clk_rate(dev); - dev->fs_hcnt = - i2c_dw_scl_hcnt(ic_clk, - 600, /* tHD;STA = tHIGH = 0.6 us */ - sda_falling_time, - 0, /* 0: DW default, 1: Ideal */ - 0); /* No offset */ - dev->fs_lcnt = - i2c_dw_scl_lcnt(ic_clk, - 1300, /* tLOW = 1.3 us */ - scl_falling_time, - 0); /* No offset */ - } - dev_dbg(dev->dev, "Fast Mode%s HCNT:LCNT = %d:%d\n", - fp_str, dev->fs_hcnt, dev->fs_lcnt); - - /* Check is high speed possible and fall back to fast mode if not */ - if ((dev->master_cfg & DW_IC_CON_SPEED_MASK) == - DW_IC_CON_SPEED_HIGH) { - if ((comp_param1 & DW_IC_COMP_PARAM_1_SPEED_MODE_MASK) - != DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH) { - dev_err(dev->dev, "High Speed not supported!\n"); - dev->master_cfg &= ~DW_IC_CON_SPEED_MASK; - dev->master_cfg |= DW_IC_CON_SPEED_FAST; - dev->hs_hcnt = 0; - dev->hs_lcnt = 0; - } else if (dev->hs_hcnt && dev->hs_lcnt) { - dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n", - dev->hs_hcnt, dev->hs_lcnt); - } - } - - ret = i2c_dw_set_sda_hold(dev); - if (ret) - goto out; - - switch (dev->master_cfg & DW_IC_CON_SPEED_MASK) { - case DW_IC_CON_SPEED_STD: - mode_str = "Standard Mode"; - break; - case DW_IC_CON_SPEED_HIGH: - mode_str = "High Speed Mode"; - break; - default: - mode_str = "Fast Mode"; - } - dev_dbg(dev->dev, "Bus speed: %s%s\n", mode_str, fp_str); - -out: - return ret; -} - -/** - * i2c_dw_init() - Initialize the designware I2C master hardware - * @dev: device private data - * - * This functions configures and enables the I2C master. - * This function is called during I2C init function, and in case of timeout at - * run time. - */ -static int i2c_dw_init_master(struct dw_i2c_dev *dev) -{ - int ret; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - return ret; - - /* Disable the adapter */ - __i2c_dw_disable(dev); - - /* Write standard speed timing parameters */ - dw_writel(dev, dev->ss_hcnt, DW_IC_SS_SCL_HCNT); - dw_writel(dev, dev->ss_lcnt, DW_IC_SS_SCL_LCNT); - - /* Write fast mode/fast mode plus timing parameters */ - dw_writel(dev, dev->fs_hcnt, DW_IC_FS_SCL_HCNT); - dw_writel(dev, dev->fs_lcnt, DW_IC_FS_SCL_LCNT); - - /* Write high speed timing parameters if supported */ - if (dev->hs_hcnt && dev->hs_lcnt) { - dw_writel(dev, dev->hs_hcnt, DW_IC_HS_SCL_HCNT); - dw_writel(dev, dev->hs_lcnt, DW_IC_HS_SCL_LCNT); - } - - /* Write SDA hold time if supported */ - if (dev->sda_hold_time) - dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); - - i2c_dw_configure_fifo_master(dev); - i2c_dw_release_lock(dev); - - return 0; -} - -static void i2c_dw_xfer_init(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - u32 ic_con, ic_tar = 0; - - /* Disable the adapter */ - __i2c_dw_disable(dev); - - /* If the slave address is ten bit address, enable 10BITADDR */ - ic_con = dw_readl(dev, DW_IC_CON); - if (msgs[dev->msg_write_idx].flags & I2C_M_TEN) { - ic_con |= DW_IC_CON_10BITADDR_MASTER; - /* - * If I2C_DYNAMIC_TAR_UPDATE is set, the 10-bit addressing - * mode has to be enabled via bit 12 of IC_TAR register. - * We set it always as I2C_DYNAMIC_TAR_UPDATE can't be - * detected from registers. - */ - ic_tar = DW_IC_TAR_10BITADDR_MASTER; - } else { - ic_con &= ~DW_IC_CON_10BITADDR_MASTER; - } - - dw_writel(dev, ic_con, DW_IC_CON); - - /* - * Set the slave (target) address and enable 10-bit addressing mode - * if applicable. - */ - dw_writel(dev, msgs[dev->msg_write_idx].addr | ic_tar, DW_IC_TAR); - - /* Enforce disabled interrupts (due to HW issues) */ - i2c_dw_disable_int(dev); - - /* Enable the adapter */ - __i2c_dw_enable(dev); - - /* Dummy read to avoid the register getting stuck on Bay Trail */ - dw_readl(dev, DW_IC_ENABLE_STATUS); - - /* Clear and enable interrupts */ - dw_readl(dev, DW_IC_CLR_INTR); - dev->intr_mask_value = DW_IC_INTR_DEFAULT_MASK; - dw_writel(dev, DW_IC_INTR_MASTER_MASK, DW_IC_INTR_MASK); -} - -/* - * Initiate (and continue) low level master read/write transaction. - * This function is only called from i2c_dw_isr, and pumping i2c_msg - * messages into the tx buffer. Even if the size of i2c_msg data is - * longer than the size of the tx buffer, it handles everything. - */ -static void -i2c_dw_xfer_msg(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - u32 intr_mask; - int tx_limit, rx_limit; - u32 addr = msgs[dev->msg_write_idx].addr; - u32 buf_len = dev->tx_buf_len; - u8 *buf = dev->tx_buf; - bool need_restart = false; - - intr_mask = DW_IC_INTR_MASTER_MASK; - - for (; dev->msg_write_idx < dev->msgs_num; dev->msg_write_idx++) { - u32 flags = msgs[dev->msg_write_idx].flags; - - /* - * If target address has changed, we need to - * reprogram the target address in the I2C - * adapter when we are done with this transfer. - */ - if (msgs[dev->msg_write_idx].addr != addr) { - dev_err(dev->dev, - "%s: invalid target address\n", __func__); - dev->msg_err = -EINVAL; - break; - } - - if (!(dev->status & STATUS_WRITE_IN_PROGRESS)) { - /* new i2c_msg */ - buf = msgs[dev->msg_write_idx].buf; - buf_len = msgs[dev->msg_write_idx].len; - - /* If both IC_EMPTYFIFO_HOLD_MASTER_EN and - * IC_RESTART_EN are set, we must manually - * set restart bit between messages. - */ - if ((dev->master_cfg & DW_IC_CON_RESTART_EN) && - (dev->msg_write_idx > 0)) - need_restart = true; - } - - tx_limit = dev->tx_fifo_depth - dw_readl(dev, DW_IC_TXFLR); - rx_limit = dev->rx_fifo_depth - dw_readl(dev, DW_IC_RXFLR); - - while (buf_len > 0 && tx_limit > 0 && rx_limit > 0) { - u32 cmd = 0; - - /* - * If IC_EMPTYFIFO_HOLD_MASTER_EN is set we must - * manually set the stop bit. However, it cannot be - * detected from the registers so we set it always - * when writing/reading the last byte. - */ - - /* - * i2c-core always sets the buffer length of - * I2C_FUNC_SMBUS_BLOCK_DATA to 1. The length will - * be adjusted when receiving the first byte. - * Thus we can't stop the transaction here. - */ - if (dev->msg_write_idx == dev->msgs_num - 1 && - buf_len == 1 && !(flags & I2C_M_RECV_LEN)) - cmd |= BIT(9); - - if (need_restart) { - cmd |= BIT(10); - need_restart = false; - } - - if (msgs[dev->msg_write_idx].flags & I2C_M_RD) { - - /* Avoid rx buffer overrun */ - if (dev->rx_outstanding >= dev->rx_fifo_depth) - break; - - dw_writel(dev, cmd | 0x100, DW_IC_DATA_CMD); - rx_limit--; - dev->rx_outstanding++; - } else - dw_writel(dev, cmd | *buf++, DW_IC_DATA_CMD); - tx_limit--; buf_len--; - } - - dev->tx_buf = buf; - dev->tx_buf_len = buf_len; - - /* - * Because we don't know the buffer length in the - * I2C_FUNC_SMBUS_BLOCK_DATA case, we can't stop - * the transaction here. - */ - if (buf_len > 0 || flags & I2C_M_RECV_LEN) { - /* more bytes to be written */ - dev->status |= STATUS_WRITE_IN_PROGRESS; - break; - } else - dev->status &= ~STATUS_WRITE_IN_PROGRESS; - } - - /* - * If i2c_msg index search is completed, we don't need TX_EMPTY - * interrupt any more. - */ - if (dev->msg_write_idx == dev->msgs_num) - intr_mask &= ~DW_IC_INTR_TX_EMPTY; - - if (dev->msg_err) - intr_mask = 0; - - dev->intr_mask_value = intr_mask; -} - -static u8 -i2c_dw_recv_len(struct dw_i2c_dev *dev, u8 len) -{ - struct i2c_msg *msgs = dev->msgs; - u32 flags = msgs[dev->msg_read_idx].flags; - - /* - * Adjust the buffer length and mask the flag - * after receiving the first byte. - */ - len += (flags & I2C_CLIENT_PEC) ? 2 : 1; - dev->tx_buf_len = len - min_t(u8, len, dev->rx_outstanding); - msgs[dev->msg_read_idx].len = len; - msgs[dev->msg_read_idx].flags &= ~I2C_M_RECV_LEN; - - return len; -} - -static void -i2c_dw_read(struct dw_i2c_dev *dev) -{ - struct i2c_msg *msgs = dev->msgs; - int rx_valid; - - for (; dev->msg_read_idx < dev->msgs_num; dev->msg_read_idx++) { - u32 len; - u8 *buf; - - if (!(msgs[dev->msg_read_idx].flags & I2C_M_RD)) - continue; - - if (!(dev->status & STATUS_READ_IN_PROGRESS)) { - len = msgs[dev->msg_read_idx].len; - buf = msgs[dev->msg_read_idx].buf; - } else { - len = dev->rx_buf_len; - buf = dev->rx_buf; - } - - rx_valid = dw_readl(dev, DW_IC_RXFLR); - - for (; len > 0 && rx_valid > 0; len--, rx_valid--) { - u32 flags = msgs[dev->msg_read_idx].flags; - - *buf = dw_readl(dev, DW_IC_DATA_CMD); - /* Ensure length byte is a valid value */ - if (flags & I2C_M_RECV_LEN && - *buf <= I2C_SMBUS_BLOCK_MAX && *buf > 0) { - len = i2c_dw_recv_len(dev, *buf); - } - buf++; - dev->rx_outstanding--; - } - - if (len > 0) { - dev->status |= STATUS_READ_IN_PROGRESS; - dev->rx_buf_len = len; - dev->rx_buf = buf; - return; - } else - dev->status &= ~STATUS_READ_IN_PROGRESS; - } -} - -/* - * Prepare controller for a transaction and call i2c_dw_xfer_msg. - */ -static int -i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - int ret; - - dev_dbg(dev->dev, "%s: msgs: %d\n", __func__, num); - - pm_runtime_get_sync(dev->dev); - - reinit_completion(&dev->cmd_complete); - dev->msgs = msgs; - dev->msgs_num = num; - dev->cmd_err = 0; - dev->msg_write_idx = 0; - dev->msg_read_idx = 0; - dev->msg_err = 0; - dev->status = STATUS_IDLE; - dev->abort_source = 0; - dev->rx_outstanding = 0; - - ret = i2c_dw_acquire_lock(dev); - if (ret) - goto done_nolock; - - ret = i2c_dw_wait_bus_not_busy(dev); - if (ret < 0) - goto done; - - /* Start the transfers */ - i2c_dw_xfer_init(dev); - - /* Wait for tx to complete */ - if (!wait_for_completion_timeout(&dev->cmd_complete, adap->timeout)) { - dev_err(dev->dev, "controller timed out\n"); - /* i2c_dw_init implicitly disables the adapter */ - i2c_recover_bus(&dev->adapter); - i2c_dw_init_master(dev); - ret = -ETIMEDOUT; - goto done; - } - - /* - * We must disable the adapter before returning and signaling the end - * of the current transfer. Otherwise the hardware might continue - * generating interrupts which in turn causes a race condition with - * the following transfer. Needs some more investigation if the - * additional interrupts are a hardware bug or this driver doesn't - * handle them correctly yet. - */ - __i2c_dw_disable_nowait(dev); - - if (dev->msg_err) { - ret = dev->msg_err; - goto done; - } - - /* No error */ - if (likely(!dev->cmd_err && !dev->status)) { - ret = num; - goto done; - } - - /* We have an error */ - if (dev->cmd_err == DW_IC_ERR_TX_ABRT) { - ret = i2c_dw_handle_tx_abort(dev); - goto done; - } - - if (dev->status) - dev_err(dev->dev, - "transfer terminated early - interrupt latency too high?\n"); - - ret = -EIO; - -done: - i2c_dw_release_lock(dev); - -done_nolock: - pm_runtime_mark_last_busy(dev->dev); - pm_runtime_put_autosuspend(dev->dev); - - return ret; -} - -static const struct i2c_algorithm i2c_dw_algo = { - .master_xfer = i2c_dw_xfer, - .functionality = i2c_dw_func, -}; - -static const struct i2c_adapter_quirks i2c_dw_quirks = { - .flags = I2C_AQ_NO_ZERO_LEN, -}; - -static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev) -{ - u32 stat; - - /* - * The IC_INTR_STAT register just indicates "enabled" interrupts. - * Ths unmasked raw version of interrupt status bits are available - * in the IC_RAW_INTR_STAT register. - * - * That is, - * stat = dw_readl(IC_INTR_STAT); - * equals to, - * stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK); - * - * The raw version might be useful for debugging purposes. - */ - stat = dw_readl(dev, DW_IC_INTR_STAT); - /* Disable all interrupts */ - dw_writel(dev, 0, DW_IC_INTR_MASK); - stat |= dw_readl(dev, DW_IC_RAW_INTR_STAT) & dev->intr_mask_value; /* Avoid to miss interrupt! */ - - - /* - * Do not use the IC_CLR_INTR register to clear interrupts, or - * you'll miss some interrupts, triggered during the period from - * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR). - * - * Instead, use the separately-prepared IC_CLR_* registers. - */ - if (stat & DW_IC_INTR_RX_UNDER) - dw_readl(dev, DW_IC_CLR_RX_UNDER); - if (stat & DW_IC_INTR_RX_OVER) - dw_readl(dev, DW_IC_CLR_RX_OVER); - if (stat & DW_IC_INTR_TX_OVER) - dw_readl(dev, DW_IC_CLR_TX_OVER); - if (stat & DW_IC_INTR_RD_REQ) - dw_readl(dev, DW_IC_CLR_RD_REQ); - if (stat & DW_IC_INTR_TX_ABRT) { - /* - * The IC_TX_ABRT_SOURCE register is cleared whenever - * the IC_CLR_TX_ABRT is read. Preserve it beforehand. - */ - dev->abort_source = dw_readl(dev, DW_IC_TX_ABRT_SOURCE); - dw_readl(dev, DW_IC_CLR_TX_ABRT); - } - if (stat & DW_IC_INTR_RX_DONE) - dw_readl(dev, DW_IC_CLR_RX_DONE); - if (stat & DW_IC_INTR_ACTIVITY) - dw_readl(dev, DW_IC_CLR_ACTIVITY); - if (stat & DW_IC_INTR_STOP_DET) - dw_readl(dev, DW_IC_CLR_STOP_DET); - if (stat & DW_IC_INTR_START_DET) - dw_readl(dev, DW_IC_CLR_START_DET); - if (stat & DW_IC_INTR_GEN_CALL) - dw_readl(dev, DW_IC_CLR_GEN_CALL); - - return stat; -} - -/* - * Interrupt service routine. This gets called whenever an I2C master interrupt - * occurs. - */ -static int i2c_dw_irq_handler_master(struct dw_i2c_dev *dev) -{ - u32 stat; - - stat = i2c_dw_read_clear_intrbits(dev); - if (stat & DW_IC_INTR_TX_ABRT) { - dev->cmd_err |= DW_IC_ERR_TX_ABRT; - dev->status = STATUS_IDLE; - - /* - * Anytime TX_ABRT is set, the contents of the tx/rx - * buffers are flushed. Make sure to skip them. - */ - dev->intr_mask_value = 0; - goto tx_aborted; - } - - if (stat & DW_IC_INTR_RX_FULL) - i2c_dw_read(dev); - - if (stat & DW_IC_INTR_TX_EMPTY) - i2c_dw_xfer_msg(dev); - - /* - * No need to modify or disable the interrupt mask here. - * i2c_dw_xfer_msg() will take care of it according to - * the current transmit status. - */ - -tx_aborted: - if ((stat & (DW_IC_INTR_TX_ABRT | DW_IC_INTR_STOP_DET)) || dev->msg_err) - complete(&dev->cmd_complete); - else if (unlikely(dev->flags & ACCESS_INTR_MASK)) { - /* Workaround to trigger pending interrupt */ - stat = dw_readl(dev, DW_IC_INTR_MASK); - i2c_dw_disable_int(dev); - dw_writel(dev, stat, DW_IC_INTR_MASK); - return 0; - } - - dw_writel(dev, dev->intr_mask_value, DW_IC_INTR_MASK); - return 0; - -} - -static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id) -{ - struct dw_i2c_dev *dev = dev_id; - u32 stat, enabled; - - enabled = dw_readl(dev, DW_IC_ENABLE); - stat = dw_readl(dev, DW_IC_RAW_INTR_STAT); - dev_dbg(dev->dev, "enabled=%#x stat=%#x\n", enabled, stat); - if (!enabled || !(stat & ~DW_IC_INTR_ACTIVITY)) - return IRQ_NONE; - - i2c_dw_irq_handler_master(dev); - - return IRQ_HANDLED; -} - -static void i2c_dw_prepare_recovery(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - i2c_dw_disable(dev); - reset_control_assert(dev->rst); - i2c_dw_prepare_clk(dev, false); -} - -static void i2c_dw_unprepare_recovery(struct i2c_adapter *adap) -{ - struct dw_i2c_dev *dev = i2c_get_adapdata(adap); - - i2c_dw_prepare_clk(dev, true); - reset_control_deassert(dev->rst); - i2c_dw_init_master(dev); -} - -static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev) -{ - struct i2c_bus_recovery_info *rinfo = &dev->rinfo; - struct i2c_adapter *adap = &dev->adapter; - struct gpio_desc *gpio; - int r; - - gpio = devm_gpiod_get(dev->dev, "scl", GPIOD_OUT_HIGH); - if (IS_ERR(gpio)) { - r = PTR_ERR(gpio); - if (r == -ENOENT || r == -ENOSYS) - return 0; - return r; - } - rinfo->scl_gpiod = gpio; - - gpio = devm_gpiod_get_optional(dev->dev, "sda", GPIOD_IN); - if (IS_ERR(gpio)) - return PTR_ERR(gpio); - rinfo->sda_gpiod = gpio; - - rinfo->recover_bus = i2c_generic_scl_recovery; - rinfo->prepare_recovery = i2c_dw_prepare_recovery; - rinfo->unprepare_recovery = i2c_dw_unprepare_recovery; - adap->bus_recovery_info = rinfo; - - dev_info(dev->dev, "running with gpio recovery mode! scl%s", - rinfo->sda_gpiod ? ",sda" : ""); - - return 0; -} - -int i2c_dw_probe(struct dw_i2c_dev *dev) -{ - struct i2c_adapter *adap = &dev->adapter; - unsigned long irq_flags; - int ret; - - init_completion(&dev->cmd_complete); - - dev->init = i2c_dw_init_master; - dev->disable = i2c_dw_disable; - dev->disable_int = i2c_dw_disable_int; - - ret = i2c_dw_set_reg_access(dev); - if (ret) - return ret; - - ret = i2c_dw_set_timings_master(dev); - if (ret) - return ret; - - ret = dev->init(dev); - if (ret) - return ret; - - snprintf(adap->name, sizeof(adap->name), - "Synopsys DesignWare I2C adapter"); - adap->retries = 3; - adap->algo = &i2c_dw_algo; - adap->quirks = &i2c_dw_quirks; - adap->dev.parent = dev->dev; - i2c_set_adapdata(adap, dev); - - if (dev->pm_disabled) - irq_flags = IRQF_NO_SUSPEND; - else - irq_flags = IRQF_SHARED | IRQF_COND_SUSPEND; - - i2c_dw_disable_int(dev); - ret = devm_request_irq(dev->dev, dev->irq, i2c_dw_isr, irq_flags, - dev_name(dev->dev), dev); - if (ret) { - dev_err(dev->dev, "failure requesting irq %i: %d\n", - dev->irq, ret); - return ret; - } - - ret = i2c_dw_init_recovery_info(dev); - if (ret) - return ret; - - /* - * Increment PM usage count during adapter registration in order to - * avoid possible spurious runtime suspend when adapter device is - * registered to the device core and immediate resume in case bus has - * registered I2C slaves that do I2C transfers in their probe. - */ - pm_runtime_get_noresume(dev->dev); - ret = i2c_add_numbered_adapter(adap); - if (ret) - dev_err(dev->dev, "failure adding adapter: %d\n", ret); - pm_runtime_put_noidle(dev->dev); - - return ret; -} -EXPORT_SYMBOL_GPL(i2c_dw_probe); - -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus master adapter"); -MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c b/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c deleted file mode 100644 index fd757e243933..000000000000 --- a/drivers/i2c/busses/i2c-sunway-sw6-platdrv.c +++ /dev/null @@ -1,498 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Synopsys DesignWare I2C adapter driver. - * - * Based on the TI DAVINCI I2C adapter driver. - * - * Copyright (C) 2006 Texas Instruments. - * Copyright (C) 2007 MontaVista Software Inc. - * Copyright (C) 2009 Provigent Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "i2c-sunway-sw6-core.h" - -static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev) -{ - return clk_get_rate(dev->clk)/1000; -} - -#ifdef CONFIG_ACPI -/* - * The HCNT/LCNT information coming from ACPI should be the most accurate - * for given platform. However, some systems get it wrong. On such systems - * we get better results by calculating those based on the input clock. - */ -static const struct dmi_system_id dw_i2c_no_acpi_params[] = { - { - .ident = "Dell Inspiron 7348", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"), - }, - }, - { } -}; - -static void dw_i2c_acpi_params(struct platform_device *pdev, char method[], - u16 *hcnt, u16 *lcnt, u32 *sda_hold) -{ - struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; - acpi_handle handle = ACPI_HANDLE(&pdev->dev); - union acpi_object *obj; - - if (dmi_check_system(dw_i2c_no_acpi_params)) - return; - - if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf))) - return; - - obj = (union acpi_object *)buf.pointer; - if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) { - const union acpi_object *objs = obj->package.elements; - - *hcnt = (u16)objs[0].integer.value; - *lcnt = (u16)objs[1].integer.value; - *sda_hold = (u32)objs[2].integer.value; - } - - kfree(buf.pointer); -} - -static int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - struct i2c_timings *t = &dev->timings; - u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0; - acpi_handle handle = ACPI_HANDLE(&pdev->dev); - const struct acpi_device_id *id; - struct acpi_device *adev; - const char *uid; - - dev->adapter.nr = -1; - dev->tx_fifo_depth = 32; - dev->rx_fifo_depth = 32; - - /* - * Try to get SDA hold time and *CNT values from an ACPI method for - * selected speed modes. - */ - dw_i2c_acpi_params(pdev, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht); - dw_i2c_acpi_params(pdev, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht); - dw_i2c_acpi_params(pdev, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht); - dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht); - - switch (t->bus_freq_hz) { - case 100000: - dev->sda_hold_time = ss_ht; - break; - case 1000000: - dev->sda_hold_time = fp_ht; - break; - case 3400000: - dev->sda_hold_time = hs_ht; - break; - case 400000: - default: - dev->sda_hold_time = fs_ht; - break; - } - - id = acpi_match_device(pdev->dev.driver->acpi_match_table, &pdev->dev); - if (id && id->driver_data) - dev->flags |= (u32)id->driver_data; - - if (acpi_bus_get_device(handle, &adev)) - return -ENODEV; - - /* - * Cherrytrail I2C7 gets used for the PMIC which gets accessed - * through ACPI opregions during late suspend / early resume - * disable pm for it. - */ - uid = adev->pnp.unique_id; - if ((dev->flags & MODEL_CHERRYTRAIL) && !strcmp(uid, "7")) - dev->pm_disabled = true; - - return 0; -} - -static const struct acpi_device_id dw_i2c_acpi_match[] = { - { "INT33C2", 0 }, - { "INT33C3", 0 }, - { "INT3432", 0 }, - { "INT3433", 0 }, - { "80860F41", 0 }, - { "808622C1", MODEL_CHERRYTRAIL }, - { "AMD0010", ACCESS_INTR_MASK }, - { "AMDI0010", ACCESS_INTR_MASK }, - { "AMDI0510", 0 }, - { "APMC0D0F", 0 }, - { "HISI02A1", 0 }, - { "HISI02A2", 0 }, - { } -}; -MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match); -#else -static inline int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - return -ENODEV; -} -#endif - -static void i2c_dw_configure_master(struct dw_i2c_dev *dev) -{ - struct i2c_timings *t = &dev->timings; - - dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE | - DW_IC_CON_RESTART_EN; - - dev->mode = DW_IC_MASTER; - - switch (t->bus_freq_hz) { - case 100000: - dev->master_cfg |= DW_IC_CON_SPEED_STD; - break; - case 3400000: - dev->master_cfg |= DW_IC_CON_SPEED_HIGH; - break; - default: - dev->master_cfg |= DW_IC_CON_SPEED_FAST; - } -} - -static void i2c_dw_configure_slave(struct dw_i2c_dev *dev) -{ - dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL | - DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED; - - dev->mode = DW_IC_SLAVE; -} - -static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev, int id) -{ - u32 param, tx_fifo_depth, rx_fifo_depth; - - /* - * Try to detect the FIFO depth if not set by interface driver, - * the depth could be from 2 to 256 from HW spec. - */ - param = i2c_dw_read_comp_param(dev); - tx_fifo_depth = ((param >> 16) & 0xff) + 1; - rx_fifo_depth = ((param >> 8) & 0xff) + 1; - if (!dev->tx_fifo_depth) { - dev->tx_fifo_depth = tx_fifo_depth; - dev->rx_fifo_depth = rx_fifo_depth; - dev->adapter.nr = id; - } else if (tx_fifo_depth >= 2) { - dev->tx_fifo_depth = min_t(u32, dev->tx_fifo_depth, - tx_fifo_depth); - dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth, - rx_fifo_depth); - } -} - -static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) -{ - pm_runtime_disable(dev->dev); - - if (dev->pm_disabled) - pm_runtime_put_noidle(dev->dev); -} - -static int dw_i2c_plat_probe(struct platform_device *pdev) -{ - struct dw_i2c_platform_data *pdata = dev_get_platdata(&pdev->dev); - struct i2c_adapter *adap; - struct dw_i2c_dev *dev; - struct i2c_timings *t; - u32 acpi_speed; - struct resource *mem; - int i, irq, ret; - static const int supported_speeds[] = { - 0, 100000, 400000, 1000000, 3400000 - }; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - dev = devm_kzalloc(&pdev->dev, sizeof(struct dw_i2c_dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->base = devm_ioremap_resource(&pdev->dev, mem); - if (IS_ERR(dev->base)) - return PTR_ERR(dev->base); - - dev->dev = &pdev->dev; - dev->irq = irq; - platform_set_drvdata(pdev, dev); - - dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); - if (IS_ERR(dev->rst)) { - if (PTR_ERR(dev->rst) == -EPROBE_DEFER) - return -EPROBE_DEFER; - } else { - reset_control_deassert(dev->rst); - } - - t = &dev->timings; - if (pdata) - t->bus_freq_hz = pdata->i2c_scl_freq; - else - i2c_parse_fw_timings(&pdev->dev, t, false); - - acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev); - /* - * Some DSTDs use a non standard speed, round down to the lowest - * standard speed. - */ - for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) { - if (acpi_speed < supported_speeds[i]) - break; - } - acpi_speed = supported_speeds[i - 1]; - - /* - * Find bus speed from the "clock-frequency" device property, ACPI - * or by using fast mode if neither is set. - */ - if (acpi_speed && t->bus_freq_hz) - t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed); - else if (acpi_speed || t->bus_freq_hz) - t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed); - else - t->bus_freq_hz = 400000; - - if (has_acpi_companion(&pdev->dev)) - dw_i2c_acpi_configure(pdev); - - /* - * Only standard mode at 100kHz, fast mode at 400kHz, - * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported. - */ - if (t->bus_freq_hz != 100000 && t->bus_freq_hz != 400000 && - t->bus_freq_hz != 1000000 && t->bus_freq_hz != 3400000) { - dev_err(&pdev->dev, - "%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n", - t->bus_freq_hz); - ret = -EINVAL; - goto exit_reset; - } - - ret = i2c_dw_probe_lock_support(dev); - if (ret) - goto exit_reset; - - if (i2c_detect_slave_mode(&pdev->dev)) - i2c_dw_configure_slave(dev); - else - i2c_dw_configure_master(dev); - - dev->clk = devm_clk_get(&pdev->dev, NULL); - if (!i2c_dw_prepare_clk(dev, true)) { - u64 clk_khz; - - dev->get_clk_rate_khz = i2c_dw_get_clk_rate_khz; - clk_khz = dev->get_clk_rate_khz(dev); - - if (!dev->sda_hold_time && t->sda_hold_ns) - dev->sda_hold_time = - div_u64(clk_khz * t->sda_hold_ns + 500000, 1000000); - } - - dw_i2c_set_fifo_size(dev, pdev->id); - - adap = &dev->adapter; - adap->owner = THIS_MODULE; - adap->class = 0;/*lm_sensors, ...*/ - ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev)); - adap->dev.of_node = pdev->dev.of_node; - - dev_pm_set_driver_flags(&pdev->dev, - DPM_FLAG_SMART_PREPARE | - DPM_FLAG_SMART_SUSPEND | - DPM_FLAG_MAY_SKIP_RESUME); - - /* The code below assumes runtime PM to be disabled. */ - WARN_ON(pm_runtime_enabled(&pdev->dev)); - - pm_runtime_set_autosuspend_delay(&pdev->dev, 1000); - pm_runtime_use_autosuspend(&pdev->dev); - pm_runtime_set_active(&pdev->dev); - - if (dev->pm_disabled) - pm_runtime_get_noresume(&pdev->dev); - - pm_runtime_enable(&pdev->dev); - - if (dev->mode == DW_IC_SLAVE) - ret = i2c_dw_probe_slave(dev); - else - ret = i2c_dw_probe(dev); - - if (ret) - goto exit_probe; - - return ret; - -exit_probe: - dw_i2c_plat_pm_cleanup(dev); -exit_reset: - if (!IS_ERR_OR_NULL(dev->rst)) - reset_control_assert(dev->rst); - return ret; -} - -static int dw_i2c_plat_remove(struct platform_device *pdev) -{ - struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - - pm_runtime_get_sync(&pdev->dev); - - i2c_del_adapter(&dev->adapter); - - dev->disable(dev); - - pm_runtime_dont_use_autosuspend(&pdev->dev); - pm_runtime_put_sync(&pdev->dev); - dw_i2c_plat_pm_cleanup(dev); - - if (!IS_ERR_OR_NULL(dev->rst)) - reset_control_assert(dev->rst); - - i2c_dw_remove_lock_support(dev); - - return 0; -} - -#ifdef CONFIG_OF -static const struct of_device_id dw_i2c_of_match[] = { - { .compatible = "snps,designware-i2c", }, - {}, -}; -MODULE_DEVICE_TABLE(of, dw_i2c_of_match); -#endif - -#ifdef CONFIG_PM_SLEEP -static int dw_i2c_plat_prepare(struct device *dev) -{ - /* - * If the ACPI companion device object is present for this device, it - * may be accessed during suspend and resume of other devices via I2C - * operation regions, so tell the PM core and middle layers to avoid - * skipping system suspend/resume callbacks for it in that case. - */ - return !has_acpi_companion(dev); -} - -static void dw_i2c_plat_complete(struct device *dev) -{ - /* - * The device can only be in runtime suspend at this point if it has not - * been resumed throughout the ending system suspend/resume cycle, so if - * the platform firmware might mess up with it, request the runtime PM - * framework to resume it. - */ - if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) - pm_request_resume(dev); -} -#else -#define dw_i2c_plat_prepare NULL -#define dw_i2c_plat_complete NULL -#endif - -#ifdef CONFIG_PM -static int dw_i2c_plat_suspend(struct device *dev) -{ - struct dw_i2c_dev *i_dev = dev_get_drvdata(dev); - - if (i_dev->pm_disabled) - return 0; - - i_dev->disable(i_dev); - i2c_dw_prepare_clk(i_dev, false); - - return 0; -} - -static int dw_i2c_plat_resume(struct device *dev) -{ - struct dw_i2c_dev *i_dev = dev_get_drvdata(dev); - - if (!i_dev->pm_disabled) - i2c_dw_prepare_clk(i_dev, true); - - i_dev->init(i_dev); - - return 0; -} - -static const struct dev_pm_ops dw_i2c_dev_pm_ops = { - .prepare = dw_i2c_plat_prepare, - .complete = dw_i2c_plat_complete, - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume) - SET_RUNTIME_PM_OPS(dw_i2c_plat_suspend, dw_i2c_plat_resume, NULL) -}; - -#define DW_I2C_DEV_PMOPS (&dw_i2c_dev_pm_ops) -#else -#define DW_I2C_DEV_PMOPS NULL -#endif - -/* Work with hotplug and coldplug */ -MODULE_ALIAS("platform:i2c_designware"); - -static struct platform_driver dw_i2c_driver = { - .probe = dw_i2c_plat_probe, - .remove = dw_i2c_plat_remove, - .driver = { - .name = "i2c_designware", - .of_match_table = of_match_ptr(dw_i2c_of_match), - .acpi_match_table = ACPI_PTR(dw_i2c_acpi_match), - .pm = DW_I2C_DEV_PMOPS, - }, -}; - -static int __init dw_i2c_init_driver(void) -{ - return platform_driver_register(&dw_i2c_driver); -} -subsys_initcall(dw_i2c_init_driver); - -static void __exit dw_i2c_exit_driver(void) -{ - platform_driver_unregister(&dw_i2c_driver); -} -module_exit(dw_i2c_exit_driver); - -MODULE_AUTHOR("Baruch Siach "); -MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter"); -MODULE_LICENSE("GPL"); -- Gitee From 748fc5cc83c7f5d7638b211e57fb1ed613d2c7d6 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Tue, 18 Jul 2023 20:11:09 +0800 Subject: [PATCH 034/150] sw64: reorganize dependency of devmap helpers Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Move pte_devmap() out of the inappropriate ifdef guard to make it independent of TRANSPARENT_HUGEPAGE. This patch also makes pmd_mkdevmap() independent of THP like x86 did, although it's not necessary. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 33 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index c9276d588e28..4b8358e7314d 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -117,12 +117,13 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_DIRTY 0x20000 #define _PAGE_ACCESSED 0x40000 -#define _PAGE_BIT_ACCESSED 18 /* bit of _PAGE_ACCESSED */ -#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ #define _PAGE_SPLITTING 0x200000 /* For Transparent Huge Page */ +#define _PAGE_DEVMAP 0x400000 /* For ZONE DEVICE page */ + +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 18 /* bit of _PAGE_ACCESSED */ #define _PAGE_BIT_SPLITTING 21 /* bit of _PAGE_SPLITTING */ #define _PAGE_BIT_DEVMAP 22 /* bit of _PAGE_DEVMAP */ - /* * NOTE! The "accessed" bit isn't necessarily exact: it can be kept exactly * by software (use the KRE/URE/KWE/UWE bits appropriately), but I'll fake it. @@ -373,6 +374,12 @@ static inline pmd_t pmd_mkdirty(pmd_t pmd) return pmd; } +static inline pmd_t pmd_mkdevmap(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_DEVMAP; + return pmd; +} + static inline pmd_t pmd_mkyoung(pmd_t pmd) { pmd_val(pmd) |= __ACCESS_BITS; @@ -521,7 +528,12 @@ static inline int pmd_protnone(pmd_t pmd) } #endif -#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) +#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP +static inline int pte_devmap(pte_t a) +{ + return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; +} +#endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -566,21 +578,8 @@ static inline int pgd_devmap(pgd_t pgd) return 0; } #endif - -static inline pmd_t pmd_mkdevmap(pmd_t pmd) -{ - pmd_val(pmd) |= _PAGE_DEVMAP; - return pmd; -} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -#ifdef CONFIG_ARCH_HAS_PTE_DEVMAP -static inline int pte_devmap(pte_t a) -{ - return (pte_val(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; -} -#endif - #define __HAVE_ARCH_PMDP_GET_AND_CLEAR static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) -- Gitee From af5b8af46912469171e0f6f7c10d87e736a251d6 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Fri, 21 Jul 2023 14:28:23 +0800 Subject: [PATCH 035/150] sw64: fix s4 sleep bug Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CB96 -------------------------------- An msi interrupt handler error occurs when the chip3 machine wakes up from s4 sleep. The root cause of the error is that the msiconfig register is not retained for recovery. To work around this problem, register a syscore_ops to do spbu, intpu and piu registers save and restore. And unify s3 sleep for these registers. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 7 ------- arch/sw_64/chip/chip3/chip.c | 32 +++++++++++++++++++++----------- arch/sw_64/include/asm/suspend.h | 1 + arch/sw_64/kernel/Makefile | 1 + arch/sw_64/kernel/pm.c | 18 ++++++++++++++++++ arch/sw_64/kernel/suspend.c | 17 +---------------- 6 files changed, 42 insertions(+), 34 deletions(-) create mode 100644 arch/sw_64/kernel/pm.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 308f55fddd23..d37818dd6458 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -654,13 +654,6 @@ config SW64_SUSPEND_DEEPSLEEP_BOOTCORE bool "SW64 bootcore suspend into deep sleep mode" default n -config SW64_SUPPORT_S3_SLEEPING_STATE - depends on SUSPEND - bool "SW64 support S3 sleeping state" - default n - help - Only SW831 support S3 sleep option and needs SROM, HMCode and BIOS support. - source "drivers/cpuidle/Kconfig" source "drivers/idle/Kconfig" diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 2f39c6b6206b..0f0451c77b5b 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "../../../../drivers/pci/pci.h" static u64 read_longtime(struct clocksource *cs) @@ -641,18 +642,20 @@ static inline void chip3_spbu_restore(void) extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); -static void chip3_suspend(bool wakeup) +static int chip3_io_suspend(void) { + chip3_spbu_save(); + chip3_intpu_save(); + chip3_pcie_save(); - if (wakeup) { - chip3_pcie_restore(); - chip3_intpu_restore(); - chip3_spbu_restore(); - } else { - chip3_spbu_save(); - chip3_intpu_save(); - chip3_pcie_save(); - } + return 0; +} + +static void chip3_io_resume(void) +{ + chip3_pcie_restore(); + chip3_intpu_restore(); + chip3_spbu_restore(); } static void chip3_hose_init(struct pci_controller *hose) @@ -737,14 +740,21 @@ static struct sw64_chip_init_ops chip3_chip_init_ops = { static struct sw64_chip_ops chip3_chip_ops = { .get_cpu_num = chip3_get_cpu_nums, - .suspend = chip3_suspend, .fixup = chip3_ops_fixup, }; +#ifdef CONFIG_PM +extern struct syscore_ops io_syscore_ops; +#endif + void __init sw64_setup_chip_ops(void) { sw64_chip_init = &chip3_chip_init_ops; sw64_chip = &chip3_chip_ops; +#ifdef CONFIG_PM + io_syscore_ops.suspend = chip3_io_suspend; + io_syscore_ops.resume = chip3_io_resume; +#endif } /* Performance counter hook. A module can override this to do something useful. */ diff --git a/arch/sw_64/include/asm/suspend.h b/arch/sw_64/include/asm/suspend.h index 7b6d7bfc9595..833e27f9d5e1 100644 --- a/arch/sw_64/include/asm/suspend.h +++ b/arch/sw_64/include/asm/suspend.h @@ -46,4 +46,5 @@ struct processor_state { }; extern void sw64_suspend_deep_sleep(struct processor_state *state); +extern const struct platform_suspend_ops native_suspend_ops; #endif /* _ASM_SW64_SUSPEND_H */ diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 615556ff7b5a..52ed1166b9bd 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PCI) += pci.o pci-sysfs.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_PCI_MSI) += msi.o +obj-$(CONFIG_PM) += pm.o obj-$(CONFIG_SUSPEND) += suspend_asm.o suspend.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HIBERNATION) += hibernate_asm.o hibernate.o diff --git a/arch/sw_64/kernel/pm.c b/arch/sw_64/kernel/pm.c new file mode 100644 index 000000000000..f0a35e5d0486 --- /dev/null +++ b/arch/sw_64/kernel/pm.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include + +struct syscore_ops io_syscore_ops; + +static int __init sw64_pm_init(void) +{ +#ifdef CONFIG_SUSPEND + suspend_set_ops(&native_suspend_ops); +#endif + register_syscore_ops(&io_syscore_ops); + + return 0; +} +device_initcall(sw64_pm_init); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index b9798baa2467..85103123ab9d 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -34,10 +34,6 @@ void sw64_suspend_enter(void) * After wake up boot processor, pc will go here */ -#ifdef CONFIG_SW64_SUPPORT_S3_SLEEPING_STATE - if (sw64_chip->suspend) - sw64_chip->suspend(false); -#endif disable_local_timer(); current_thread_info()->pcb.tp = rtid(); @@ -50,10 +46,6 @@ void sw64_suspend_enter(void) #endif wrtp(current_thread_info()->pcb.tp); -#ifdef CONFIG_SW64_SUPPORT_S3_SLEEPING_STATE - if (sw64_chip->suspend) - sw64_chip->suspend(true); -#endif disable_local_timer(); } @@ -65,14 +57,7 @@ static int native_suspend_enter(suspend_state_t state) return 0; } -static const struct platform_suspend_ops native_suspend_ops = { +const struct platform_suspend_ops native_suspend_ops = { .valid = native_suspend_state_valid, .enter = native_suspend_enter, }; - -static int __init sw64_pm_init(void) -{ - suspend_set_ops(&native_suspend_ops); - return 0; -} -arch_initcall(sw64_pm_init); -- Gitee From ed44ba9001d6e204fe196d194587dc62cdfea76a Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Thu, 20 Jul 2023 13:40:02 +0800 Subject: [PATCH 036/150] sw64: remove C3A support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- C3A support has been deprecated for a long time, and this patch removes related codes. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 4 ---- arch/sw_64/include/asm/atomic.h | 21 --------------------- arch/sw_64/include/asm/bitops.h | 21 --------------------- arch/sw_64/include/asm/cacheflush.h | 2 +- arch/sw_64/include/asm/futex.h | 12 ------------ arch/sw_64/include/asm/xchg.h | 24 ------------------------ arch/sw_64/kernel/smp.c | 4 ---- 7 files changed, 1 insertion(+), 87 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d37818dd6458..7f740d53be19 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -256,10 +256,6 @@ config MIGHT_HAVE_PC_SERIO endmenu -config LOCK_MEMB - bool "Insert mem barrier before lock instruction" - default y - menu "CPU Power Management" source "drivers/cpufreq/Kconfig" diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h index 66f4437be103..0cfebacef080 100644 --- a/arch/sw_64/include/asm/atomic.h +++ b/arch/sw_64/include/asm/atomic.h @@ -50,9 +50,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) unsigned long addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldw %0, 0(%3)\n" " cmpeq %0, %5, %4\n" @@ -90,9 +87,6 @@ static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) unsigned long addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldl %0, 0(%3)\n" " cmpeq %0, %5, %4\n" @@ -127,9 +121,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) unsigned long old, temp1, addr, temp2; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %2\n" "1: lldl %4, 0(%3)\n" " cmple %4, 0, %0\n" @@ -155,12 +146,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_dec_if_positive atomic64_dec_if_positive -#ifdef CONFIG_LOCK_MEMB -#define LOCK_MEMB "memb\n" -#else -#define LOCK_MEMB -#endif - #ifdef CONFIG_LOCK_FIXUP #define LOCK_FIXUP "memb\n" #else @@ -173,7 +158,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ { \ unsigned long temp1, temp2, addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ @@ -197,7 +181,6 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ int temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ @@ -224,7 +207,6 @@ static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ int temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldw %0, 0(%3)\n" \ " ldi %1, 1\n" \ @@ -248,7 +230,6 @@ static inline void atomic64_##op(long i, atomic64_t *v) \ { \ unsigned long temp1, temp2, addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ @@ -272,7 +253,6 @@ static inline long atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ long temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ @@ -297,7 +277,6 @@ static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ long temp1, temp2; \ unsigned long addr; \ __asm__ __volatile__( \ - LOCK_MEMB \ " ldi %3, %2\n" \ "1: lldl %0, 0(%3)\n" \ " ldi %1, 1\n" \ diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h index add1b3c11366..e69b0b31dc79 100644 --- a/arch/sw_64/include/asm/bitops.h +++ b/arch/sw_64/include/asm/bitops.h @@ -32,9 +32,6 @@ set_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" @@ -74,9 +71,6 @@ clear_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" @@ -127,9 +121,6 @@ change_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" @@ -168,9 +159,6 @@ test_and_set_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" @@ -202,9 +190,6 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" @@ -250,9 +235,6 @@ test_and_clear_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %4, %6\n" "1: lldw %0, 0(%4)\n" " and %0, %5, %3\n" @@ -298,9 +280,6 @@ test_and_change_bit(unsigned long nr, volatile void *addr) int *m = ((int *) addr) + (nr >> 5); __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %2, 1\n" diff --git a/arch/sw_64/include/asm/cacheflush.h b/arch/sw_64/include/asm/cacheflush.h index 536b0b7b78bd..0d49830b8493 100644 --- a/arch/sw_64/include/asm/cacheflush.h +++ b/arch/sw_64/include/asm/cacheflush.h @@ -5,7 +5,7 @@ /* * DCache: PIPT * ICache: - * - C3A/B is VIVT with ICTAG, support coherence. + * - C3B is VIVT with ICTAG, support coherence. * - C4 is VIPT */ #include diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h index 344ce5e4da28..f4806b856804 100644 --- a/arch/sw_64/include/asm/futex.h +++ b/arch/sw_64/include/asm/futex.h @@ -9,14 +9,6 @@ #include #include -#ifndef LOCK_MEMB -#ifdef CONFIG_LOCK_MEMB -#define LOCK_MEMB "memb\n" -#else -#define LOCK_MEMB -#endif -#endif - #ifndef LOCK_FIXUP #ifdef CONFIG_LOCK_FIXUP #define LOCK_FIXUP "memb\n" @@ -27,7 +19,6 @@ #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ __asm__ __volatile__( \ - LOCK_MEMB \ "1: lldw %0, 0(%3)\n" \ " ldi %2, 1\n" \ " wr_f %2\n" \ @@ -97,9 +88,6 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; __asm__ __volatile__ ( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif "1: lldw %1, 0(%4)\n" " cmpeq %1, %5, %2\n" " wr_f %2\n" diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index ba4e6d1a27ad..c0f632c2840e 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -23,9 +23,6 @@ ____xchg(_u8, volatile char *m, unsigned long val) unsigned long ret, tmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %4, 7, %3\n" " inslb %1, %4, %1\n" @@ -56,9 +53,6 @@ ____xchg(_u16, volatile short *m, unsigned long val) unsigned long ret, tmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %4, 7, %3\n" " inslh %1, %4, %1\n" "1: lldl %2, 0(%3)\n" @@ -88,9 +82,6 @@ ____xchg(_u32, volatile int *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" " ldi %1, 1\n" @@ -117,9 +108,6 @@ ____xchg(_u64, volatile long *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %5\n" "1: lldl %0, 0(%3)\n" " ldi %1, 1\n" @@ -180,9 +168,6 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %5, 7, %4\n" " inslb %1, %5, %1\n" "1: lldl %2, 0(%4)\n" @@ -214,9 +199,6 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) unsigned long prev, tmp, cmp, addr64; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " andnot %5, 7, %4\n" " inslh %1, %5, %1\n" "1: lldl %2, 0(%4)\n" @@ -248,9 +230,6 @@ ____cmpxchg(_u32, volatile int *m, int old, int new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %7\n" "1: lldw %0, 0(%3)\n" " cmpeq %0, %5, %1\n" @@ -279,9 +258,6 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( -#ifdef CONFIG_LOCK_MEMB - " memb\n" -#endif " ldi %3, %7\n" "1: lldl %0, 0(%3)\n" " cmpeq %0, %5, %1\n" diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 78ef01b54da9..83e4141421b1 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -56,8 +56,6 @@ EXPORT_SYMBOL(smp_num_cpus); #define send_sleep_interrupt(cpu) send_ipi((cpu), II_SLEEP) #define send_wakeup_interrupt(cpu) send_ipi((cpu), II_WAKE) -void __weak enable_chip_int(void) { } - /* * Where secondaries begin a life of C. */ @@ -67,8 +65,6 @@ void smp_callin(void) local_irq_disable(); - enable_chip_int(); - if (cpu_online(cpuid)) { printk("??, cpu 0x%x already present??\n", cpuid); BUG(); -- Gitee From 7439cfd1c48cabffb4654088479927f9a013d51b Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Thu, 27 Jul 2023 17:15:33 +0800 Subject: [PATCH 037/150] sw64: do not set __ARCH_SI_TRAPNO Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- If __ARCH_SI_TRAPNO is defined, struct siginfo would include member int _trapno. But struct siginfo defined in libc does not have this member. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/siginfo.h | 1 - arch/sw_64/kernel/ptrace.c | 2 +- arch/sw_64/kernel/signal.c | 4 ++-- arch/sw_64/kernel/sys_sw64.c | 2 +- arch/sw_64/kernel/traps.c | 12 ++++++------ arch/sw_64/mm/fault.c | 4 ++-- 6 files changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/include/uapi/asm/siginfo.h b/arch/sw_64/include/uapi/asm/siginfo.h index 28c656c28313..f47fb917c9b2 100644 --- a/arch/sw_64/include/uapi/asm/siginfo.h +++ b/arch/sw_64/include/uapi/asm/siginfo.h @@ -3,7 +3,6 @@ #define _UAPI_ASM_SW64_SIGINFO_H #define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) -#define __ARCH_SI_TRAPNO #include diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index b509576257cc..01fdfc2babc9 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -522,7 +522,7 @@ int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_r task_thread_info(current)->pcb.dv_match = 0; task_thread_info(current)->pcb.dc_ctl = 0; printk("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void *) address, 0); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void *) address); return 1; case MMCSR__IA_MATCH: diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 0b7db801a7e8..46816e9303fb 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -168,7 +168,7 @@ do_sigreturn(struct sigcontext __user *sc) /* Send SIGTRAP if we're single-stepping: */ if (ptrace_cancel_bpt(current)) { force_sig_fault(SIGTRAP, TRAP_BRKPT, - (void __user *)regs->pc, 0); + (void __user *)regs->pc); } return; @@ -199,7 +199,7 @@ do_rt_sigreturn(struct rt_sigframe __user *frame) /* Send SIGTRAP if we're single-stepping: */ if (ptrace_cancel_bpt(current)) { force_sig_fault(SIGTRAP, TRAP_BRKPT, - (void __user *)regs->pc, 0); + (void __user *)regs->pc); } return; diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 470c74853d47..9793b6a89965 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -93,7 +93,7 @@ SYSCALL_DEFINE5(setsysinfo, unsigned long, op, void __user *, buffer, if (fex & IEEE_TRAP_ENABLE_INV) si_code = FPE_FLTINV; - send_sig_fault(SIGFPE, si_code, (void __user *)NULL, 0, current); + send_sig_fault(SIGFPE, si_code, (void __user *)NULL, current); } return 0; } diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index cdbd6621e79c..76c476bed319 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -164,7 +164,7 @@ do_entArith(unsigned long summary, unsigned long write_mask, if (!user_mode(regs)) die("Arithmetic fault", regs, 0); - force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc, 0); + force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc); } void simd_emulate(unsigned int inst, unsigned long va) @@ -217,7 +217,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) switch (type) { case IF_BREAKPOINT: /* gdb do pc-4 for sigtrap */ - force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc, 0); + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); return; case IF_GENTRAP: @@ -280,7 +280,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) break; } - force_sig_fault(signo, code, (void __user *)regs->pc, regs->r16); + force_sig_fault(signo, code, (void __user *)regs->pc); return; case IF_FEN: @@ -318,7 +318,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) break; } - force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc, 0); + force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)regs->pc); } asmlinkage void @@ -1481,12 +1481,12 @@ do_entUnaUser(void __user *va, unsigned long opcode, si_code = SEGV_MAPERR; up_read(&mm->mmap_lock); } - force_sig_fault(SIGSEGV, si_code, va, 0); + force_sig_fault(SIGSEGV, si_code, va); return; give_sigbus: regs->pc -= 4; - force_sig_fault(SIGBUS, BUS_ADRALN, va, 0); + force_sig_fault(SIGBUS, BUS_ADRALN, va); } void diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index 574fe7930aac..0f4ec9dac8d0 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -296,13 +296,13 @@ do_page_fault(unsigned long address, unsigned long mmcsr, * Send a sigbus, regardless of whether we were in kernel * or user mode. */ - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address, 0); + force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *) address); if (!user_mode(regs)) goto no_context; return; do_sigsegv: - force_sig_fault(SIGSEGV, si_code, (void __user *) address, 0); + force_sig_fault(SIGSEGV, si_code, (void __user *) address); if (unlikely(segv_debug_enabled)) { pr_info("fault: want to send_segv: pid %d, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", -- Gitee From c8e878ec0fc61d73dfc38f7e6fe6bed498dc9a28 Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Thu, 27 Jul 2023 17:23:55 +0800 Subject: [PATCH 038/150] sw64: remove arch overridden RLIM_INFINITY Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- RLIM_INFINITY in libc is the generic version, so here remove arch specific definition, and use the generic one to keep compatible with libc. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/resource.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/sw_64/include/uapi/asm/resource.h b/arch/sw_64/include/uapi/asm/resource.h index fecca2214849..2e1ce8f6ee64 100644 --- a/arch/sw_64/include/uapi/asm/resource.h +++ b/arch/sw_64/include/uapi/asm/resource.h @@ -11,13 +11,6 @@ #define RLIMIT_NPROC 8 /* max number of processes */ #define RLIMIT_MEMLOCK 9 /* max locked-in-memory address space */ -/* - * SuS says limits have to be unsigned. Fine, it's unsigned, but - * we retain the old value for compatibility, especially with DU. - * When you run into the 2^63 barrier, you call me. - */ -#define RLIM_INFINITY 0x7ffffffffffffffful - #include #endif /* _UAPI_ASM_SW64_RESOURCE_H */ -- Gitee From 1b6f66587e8e1424f87fe2c6272a6a4295ff3d7f Mon Sep 17 00:00:00 2001 From: Min Fanlei Date: Mon, 31 Jul 2023 10:34:57 +0800 Subject: [PATCH 039/150] sw64: fix incorrect hmcalls in the virtual machine Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CBE6 -------------------------------- It's a mistake for guest to call hmcall sleepen and sendii during the suspend cycle, so we fix it. And suspend mode is not currently supported on guest. Signed-off-by: Min Fanlei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/suspend.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 85103123ab9d..94fb35fc9b47 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -52,6 +52,8 @@ void sw64_suspend_enter(void) static int native_suspend_enter(suspend_state_t state) { + if (is_in_guest()) + return 0; /* processor specific suspend */ sw64_suspend_enter(); return 0; -- Gitee From 1d5ecc0960135085d6c1045342ba00c52d371ffb Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Wed, 9 Aug 2023 15:50:39 +0800 Subject: [PATCH 040/150] sw64: fix frequency in cpufreq_frequency_table Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CBJ2 -------------------------------- The unit of this frequency is kHz, not mHz. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/cpufreq_platform.c | 6 +++--- arch/sw_64/include/asm/hw_init.h | 4 ++-- drivers/cpufreq/sw64_cpufreq.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/chip/chip3/cpufreq_platform.c b/arch/sw_64/chip/chip3/cpufreq_platform.c index a37e524f7b14..4dcaea7193ce 100644 --- a/arch/sw_64/chip/chip3/cpufreq_platform.c +++ b/arch/sw_64/chip/chip3/cpufreq_platform.c @@ -42,14 +42,14 @@ static int __init sw64_cpufreq_init(void) int i; unsigned char external_clk; unsigned long max_rate, freq_off; - max_rate = get_cpu_freq() / 1000000; + max_rate = get_cpu_freq() / 1000; external_clk = *((unsigned char *)__va(0x908011)); if (external_clk == 240) - freq_off = 60; + freq_off = 60000; else - freq_off = 50; + freq_off = 50000; /* clock table init */ for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 65ced5bbdc72..612ef83b9f52 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -83,9 +83,9 @@ static inline unsigned long get_cpu_freq(void) return cpu_desc.frequency; } -static inline void update_cpu_freq(unsigned long freq) +static inline void update_cpu_freq(unsigned long khz) { - cpu_desc.frequency = freq * 1000000; + cpu_desc.frequency = khz * 1000; } #define EMUL_FLAG (0x1UL << 63) diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index 91bd05f245e9..0ebab111e812 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -42,7 +42,7 @@ static int sw64_cpu_freq_notifier(struct notifier_block *nb, unsigned long cpu = freqs->policy->cpu; if (val == CPUFREQ_POSTCHANGE) - sw64_update_clockevents(cpu, freqs->new * 1000000); + sw64_update_clockevents(cpu, freqs->new * 1000); return 0; } -- Gitee From 026aa45c89804e82f2b2753e71a4b9ce26447894 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Thu, 10 Aug 2023 20:12:32 +0800 Subject: [PATCH 041/150] sw64: Kconfig: remove out-of-date rwsem options Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 7f740d53be19..8be83d09eb2b 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -128,13 +128,6 @@ config PGTABLE_LEVELS config SYS_SUPPORTS_HUGETLBFS def_bool y -config RWSEM_GENERIC_SPINLOCK - bool - -config RWSEM_XCHGADD_ALGORITHM - bool - default y - config ARCH_ENABLE_MEMORY_HOTPLUG bool default y -- Gitee From 6db42326bbf7477ea591ca7b74df891d1df3fa66 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 10:29:39 +0800 Subject: [PATCH 042/150] sw64: add missing break Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Add missing break statements to avoid problems in rare conditions. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/smp.c | 2 ++ arch/sw_64/kernel/traps.c | 3 +++ 2 files changed, 5 insertions(+) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 83e4141421b1..f145556d879d 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -373,6 +373,8 @@ void handle_ipi(struct pt_regs *regs) case IPI_CPU_STOP: ipi_cpu_stop(cpu); + break; + default: pr_crit("Unknown IPI on CPU %d: %lu\n", cpu, which); break; diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 76c476bed319..934559d0d125 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -293,14 +293,17 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) case BREAK_KPROBE: if (notify_die(DIE_BREAK, "kprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; + break; case BREAK_KPROBE_SS: if (notify_die(DIE_SSTEPBP, "single_step", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; + break; #endif #ifdef CONFIG_UPROBES case UPROBE_BRK_UPROBE: if (notify_die(DIE_UPROBE, "uprobe", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; + break; case UPROBE_BRK_UPROBE_XOL: if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, 0, 0, SIGTRAP) == NOTIFY_STOP) return; -- Gitee From a5c7ccef7cd8aece8984b1c8d6ddef25b519062c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 10:29:40 +0800 Subject: [PATCH 043/150] sw64: add missing fallthrough Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Add missing fallthrough statements to avoid compiler warning. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/signal.c | 2 +- arch/sw_64/net/bpf_jit_comp.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 46816e9303fb..037162e4e844 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -344,7 +344,7 @@ syscall_restart(unsigned long r0, unsigned long r19, regs->r0 = EINTR; break; } - /* else: fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->r0 = r0; /* reset v0 and a3 and replay syscall */ regs->r19 = r19; diff --git a/arch/sw_64/net/bpf_jit_comp.c b/arch/sw_64/net/bpf_jit_comp.c index 43d548f7f755..91bcb266f126 100644 --- a/arch/sw_64/net/bpf_jit_comp.c +++ b/arch/sw_64/net/bpf_jit_comp.c @@ -727,6 +727,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) break; case BPF_ALU | BPF_RSH | BPF_X: emit(SW64_BPF_ZAP_IMM(dst, 0xf0, dst), ctx); + fallthrough; case BPF_ALU64 | BPF_RSH | BPF_X: emit(SW64_BPF_SRL_REG(dst, src, dst), ctx); break; @@ -1010,6 +1011,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) src = tmp1; emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); dst = tmp2; + fallthrough; case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP | BPF_JLT | BPF_X: @@ -1080,6 +1082,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) case BPF_JMP32 | BPF_JSET | BPF_K: emit(SW64_BPF_ADDW_REG(SW64_BPF_REG_ZR, dst, tmp2), ctx); dst = tmp2; + fallthrough; case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP | BPF_JLT | BPF_K: -- Gitee From 6095539b895833d6d1a1f473a6ff121998f67d9a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 10:29:40 +0800 Subject: [PATCH 044/150] sw64: fix header include guards Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Fix incorrect header include guards and remove unnecessary ones. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/asm-offsets.h | 6 ------ arch/sw_64/include/asm/cpu.h | 4 ++++ arch/sw_64/include/asm/cpufreq.h | 6 +++--- arch/sw_64/include/asm/idle.h | 2 +- arch/sw_64/include/uapi/asm/bpf_perf_event.h | 6 +++--- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/arch/sw_64/include/asm/asm-offsets.h b/arch/sw_64/include/asm/asm-offsets.h index 5ddfd96ccb79..d370ee36a182 100644 --- a/arch/sw_64/include/asm/asm-offsets.h +++ b/arch/sw_64/include/asm/asm-offsets.h @@ -1,7 +1 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_ASM_OFFSETS_H -#define _ASM_SW64_ASM_OFFSETS_H - #include - -#endif /* _ASM_SW64_ASM_OFFSETS_H */ diff --git a/arch/sw_64/include/asm/cpu.h b/arch/sw_64/include/asm/cpu.h index ea32a7d3cf1b..4da30bb91d89 100644 --- a/arch/sw_64/include/asm/cpu.h +++ b/arch/sw_64/include/asm/cpu.h @@ -1 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CPU_H +#define _ASM_SW64_CPU_H + +#endif /* _ASM_SW64_CPU_H */ diff --git a/arch/sw_64/include/asm/cpufreq.h b/arch/sw_64/include/asm/cpufreq.h index 0741282e8d4c..bb3f2ada7960 100644 --- a/arch/sw_64/include/asm/cpufreq.h +++ b/arch/sw_64/include/asm/cpufreq.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_CLOCK_H -#define _ASM_SW64_CLOCK_H +#ifndef _ASM_SW64_CPUFREQ_H +#define _ASM_SW64_CPUFREQ_H #include #include @@ -65,4 +65,4 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq); void sw64_store_policy(struct cpufreq_policy *policy); unsigned int __sw64_cpufreq_get(struct cpufreq_policy *policy); -#endif /* _ASM_SW64_CLOCK_H */ +#endif /* _ASM_SW64_CPUFREQ_H */ diff --git a/arch/sw_64/include/asm/idle.h b/arch/sw_64/include/asm/idle.h index 7e88af18ba39..95e145f25306 100644 --- a/arch/sw_64/include/asm/idle.h +++ b/arch/sw_64/include/asm/idle.h @@ -4,4 +4,4 @@ extern void arch_cpu_idle(void); -#endif /* _ASM_SW64_IDLE_H */ +#endif /* _ASM_SW64_IDLE_H */ diff --git a/arch/sw_64/include/uapi/asm/bpf_perf_event.h b/arch/sw_64/include/uapi/asm/bpf_perf_event.h index 5e1e648aeec4..52f6f1e555f1 100644 --- a/arch/sw_64/include/uapi/asm/bpf_perf_event.h +++ b/arch/sw_64/include/uapi/asm/bpf_perf_event.h @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ -#define _UAPI__ASM_BPF_PERF_EVENT_H__ +#ifndef _UAPI_ASM_SW64_BPF_PERF_EVENT_H +#define _UAPI_ASM_SW64_BPF_PERF_EVENT_H #include typedef struct user_pt_regs bpf_user_pt_regs_t; -#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ +#endif /* _UAPI_ASM_SW64_BPF_PERF_EVENT_H */ -- Gitee From 141b752a3f31007ed78aecc195d2f982175a29ec Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 10:29:41 +0800 Subject: [PATCH 045/150] sw64: fix indent of inline assembly Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- We use tab after instruction. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/xchg.h | 76 +++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index c0f632c2840e..34acfc1699b0 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -82,19 +82,19 @@ ____xchg(_u32, volatile int *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( - " ldi %3, %5\n" - "1: lldw %0, 0(%3)\n" - " ldi %1, 1\n" - " wr_f %1\n" - " bis $31, %4, %1\n" + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" #ifdef CONFIG_LOCK_FIXUP " memb\n" #endif - " lstw %1, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " lstw %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" ".subsection 2\n" - "2: br 1b\n" + "2: br 1b\n" ".previous" : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) : "rI" (val), "m" (*m) : "memory"); @@ -108,19 +108,19 @@ ____xchg(_u64, volatile long *m, unsigned long val) unsigned long dummy, addr; __asm__ __volatile__( - " ldi %3, %5\n" - "1: lldl %0, 0(%3)\n" - " ldi %1, 1\n" - " wr_f %1\n" - " bis $31, %4, %1\n" + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " ldi %1, 1\n" + " wr_f %1\n" + " bis $31, %4, %1\n" #ifdef CONFIG_LOCK_FIXUP " memb\n" #endif - " lstl %1, 0(%3)\n" - " rd_f %1\n" - " beq %1, 2f\n" + " lstl %1, 0(%3)\n" + " rd_f %1\n" + " beq %1, 2f\n" ".subsection 2\n" - "2: br 1b\n" + "2: br 1b\n" ".previous" : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) : "rI" (val), "m" (*m) : "memory"); @@ -230,21 +230,21 @@ ____cmpxchg(_u32, volatile int *m, int old, int new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( - " ldi %3, %7\n" - "1: lldw %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " wr_f %1\n" - " bis $31, %6, %4\n" + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" #ifdef CONFIG_LOCK_FIXUP " memb\n" #endif - " lstw %4, 0(%3)\n" - " rd_f %4\n" - " beq %1, 2f\n" - " beq %4, 3f\n" + " lstw %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" "2:\n" ".subsection 2\n" - "3: br 1b\n" + "3: br 1b\n" ".previous" : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) : "r"((long) old), "r"(new), "m"(*m) : "memory"); @@ -258,21 +258,21 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) unsigned long prev, cmp, addr, tmp; __asm__ __volatile__( - " ldi %3, %7\n" - "1: lldl %0, 0(%3)\n" - " cmpeq %0, %5, %1\n" - " wr_f %1\n" - " bis $31, %6, %4\n" + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " wr_f %1\n" + " bis $31, %6, %4\n" #ifdef CONFIG_LOCK_FIXUP " memb\n" #endif - " lstl %4, 0(%3)\n" - " rd_f %4\n" - " beq %1, 2f\n" - " beq %4, 3f\n" + " lstl %4, 0(%3)\n" + " rd_f %4\n" + " beq %1, 2f\n" + " beq %4, 3f\n" "2:\n" ".subsection 2\n" - "3: br 1b\n" + "3: br 1b\n" ".previous" : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) : "r"((long) old), "r"(new), "m"(*m) : "memory"); -- Gitee From da9f28d4d4948f7e5814388e0e628c55b79e77ab Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 10:34:41 +0800 Subject: [PATCH 046/150] sw64: drop fpga and simulator support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- These code are for debugging purpose, remove them. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 28 -------------------- arch/sw_64/chip/chip3/chip.c | 43 +------------------------------ arch/sw_64/include/asm/chip3_io.h | 4 --- arch/sw_64/kernel/setup.c | 11 -------- 4 files changed, 1 insertion(+), 85 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 8be83d09eb2b..956a37a3388f 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -200,34 +200,6 @@ config SW64_CHIP3 depends on SUBARCH_C3B endchoice -choice - prompt "Runtime System" - depends on SW64_CHIP3 - default SW64_ASIC - -config SW64_FPGA - bool "FPGA" - help - Support for chip3 FPGA. - -config SW64_SIM - bool "Hardware Simulator" - help - Support for chip3 hardware simulator. - -config SW64_ASIC - bool "ASIC" - help - Support for chip3 asic. - -endchoice - -config SW64_CHIP3_ASIC_DEBUG - bool "Debug Support for Chip3 Asic" - depends on SW64_ASIC - help - Used for debug - choice prompt "Platform Type" diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 0f0451c77b5b..337f392dc4cc 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -15,10 +15,7 @@ static u64 read_longtime(struct clocksource *cs) u64 result; unsigned long node; - if (IS_ENABLED(CONFIG_SW64_FPGA) || IS_ENABLED(CONFIG_SW64_SIM)) - node = 0; - else - node = __this_cpu_read(hard_node_id); + node = __this_cpu_read(hard_node_id); result = sw64_io_read(node, LONG_TIME); return result; @@ -79,16 +76,10 @@ static struct clocksource clocksource_vtime = { void setup_chip_clocksource(void) { -#ifdef CONFIG_SW64_SIM - clocksource_register_khz(&clocksource_longtime, 400000); /* Hardware Simulator 400Mhz */ -#elif defined(CONFIG_SW64_FPGA) - clocksource_register_khz(&clocksource_longtime, 1000); /* FPGA 1Mhz */ -#else if (is_in_host()) clocksource_register_khz(&clocksource_longtime, 25000); else clocksource_register_khz(&clocksource_vtime, 25000); -#endif } void set_devint_wken(int node) @@ -335,28 +326,6 @@ static int chip3_check_pci_linkup(unsigned long node, unsigned long index) { unsigned long rc_debug; -#ifdef CONFIG_SW64_FPGA //for PCIE4.0 - printk("waiting for link up...\n"); - if (index == 0) - sw64_io_write(node, PIU_TOP0_CONFIG, 0x10011); - else - sw64_io_write(node, PIU_TOP1_CONFIG, 0x10011); - mdelay(10); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - while (!(rc_debug & 0x1)) { - udelay(10); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - } - mdelay(10); -#endif -#ifdef CONFIG_SW64_SIM - printk("waiting for link up...\n"); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - while (!(rc_debug & 0x1)) { - udelay(10); - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); - } -#endif rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); return !(rc_debug & 0x1); @@ -377,12 +346,6 @@ static void chip3_set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_EXP_DEVCTL2, 0x6); write_rc_conf(node, index, RC_ORDER_RULE_CTL, 0x0100); - if (IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_SW64_SIM)) { - value = read_rc_conf(node, index, RC_LINK_STAT); - value |= 0x3; - write_rc_conf(node, index, RC_LINK_STAT, value); - } - /* enable DBI_RO_WR_EN */ rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); @@ -431,10 +394,6 @@ static unsigned long chip3_get_rc_enable(unsigned long node) if (is_guest_or_emul()) return 1; - if (!IS_ENABLED(CONFIG_SW64_ASIC)) { - rc_enable = 0x1; - sw64_io_write(node, IO_START, rc_enable); - } rc_enable = sw64_io_read(node, IO_START); return rc_enable; diff --git a/arch/sw_64/include/asm/chip3_io.h b/arch/sw_64/include/asm/chip3_io.h index 7f64e8816ed3..013730235b83 100644 --- a/arch/sw_64/include/asm/chip3_io.h +++ b/arch/sw_64/include/asm/chip3_io.h @@ -9,11 +9,7 @@ #define PCI_IOR0_BASE (0x2UL << 32) #define PCI_IOR1_BASE (0x3UL << 32) -#ifdef CONFIG_SW64_FPGA -#define PCI_RC_CFG (0x4UL << 32) -#else #define PCI_RC_CFG (0x5UL << 32) -#endif #define PCI_EP_CFG (0x3UL << 33) #define PCI_LEGACY_IO (0x1UL << 32) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index aa90d284de9b..a6f173b64977 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -795,17 +795,6 @@ setup_arch(char **cmdline_p) } #endif /* CMDLINE_EXTEND */ #endif - if (IS_ENABLED(CONFIG_SW64_CHIP3_ASIC_DEBUG) && - IS_ENABLED(CONFIG_SW64_CHIP3)) { - unsigned long bmc, cpu_online, node; - - bmc = *(unsigned long *)__va(0x800000); - pr_info("bmc = %ld\n", bmc); - cpu_online = sw64_chip->get_cpu_num(); - for (node = 0; node < cpu_online; node++) - sw64_io_write(node, SI_FAULT_INT_EN, 0); - sprintf(boot_command_line, "root=/dev/sda2 ip=172.16.137.%ld::172.16.137.254:255.255.255.0::eth0:off", 180+bmc); - } strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; -- Gitee From 352dca8f987f302eeb2e3402e74a8cfffa3680e0 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 15:02:11 +0800 Subject: [PATCH 047/150] sw64: cleanup hugetlb code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Remove unnecessary hugetlb code. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hugetlb.h | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/arch/sw_64/include/asm/hugetlb.h b/arch/sw_64/include/asm/hugetlb.h index 11565a8f86cb..246d12c2103d 100644 --- a/arch/sw_64/include/asm/hugetlb.h +++ b/arch/sw_64/include/asm/hugetlb.h @@ -5,21 +5,4 @@ #include #include -static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ -} - -void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, - unsigned long end, unsigned long floor, - unsigned long ceiling); - -static inline int arch_prepare_hugepage(struct page *page) -{ - return 0; -} - -static inline void arch_release_hugepage(struct page *page) -{ -} - #endif /* _ASM_SW64_HUGETLB_H */ -- Gitee From 6b2790ec95a96e75bf00a7ecf027e3c97ee5ddbb Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 14 Aug 2023 17:19:59 +0800 Subject: [PATCH 048/150] sw64: fix compile warnings Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56QAM -------------------------------- Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/cpuautoplug.c | 10 +++++++--- drivers/firmware/efi/sunway-init.c | 5 +++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/sw_64/kernel/cpuautoplug.c b/arch/sw_64/kernel/cpuautoplug.c index e867d5f544c0..16017b6d1254 100644 --- a/arch/sw_64/kernel/cpuautoplug.c +++ b/arch/sw_64/kernel/cpuautoplug.c @@ -275,7 +275,7 @@ static int find_min_busy_cpu(void) int nr_all_cpus = num_possible_cpus(); unsigned int cpus, target_cpu; cputime64_t busy_time; - cputime64_t b_time[nr_all_cpus]; + cputime64_t b_time[NR_CPUS]; memset(b_time, 0, sizeof(b_time)); for_each_online_cpu(cpus) { @@ -288,12 +288,14 @@ static int find_min_busy_cpu(void) static void increase_cores(int cur_cpus) { + struct device *dev; + if (cur_cpus == ap_info.maxcpus) return; cur_cpus = cpumask_next_zero(0, cpu_online_mask); - struct device *dev = get_cpu_device(cur_cpus); + dev = get_cpu_device(cur_cpus); per_cpu(cpu_adjusting, dev->id) = 1; lock_device_hotplug(); @@ -307,12 +309,14 @@ static void increase_cores(int cur_cpus) static void decrease_cores(int cur_cpus) { + struct device *dev; + if (cur_cpus == ap_info.mincpus) return; cur_cpus = find_min_busy_cpu(); - struct device *dev = get_cpu_device(cur_cpus); + dev = get_cpu_device(cur_cpus); if (dev->id > 0) { per_cpu(cpu_adjusting, dev->id) = -1; diff --git a/drivers/firmware/efi/sunway-init.c b/drivers/firmware/efi/sunway-init.c index d3af23feb8bb..870abc2f5afe 100644 --- a/drivers/firmware/efi/sunway-init.c +++ b/drivers/firmware/efi/sunway-init.c @@ -36,9 +36,10 @@ static int __init is_memory(efi_memory_desc_t *md) return 0; } static efi_config_table_type_t arch_tables[] __initdata = { - {SMBIOS3_TABLE_GUID, NULL, NULL}, + {SMBIOS3_TABLE_GUID, NULL, ""}, {SLEEP_ENTRY_GUID, &entSuspend, "SLEEP ENTRY"}, - {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"} + {BIOS_VERSION_GUID, &bios_version, "BIOS VERSION"}, + {}, }; static int __init uefi_init(u64 efi_system_table) -- Gitee From 5358807c4e1e72fe8470c529f7795f04ec8b282c Mon Sep 17 00:00:00 2001 From: Dai Xin Date: Fri, 25 Aug 2023 14:23:19 +0000 Subject: [PATCH 049/150] i2c: designware: add parameter MODEL_SUNWAY Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC4P -------------------------------- To be compatible with the device tree blob of previous BIOS firmware, we add parameter MODEL_SUNWAY to match i2c driver on sw64 platform. Signed-off-by: Dai Xin Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/i2c/busses/i2c-designware-platdrv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 38f5c88fb431..d88a083fd82c 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -151,7 +151,11 @@ static int dw_i2c_of_configure(struct platform_device *pdev) } static const struct of_device_id dw_i2c_of_match[] = { +#ifdef CONFIG_SW64 + { .compatible = "snps,designware-i2c", .data = (void *)MODEL_SUNWAY }, +#else { .compatible = "snps,designware-i2c", }, +#endif { .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT }, { .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 }, { .compatible = "sunway,suntai-i2c", .data = (void *)MODEL_SUNWAY }, -- Gitee From 7e98731f2cbc67cae96c57140becd58d95567f2b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:45 +0800 Subject: [PATCH 050/150] sw64: reorganize cpu frequency related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Patch series "sw64: reorganize code structure". Reorganize code structure to make it easier to add new cpu support. This patch (of 10): Move cpu frequency related code to proper location. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 8 -- arch/sw_64/chip/chip3/Makefile | 2 - arch/sw_64/chip/chip3/cpufreq_platform.c | 69 ----------------- arch/sw_64/platform/Makefile | 2 +- .../cpufreq.c => platform/cpufreq_xuelang.c} | 75 ++++++++++++++++--- drivers/cpufreq/Kconfig | 13 +++- drivers/cpufreq/Makefile | 1 + .../cpufreq/sw64_cpufreq_debugfs.c | 0 8 files changed, 79 insertions(+), 91 deletions(-) delete mode 100644 arch/sw_64/chip/chip3/cpufreq_platform.c rename arch/sw_64/{chip/chip3/cpufreq.c => platform/cpufreq_xuelang.c} (56%) rename arch/sw_64/chip/chip3/cpufreq_debugfs.c => drivers/cpufreq/sw64_cpufreq_debugfs.c (100%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 956a37a3388f..ff95c5937af1 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -231,14 +231,6 @@ config SW64_CPUAUTOPLUG help Turns on the interface for SW64_CPU CPUAUTOPLUG. -config CPUFREQ_DEBUGFS - bool "CPU Frequency debugfs interface for Chip3 Asic" - depends on SW64_CHIP3 && DEBUG_FS && CPU_FREQ - help - Turns on the DebugFS interface for CPU Frequency. - - If you don't know what to do here, say N. - endmenu # clear all implied options (don't want default values for those): # Most of these machines have ISA slots; not exactly sure which don't, diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile index bc262e8f12ef..17769a1fefa3 100644 --- a/arch/sw_64/chip/chip3/Makefile +++ b/arch/sw_64/chip/chip3/Makefile @@ -4,5 +4,3 @@ obj-y := chip.o i2c-lib.o obj-$(CONFIG_PCI) += pci-quirks.o obj-$(CONFIG_PCI_MSI) += msi.o vt_msi.o -obj-$(CONFIG_CPU_FREQ) += cpufreq_platform.o cpufreq.o -obj-$(CONFIG_CPUFREQ_DEBUGFS) += cpufreq_debugfs.o diff --git a/arch/sw_64/chip/chip3/cpufreq_platform.c b/arch/sw_64/chip/chip3/cpufreq_platform.c deleted file mode 100644 index 4dcaea7193ce..000000000000 --- a/arch/sw_64/chip/chip3/cpufreq_platform.c +++ /dev/null @@ -1,69 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include -#include - -/* Minimum CLK support */ -enum { - DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, - DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV -}; - -struct cpufreq_frequency_table freq_table[] = { - {0, 200, CPUFREQ_ENTRY_INVALID}, - {0, DC_1, CPUFREQ_ENTRY_INVALID}, - {0, DC_2, 0}, - {0, DC_3, 0}, - {0, DC_4, 0}, - {0, DC_5, 0}, - {0, DC_6, 0}, - {0, DC_7, 0}, - {0, DC_8, 0}, - {0, DC_9, 0}, - {0, DC_10, 0}, - {0, DC_11, 0}, - {0, DC_12, 0}, - {0, DC_13, 0}, - {0, DC_14, 0}, - {0, DC_15, 0}, - {-1, DC_RESV, CPUFREQ_TABLE_END}, -}; - - -static struct platform_device sw64_cpufreq_device = { - .name = "sw64_cpufreq", - .id = -1, -}; - -static int __init sw64_cpufreq_init(void) -{ - int i; - unsigned char external_clk; - unsigned long max_rate, freq_off; - max_rate = get_cpu_freq() / 1000; - - external_clk = *((unsigned char *)__va(0x908011)); - - if (external_clk == 240) - freq_off = 60000; - else - freq_off = 50000; - - /* clock table init */ - for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { - if (i == 1) - freq_table[i].driver_data = freq_off * 24; - if (i == 2) - freq_table[i].frequency = freq_off * 36; - if (i > 2) - freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); - - if (freq_table[i].frequency == max_rate) - freq_table[i + 1].frequency = CPUFREQ_TABLE_END; - } - - return platform_device_register(&sw64_cpufreq_device); -} -arch_initcall(sw64_cpufreq_init); diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile index a972b931dea2..7e89ab09624f 100644 --- a/arch/sw_64/platform/Makefile +++ b/arch/sw_64/platform/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PLATFORM_XUELANG) += platform_xuelang.o +obj-$(CONFIG_PLATFORM_XUELANG) += platform_xuelang.o cpufreq_xuelang.o diff --git a/arch/sw_64/chip/chip3/cpufreq.c b/arch/sw_64/platform/cpufreq_xuelang.c similarity index 56% rename from arch/sw_64/chip/chip3/cpufreq.c rename to arch/sw_64/platform/cpufreq_xuelang.c index e859d822870c..04b412c5f755 100644 --- a/arch/sw_64/chip/chip3/cpufreq.c +++ b/arch/sw_64/platform/cpufreq_xuelang.c @@ -1,17 +1,74 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include +#include -#include -#include -#include -#include #include +#include +#include + +/* Minimum CLK support */ +enum { + DC_0, DC_1, DC_2, DC_3, DC_4, DC_5, DC_6, DC_7, DC_8, + DC_9, DC_10, DC_11, DC_12, DC_13, DC_14, DC_15, DC_RESV +}; + +struct cpufreq_frequency_table freq_table[] = { + {0, 200, CPUFREQ_ENTRY_INVALID}, + {0, DC_1, CPUFREQ_ENTRY_INVALID}, + {0, DC_2, 0}, + {0, DC_3, 0}, + {0, DC_4, 0}, + {0, DC_5, 0}, + {0, DC_6, 0}, + {0, DC_7, 0}, + {0, DC_8, 0}, + {0, DC_9, 0}, + {0, DC_10, 0}, + {0, DC_11, 0}, + {0, DC_12, 0}, + {0, DC_13, 0}, + {0, DC_14, 0}, + {0, DC_15, 0}, + {-1, DC_RESV, CPUFREQ_TABLE_END}, +}; + +static struct platform_device sw64_cpufreq_device = { + .name = "sw64_cpufreq", + .id = -1, +}; + +static int __init sw64_cpufreq_init(void) +{ + int i; + unsigned char external_clk; + unsigned long max_rate, freq_off; + + max_rate = get_cpu_freq() / 1000; + + external_clk = *((unsigned char *)__va(0x908011)); + + if (external_clk == 240) + freq_off = 60000; + else + freq_off = 50000; + + /* clock table init */ + for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { + if (i == 1) + freq_table[i].driver_data = freq_off * 24; + if (i == 2) + freq_table[i].frequency = freq_off * 36; + if (i > 2) + freq_table[i].frequency = freq_off * 38 + ((i - 3) * freq_off); + + if (freq_table[i].frequency == max_rate) + freq_table[i + 1].frequency = CPUFREQ_TABLE_END; + } + + return platform_device_register(&sw64_cpufreq_device); +} +arch_initcall(sw64_cpufreq_init); char curruent_policy[CPUFREQ_NAME_LEN]; diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index ffa99a9d1299..43682f9b6e09 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -356,16 +356,25 @@ endif if SW64 config SW64_CPUFREQ - bool "sw64 CPU Frequency interface for Chip3 Asic" + bool "SW64 CPU Frequency interface" depends on SW64_CHIP3 default y help - This adds the CPUFreq driver for SW6B processor which supports + This adds the CPUFreq driver for SW64 processor which supports software configurable cpu frequency. For details, take a look at . If unsure, say N. + +config SW64_CPUFREQ_DEBUGFS + bool "SW64 CPU Frequency debugfs interface" + depends on SW64_CPUFREQ && DEBUG_FS + default y + help + Turns on the DebugFS interface for CPU Frequency. + + If you don't know what to do here, say N. endif config QORIQ_CPUFREQ diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index a02e0e518a05..d6c180ca39a7 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -112,3 +112,4 @@ obj-$(CONFIG_SH_CPU_FREQ) += sh-cpufreq.o obj-$(CONFIG_SPARC_US2E_CPUFREQ) += sparc-us2e-cpufreq.o obj-$(CONFIG_SPARC_US3_CPUFREQ) += sparc-us3-cpufreq.o obj-$(CONFIG_SW64_CPUFREQ) += sw64_cpufreq.o +obj-$(CONFIG_SW64_CPUFREQ_DEBUGFS) += sw64_cpufreq_debugfs.o diff --git a/arch/sw_64/chip/chip3/cpufreq_debugfs.c b/drivers/cpufreq/sw64_cpufreq_debugfs.c similarity index 100% rename from arch/sw_64/chip/chip3/cpufreq_debugfs.c rename to drivers/cpufreq/sw64_cpufreq_debugfs.c -- Gitee From cad5182143c91862037b810257d69e9e642c5c28 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:46 +0800 Subject: [PATCH 051/150] sw64: reorganize timer related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move timer related code to drivers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/chip/chip3/chip.c | 72 ------ arch/sw_64/include/asm/sw64io.h | 2 - arch/sw_64/include/asm/timer.h | 11 + arch/sw_64/kernel/Makefile | 2 +- arch/sw_64/kernel/proto.h | 8 - arch/sw_64/kernel/setup.c | 4 +- arch/sw_64/kernel/smp.c | 3 +- arch/sw_64/kernel/time.c | 165 +------------ drivers/clocksource/Kconfig | 3 + drivers/clocksource/Makefile | 1 + .../clocksource/timer-sw64.c | 231 +++++++++++++++++- 12 files changed, 246 insertions(+), 257 deletions(-) create mode 100644 arch/sw_64/include/asm/timer.h rename arch/sw_64/kernel/timer.c => drivers/clocksource/timer-sw64.c (39%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index ff95c5937af1..4497f43f4484 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -108,6 +108,7 @@ config SW64 select PCI_MSI_ARCH_FALLBACKS select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM + select SW64_TIMER select SWIOTLB select THREAD_INFO_IN_TASK diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 337f392dc4cc..6eb1f4fcb7d3 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -10,78 +10,6 @@ #include #include "../../../../drivers/pci/pci.h" -static u64 read_longtime(struct clocksource *cs) -{ - u64 result; - unsigned long node; - - node = __this_cpu_read(hard_node_id); - result = sw64_io_read(node, LONG_TIME); - - return result; -} - -static int longtime_enable(struct clocksource *cs) -{ - switch (cpu_desc.model) { - case CPU_SW3231: - sw64_io_write(0, GPIO_SWPORTA_DR, 0); - sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); - break; - case CPU_SW831: - sw64_io_write(0, LONG_TIME_START_EN, 0x1); - break; - default: - break; - } - - return 0; -} - -static struct clocksource clocksource_longtime = { - .name = "longtime", - .rating = 100, - .enable = longtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_longtime, -}; - -static u64 read_vtime(struct clocksource *cs) -{ - u64 result; - unsigned long vtime_addr = IO_BASE | LONG_TIME; - - result = rdio64(vtime_addr); - return result; -} - -static int vtime_enable(struct clocksource *cs) -{ - return 0; -} - -static struct clocksource clocksource_vtime = { - .name = "vtime", - .rating = 100, - .enable = vtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_vtime, -}; - -void setup_chip_clocksource(void) -{ - if (is_in_host()) - clocksource_register_khz(&clocksource_longtime, 25000); - else - clocksource_register_khz(&clocksource_vtime, 25000); -} - void set_devint_wken(int node) { unsigned long val; diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index 0892356b8e6b..281b0b3da32c 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -5,8 +5,6 @@ #include #include -extern void setup_chip_clocksource(void); - #if defined(CONFIG_SW64_CHIP3) #include #endif diff --git a/arch/sw_64/include/asm/timer.h b/arch/sw_64/include/asm/timer.h new file mode 100644 index 000000000000..9ea9e0a538d0 --- /dev/null +++ b/arch/sw_64/include/asm/timer.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_TIMER_H +#define _ASM_SW64_TIMER_H + +extern void sw64_setup_clocksource(void); + +extern void sw64_setup_timer(void); + +extern void __init setup_sched_clock(void); + +#endif /* _ASM_SW64_TIMER_H */ diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 52ed1166b9bd..effe855aa5e0 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -15,7 +15,7 @@ endif obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ irq_sw64.o signal.o setup.o ptrace.o time.o \ - systbls.o dup_print.o tc.o timer.o \ + systbls.o dup_print.o tc.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o diff --git a/arch/sw_64/kernel/proto.h b/arch/sw_64/kernel/proto.h index 7b2564f47a9d..d7222334d1b9 100644 --- a/arch/sw_64/kernel/proto.h +++ b/arch/sw_64/kernel/proto.h @@ -15,12 +15,4 @@ extern int ptrace_cancel_bpt(struct task_struct *child); extern void show_regs(struct pt_regs *regs); extern void die(char *str, struct pt_regs *regs, long err); -/* timer.c */ -extern void setup_timer(void); - -extern void __init setup_sched_clock(void); -#ifdef CONFIG_GENERIC_SCHED_CLOCK -extern void __init sw64_sched_clock_init(void); -#endif - #endif /* _SW64_KERNEL_PROTO_H */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index a6f173b64977..73fcc91c56f0 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "proto.h" #include "pci_impl.h" @@ -765,9 +766,6 @@ setup_arch(char **cmdline_p) sw64_chip_init->early_init.get_smp_info(); setup_sched_clock(); -#ifdef CONFIG_GENERIC_SCHED_CLOCK - sw64_sched_clock_init(); -#endif setup_machine_fdt(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index f145556d879d..08c0c9c76328 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "proto.h" @@ -82,7 +83,7 @@ void smp_callin(void) wrent(entInt, 0); /* Get our local ticker going. */ - setup_timer(); + sw64_setup_timer(); /* All kernel threads share the same mm context. */ mmgrab(&init_mm); diff --git a/arch/sw_64/kernel/time.c b/arch/sw_64/kernel/time.c index 872a19802ade..533a6a14c200 100644 --- a/arch/sw_64/kernel/time.c +++ b/arch/sw_64/kernel/time.c @@ -4,19 +4,15 @@ #include #include #include -#ifndef CONFIG_SMP -#include -#endif #include +#include #include "proto.h" DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL(rtc_lock); -DECLARE_PER_CPU(u64, tc_offset); - #define TICK_SIZE (tick_nsec / 1000) /* @@ -28,25 +24,6 @@ DECLARE_PER_CPU(u64, tc_offset); unsigned long est_cycle_freq; -static u64 sc_start; -static u64 sc_shift; -static u64 sc_multi; - -DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); -static int __init sched_clock_setup(char *opt) -{ - if (!opt) - return -EINVAL; - - if (!strncmp(opt, "on", 2)) { - static_branch_enable(&use_tc_as_sched_clock); - pr_info("Using TC instead of jiffies as source of sched_clock()\n"); - } - - return 0; -} -early_param("tc_sched_clock", sched_clock_setup); - #ifdef CONFIG_IRQ_WORK DEFINE_PER_CPU(u8, irq_work_pending); @@ -67,36 +44,6 @@ void arch_irq_work_raise(void) #endif /* CONFIG_IRQ_WORK */ -#ifndef CONFIG_SMP -static u64 read_tc(struct clocksource *cs) -{ - return rdtc(); -} - -static struct clocksource clocksource_tc = { - .name = "tc", - .rating = 300, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 22, - .mult = 0, /* To be filled in */ - .read = read_tc, -}; - -void setup_clocksource(void) -{ - clocksource_register_hz(&clocksource_tc, get_cpu_freq()); - pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); -} - -#else /* !CONFIG_SMP */ -void setup_clocksource(void) -{ - setup_chip_clocksource(); -} -#endif /* !CONFIG_SMP */ - - void __init time_init(void) { @@ -107,116 +54,10 @@ time_init(void) pr_info("CPU Cycle frequency = %ld Hz\n", cycle_freq); /* Register clocksource */ - setup_clocksource(); + sw64_setup_clocksource(); of_clk_init(NULL); /* Startup the timer source. */ - setup_timer(); + sw64_setup_timer(); /* Calibrate the delay loop directly */ lpj_fine = cycle_freq / HZ; } - -static void __init calibrate_sched_clock(void) -{ - sc_start = rdtc(); -} - -void __init setup_sched_clock(void) -{ - unsigned long step; - - sc_shift = 7; - step = 1UL << sc_shift; - sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); - calibrate_sched_clock(); - - pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); -} - -#ifdef CONFIG_GENERIC_SCHED_CLOCK -#include -static u64 notrace sched_clock_read(void) -{ - return (rdtc() - sc_start) >> sc_shift; -} - -void __init sw64_sched_clock_init(void) -{ - sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); -} -#else -unsigned long long notrace sched_clock(void) -{ - if (static_branch_likely(&use_tc_as_sched_clock)) - return ((rdtc() - sc_start + __this_cpu_read(tc_offset)) >> sc_shift) * sc_multi; - else - return (jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); -} - -#ifdef CONFIG_DEBUG_FS -static ssize_t sched_clock_status_read(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - char buf[2]; - - if (static_key_enabled(&use_tc_as_sched_clock)) - buf[0] = 'Y'; - else - buf[0] = 'N'; - buf[1] = '\n'; - return simple_read_from_buffer(user_buf, count, ppos, buf, 2); -} - -static ssize_t sched_clock_status_write(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - int r; - bool bv; - bool val = static_key_enabled(&use_tc_as_sched_clock); - - r = kstrtobool_from_user(user_buf, count, &bv); - if (!r) { - if (val != bv) { - if (bv) { - static_branch_enable(&use_tc_as_sched_clock); - pr_info("source of sched_clock() switched from jiffies to TC\n"); - } else { - static_branch_disable(&use_tc_as_sched_clock); - pr_info("source of sched_clock() switched from TC to jiffies\n"); - } - } else { - if (val) - pr_info("source of sched_clock() unchanged (using TC)\n"); - else - pr_info("source of sched_clock() unchanged (using jiffies)\n"); - } - } - - return count; -} - -static const struct file_operations sched_clock_status_fops = { - .read = sched_clock_status_read, - .write = sched_clock_status_write, - .open = nonseekable_open, - .llseek = no_llseek, -}; - -static int __init sched_clock_debug_init(void) -{ - struct dentry *sched_clock_status; - - if (!sw64_debugfs_dir) - return -ENODEV; - - sched_clock_status = debugfs_create_file("tc_sched_clock", - 0644, sw64_debugfs_dir, NULL, - &sched_clock_status_fops); - - if (!sched_clock_status) - return -ENOMEM; - - return 0; -} -late_initcall(sched_clock_debug_init); -#endif /* CONFIG_DEBUG_FS */ -#endif /* CONFIG_GENERIC_SCHED_CLOCK */ diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index a0c6e88bebe0..dc3ebb974eab 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -740,4 +740,7 @@ config MICROCHIP_PIT64B modes and high resolution. It is used as a clocksource and a clockevent. +config SW64_TIMER + bool + endmenu diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 1c444cc3bb44..70a9355f23a4 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -94,3 +94,4 @@ obj-$(CONFIG_CSKY_MP_TIMER) += timer-mp-csky.o obj-$(CONFIG_GX6605S_TIMER) += timer-gx6605s.o obj-$(CONFIG_HYPERV_TIMER) += hyperv_timer.o obj-$(CONFIG_MICROCHIP_PIT64B) += timer-microchip-pit64b.o +obj-$(CONFIG_SW64_TIMER) += timer-sw64.o diff --git a/arch/sw_64/kernel/timer.c b/drivers/clocksource/timer-sw64.c similarity index 39% rename from arch/sw_64/kernel/timer.c rename to drivers/clocksource/timer-sw64.c index f4745c1c6a78..456f2becb182 100644 --- a/arch/sw_64/kernel/timer.c +++ b/drivers/clocksource/timer-sw64.c @@ -1,14 +1,229 @@ // SPDX-License-Identifier: GPL-2.0 -/* - * Filename: timer.c - * Description: percpu local timer, based on arch/x86/kernel/apic/apic.c - */ -#include #include +#include +#include +#include #include -#include +#include + +DECLARE_PER_CPU(u64, tc_offset); + +static u64 sc_start; +static u64 sc_shift; +static u64 sc_multi; + +DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); +static int __init sched_clock_setup(char *opt) +{ + if (!opt) + return -EINVAL; + + if (!strncmp(opt, "on", 2)) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("Using TC instead of jiffies as source of sched_clock()\n"); + } + + return 0; +} +early_param("tc_sched_clock", sched_clock_setup); + +static void __init calibrate_sched_clock(void) +{ + sc_start = rdtc(); +} + +void __init setup_sched_clock(void) +{ + unsigned long step; + + sc_shift = 7; + step = 1UL << sc_shift; + sc_multi = step * NSEC_PER_SEC / get_cpu_freq(); + calibrate_sched_clock(); + + pr_info("sched_clock: sc_multi=%llu, sc_shift=%llu\n", sc_multi, sc_shift); +} + +#ifdef CONFIG_GENERIC_SCHED_CLOCK +static u64 notrace sched_clock_read(void) +{ + return (rdtc() - sc_start) >> sc_shift; +} +#else /* !CONFIG_GENERIC_SCHED_CLOCK */ +unsigned long long notrace sched_clock(void) +{ + if (static_branch_likely(&use_tc_as_sched_clock)) + return ((rdtc() - sc_start + __this_cpu_read(tc_offset)) >> sc_shift) * sc_multi; + else + return (jiffies - INITIAL_JIFFIES) * (NSEC_PER_SEC / HZ); +} + +#ifdef CONFIG_DEBUG_FS +static ssize_t sched_clock_status_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[2]; + + if (static_key_enabled(&use_tc_as_sched_clock)) + buf[0] = 'Y'; + else + buf[0] = 'N'; + buf[1] = '\n'; + return simple_read_from_buffer(user_buf, count, ppos, buf, 2); +} + +static ssize_t sched_clock_status_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int r; + bool bv; + bool val = static_key_enabled(&use_tc_as_sched_clock); + + r = kstrtobool_from_user(user_buf, count, &bv); + if (!r) { + if (val != bv) { + if (bv) { + static_branch_enable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from jiffies to TC\n"); + } else { + static_branch_disable(&use_tc_as_sched_clock); + pr_info("source of sched_clock() switched from TC to jiffies\n"); + } + } else { + if (val) + pr_info("source of sched_clock() unchanged (using TC)\n"); + else + pr_info("source of sched_clock() unchanged (using jiffies)\n"); + } + } + + return count; +} + +static const struct file_operations sched_clock_status_fops = { + .read = sched_clock_status_read, + .write = sched_clock_status_write, + .open = nonseekable_open, + .llseek = no_llseek, +}; + +static int __init sched_clock_debug_init(void) +{ + struct dentry *sched_clock_status; + + if (!sw64_debugfs_dir) + return -ENODEV; + + sched_clock_status = debugfs_create_file("tc_sched_clock", + 0644, sw64_debugfs_dir, NULL, + &sched_clock_status_fops); + + if (!sched_clock_status) + return -ENOMEM; + + return 0; +} +late_initcall(sched_clock_debug_init); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_GENERIC_SCHED_CLOCK */ + +static u64 read_tc(struct clocksource *cs) +{ + return rdtc(); +} + +static struct clocksource clocksource_tc = { + .name = "tc", + .rating = 300, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 22, + .mult = 0, /* To be filled in */ + .read = read_tc, +}; + +static u64 read_longtime(struct clocksource *cs) +{ + u64 result; + unsigned long node; + + node = __this_cpu_read(hard_node_id); + result = sw64_io_read(node, LONG_TIME); + + return result; +} + +static int longtime_enable(struct clocksource *cs) +{ + switch (cpu_desc.model) { + case CPU_SW3231: + sw64_io_write(0, GPIO_SWPORTA_DR, 0); + sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); + break; + case CPU_SW831: + sw64_io_write(0, LONG_TIME_START_EN, 0x1); + break; + default: + break; + } + + return 0; +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .enable = longtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 read_vtime(struct clocksource *cs) +{ + u64 result; + unsigned long vtime_addr = IO_BASE | LONG_TIME; + + result = rdio64(vtime_addr); + return result; +} + +static int vtime_enable(struct clocksource *cs) +{ + return 0; +} + +static struct clocksource clocksource_vtime = { + .name = "vtime", + .rating = 100, + .enable = vtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_vtime, +}; + +void __init sw64_setup_clocksource(void) +{ + if (!IS_ENABLED(CONFIG_SMP)) { + clocksource_register_hz(&clocksource_tc, get_cpu_freq()); + pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); + } else { + if (is_in_host()) + clocksource_register_khz(&clocksource_longtime, 25000); + else + clocksource_register_khz(&clocksource_vtime, 25000); + } + +#ifdef CONFIG_GENERIC_SCHED_CLOCK + sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); +#endif +} static int timer_next_event(unsigned long delta, struct clock_event_device *evt); @@ -96,10 +311,10 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq) } /* - * Setup the local timer for this CPU. Copy the initilized values + * Setup the local timer for this CPU. Copy the initialized values * of the boot CPU and register the clock event in the framework. */ -void setup_timer(void) +void sw64_setup_timer(void) { int cpu = smp_processor_id(); struct clock_event_device *swevt = &per_cpu(timer_events, cpu); -- Gitee From 32280da0b027649c3ee56b73dda113b0995a824a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:46 +0800 Subject: [PATCH 052/150] sw64: reorganize i2c related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move i2c related code to drivers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/chip/chip3/Makefile | 2 +- drivers/i2c/busses/Kconfig | 11 +++++++++++ drivers/i2c/busses/Makefile | 1 + .../i2c-lib.c => drivers/i2c/busses/i2c-sunway.c | 2 -- 5 files changed, 14 insertions(+), 3 deletions(-) rename arch/sw_64/chip/chip3/i2c-lib.c => drivers/i2c/busses/i2c-sunway.c (99%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 4497f43f4484..d375be503f14 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -210,6 +210,7 @@ config PLATFORM_XUELANG select SPARSE_IRQ select SYS_HAS_EARLY_PRINTK select SW64_INTC_V2 + select I2C_SUNWAY if I2C help Sunway chip3 board chipset diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile index 17769a1fefa3..17f3574c0331 100644 --- a/arch/sw_64/chip/chip3/Makefile +++ b/arch/sw_64/chip/chip3/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y := chip.o i2c-lib.o +obj-y := chip.o obj-$(CONFIG_PCI) += pci-quirks.o obj-$(CONFIG_PCI_MSI) += msi.o vt_msi.o diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index a490ba486474..a5f18dfa896e 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -323,6 +323,17 @@ config I2C_VIAPRO This driver can also be built as a module. If so, the module will be called i2c-viapro. +config I2C_SUNWAY + tristate "Sunway i2c lib" + depends on UNCORE_XUELANG + help + If you say yes to this option, support will be included for the + Sunway Soc I2C interface on SW64 platfrom. + + This driver can also be built as a module. If so, the module + will be called i2c-sunway. + + if ACPI comment "ACPI drivers" diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index d274db87ecd1..e2a0df0fd858 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_I2C_SIS630) += i2c-sis630.o obj-$(CONFIG_I2C_SIS96X) += i2c-sis96x.o obj-$(CONFIG_I2C_VIA) += i2c-via.o obj-$(CONFIG_I2C_VIAPRO) += i2c-viapro.o +obj-$(CONFIG_I2C_SUNWAY) += i2c-sunway.o # Mac SMBus host controller drivers obj-$(CONFIG_I2C_HYDRA) += i2c-hydra.o diff --git a/arch/sw_64/chip/chip3/i2c-lib.c b/drivers/i2c/busses/i2c-sunway.c similarity index 99% rename from arch/sw_64/chip/chip3/i2c-lib.c rename to drivers/i2c/busses/i2c-sunway.c index e70f0f0c9a56..f99955ac7ae3 100644 --- a/arch/sw_64/chip/chip3/i2c-lib.c +++ b/drivers/i2c/busses/i2c-sunway.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 /* - * arch/sw_64/chip/chip3/i2c-lib.c - * * Copyright (C) 2020 Platform Software * * This program is free software; you can redistribute it and/or -- Gitee From bcbb38650b9c39af70f48f02022d5fb3c3affaf0 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:46 +0800 Subject: [PATCH 053/150] sw64: reorganize usb quirks related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move usb quirks related code to drivers and remove duplicated code. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/pci-quirks.c | 192 ----------------------------- drivers/usb/host/pci-quirks.c | 128 +++++++++++++++++++ 2 files changed, 128 insertions(+), 192 deletions(-) diff --git a/arch/sw_64/chip/chip3/pci-quirks.c b/arch/sw_64/chip/chip3/pci-quirks.c index 22887d269fe3..1700cecdf659 100644 --- a/arch/sw_64/chip/chip3/pci-quirks.c +++ b/arch/sw_64/chip/chip3/pci-quirks.c @@ -4,198 +4,6 @@ #include -static int handshake(void __iomem *ptr, u32 mask, u32 done, - int wait_usec, int delay_usec) -{ - u32 result; - - do { - result = readl(ptr); - result &= mask; - if (result == done) - return 0; - udelay(delay_usec); - wait_usec -= delay_usec; - } while (wait_usec > 0); - return -ETIMEDOUT; -} - -#define XHCI_HCC_EXT_CAPS(p) (((p) >> 16) & 0xffff) -#define XHCI_EXT_CAPS_ID(p) (((p) >> 0) & 0xff) -#define XHCI_EXT_CAPS_NEXT(p) (((p) >> 8) & 0xff) -#define XHCI_HC_LENGTH(p) (((p) >> 0) & 0x00ff) -#define XHCI_CMD_OFFSET (0x00) -#define XHCI_STS_OFFSET (0x04) -#define XHCI_EXT_CAPS_LEGACY (1) -#define XHCI_HCC_PARAMS_OFFSET (0x10) -#define XHCI_LEGACY_CONTROL_OFFSET (0x04) -#define XHCI_LEGACY_DISABLE_SMI ((0x7 << 1) + (0xff << 5) + (0x7 << 17)) -#define XHCI_LEGACY_SMI_EVENTS (0x7 << 29) -#define XHCI_HC_BIOS_OWNED (1 << 16) -#define XHCI_HC_OS_OWNED (1 << 24) -#define XHCI_CMD_RUN (1 << 0) -#define XHCI_STS_HALT (1 << 0) -#define XHCI_MAX_HALT_USEC (16 * 1000) -#define XHCI_CMD_EIE (1 << 2) -#define XHCI_CMD_HSEIE (1 << 3) -#define XHCI_CMD_EWE (1 << 10) -#define XHCI_IRQS (XHCI_CMD_EIE | XHCI_CMD_HSEIE | XHCI_CMD_EWE) -#define XHCI_STS_CNR (1 << 11) -#define STS_FATAL (1 << 2) -#define STS_EINT (1 << 3) -#define STS_PORT (1 << 4) -#define STS_SRE (1 << 10) -#define STS_RW1C_BITS (STS_FATAL | STS_EINT | STS_PORT | STS_SRE) - -static inline int xhci_find_next_ext_cap(void __iomem *base, u32 start, int id) -{ - u32 val; - u32 next; - u32 offset; - - offset = start; - if (!start || start == XHCI_HCC_PARAMS_OFFSET) { - val = readl(base + XHCI_HCC_PARAMS_OFFSET); - if (val == ~0) - return 0; - offset = XHCI_HCC_EXT_CAPS(val) << 2; - if (!offset) - return 0; - } - do { - val = readl(base + offset); - if (val == ~0) - return 0; - if (offset != start && (id == 0 || XHCI_EXT_CAPS_ID(val) == id)) - return offset; - - next = XHCI_EXT_CAPS_NEXT(val); - offset += next << 2; - } while (next); - - return 0; -} - -extern void usb_enable_intel_xhci_ports(struct pci_dev *xhci_pdev); - -static void -fixup_usb_xhci_reset(struct pci_dev *dev) -{ - void __iomem *op_reg_base; - int timeout; - u32 xhci_command; - u32 tmp, val; - void __iomem *base; - struct pci_controller *hose = dev->sysdata; - unsigned long offset; - int ext_cap_offset; - int retries = 3; - - pci_read_config_dword(dev, PCI_COMMAND, &tmp); - tmp |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); - pci_write_config_dword(dev, PCI_COMMAND, tmp); - - pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &tmp); - if (tmp & PCI_BASE_ADDRESS_MEM_TYPE_MASK) { - pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val); - offset = (unsigned long)(val) << 32 | (tmp & (~0xf)); - } else - offset = (unsigned long)(tmp & (~0xf)); - - if (offset == 0) - return; - - base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); - - ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); - if (!ext_cap_offset) - goto hc_init; - - val = readl(base + ext_cap_offset); - - if ((dev->vendor == PCI_VENDOR_ID_TI && dev->device == 0x8241) || - (dev->vendor == PCI_VENDOR_ID_RENESAS - && dev->device == 0x0014)) { - val = (val | XHCI_HC_OS_OWNED) & ~XHCI_HC_BIOS_OWNED; - writel(val, base + ext_cap_offset); - } - - if (val & XHCI_HC_BIOS_OWNED) { - writel(val | XHCI_HC_OS_OWNED, base + ext_cap_offset); - - timeout = handshake(base + ext_cap_offset, XHCI_HC_BIOS_OWNED, - 0, 1000000, 10); - if (timeout) { - pr_err("xHCI BIOS handoff failed (BIOS bug ?) %08x\n", val); - writel(val & ~XHCI_HC_BIOS_OWNED, base + ext_cap_offset); - } - } - - val = readl(base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); - val &= XHCI_LEGACY_DISABLE_SMI; - val |= XHCI_LEGACY_SMI_EVENTS; - writel(val, base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); - -hc_init: - if (dev->vendor == PCI_VENDOR_ID_INTEL) - usb_enable_intel_xhci_ports(dev); - - op_reg_base = base + XHCI_HC_LENGTH(readl(base)); - - timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_CNR, 0, - 5000000, 10); - if (timeout) { - val = readl(op_reg_base + XHCI_STS_OFFSET); - pr_err("xHCI HW not ready after 5 sec (HC bug?) status = 0x%x\n", val); - } - - xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); - xhci_command |= 0x2; - writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); - - timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, - 0x2, 0, 10 * 1000 * 1000, 125); - if (timeout) - pr_err("xHCI BIOS handoff time out\n"); - -retry: - val = readl(op_reg_base + XHCI_STS_OFFSET); - val |= STS_RW1C_BITS; - writel(val, op_reg_base + XHCI_STS_OFFSET); - val = readl(op_reg_base + XHCI_STS_OFFSET); - - if ((val & STS_RW1C_BITS) && retries--) { - pr_err("clear USB Status Register (status = %#x) failed, retry\n", val); - goto retry; - } - - val = readl(op_reg_base + XHCI_CMD_OFFSET); - val &= ~(XHCI_CMD_RUN | XHCI_IRQS); - writel(val, op_reg_base + XHCI_CMD_OFFSET); - timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_HALT, 1, - XHCI_MAX_HALT_USEC, 125); - if (timeout) { - val = readl(op_reg_base + XHCI_STS_OFFSET); - pr_err("xHCI HW did not halt within %d usec status = 0x%x\n", - XHCI_MAX_HALT_USEC, val); - } - - xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); - xhci_command |= 0x2; - writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); - - timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, - 0x2, 0, 10 * 1000 * 1000, 125); - if (timeout) - pr_err("xHCI BIOS handoff time out\n"); - - pci_read_config_dword(dev, PCI_COMMAND, &tmp); - tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); - pci_write_config_dword(dev, PCI_COMMAND, tmp); -} -DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, - PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); - #ifdef CONFIG_DCA static void enable_sw_dca(struct pci_dev *dev) { diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index ef08d68b9714..6bc756a6b63e 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1285,3 +1285,131 @@ static void quirk_usb_early_handoff(struct pci_dev *pdev) } DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_SERIAL_USB, 8, quirk_usb_early_handoff); + +#ifdef CONFIG_SW64 +#include + +#define xhci_find_next_ext_cap xhci_find_next_ext_cap_2 // avoid redefinition +#include "xhci.h" +#undef xhci_find_next_ext_cap + +#define STS_RW1C_BITS (STS_FATAL | STS_EINT | STS_PORT | STS_SRE) + +static void +fixup_usb_xhci_reset(struct pci_dev *dev) +{ + void __iomem *op_reg_base; + int timeout; + u32 xhci_command; + u32 tmp, val; + void __iomem *base; + struct pci_controller *hose = dev->sysdata; + unsigned long offset; + int ext_cap_offset; + int retries = 3; + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); + + pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &tmp); + if (tmp & PCI_BASE_ADDRESS_MEM_TYPE_MASK) { + pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val); + offset = (unsigned long)(val) << 32 | (tmp & (~0xf)); + } else + offset = (unsigned long)(tmp & (~0xf)); + + if (offset == 0) + return; + + base = (void *)__va(SW64_PCI_IO_BASE(hose->node, hose->index) | offset); + + ext_cap_offset = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_LEGACY); + if (!ext_cap_offset) + goto hc_init; + + val = readl(base + ext_cap_offset); + + if ((dev->vendor == PCI_VENDOR_ID_TI && dev->device == 0x8241) || + (dev->vendor == PCI_VENDOR_ID_RENESAS + && dev->device == 0x0014)) { + val = (val | XHCI_HC_OS_OWNED) & ~XHCI_HC_BIOS_OWNED; + writel(val, base + ext_cap_offset); + } + + if (val & XHCI_HC_BIOS_OWNED) { + writel(val | XHCI_HC_OS_OWNED, base + ext_cap_offset); + + timeout = handshake(base + ext_cap_offset, XHCI_HC_BIOS_OWNED, + 0, 1000000, 10); + if (timeout) { + pr_err("xHCI BIOS handoff failed (BIOS bug ?) %08x\n", val); + writel(val & ~XHCI_HC_BIOS_OWNED, base + ext_cap_offset); + } + } + + val = readl(base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + val &= XHCI_LEGACY_DISABLE_SMI; + val |= XHCI_LEGACY_SMI_EVENTS; + writel(val, base + ext_cap_offset + XHCI_LEGACY_CONTROL_OFFSET); + +hc_init: + if (dev->vendor == PCI_VENDOR_ID_INTEL) + usb_enable_intel_xhci_ports(dev); + + op_reg_base = base + XHCI_HC_LENGTH(readl(base)); + + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_CNR, 0, + 5000000, 10); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW not ready after 5 sec (HC bug?) status = 0x%x\n", val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + +retry: + val = readl(op_reg_base + XHCI_STS_OFFSET); + val |= STS_RW1C_BITS; + writel(val, op_reg_base + XHCI_STS_OFFSET); + val = readl(op_reg_base + XHCI_STS_OFFSET); + + if ((val & STS_RW1C_BITS) && retries--) { + pr_err("clear USB Status Register (status = %#x) failed, retry\n", val); + goto retry; + } + + val = readl(op_reg_base + XHCI_CMD_OFFSET); + val &= ~(XHCI_CMD_RUN | XHCI_IRQS); + writel(val, op_reg_base + XHCI_CMD_OFFSET); + timeout = handshake(op_reg_base + XHCI_STS_OFFSET, XHCI_STS_HALT, 1, + XHCI_MAX_HALT_USEC, 125); + if (timeout) { + val = readl(op_reg_base + XHCI_STS_OFFSET); + pr_err("xHCI HW did not halt within %d usec status = 0x%x\n", + XHCI_MAX_HALT_USEC, val); + } + + xhci_command = readl(op_reg_base + XHCI_CMD_OFFSET); + xhci_command |= 0x2; + writel(xhci_command, op_reg_base + XHCI_CMD_OFFSET); + + timeout = handshake(op_reg_base + XHCI_CMD_OFFSET, + 0x2, 0, 10 * 1000 * 1000, 125); + if (timeout) + pr_err("xHCI BIOS handoff time out\n"); + + pci_read_config_dword(dev, PCI_COMMAND, &tmp); + tmp &= ~(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + pci_write_config_dword(dev, PCI_COMMAND, tmp); +} +DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_ANY_ID, PCI_ANY_ID, + PCI_CLASS_SERIAL_USB_XHCI, 0, fixup_usb_xhci_reset); +#endif -- Gitee From ebd839fda0c47c0b2995a12cbaed16aa90016b40 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:47 +0800 Subject: [PATCH 054/150] sw64: reorganize pci related quirks Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move pci related quirks to proper location. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/chip.c | 33 ------------------------------ arch/sw_64/chip/chip3/pci-quirks.c | 32 ----------------------------- arch/sw_64/kernel/pci.c | 33 ++++++++++++++++++++++++++++++ arch/sw_64/kernel/pci_common.c | 32 +++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 65 deletions(-) diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 6eb1f4fcb7d3..d153a3d08143 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -804,36 +804,3 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); } EXPORT_SYMBOL(do_entInt); - -/* - * Early fix up the chip3 Root Complex settings - */ -static void chip3_pci_fixup_root_complex(struct pci_dev *dev) -{ - int i; - struct pci_bus *bus = dev->bus; - struct pci_controller *hose = bus->sysdata; - - hose->self_busno = hose->busn_space->start; - - if (likely(bus->number == hose->self_busno)) { - if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { - /* Check Root Complex port again */ - dev->is_hotplug_bridge = 0; - dev->current_state = PCI_D0; - } - - dev->class &= 0xff; - dev->class |= PCI_CLASS_BRIDGE_PCI << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = IORESOURCE_PCI_FIXED; - } - } - atomic_inc(&dev->enable_cnt); - - dev->no_msi = 1; -} - -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_CHIP3, chip3_pci_fixup_root_complex); diff --git a/arch/sw_64/chip/chip3/pci-quirks.c b/arch/sw_64/chip/chip3/pci-quirks.c index 1700cecdf659..0549efc07d0c 100644 --- a/arch/sw_64/chip/chip3/pci-quirks.c +++ b/arch/sw_64/chip/chip3/pci-quirks.c @@ -4,38 +4,6 @@ #include -#ifdef CONFIG_DCA -static void enable_sw_dca(struct pci_dev *dev) -{ - struct pci_controller *hose = (struct pci_controller *)dev->sysdata; - unsigned long node, rc_index, dca_ctl, dca_conf; - int i; - - if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) - return; - node = hose->node; - rc_index = hose->index; - for (i = 0; i < 256; i++) { - dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); - if (dca_conf >> 63) - continue; - else { - dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; - printk("dca device index %d, dca_conf = %#lx\n", i, dca_conf); - write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); - break; - } - } - dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); - if (dca_ctl & 0x1) { - dca_ctl = 0x2; - write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); - printk("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); - } -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); -#endif - void __init reserve_mem_for_pci(void) { int ret; diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index efa1625cf895..1744efb41ae8 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -57,6 +57,39 @@ static void quirk_isa_bridge(struct pci_dev *dev) } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82378, quirk_isa_bridge); +/* + * Early fix up the Root Complex settings + */ +static void fixup_root_complex(struct pci_dev *dev) +{ + int i; + struct pci_bus *bus = dev->bus; + struct pci_controller *hose = bus->sysdata; + + hose->self_busno = hose->busn_space->start; + + if (likely(bus->number == hose->self_busno)) { + if (IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)) { + /* Check Root Complex port again */ + dev->is_hotplug_bridge = 0; + dev->current_state = PCI_D0; + } + + dev->class &= 0xff; + dev->class |= PCI_CLASS_BRIDGE_PCI << 8; + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + dev->resource[i].start = 0; + dev->resource[i].end = 0; + dev->resource[i].flags = IORESOURCE_PCI_FIXED; + } + } + atomic_inc(&dev->enable_cnt); + + dev->no_msi = 1; +} + +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_CHIP3, fixup_root_complex); + /* Just declaring that the power-of-ten prefixes are actually the * power-of-two ones doesn't make it true :) */ diff --git a/arch/sw_64/kernel/pci_common.c b/arch/sw_64/kernel/pci_common.c index f996baca9d93..56c9be523461 100644 --- a/arch/sw_64/kernel/pci_common.c +++ b/arch/sw_64/kernel/pci_common.c @@ -144,3 +144,35 @@ const struct dma_map_ops sw64_dma_direct_ops = { const struct dma_map_ops *dma_ops = &sw64_dma_direct_ops; EXPORT_SYMBOL(dma_ops); + +#ifdef CONFIG_DCA +static void enable_sw_dca(struct pci_dev *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev->sysdata; + unsigned long node, rc_index, dca_ctl, dca_conf; + int i; + + if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) + return; + node = hose->node; + rc_index = hose->index; + for (i = 0; i < 256; i++) { + dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); + if (dca_conf >> 63) + continue; + else { + dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; + pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); + write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); + break; + } + } + dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); + if (dca_ctl & 0x1) { + dca_ctl = 0x2; + write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); + pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); +#endif -- Gitee From 71d8e2e464fd4c058f54f0a04923839df61671c7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:47 +0800 Subject: [PATCH 055/150] sw64: reorganize basic irq handling code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move basic irq handling to drivers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/chip/chip3/chip.c | 217 ------------------------------- arch/sw_64/include/asm/msi.h | 9 ++ arch/sw_64/kernel/entry.S | 2 + drivers/irqchip/Kconfig | 5 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-sunway-cpu.c | 213 ++++++++++++++++++++++++++++++ 7 files changed, 231 insertions(+), 217 deletions(-) create mode 100644 drivers/irqchip/irq-sunway-cpu.c diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index d375be503f14..74091502baf6 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -108,6 +108,7 @@ config SW64 select PCI_MSI_ARCH_FALLBACKS select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM + select SW64_IRQ_CPU select SW64_TIMER select SWIOTLB select THREAD_INFO_IN_TASK diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index d153a3d08143..10ecfdf12c06 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -337,62 +337,6 @@ static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return hose->int_irq; } -extern struct pci_controller *hose_head, **hose_tail; -static void sw6_handle_intx(unsigned int offset) -{ - struct pci_controller *hose; - unsigned long value; - - hose = hose_head; - for (hose = hose_head; hose; hose = hose->next) { - value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); - if (value >> 63) { - value = value & (~(1UL << 62)); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); - handle_irq(hose->int_irq); - value = value | (1UL << 62); - write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); - } - - if (IS_ENABLED(CONFIG_PCIE_PME)) { - value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); - } - } - - if (IS_ENABLED(CONFIG_PCIEAER)) { - value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); - if (value >> 63) { - handle_irq(hose->service_irq); - write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); - } - } - - if (hose->iommu_enable) { - value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); - if (value >> 63) - handle_irq(hose->int_irq); - } - } -} - -static void chip3_device_interrupt(unsigned long irq_info) -{ - unsigned int i; - - if (is_guest_or_emul()) { - handle_irq(irq_info); - return; - } - - for (i = 0; i < 4; i++) { - if ((irq_info >> i) & 0x1) - sw6_handle_intx(i); - } -} - static void chip3_i2c_srst(void) { sw64_io_write(0, I2C0_SRST_L, 0x0); @@ -643,164 +587,3 @@ void __init sw64_setup_chip_ops(void) io_syscore_ops.resume = chip3_io_resume; #endif } - -/* Performance counter hook. A module can override this to do something useful. */ -static void dummy_perf(unsigned long vector, struct pt_regs *regs) -{ - irq_err_count++; - pr_crit("Performance counter interrupt!\n"); -} - -void (*perf_irq)(unsigned long, struct pt_regs*) = dummy_perf; -EXPORT_SYMBOL(perf_irq); - -#ifdef CONFIG_PCI_MSI -extern void handle_pci_msi_interrupt(unsigned long type, - unsigned long vector, - unsigned long pci_msi1_addr); -#else -void handle_pci_msi_interrupt(unsigned long type, - unsigned long vector, unsigned long pci_msi1_addr) -{ - pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); -} -#endif - -static void handle_fault_int(void) -{ - int node; - - node = __this_cpu_read(hard_node_id); - printk("enter fault int, si_fault_stat = %#lx\n", - sw64_io_read(node, SI_FAULT_STAT)); - sw64_io_write(node, SI_FAULT_INT_EN, 0); - sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG0_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG1_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG2_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG3_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG4_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG5_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG6_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG7_FAULT_INTEN, 0); -} - -static void handle_mt_int(void) -{ - printk("enter mt int\n"); -} - -static void handle_nmi_int(void) -{ - printk("enter nmi int\n"); -} - -static void handle_dev_int(struct pt_regs *regs) -{ - unsigned long config_val, val, stat; - int node = 0; - unsigned int hwirq; - - config_val = sw64_io_read(node, DEV_INT_CONFIG); - val = config_val & (~(1UL << 8)); - sw64_io_write(node, DEV_INT_CONFIG, val); - stat = sw64_io_read(node, MCU_DVC_INT); - - while (stat) { - hwirq = ffs(stat) - 1; - handle_domain_irq(NULL, hwirq, regs); - stat &= ~(1UL << hwirq); - } - /*do handle irq */ - - sw64_io_write(node, DEV_INT_CONFIG, config_val); -} - -asmlinkage void do_entInt(unsigned long type, unsigned long vector, - unsigned long irq_arg, struct pt_regs *regs) -{ - struct pt_regs *old_regs; - extern char __idle_start[], __idle_end[]; - - if (is_guest_or_emul()) { - if ((type & 0xffff) > 15) { - vector = type; - if (vector == 16) - type = INT_INTx; - else - type = INT_MSI; - } - } - - /* restart idle routine if it is interrupted */ - if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) - regs->pc = (u64)__idle_start; - - switch (type & 0xffff) { - case INT_MSI: - old_regs = set_irq_regs(regs); - handle_pci_msi_interrupt(type, vector, irq_arg); - set_irq_regs(old_regs); - return; - case INT_INTx: - old_regs = set_irq_regs(regs); - chip3_device_interrupt(vector); - set_irq_regs(old_regs); - return; - - case INT_IPI: -#ifdef CONFIG_SMP - handle_ipi(regs); - return; -#else - irq_err_count++; - pr_crit("Interprocessor interrupt? You must be kidding!\n"); -#endif - break; - case INT_RTC: - old_regs = set_irq_regs(regs); - sw64_timer_interrupt(); - set_irq_regs(old_regs); - return; - case INT_VT_SERIAL: - old_regs = set_irq_regs(regs); - handle_irq(type); - set_irq_regs(old_regs); - return; - case INT_VT_HOTPLUG: - old_regs = set_irq_regs(regs); - handle_irq(type); - set_irq_regs(old_regs); - return; - case INT_PC0: - perf_irq(PERFMON_PC0, regs); - return; - case INT_PC1: - perf_irq(PERFMON_PC1, regs); - return; - case INT_DEV: - old_regs = set_irq_regs(regs); - handle_dev_int(regs); - set_irq_regs(old_regs); - return; - case INT_FAULT: - old_regs = set_irq_regs(regs); - handle_fault_int(); - set_irq_regs(old_regs); - return; - case INT_MT: - old_regs = set_irq_regs(regs); - handle_mt_int(); - set_irq_regs(old_regs); - return; - case INT_NMI: - old_regs = set_irq_regs(regs); - handle_nmi_int(); - set_irq_regs(old_regs); - return; - default: - pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); - } - pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); -} -EXPORT_SYMBOL(do_entInt); diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index f27a1fcf6a20..554ab7289fb1 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -32,6 +32,9 @@ extern bool find_free_cpu_vector(const struct cpumask *search_mask, extern int msi_compose_msg(unsigned int irq, struct msi_msg *msg); extern void sw64_irq_noop(struct irq_data *d); extern struct irq_chip sw64_irq_chip; +extern void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, + unsigned long pci_msi1_addr); #ifdef CONFIG_PCI_MSI_IRQ_DOMAIN #define MSI_ADDR_BASE_HI 0 @@ -64,5 +67,11 @@ struct irq_alloc_info { }; typedef struct irq_alloc_info msi_alloc_info_t; #endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */ +#else /* !CONFIG_PCI_MSI */ +static inline void handle_pci_msi_interrupt(unsigned long type, + unsigned long vector, unsigned long pci_msi1_addr) +{ + pr_warn("SW arch disable CONFIG_PCI_MSI option.\n"); +} #endif /* CONFIG_PCI_MSI */ #endif /* _ASM_SW64_MSI_H */ diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index 77eab978b90d..d3c961ba53e2 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -225,6 +225,8 @@ ret_from_sys_call: fillcs 0($sp) /* prefetch */ fillcs 128($sp) /* prefetch */ #endif + br $27, 1f +1: ldgp $29, 0($27) selne $26, 0, $18, $18 /* $18 = 0 => non-restartable */ ldl $0, PT_REGS_PS($sp) and $0, 8, $0 diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 4f7d22b1ef42..50207b2b69d6 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -29,6 +29,11 @@ config SW64_LPC_INTC Say yes here to add support for the SW64 cpu builtin LPC IRQ controller. +config SW64_IRQ_CPU + bool + depends on SW64 + default y + config ARM_GIC_PM bool depends on PM diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 0e69f31b4508..e8d8092d0091 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o +obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c new file mode 100644 index 000000000000..f43b327cc291 --- /dev/null +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include +#include +#include + +static void handle_intx(unsigned int offset) +{ + struct pci_controller *hose; + unsigned long value; + + hose = hose_head; + for (hose = hose_head; hose; hose = hose->next) { + value = read_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7)); + if (value >> 63) { + value = value & (~(1UL << 62)); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + handle_irq(hose->int_irq); + value = value | (1UL << 62); + write_piu_ior0(hose->node, hose->index, INTACONFIG + (offset << 7), value); + } + + if (IS_ENABLED(CONFIG_PCIE_PME)) { + value = read_piu_ior0(hose->node, hose->index, PMEINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, PMEINTCONFIG, value); + } + } + + if (IS_ENABLED(CONFIG_PCIEAER)) { + value = read_piu_ior0(hose->node, hose->index, AERERRINTCONFIG); + if (value >> 63) { + handle_irq(hose->service_irq); + write_piu_ior0(hose->node, hose->index, AERERRINTCONFIG, value); + } + } + + if (hose->iommu_enable) { + value = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (value >> 63) + handle_irq(hose->int_irq); + } + } +} + +static void handle_device_interrupt(unsigned long irq_info) +{ + unsigned int i; + + if (is_guest_or_emul()) { + handle_irq(irq_info); + return; + } + + for (i = 0; i < 4; i++) { + if ((irq_info >> i) & 0x1) + handle_intx(i); + } +} + +/* Performance counter hook. A module can override this to do something useful. */ +static void dummy_perf(unsigned long vector, struct pt_regs *regs) +{ + irq_err_count++; + pr_crit("Performance counter interrupt!\n"); +} + +void (*perf_irq)(unsigned long, struct pt_regs*) = dummy_perf; +EXPORT_SYMBOL(perf_irq); + +static void handle_fault_int(void) +{ + int node; + + node = __this_cpu_read(hard_node_id); + pr_info("enter fault int, si_fault_stat = %#lx\n", + sw64_io_read(node, SI_FAULT_STAT)); + sw64_io_write(node, SI_FAULT_INT_EN, 0); + sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG0_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG1_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG2_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG3_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG4_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG5_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG6_FAULT_INTEN, 0); + sw64_io_write(node, DUAL_CG7_FAULT_INTEN, 0); +} + +static void handle_mt_int(void) +{ + pr_info("enter mt int\n"); +} + +static void handle_nmi_int(void) +{ + pr_info("enter nmi int\n"); +} + +static void handle_dev_int(struct pt_regs *regs) +{ + unsigned long config_val, val, stat; + int node = 0; + unsigned int hwirq; + + config_val = sw64_io_read(node, DEV_INT_CONFIG); + val = config_val & (~(1UL << 8)); + sw64_io_write(node, DEV_INT_CONFIG, val); + stat = sw64_io_read(node, MCU_DVC_INT); + + while (stat) { + hwirq = ffs(stat) - 1; + handle_domain_irq(NULL, hwirq, regs); + stat &= ~(1UL << hwirq); + } + /*do handle irq */ + + sw64_io_write(node, DEV_INT_CONFIG, config_val); +} + +asmlinkage void do_entInt(unsigned long type, unsigned long vector, + unsigned long irq_arg, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + extern char __idle_start[], __idle_end[]; + + if (is_guest_or_emul()) { + if ((type & 0xffff) > 15) { + vector = type; + if (vector == 16) + type = INT_INTx; + else + type = INT_MSI; + } + } + + /* restart idle routine if it is interrupted */ + if (regs->pc > (u64)__idle_start && regs->pc < (u64)__idle_end) + regs->pc = (u64)__idle_start; + + switch (type & 0xffff) { + case INT_MSI: + old_regs = set_irq_regs(regs); + handle_pci_msi_interrupt(type, vector, irq_arg); + set_irq_regs(old_regs); + return; + case INT_INTx: + old_regs = set_irq_regs(regs); + handle_device_interrupt(vector); + set_irq_regs(old_regs); + return; + + case INT_IPI: +#ifdef CONFIG_SMP + handle_ipi(regs); + return; +#else + irq_err_count++; + pr_crit("Interprocessor interrupt? You must be kidding!\n"); +#endif + break; + case INT_RTC: + old_regs = set_irq_regs(regs); + sw64_timer_interrupt(); + set_irq_regs(old_regs); + return; + case INT_VT_SERIAL: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_VT_HOTPLUG: + old_regs = set_irq_regs(regs); + handle_irq(type); + set_irq_regs(old_regs); + return; + case INT_PC0: + perf_irq(PERFMON_PC0, regs); + return; + case INT_PC1: + perf_irq(PERFMON_PC1, regs); + return; + case INT_DEV: + old_regs = set_irq_regs(regs); + handle_dev_int(regs); + set_irq_regs(old_regs); + return; + case INT_FAULT: + old_regs = set_irq_regs(regs); + handle_fault_int(); + set_irq_regs(old_regs); + return; + case INT_MT: + old_regs = set_irq_regs(regs); + handle_mt_int(); + set_irq_regs(old_regs); + return; + case INT_NMI: + old_regs = set_irq_regs(regs); + handle_nmi_int(); + set_irq_regs(old_regs); + return; + default: + pr_crit("Hardware intr %ld %lx? uh?\n", type, vector); + } + pr_crit("PC = %016lx PS = %04lx\n", regs->pc, regs->ps); +} +EXPORT_SYMBOL(do_entInt); -- Gitee From 73e156a409a705567a3424c1c0e8963064f80668 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:48 +0800 Subject: [PATCH 056/150] sw64: reorganize msi related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move msi related code to drivers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 ++ arch/sw_64/chip/chip3/Makefile | 1 - drivers/irqchip/Kconfig | 10 ++++++++++ drivers/irqchip/Makefile | 2 ++ .../vt_msi.c => drivers/irqchip/irq-sunway-msi-vt.c | 0 .../chip3/msi.c => drivers/irqchip/irq-sunway-msi.c | 0 6 files changed, 14 insertions(+), 1 deletion(-) rename arch/sw_64/chip/chip3/vt_msi.c => drivers/irqchip/irq-sunway-msi-vt.c (100%) rename arch/sw_64/chip/chip3/msi.c => drivers/irqchip/irq-sunway-msi.c (100%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 74091502baf6..81cbbd1d6db7 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -109,6 +109,8 @@ config SW64 select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM select SW64_IRQ_CPU + select SW64_IRQ_MSI if PCI_MSI + select SW64_IRQ_MSI_VT if PCI_MSI select SW64_TIMER select SWIOTLB select THREAD_INFO_IN_TASK diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile index 17f3574c0331..895749957ebc 100644 --- a/arch/sw_64/chip/chip3/Makefile +++ b/arch/sw_64/chip/chip3/Makefile @@ -3,4 +3,3 @@ obj-y := chip.o obj-$(CONFIG_PCI) += pci-quirks.o -obj-$(CONFIG_PCI_MSI) += msi.o vt_msi.o diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 50207b2b69d6..52b476eda53b 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -34,6 +34,16 @@ config SW64_IRQ_CPU depends on SW64 default y +config SW64_IRQ_MSI + bool + depends on SW64 && PCI_MSI + default y + +config SW64_IRQ_MSI_VT + bool + depends on SW64_IRQ_MSI + default y + config ARM_GIC_PM bool depends on PM diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index e8d8092d0091..b40c75f5b4cf 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -30,6 +30,8 @@ obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi.o +obj-$(CONFIG_SW64_IRQ_MSI_VT) += irq-sunway-msi-vt.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o diff --git a/arch/sw_64/chip/chip3/vt_msi.c b/drivers/irqchip/irq-sunway-msi-vt.c similarity index 100% rename from arch/sw_64/chip/chip3/vt_msi.c rename to drivers/irqchip/irq-sunway-msi-vt.c diff --git a/arch/sw_64/chip/chip3/msi.c b/drivers/irqchip/irq-sunway-msi.c similarity index 100% rename from arch/sw_64/chip/chip3/msi.c rename to drivers/irqchip/irq-sunway-msi.c -- Gitee From 57bc5983acfe85291ac0185c47110edd75307223 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:48 +0800 Subject: [PATCH 057/150] sw64: reorganize chip setup code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move chip setup code to a dedicate file and remove uncessary functions. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/chip/chip3/chip.c | 299 ++--------------------------- arch/sw_64/include/asm/chip3_io.h | 3 - arch/sw_64/include/asm/pci.h | 1 + arch/sw_64/include/asm/sw64_init.h | 5 +- arch/sw_64/include/asm/topology.h | 3 + arch/sw_64/kernel/Makefile | 2 +- arch/sw_64/kernel/chip_setup.c | 237 +++++++++++++++++++++++ arch/sw_64/kernel/early_init.c | 3 - arch/sw_64/kernel/setup.c | 47 ++--- arch/sw_64/kernel/topology.c | 13 ++ 10 files changed, 301 insertions(+), 312 deletions(-) create mode 100644 arch/sw_64/kernel/chip_setup.c diff --git a/arch/sw_64/chip/chip3/chip.c b/arch/sw_64/chip/chip3/chip.c index 10ecfdf12c06..7730a2441461 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/arch/sw_64/chip/chip3/chip.c @@ -29,81 +29,6 @@ void set_pcieport_service_irq(int node, int index) write_piu_ior0(node, index, AERERRINTCONFIG, AER_ENABLE_INTD_CORE0); } -static int chip3_get_cpu_nums(void) -{ - unsigned long trkmode; - int cpus; - - if (is_guest_or_emul()) - return 1; - - trkmode = sw64_io_read(0, TRKMODE); - trkmode = (trkmode >> 6) & 0x3; - cpus = 1 << trkmode; - - return cpus; -} - -static void chip3_get_vt_smp_info(void) -{ - unsigned long smp_info; - - smp_info = sw64_io_read(0, SMP_INFO); - if (smp_info == -1UL) - smp_info = 0; - topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; - topo_nr_cores = (smp_info >> VT_CORES_SHIFT) & VT_CORES_MASK; - topo_nr_maxcpus = (smp_info >> VT_MAX_CPUS_SHIFT) & VT_MAX_CPUS_MASK; -} - -static unsigned long chip3_get_vt_node_mem(int nodeid) -{ - return *(unsigned long *)MMSIZE & MMSIZE_MASK; -} - -static unsigned long chip3_get_node_mem(int nodeid) -{ - unsigned long mc_config, mc_online, mc_cap, mc_num; - unsigned long node_mem; - - mc_config = sw64_io_read(nodeid, MC_CAP_CFG) & 0xf; - mc_cap = (1UL << mc_config) << 28; - mc_online = sw64_io_read(nodeid, MC_ONLINE) & 0xff; - mc_num = __kernel_ctpop(mc_online); - node_mem = mc_cap * mc_num; - - return node_mem; -} - -static void chip3_setup_vt_core_start(struct cpumask *cpumask) -{ - int i; - unsigned long coreonline; - - coreonline = sw64_io_read(0, CORE_ONLINE); - - for (i = 0; i < 64 ; i++) { - if (coreonline & (1UL << i)) - cpumask_set_cpu(i, cpumask); - } -} - -static void chip3_setup_core_start(struct cpumask *cpumask) -{ - int i, j, cpus; - unsigned long coreonline; - - cpus = chip3_get_cpu_nums(); - for (i = 0; i < cpus; i++) { - coreonline = sw64_io_read(i, CORE_ONLINE); - for (j = 0; j < 32 ; j++) { - if (coreonline & (1UL << j)) - cpumask_set_cpu(i * 32 + j, cpumask); - } - } - -} - int chip_pcie_configure(struct pci_controller *hose) { struct pci_dev *dev; @@ -242,19 +167,18 @@ int chip_pcie_configure(struct pci_controller *hose) return bus_max_num; } -static int chip3_check_pci_vt_linkup(unsigned long node, unsigned long index) -{ - if (node == 0 && index == 0) - return 0; - else - return 1; -} - static int chip3_check_pci_linkup(unsigned long node, unsigned long index) { unsigned long rc_debug; - rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); + if (is_guest_or_emul()) { + if (node == 0 && index == 0) + return 0; + else + return 1; + } else { + rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); + } return !(rc_debug & 0x1); } @@ -337,158 +261,6 @@ static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return hose->int_irq; } -static void chip3_i2c_srst(void) -{ - sw64_io_write(0, I2C0_SRST_L, 0x0); - sw64_io_write(0, I2C0_SRST_L, 0x1); - - sw64_io_write(0, I2C1_SRST_L, 0x0); - sw64_io_write(0, I2C1_SRST_L, 0x1); - - sw64_io_write(0, I2C2_SRST_L, 0x0); - sw64_io_write(0, I2C2_SRST_L, 0x1); -} - -static void chip3_pcie_save(void) -{ - struct pci_controller *hose; - struct piu_saved *piu_save; - unsigned long node, index; - unsigned long i; - - for (hose = hose_head; hose; hose = hose->next) { - piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); - - node = hose->node; - index = hose->index; - hose->sysdata = piu_save; - - piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); - piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); - piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); - piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); - - for (i = 0; i < 256; i++) { - piu_save->msiconfig[i] = read_piu_ior0(node, index, - MSICONFIG0 + (i << 7)); - } - - piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); - piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); - - piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); - piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); - piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); - piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); - piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); - piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); - piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); - - } -} - -static void chip3_pcie_restore(void) -{ - struct pci_controller *hose; - struct piu_saved *piu_save; - unsigned long node, index; - u32 rc_misc_ctrl; - unsigned int value; - unsigned long i; - - for (hose = hose_head; hose; hose = hose->next) { - node = hose->node; - index = hose->index; - piu_save = hose->sysdata; - - write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); - write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); - write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); - write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); - - for (i = 0; i < 256; i++) { - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), - piu_save->msiconfig[i]); - } - - write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); - write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); - - write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); - write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); - write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); - write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); - write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); - write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); - write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); - - /* Enable DBI_RO_WR_EN */ - rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); - - /* Fix up DEVICE_ID_VENDOR_ID register */ - value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; - write_rc_conf(node, index, RC_VENDOR_ID, value); - - /* Set PCI-E root class code */ - value = read_rc_conf(node, index, RC_REVISION_ID); - write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); - - /* Disable DBI_RO_WR_EN */ - write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); - } - -} - -static unsigned long saved_dvc_int, saved_long_time; - -static inline void chip3_intpu_save(void) -{ - saved_long_time = sw64_io_read(0, LONG_TIME); -} - -static inline void chip3_intpu_restore(void) -{ - switch (cpu_desc.model) { - case CPU_SW831: - sw64_io_write(0, LONG_TIME, saved_long_time); - sw64_io_write(0, LONG_TIME_START_EN, 0x1); - break; - default: - pr_info("long time start is disable!"); - break; - } -} - -static inline void chip3_spbu_save(void) -{ - saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); -} - -static inline void chip3_spbu_restore(void) -{ - chip3_i2c_srst(); - sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); -} - -extern void cpld_write(uint8_t slave_addr, uint8_t reg, uint8_t data); - -static int chip3_io_suspend(void) -{ - chip3_spbu_save(); - chip3_intpu_save(); - chip3_pcie_save(); - - return 0; -} - -static void chip3_io_resume(void) -{ - chip3_pcie_restore(); - chip3_intpu_restore(); - chip3_spbu_restore(); -} - static void chip3_hose_init(struct pci_controller *hose) { unsigned long pci_io_base; @@ -537,53 +309,16 @@ static void chip3_hose_init(struct pci_controller *hose) } }; -static void chip3_init_ops_fixup(void) -{ - if (is_guest_or_emul()) { - sw64_chip_init->early_init.setup_core_start = chip3_setup_vt_core_start; - sw64_chip_init->early_init.get_node_mem = chip3_get_vt_node_mem; - sw64_chip_init->early_init.get_smp_info = chip3_get_vt_smp_info; - sw64_chip_init->pci_init.check_pci_linkup = chip3_check_pci_vt_linkup; - } -}; - -static void chip3_ops_fixup(void) -{ - if (is_guest_or_emul()) - sw64_chip->suspend = NULL; -}; - -static struct sw64_chip_init_ops chip3_chip_init_ops = { - .early_init = { - .setup_core_start = chip3_setup_core_start, - .get_node_mem = chip3_get_node_mem, - }, - .pci_init = { - .map_irq = chip3_map_irq, - .get_rc_enable = chip3_get_rc_enable, - .hose_init = chip3_hose_init, - .set_rc_piu = chip3_set_rc_piu, - .check_pci_linkup = chip3_check_pci_linkup, - .set_intx = chip3_set_intx, - }, - .fixup = chip3_init_ops_fixup, +static struct sw64_pci_init_ops chip_pci_init_ops = { + .map_irq = chip3_map_irq, + .get_rc_enable = chip3_get_rc_enable, + .hose_init = chip3_hose_init, + .set_rc_piu = chip3_set_rc_piu, + .check_pci_linkup = chip3_check_pci_linkup, + .set_intx = chip3_set_intx, }; -static struct sw64_chip_ops chip3_chip_ops = { - .get_cpu_num = chip3_get_cpu_nums, - .fixup = chip3_ops_fixup, -}; - -#ifdef CONFIG_PM -extern struct syscore_ops io_syscore_ops; -#endif - -void __init sw64_setup_chip_ops(void) +void __init setup_chip_pci_ops(void) { - sw64_chip_init = &chip3_chip_init_ops; - sw64_chip = &chip3_chip_ops; -#ifdef CONFIG_PM - io_syscore_ops.suspend = chip3_io_suspend; - io_syscore_ops.resume = chip3_io_resume; -#endif + sw64_chip_init->pci_init = chip_pci_init_ops; } diff --git a/arch/sw_64/include/asm/chip3_io.h b/arch/sw_64/include/asm/chip3_io.h index 013730235b83..6661e41057d1 100644 --- a/arch/sw_64/include/asm/chip3_io.h +++ b/arch/sw_64/include/asm/chip3_io.h @@ -24,9 +24,6 @@ #define IO_NODE_SHIFT 44 #define IO_MARK_BIT 47 -extern int topo_nr_threads; -extern int topo_nr_cores; -extern int topo_nr_maxcpus; #define VT_MAX_CPUS_SHIFT 0 #define VT_MAX_CPUS_MASK 0x3ff #define VT_CORES_SHIFT 10 diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 1a931cc29195..1262d812086a 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -89,6 +89,7 @@ extern void __init sw64_init_arch(void); extern struct pci_ops sw64_pci_ops; extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; +extern void __init setup_chip_pci_ops(void); #ifdef CONFIG_SUNWAY_IOMMU extern struct syscore_ops iommu_cpu_syscore_ops; diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index 8b437ef73cde..b7ca9cb89478 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -38,9 +38,12 @@ struct sw64_chip_ops { }; extern void sw64_init_noop(void); -extern void sw64_setup_chip_ops(void); +extern void setup_chip_ops(void); extern struct sw64_chip_ops *sw64_chip; extern struct sw64_chip_init_ops *sw64_chip_init; +#ifdef CONFIG_PM +extern struct syscore_ops io_syscore_ops; +#endif DECLARE_PER_CPU(unsigned long, hard_node_id); diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h index c39ca5b4beb7..74bd81524fe9 100644 --- a/arch/sw_64/include/asm/topology.h +++ b/arch/sw_64/include/asm/topology.h @@ -54,6 +54,9 @@ static inline void numa_add_cpu(unsigned int cpu) { } static inline void numa_remove_cpu(unsigned int cpu) { } static inline void numa_store_cpu_info(unsigned int cpu) { } #endif /* CONFIG_NUMA */ + +extern void get_vt_smp_info(void); + #include static inline void arch_fix_phys_package_id(int num, u32 slot) { } diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index effe855aa5e0..0857acbf60ef 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -15,7 +15,7 @@ endif obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ irq_sw64.o signal.o setup.o ptrace.o time.o \ - systbls.o dup_print.o tc.o \ + systbls.o dup_print.o tc.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c new file mode 100644 index 000000000000..88d3ef2a9bf2 --- /dev/null +++ b/arch/sw_64/kernel/chip_setup.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +struct sw64_chip_ops *sw64_chip; +struct sw64_chip_init_ops *sw64_chip_init; + +static int get_cpu_nums(void) +{ + unsigned long trkmode; + int cpus; + + if (is_guest_or_emul()) + return 1; + + trkmode = sw64_io_read(0, TRKMODE); + trkmode = (trkmode >> 6) & 0x3; + cpus = 1 << trkmode; + + return cpus; +} + +static unsigned long __init get_node_mem(int nodeid) +{ + unsigned long mc_config, mc_online, mc_cap, mc_num; + unsigned long node_mem; + + if (is_guest_or_emul()) + return *(unsigned long *)MMSIZE & MMSIZE_MASK; + + mc_config = sw64_io_read(nodeid, MC_CAP_CFG) & 0xf; + mc_cap = (1UL << mc_config) << 28; + mc_online = sw64_io_read(nodeid, MC_ONLINE) & 0xff; + mc_num = __kernel_ctpop(mc_online); + node_mem = mc_cap * mc_num; + + return node_mem; +} + +static void __init setup_core_start(struct cpumask *cpumask) +{ + int i, j, cpus; + unsigned long coreonline; + + if (is_guest_or_emul()) { + coreonline = sw64_io_read(0, CORE_ONLINE); + for (i = 0; i < 64 ; i++) { + if (coreonline & (1UL << i)) + cpumask_set_cpu(i, cpumask); + } + } else { + cpus = get_cpu_nums(); + for (i = 0; i < cpus; i++) { + coreonline = sw64_io_read(i, CORE_ONLINE); + for (j = 0; j < 32 ; j++) { + if (coreonline & (1UL << j)) + cpumask_set_cpu(i * 32 + j, cpumask); + } + } + } +} + +static void i2c_srst(void) +{ + sw64_io_write(0, I2C0_SRST_L, 0x0); + sw64_io_write(0, I2C0_SRST_L, 0x1); + + sw64_io_write(0, I2C1_SRST_L, 0x0); + sw64_io_write(0, I2C1_SRST_L, 0x1); + + sw64_io_write(0, I2C2_SRST_L, 0x0); + sw64_io_write(0, I2C2_SRST_L, 0x1); +} + +static void pcie_save(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + piu_save = kzalloc(sizeof(*piu_save), GFP_KERNEL); + + node = hose->node; + index = hose->index; + hose->sysdata = piu_save; + + piu_save->piuconfig0 = read_piu_ior0(node, index, PIUCONFIG0); + piu_save->piuconfig1 = read_piu_ior1(node, index, PIUCONFIG1); + piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); + piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); + + for (i = 0; i < 256; i++) { + piu_save->msiconfig[i] = read_piu_ior0(node, index, + MSICONFIG0 + (i << 7)); + } + + piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); + piu_save->dtbaseaddr = read_piu_ior0(node, index, DTBASEADDR); + + piu_save->intaconfig = read_piu_ior0(node, index, INTACONFIG); + piu_save->intbconfig = read_piu_ior0(node, index, INTBCONFIG); + piu_save->intcconfig = read_piu_ior0(node, index, INTCCONFIG); + piu_save->intdconfig = read_piu_ior0(node, index, INTDCONFIG); + piu_save->pmeintconfig = read_piu_ior0(node, index, PMEINTCONFIG); + piu_save->aererrintconfig = read_piu_ior0(node, index, AERERRINTCONFIG); + piu_save->hpintconfig = read_piu_ior0(node, index, HPINTCONFIG); + + } +} + +static void pcie_restore(void) +{ + struct pci_controller *hose; + struct piu_saved *piu_save; + unsigned long node, index; + u32 rc_misc_ctrl; + unsigned int value; + unsigned long i; + + for (hose = hose_head; hose; hose = hose->next) { + node = hose->node; + index = hose->index; + piu_save = hose->sysdata; + + write_piu_ior0(node, index, PIUCONFIG0, piu_save->piuconfig0); + write_piu_ior1(node, index, PIUCONFIG1, piu_save->piuconfig1); + write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); + write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); + + for (i = 0; i < 256; i++) { + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), + piu_save->msiconfig[i]); + } + + write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); + write_piu_ior0(node, index, DTBASEADDR, piu_save->dtbaseaddr); + + write_piu_ior0(node, index, INTACONFIG, piu_save->intaconfig); + write_piu_ior0(node, index, INTBCONFIG, piu_save->intbconfig); + write_piu_ior0(node, index, INTCCONFIG, piu_save->intcconfig); + write_piu_ior0(node, index, INTDCONFIG, piu_save->intdconfig); + write_piu_ior0(node, index, PMEINTCONFIG, piu_save->pmeintconfig); + write_piu_ior0(node, index, AERERRINTCONFIG, piu_save->aererrintconfig); + write_piu_ior0(node, index, HPINTCONFIG, piu_save->hpintconfig); + + /* Enable DBI_RO_WR_EN */ + rc_misc_ctrl = read_rc_conf(node, index, RC_MISC_CONTROL_1); + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); + + /* Fix up DEVICE_ID_VENDOR_ID register */ + value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; + write_rc_conf(node, index, RC_VENDOR_ID, value); + + /* Set PCI-E root class code */ + value = read_rc_conf(node, index, RC_REVISION_ID); + write_rc_conf(node, index, RC_REVISION_ID, (PCI_CLASS_BRIDGE_HOST << 16) | value); + + /* Disable DBI_RO_WR_EN */ + write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); + } + +} + +static unsigned long saved_dvc_int, saved_long_time; + +static inline void intpu_save(void) +{ + saved_long_time = sw64_io_read(0, LONG_TIME); +} + +static inline void intpu_restore(void) +{ + switch (cpu_desc.model) { + case CPU_SW831: + sw64_io_write(0, LONG_TIME, saved_long_time); + sw64_io_write(0, LONG_TIME_START_EN, 0x1); + break; + default: + pr_info("long time start is disable!"); + break; + } +} + +static inline void spbu_save(void) +{ + saved_dvc_int = sw64_io_read(0, MCU_DVC_INT_EN); +} + +static inline void spbu_restore(void) +{ + i2c_srst(); + sw64_io_write(0, MCU_DVC_INT_EN, saved_dvc_int); +} + +static int io_suspend(void) +{ + spbu_save(); + intpu_save(); + pcie_save(); + + return 0; +} + +static void io_resume(void) +{ + pcie_restore(); + intpu_restore(); + spbu_restore(); +} + +static struct sw64_chip_init_ops chip_init_ops = { + .early_init = { + .setup_core_start = setup_core_start, + .get_node_mem = get_node_mem, + }, +}; + +static struct sw64_chip_ops chip_ops = { + .get_cpu_num = get_cpu_nums, +}; + +void __init setup_chip_ops(void) +{ + sw64_chip_init = &chip_init_ops; + sw64_chip = &chip_ops; + setup_chip_pci_ops(); +#ifdef CONFIG_PM + io_syscore_ops.suspend = io_suspend; + io_syscore_ops.resume = io_resume; +#endif +} diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c index bcd458a9bdad..7f4fdc43a43d 100644 --- a/arch/sw_64/kernel/early_init.c +++ b/arch/sw_64/kernel/early_init.c @@ -6,8 +6,6 @@ void sw64_init_noop(void) { } struct sw64_platform_ops *sw64_platform; EXPORT_SYMBOL(sw64_platform); -struct sw64_chip_ops *sw64_chip; -struct sw64_chip_init_ops *sw64_chip_init; static void __init sw64_setup_platform_ops(void) { @@ -25,7 +23,6 @@ asmlinkage __visible void __init sw64_start_kernel(void) { fixup_hmcall(); save_ktp(); - sw64_setup_chip_ops(); sw64_setup_platform_ops(); sw64_platform->ops_fixup(); start_kernel(); diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 73fcc91c56f0..018feb25130b 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -633,25 +633,6 @@ static void __init setup_cpu_info(void) cpu_desc.pa_bits = CPUID_PA_BITS(val); cpu_desc.va_bits = CPUID_VA_BITS(val); - if (*(unsigned long *)MMSIZE) { - static_branch_disable(&run_mode_host_key); - if (*(unsigned long *)MMSIZE & EMUL_FLAG) { - pr_info("run mode: emul\n"); - static_branch_disable(&run_mode_guest_key); - static_branch_enable(&run_mode_emul_key); - - } else { - pr_info("run mode: guest\n"); - static_branch_enable(&run_mode_guest_key); - static_branch_disable(&run_mode_emul_key); - } - } else { - pr_info("run mode: host\n"); - static_branch_enable(&run_mode_host_key); - static_branch_disable(&run_mode_guest_key); - static_branch_disable(&run_mode_emul_key); - } - for (i = 0; i < VENDOR_ID_MAX; i++) { val = cpuid(GET_VENDOR_ID, i); memcpy(cpu_desc.vendor_id + (i * 8), &val, 8); @@ -695,6 +676,28 @@ static void __init setup_cpu_info(void) } } +static void __init setup_run_mode(void) +{ + if (*(unsigned long *)MMSIZE) { + static_branch_disable(&run_mode_host_key); + if (*(unsigned long *)MMSIZE & EMUL_FLAG) { + pr_info("run mode: emul\n"); + static_branch_disable(&run_mode_guest_key); + static_branch_enable(&run_mode_emul_key); + + } else { + pr_info("run mode: guest\n"); + static_branch_enable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } + } else { + pr_info("run mode: host\n"); + static_branch_enable(&run_mode_host_key); + static_branch_disable(&run_mode_guest_key); + static_branch_disable(&run_mode_emul_key); + } +} + static void __init setup_socket_info(void) { int i; @@ -757,13 +760,13 @@ setup_arch(char **cmdline_p) { jump_label_init(); setup_cpu_info(); - sw64_chip->fixup(); - sw64_chip_init->fixup(); + setup_run_mode(); + setup_chip_ops(); setup_socket_info(); show_socket_mem_layout(); sw64_chip_init->early_init.setup_core_start(&core_start); if (is_guest_or_emul()) - sw64_chip_init->early_init.get_smp_info(); + get_vt_smp_info(); setup_sched_clock(); diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index d1037e33480e..26200cf68cc8 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -3,6 +3,7 @@ #include #include #include +#include #include static int __init parse_dt_topology(void) @@ -23,6 +24,18 @@ static int topo_threads[NR_CPUS]; static int topo_cores[NR_CPUS]; static int topo_packages[NR_CPUS]; +void __init get_vt_smp_info(void) +{ + unsigned long smp_info; + + smp_info = sw64_io_read(0, SMP_INFO); + if (smp_info == -1UL) + smp_info = 0; + topo_nr_threads = (smp_info >> VT_THREADS_SHIFT) & VT_THREADS_MASK; + topo_nr_cores = (smp_info >> VT_CORES_SHIFT) & VT_CORES_MASK; + topo_nr_maxcpus = (smp_info >> VT_MAX_CPUS_SHIFT) & VT_MAX_CPUS_MASK; +} + static void __init init_topo_threads(void) { int i, j; -- Gitee From 6d02d6e58c274f8a5379c62a1f2a0e5c549e86e7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:48 +0800 Subject: [PATCH 058/150] sw64: reorganize pci setup related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- Move pci setup related code to drivers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 1 + arch/sw_64/chip/chip3/Makefile | 3 -- arch/sw_64/chip/chip3/pci-quirks.c | 20 ------------ arch/sw_64/include/asm/pci.h | 4 +++ arch/sw_64/kernel/pci.c | 15 +++++++++ drivers/pci/controller/Kconfig | 4 +++ drivers/pci/controller/Makefile | 1 + .../pci/controller/pci-sunway.c | 32 +++++++------------ 8 files changed, 37 insertions(+), 43 deletions(-) delete mode 100644 arch/sw_64/chip/chip3/pci-quirks.c rename arch/sw_64/chip/chip3/chip.c => drivers/pci/controller/pci-sunway.c (91%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 81cbbd1d6db7..7468871001bb 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -106,6 +106,7 @@ config SW64 select OF_EARLY_FLATTREE if OF select OLD_SIGSUSPEND select PCI_MSI_ARCH_FALLBACKS + select PCI_SW64 if PCI select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM select SW64_IRQ_CPU diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile index 895749957ebc..d1558d84eeae 100644 --- a/arch/sw_64/chip/chip3/Makefile +++ b/arch/sw_64/chip/chip3/Makefile @@ -1,5 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y := chip.o - -obj-$(CONFIG_PCI) += pci-quirks.o diff --git a/arch/sw_64/chip/chip3/pci-quirks.c b/arch/sw_64/chip/chip3/pci-quirks.c deleted file mode 100644 index 0549efc07d0c..000000000000 --- a/arch/sw_64/chip/chip3/pci-quirks.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -#include - -void __init reserve_mem_for_pci(void) -{ - int ret; - unsigned long base = PCI_32BIT_MEMIO; - - ret = add_memmap_region(base, PCI_32BIT_MEMIO_SIZE, memmap_pci); - if (ret) { - pr_err("reserved pages for pcie memory space failed\n"); - return; - } - - pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, - (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); -} diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index 1262d812086a..c203390f4032 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -89,7 +89,11 @@ extern void __init sw64_init_arch(void); extern struct pci_ops sw64_pci_ops; extern int sw64_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); extern struct pci_controller *hose_head; +#ifdef CONFIG_PCI_SW64 extern void __init setup_chip_pci_ops(void); +#else +#define setup_chip_pci_ops() do { } while (0) +#endif #ifdef CONFIG_SUNWAY_IOMMU extern struct syscore_ops iommu_cpu_syscore_ops; diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 1744efb41ae8..6c6a64cca9ad 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -728,6 +728,21 @@ sw64_init_pci(void) pci_clear_flags(PCI_REASSIGN_ALL_BUS); } +void __init reserve_mem_for_pci(void) +{ + int ret; + unsigned long base = PCI_32BIT_MEMIO; + + ret = add_memmap_region(base, PCI_32BIT_MEMIO_SIZE, memmap_pci); + if (ret) { + pr_err("reserved pages for pcie memory space failed\n"); + return; + } + + pr_info("reserved pages for pcie memory space %lx:%lx\n", base >> PAGE_SHIFT, + (base + PCI_32BIT_MEMIO_SIZE) >> PAGE_SHIFT); +} + static int setup_bus_dma_cb(struct pci_dev *pdev, void *data) { pdev->dev.bus_dma_limit = DMA_BIT_MASK(32); diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 5c4e184ac9dc..215dcfbbbda5 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -305,6 +305,10 @@ config PCIE_HISI_ERR Say Y here if you want error handling support for the PCIe controller's errors on HiSilicon HIP SoCs +config PCI_SW64 + bool + depends on SW64 && PCI + source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" source "drivers/pci/controller/cadence/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 04c6edc285c5..e633f0a38d99 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o +obj-$(CONFIG_PCI_SW64) += pci-sunway.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ obj-y += mobiveil/ diff --git a/arch/sw_64/chip/chip3/chip.c b/drivers/pci/controller/pci-sunway.c similarity index 91% rename from arch/sw_64/chip/chip3/chip.c rename to drivers/pci/controller/pci-sunway.c index 7730a2441461..92adc5badbb9 100644 --- a/arch/sw_64/chip/chip3/chip.c +++ b/drivers/pci/controller/pci-sunway.c @@ -1,14 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -#include -#include #include -#include -#include -#include -#include -#include -#include "../../../../drivers/pci/pci.h" void set_devint_wken(int node) { @@ -167,7 +159,7 @@ int chip_pcie_configure(struct pci_controller *hose) return bus_max_num; } -static int chip3_check_pci_linkup(unsigned long node, unsigned long index) +static int check_pci_linkup(unsigned long node, unsigned long index) { unsigned long rc_debug; @@ -183,7 +175,7 @@ static int chip3_check_pci_linkup(unsigned long node, unsigned long index) return !(rc_debug & 0x1); } -static void chip3_set_rc_piu(unsigned long node, unsigned long index) +static void set_rc_piu(unsigned long node, unsigned long index) { unsigned int i, value; u32 rc_misc_ctrl; @@ -227,7 +219,7 @@ static void chip3_set_rc_piu(unsigned long node, unsigned long index) } } -static void chip3_set_intx(unsigned long node, unsigned long index, +static void set_intx(unsigned long node, unsigned long index, unsigned long int_conf) { if (is_guest_or_emul()) @@ -239,7 +231,7 @@ static void chip3_set_intx(unsigned long node, unsigned long index, write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x1UL << 10)); } -static unsigned long chip3_get_rc_enable(unsigned long node) +static unsigned long get_rc_enable(unsigned long node) { unsigned long rc_enable; @@ -251,7 +243,7 @@ static unsigned long chip3_get_rc_enable(unsigned long node) return rc_enable; } -static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +static int map_irq(const struct pci_dev *dev, u8 slot, u8 pin) { struct pci_controller *hose = dev->sysdata; @@ -261,7 +253,7 @@ static int chip3_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) return hose->int_irq; } -static void chip3_hose_init(struct pci_controller *hose) +static void hose_init(struct pci_controller *hose) { unsigned long pci_io_base; @@ -310,12 +302,12 @@ static void chip3_hose_init(struct pci_controller *hose) }; static struct sw64_pci_init_ops chip_pci_init_ops = { - .map_irq = chip3_map_irq, - .get_rc_enable = chip3_get_rc_enable, - .hose_init = chip3_hose_init, - .set_rc_piu = chip3_set_rc_piu, - .check_pci_linkup = chip3_check_pci_linkup, - .set_intx = chip3_set_intx, + .map_irq = map_irq, + .get_rc_enable = get_rc_enable, + .hose_init = hose_init, + .set_rc_piu = set_rc_piu, + .check_pci_linkup = check_pci_linkup, + .set_intx = set_intx, }; void __init setup_chip_pci_ops(void) -- Gitee From 14882598bb18eee0b6c85ea7dd1ca83c419da8c6 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:49 +0800 Subject: [PATCH 059/150] sw64: remove chip directory Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CC6M -------------------------------- The chip directory is now empty, so remove it. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 1 - arch/sw_64/chip/Makefile | 2 -- arch/sw_64/chip/chip3/Makefile | 2 -- 3 files changed, 5 deletions(-) delete mode 100644 arch/sw_64/chip/Makefile delete mode 100644 arch/sw_64/chip/chip3/Makefile diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 3175b0fa2a19..678880cee7fd 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -37,7 +37,6 @@ head-y := arch/sw_64/kernel/head.o core-y += arch/sw_64/kernel/ arch/sw_64/mm/ core-y += arch/sw_64/platform/ -core-y += arch/sw_64/chip/ core-$(CONFIG_MATHEMU) += arch/sw_64/math-emu/ drivers-$(CONFIG_OPROFILE) += arch/sw_64/oprofile/ libs-y += arch/sw_64/lib/ diff --git a/arch/sw_64/chip/Makefile b/arch/sw_64/chip/Makefile deleted file mode 100644 index a64818cdf35b..000000000000 --- a/arch/sw_64/chip/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SW64_CHIP3) := chip3/ diff --git a/arch/sw_64/chip/chip3/Makefile b/arch/sw_64/chip/chip3/Makefile deleted file mode 100644 index d1558d84eeae..000000000000 --- a/arch/sw_64/chip/chip3/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -- Gitee From 433e5c141c0c3ad42105a6cf0dc0da15abf863c0 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 15 Aug 2023 08:50:49 +0800 Subject: [PATCH 060/150] sw64: remove sw64_platform_ops Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Remove useless sw64_platform_ops. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/platform.h | 9 --------- arch/sw_64/kernel/early_init.c | 18 ------------------ arch/sw_64/lib/iomap.c | 9 ++++++++- arch/sw_64/platform/Makefile | 2 +- arch/sw_64/platform/platform_xuelang.c | 20 -------------------- 5 files changed, 9 insertions(+), 49 deletions(-) delete mode 100644 arch/sw_64/platform/platform_xuelang.c diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index 03f098bb4cad..f3d1fc6774f8 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -5,12 +5,6 @@ #include #include -struct sw64_platform_ops { - void __iomem *(*ioportmap)(unsigned long); - void (*register_platform_devices)(void); - void (*ops_fixup)(void); -}; - #ifdef CONFIG_EFI #define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) @@ -20,9 +14,6 @@ extern unsigned long bios_version; #endif -extern struct sw64_platform_ops *sw64_platform; - -extern struct sw64_platform_ops xuelang_ops; extern struct boot_params *sunway_boot_params; extern void sw64_halt(void); diff --git a/arch/sw_64/kernel/early_init.c b/arch/sw_64/kernel/early_init.c index 7f4fdc43a43d..2ec7a3e99443 100644 --- a/arch/sw_64/kernel/early_init.c +++ b/arch/sw_64/kernel/early_init.c @@ -3,27 +3,9 @@ #include -void sw64_init_noop(void) { } -struct sw64_platform_ops *sw64_platform; -EXPORT_SYMBOL(sw64_platform); - -static void __init sw64_setup_platform_ops(void) -{ - /* - * FIXME: set platform operation depending on CONFIG now. - * SMBIOS will help use to determin actual board. - */ -#ifdef CONFIG_PLATFORM_XUELANG - sw64_platform = &xuelang_ops; -#endif -} - - asmlinkage __visible void __init sw64_start_kernel(void) { fixup_hmcall(); save_ktp(); - sw64_setup_platform_ops(); - sw64_platform->ops_fixup(); start_kernel(); } diff --git a/arch/sw_64/lib/iomap.c b/arch/sw_64/lib/iomap.c index 3a8d879ef070..34cdedcb54de 100644 --- a/arch/sw_64/lib/iomap.c +++ b/arch/sw_64/lib/iomap.c @@ -460,7 +460,14 @@ EXPORT_SYMBOL(_memset_c_io); void __iomem *ioport_map(unsigned long port, unsigned int size) { - return sw64_platform->ioportmap(port); + unsigned long io_offset; + + if (port < 0x100000) { + io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; + port = port | io_offset; + } + + return __va(port); } EXPORT_SYMBOL(ioport_map); diff --git a/arch/sw_64/platform/Makefile b/arch/sw_64/platform/Makefile index 7e89ab09624f..4c0edceb4a2c 100644 --- a/arch/sw_64/platform/Makefile +++ b/arch/sw_64/platform/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PLATFORM_XUELANG) += platform_xuelang.o cpufreq_xuelang.o +obj-$(CONFIG_PLATFORM_XUELANG) += cpufreq_xuelang.o diff --git a/arch/sw_64/platform/platform_xuelang.c b/arch/sw_64/platform/platform_xuelang.c deleted file mode 100644 index 28c30cddcff2..000000000000 --- a/arch/sw_64/platform/platform_xuelang.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include - -static inline void __iomem *xuelang_ioportmap(unsigned long addr) -{ - unsigned long io_offset; - - if (addr < 0x100000) { - io_offset = is_in_host() ? LPC_LEGACY_IO : PCI_VT_LEGACY_IO; - addr = addr | io_offset; - } - - return __va(addr); -} - -struct sw64_platform_ops xuelang_ops = { - .ioportmap = xuelang_ioportmap, - .ops_fixup = sw64_init_noop, -}; -- Gitee From df1d73499e9d622877d13eb75c9ac21dac3ffcbf Mon Sep 17 00:00:00 2001 From: He Sheng Date: Thu, 31 Aug 2023 13:51:17 +0800 Subject: [PATCH 061/150] sw64: simplify PMC interfaces Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Rename PMC related macros and header file to make them easier to understand, and after that, some operations can be simplified. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/perf_event.h | 2 +- arch/sw_64/include/asm/{wrperfmon.h => pmc.h} | 53 +++++++-------- arch/sw_64/kernel/perf_event.c | 65 +++++++------------ drivers/irqchip/irq-sunway-cpu.c | 6 +- 4 files changed, 49 insertions(+), 77 deletions(-) rename arch/sw_64/include/asm/{wrperfmon.h => pmc.h} (41%) diff --git a/arch/sw_64/include/asm/perf_event.h b/arch/sw_64/include/asm/perf_event.h index 382a74e85011..dc55a361babd 100644 --- a/arch/sw_64/include/asm/perf_event.h +++ b/arch/sw_64/include/asm/perf_event.h @@ -2,7 +2,7 @@ #ifndef _ASM_SW64_PERF_EVENT_H #define _ASM_SW64_PERF_EVENT_H -#include +#include #include #ifdef CONFIG_PERF_EVENTS diff --git a/arch/sw_64/include/asm/wrperfmon.h b/arch/sw_64/include/asm/pmc.h similarity index 41% rename from arch/sw_64/include/asm/wrperfmon.h rename to arch/sw_64/include/asm/pmc.h index c06a05121a68..d5672dd940a7 100644 --- a/arch/sw_64/include/asm/wrperfmon.h +++ b/arch/sw_64/include/asm/pmc.h @@ -1,44 +1,35 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Definitions for use with the sw64 wrperfmon HMCODE call. + * Definitions for use with the sw64 PMC interface. */ -#ifndef _ASM_SW64_WRPERFMON_H -#define _ASM_SW64_WRPERFMON_H +#ifndef _ASM_SW64_PMC_H +#define _ASM_SW64_PMC_H -#define PERFMON_PC0 0 -#define PERFMON_PC1 1 +#define PMC_PC0 0 +#define PMC_PC1 1 /* Following commands are implemented on all CPUs */ -#define PERFMON_CMD_DISABLE 0 -#define PERFMON_CMD_ENABLE 1 -#define PERFMON_CMD_EVENT_PC0 2 -#define PERFMON_CMD_EVENT_PC1 3 -#define PERFMON_CMD_PM 4 -#define PERFMON_CMD_READ 5 -#define PERFMON_CMD_READ_CLEAR 6 -#define PERFMON_CMD_WRITE_PC0 7 -#define PERFMON_CMD_WRITE_PC1 8 +#define PMC_CMD_DISABLE 0 +#define PMC_CMD_ENABLE 1 +#define PMC_CMD_EVENT_BASE 2 +#define PMC_CMD_PM 4 +#define PMC_CMD_READ 5 +#define PMC_CMD_READ_CLEAR 6 +#define PMC_CMD_WRITE_BASE 7 -#define PERFMON_DISABLE_ARGS_PC0 1 -#define PERFMON_DISABLE_ARGS_PC1 2 -#define PERFMON_DISABLE_ARGS_PC 3 +#define PMC_DISABLE_BASE 1 -#define PERFMON_ENABLE_ARGS_PC0 1 -#define PERFMON_ENABLE_ARGS_PC1 2 -#define PERFMON_ENABLE_ARGS_PC 3 +#define PMC_ENABLE_BASE 1 -#define PERFMON_READ_PC0 0 -#define PERFMON_READ_PC1 1 +#define PC0_RAW_BASE 0x0 +#define PC1_RAW_BASE 0x100 +#define PC0_MAX 0xF +#define PC1_MAX 0x3D -#define PC0_RAW_BASE 0x0 -#define PC1_RAW_BASE 0x100 -#define PC0_MAX 0xF -#define PC1_MAX 0x3D - -#define SW64_PERFCTRL_KM 2 -#define SW64_PERFCTRL_UM 3 -#define SW64_PERFCTRL_AM 4 +#define SW64_PERFCTRL_KM 2 +#define SW64_PERFCTRL_UM 3 +#define SW64_PERFCTRL_AM 4 /* pc0 events */ #define PC0_INSTRUCTIONS 0x0 @@ -61,4 +52,4 @@ #define MAX_HWEVENTS 2 #define PMC_COUNT_MASK ((1UL << 58) - 1) -#endif /* _ASM_SW64_WRPERFMON_H */ +#endif /* _ASM_SW64_PMC_H */ diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index a30817c47e66..72c5e506aa8b 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -91,12 +91,12 @@ static const struct sw64_pmu_t *sw64_pmu; /* Mapping of the hw event types to the perf tool interface */ static const struct sw64_perf_event core3_hw_event_map[] = { - [PERF_COUNT_HW_CPU_CYCLES] = {PERFMON_PC0, PC0_CPU_CYCLES}, - [PERF_COUNT_HW_INSTRUCTIONS] = {PERFMON_PC0, PC0_INSTRUCTIONS}, - [PERF_COUNT_HW_CACHE_REFERENCES] = {PERFMON_PC0, PC0_SCACHE_REFERENCES}, - [PERF_COUNT_HW_CACHE_MISSES] = {PERFMON_PC1, PC1_SCACHE_MISSES}, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {PERFMON_PC0, PC0_BRANCH_INSTRUCTIONS}, - [PERF_COUNT_HW_BRANCH_MISSES] = {PERFMON_PC1, PC1_BRANCH_MISSES}, + [PERF_COUNT_HW_CPU_CYCLES] = {PMC_PC0, PC0_CPU_CYCLES}, + [PERF_COUNT_HW_INSTRUCTIONS] = {PMC_PC0, PC0_INSTRUCTIONS}, + [PERF_COUNT_HW_CACHE_REFERENCES] = {PMC_PC0, PC0_SCACHE_REFERENCES}, + [PERF_COUNT_HW_CACHE_MISSES] = {PMC_PC1, PC1_SCACHE_MISSES}, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = {PMC_PC0, PC0_BRANCH_INSTRUCTIONS}, + [PERF_COUNT_HW_BRANCH_MISSES] = {PMC_PC1, PC1_BRANCH_MISSES}, }; /* Mapping of the hw cache event types to the perf tool interface */ @@ -107,8 +107,8 @@ static const struct sw64_perf_event core3_cache_event_map [PERF_COUNT_HW_CACHE_RESULT_MAX] = { [C(L1D)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_DCACHE_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_DCACHE_MISSES} + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DCACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DCACHE_MISSES} }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -121,8 +121,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(L1I)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_ICACHE_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_ICACHE_READ_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ICACHE_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ICACHE_READ_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -149,8 +149,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(DTLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_DTB_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_DTB_SINGLE_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_DTB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_DTB_SINGLE_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -163,8 +163,8 @@ static const struct sw64_perf_event core3_cache_event_map }, [C(ITLB)] = { [C(OP_READ)] = { - [C(RESULT_ACCESS)] = {PERFMON_PC0, PC0_ITB_READ}, - [C(RESULT_MISS)] = {PERFMON_PC1, PC1_ITB_MISSES}, + [C(RESULT_ACCESS)] = {PMC_PC0, PC0_ITB_READ}, + [C(RESULT_MISS)] = {PMC_PC1, PC1_ITB_MISSES}, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = SW64_OP_UNSUP, @@ -267,21 +267,12 @@ static const struct sw64_pmu_t core3_pmu = { */ static void sw64_write_pmc(int idx, unsigned long val) { - if (idx == PERFMON_PC0) - wrperfmon(PERFMON_CMD_WRITE_PC0, val); - else - wrperfmon(PERFMON_CMD_WRITE_PC1, val); + wrperfmon(PMC_CMD_WRITE_BASE + idx, val); } static unsigned long sw64_read_pmc(int idx) { - unsigned long val; - - if (idx == PERFMON_PC0) - val = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC0); - else - val = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC1); - return val; + return wrperfmon(PMC_CMD_READ, idx); } /* Set a new period to sample over */ @@ -386,14 +377,9 @@ static void sw64_pmu_start(struct perf_event *event, int flags) hwc->state = 0; /* counting in selected modes, for both counters */ - wrperfmon(PERFMON_CMD_PM, hwc->config_base); - if (hwc->idx == PERFMON_PC0) { - wrperfmon(PERFMON_CMD_EVENT_PC0, hwc->event_base); - wrperfmon(PERFMON_CMD_ENABLE, PERFMON_ENABLE_ARGS_PC0); - } else { - wrperfmon(PERFMON_CMD_EVENT_PC1, hwc->event_base); - wrperfmon(PERFMON_CMD_ENABLE, PERFMON_ENABLE_ARGS_PC1); - } + wrperfmon(PMC_CMD_PM, hwc->config_base); + wrperfmon(PMC_CMD_EVENT_BASE + hwc->idx, hwc->event_base); + wrperfmon(PMC_CMD_ENABLE, PMC_ENABLE_BASE + hwc->idx); } /* @@ -404,9 +390,7 @@ static void sw64_pmu_stop(struct perf_event *event, int flags) struct hw_perf_event *hwc = &event->hw; if (!(hwc->state & PERF_HES_STOPPED)) { - wrperfmon(PERFMON_CMD_DISABLE, hwc->idx == 0 ? - PERFMON_DISABLE_ARGS_PC0 : - PERFMON_DISABLE_ARGS_PC1); + wrperfmon(PMC_CMD_DISABLE, PMC_DISABLE_BASE + hwc->idx); hwc->state |= PERF_HES_STOPPED; barrier(); } @@ -610,28 +594,25 @@ void perf_event_print_debug(void) cpu = smp_processor_id(); - pcr0 = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC0); - pcr1 = wrperfmon(PERFMON_CMD_READ, PERFMON_READ_PC1); + pcr0 = wrperfmon(PMC_CMD_READ, PMC_PC0); + pcr1 = wrperfmon(PMC_CMD_READ, PMC_PC1); pr_info("CPU#%d: PCTR0[%lx] PCTR1[%lx]\n", cpu, pcr0, pcr1); local_irq_restore(flags); } -static void sw64_perf_event_irq_handler(unsigned long perfmon_num, +static void sw64_perf_event_irq_handler(unsigned long idx, struct pt_regs *regs) { struct cpu_hw_events *cpuc; struct perf_sample_data data; struct perf_event *event; struct hw_perf_event *hwc; - int idx; __this_cpu_inc(irq_pmi_count); cpuc = this_cpu_ptr(&cpu_hw_events); - idx = perfmon_num; - event = cpuc->event[idx]; if (unlikely(!event)) { diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index f43b327cc291..05ed5a045300 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -5,8 +5,8 @@ #include #include +#include #include -#include static void handle_intx(unsigned int offset) { @@ -180,10 +180,10 @@ asmlinkage void do_entInt(unsigned long type, unsigned long vector, set_irq_regs(old_regs); return; case INT_PC0: - perf_irq(PERFMON_PC0, regs); + perf_irq(PMC_PC0, regs); return; case INT_PC1: - perf_irq(PERFMON_PC1, regs); + perf_irq(PMC_PC1, regs); return; case INT_DEV: old_regs = set_irq_regs(regs); -- Gitee From ed26b5c90c44a0c6d59bd174980887981b76c369 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Thu, 7 Sep 2023 16:01:42 +0800 Subject: [PATCH 062/150] sw64: fix excessive timer interrupts Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCG0 -------------------------------- The architecture code did not set max_delta_ticks of clock_event_device. This make max_delta_ns always be calculated to 1000 when it updates clock event, leading to one cpu suffering abnormal frequent timer interrupts. This patch set it to fix this problem. Besides, delta_ns and mult can be calcutated from clockevents_config(), so remove them. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 456f2becb182..7d7e0fff2615 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -316,9 +316,13 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq) */ void sw64_setup_timer(void) { + unsigned long min_delta; int cpu = smp_processor_id(); struct clock_event_device *swevt = &per_cpu(timer_events, cpu); + /* min_delta ticks => 100ns */ + min_delta = get_cpu_freq()/1000/1000/10; + if (is_in_guest()) { memcpy(swevt, &vtimer_clockevent, sizeof(*swevt)); /* @@ -326,19 +330,13 @@ void sw64_setup_timer(void) * If it's too small, the timer will timeout when the IER * haven't been opened. */ - swevt->min_delta_ns = 400; + min_delta *= 4; } else { memcpy(swevt, &timer_clockevent, sizeof(*swevt)); - swevt->min_delta_ns = 100; } - swevt->cpumask = cpumask_of(cpu); - swevt->mult = div_sc(get_cpu_freq(), NSEC_PER_SEC, swevt->shift); - swevt->max_delta_ns = clockevent_delta2ns(0xFFFFFFFFFFFFFFFF, swevt); - swevt->set_state_shutdown(swevt); - - clockevents_register_device(swevt); + clockevents_config_and_register(swevt, get_cpu_freq(), min_delta, ULONG_MAX); } void sw64_timer_interrupt(void) -- Gitee From abba5d9503b53ae0b4275c1251c6a9f859ad6e79 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Thu, 7 Sep 2023 16:44:56 +0800 Subject: [PATCH 063/150] sw64: cpufreq: fix clockevents update method Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCGC -------------------------------- It used to assume that each CPU would trigger its own cpufreq notifier callback, and only the CPU that received the callback will update clock event. However, a single callback is issued for all CPUs which share policy, and only one clockevent update. This patch fixes frequency update code to account for share policies. All clockevents involved are updated to ensure consistence across system. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 7d7e0fff2615..8f2ab450cf40 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -308,6 +308,11 @@ void sw64_update_clockevents(unsigned long cpu, u32 freq) if (cpu == smp_processor_id()) clockevents_update_freq(swevt, freq); + else { + clockevents_calc_mult_shift(swevt, freq, 4); + swevt->min_delta_ns = clockevent_delta2ns(swevt->min_delta_ticks, swevt); + swevt->max_delta_ns = clockevent_delta2ns(swevt->max_delta_ticks, swevt); + } } /* -- Gitee From c7d4b1162829baea26836be49c312dae6f49fed4 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 16:27:33 +0800 Subject: [PATCH 064/150] sw64: kconfig: fix config error when PCI_MSI is not set Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Select PCI_MSI_ARCH_FALLBACKS only if PCI_MSI is selected. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 7468871001bb..daf535344845 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -105,7 +105,7 @@ config SW64 select NO_BOOTMEM select OF_EARLY_FLATTREE if OF select OLD_SIGSUSPEND - select PCI_MSI_ARCH_FALLBACKS + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_SW64 if PCI select SET_FS select SPARSEMEM_EXTREME if SPARSEMEM -- Gitee From bbff54009296b0e50845748cc0ed954306d0b9fc Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 16:20:57 +0800 Subject: [PATCH 065/150] sw64: fix compile error when CONFIG_PM is not set Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Enable io_suspend() and io_resume() related code only if CONFIG_PM is defined. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/chip_setup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 88d3ef2a9bf2..03f073ec9bca 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -64,6 +64,7 @@ static void __init setup_core_start(struct cpumask *cpumask) } } +#ifdef CONFIG_PM static void i2c_srst(void) { sw64_io_write(0, I2C0_SRST_L, 0x0); @@ -213,6 +214,7 @@ static void io_resume(void) intpu_restore(); spbu_restore(); } +#endif /* CONFIG_PM */ static struct sw64_chip_init_ops chip_init_ops = { .early_init = { -- Gitee From 17ce1dc682d98c31783d14f379e94e7d820481e3 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Thu, 31 Aug 2023 08:57:24 +0800 Subject: [PATCH 066/150] sw64: remove out-of-date LOCK_FIXUP Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- This option was introduced for SW1631 which is no longer supported. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 7 ------- arch/sw_64/include/asm/atomic.h | 22 ---------------------- arch/sw_64/include/asm/bitops.h | 21 --------------------- arch/sw_64/include/asm/futex.h | 12 ------------ arch/sw_64/include/asm/xchg.h | 24 ------------------------ 5 files changed, 86 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index daf535344845..7fa38a675af1 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -322,13 +322,6 @@ config GENERIC_HWEIGHT bool default y -config LOCK_FIXUP - bool "fix up the lock" - depends on SW64 - help - Add an instruction("memb\n") to ensure the correctness of the lock. - - config SMP bool "Symmetric multi-processing support" depends on SW64 diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h index 0cfebacef080..cad03ba13076 100644 --- a/arch/sw_64/include/asm/atomic.h +++ b/arch/sw_64/include/asm/atomic.h @@ -56,9 +56,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) " seleq %4, 1, $31, %4\n" " wr_f %4\n" " addw %0, %6, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %1, 0(%3)\n" " rd_f %1\n" " beq %4, 2f\n" @@ -93,9 +90,6 @@ static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) " seleq %4, 1, $31, %4\n" " wr_f %4\n" " addl %0, %6, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %1, 0(%3)\n" " rd_f %1\n" " beq %4, 2f\n" @@ -127,9 +121,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) " seleq %0, 1, $31, %0\n" " wr_f %0\n" " subl %4, 1, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %1, 0(%3)\n" " rd_f %1\n" " beq %0, 2f\n" @@ -146,13 +137,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) #define atomic64_dec_if_positive atomic64_dec_if_positive -#ifdef CONFIG_LOCK_FIXUP -#define LOCK_FIXUP "memb\n" -#else -#define LOCK_FIXUP -#endif - - #define ATOMIC_OP(op, asm_op) \ static inline void atomic_##op(int i, atomic_t *v) \ { \ @@ -163,7 +147,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstw %0, 0(%3)\n" \ " rd_f %0\n" \ " beq %0, 2f\n" \ @@ -187,7 +170,6 @@ static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstw %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -212,7 +194,6 @@ static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ - LOCK_FIXUP \ " lstw %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -235,7 +216,6 @@ static inline void atomic64_##op(long i, atomic64_t *v) \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstl %0, 0(%3)\n" \ " rd_f %0\n" \ " beq %0, 2f\n" \ @@ -259,7 +239,6 @@ static inline long atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ " " #asm_op " %0, %4, %0\n" \ - LOCK_FIXUP \ " lstl %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ @@ -282,7 +261,6 @@ static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ " ldi %1, 1\n" \ " wr_f %1\n" \ " " #asm_op " %0, %4, %1\n" \ - LOCK_FIXUP \ " lstl %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 2f\n" \ diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h index e69b0b31dc79..8d2865ab7b74 100644 --- a/arch/sw_64/include/asm/bitops.h +++ b/arch/sw_64/include/asm/bitops.h @@ -37,9 +37,6 @@ set_bit(unsigned long nr, volatile void *addr) " ldi %1, 1\n" " wr_f %1\n" " bis %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" " rd_f %0\n" " beq %0, 2f\n" @@ -76,9 +73,6 @@ clear_bit(unsigned long nr, volatile void *addr) " ldi %1, 1\n" " wr_f %1\n" " bic %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" " rd_f %0\n" " beq %0, 2f\n" @@ -126,9 +120,6 @@ change_bit(unsigned long nr, volatile void *addr) " ldi %1, 1\n" " wr_f %1\n" " xor %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" " rd_f %0\n" " beq %0, 2f\n" @@ -165,9 +156,6 @@ test_and_set_bit(unsigned long nr, volatile void *addr) " seleq %3, 1, $31, %1\n" " wr_f %1\n" " bis %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " bne %3, 2f\n" // %3 is not zero, no need to set, return @@ -196,9 +184,6 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) " seleq %3, 1, $31, %1\n" " wr_f %1\n" " bis %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " bne %3, 2f\n" // %3 is not zero, no need to set, return @@ -241,9 +226,6 @@ test_and_clear_bit(unsigned long nr, volatile void *addr) " selne %3, 1, $31, %1\n" //Note: here is SELNE!!! " wr_f %1\n" " bic %0, %5, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%4)\n" " rd_f %0\n" " beq %3, 2f\n" // %3 is zero, no need to set, return @@ -286,9 +268,6 @@ test_and_change_bit(unsigned long nr, volatile void *addr) " wr_f %2\n" " and %0, %4, %2\n" " xor %0, %4, %0\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %0, 0(%3)\n" " rd_f %0\n" " beq %0, 3f\n" diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h index f4806b856804..2a814d57991a 100644 --- a/arch/sw_64/include/asm/futex.h +++ b/arch/sw_64/include/asm/futex.h @@ -9,21 +9,12 @@ #include #include -#ifndef LOCK_FIXUP -#ifdef CONFIG_LOCK_FIXUP -#define LOCK_FIXUP "memb\n" -#else -#define LOCK_FIXUP -#endif -#endif - #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ __asm__ __volatile__( \ "1: lldw %0, 0(%3)\n" \ " ldi %2, 1\n" \ " wr_f %2\n" \ insn \ - LOCK_FIXUP \ "2: lstw %1, 0(%3)\n" \ " rd_f %1\n" \ " beq %1, 4f\n" \ @@ -92,9 +83,6 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, " cmpeq %1, %5, %2\n" " wr_f %2\n" " bis $31, %6, %3\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif "2: lstw %3, 0(%4)\n" " rd_f %3\n" " beq %2, 3f\n" diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index 34acfc1699b0..154467ef30e4 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -32,9 +32,6 @@ ____xchg(_u8, volatile char *m, unsigned long val) " extlb %2, %4, %0\n" " masklb %2, %4, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%3)\n" " rd_f %2\n" " beq %2, 2f\n" @@ -61,9 +58,6 @@ ____xchg(_u16, volatile short *m, unsigned long val) " extlh %2, %4, %0\n" " masklh %2, %4, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%3)\n" " rd_f %2\n" " beq %2, 2f\n" @@ -87,9 +81,6 @@ ____xchg(_u32, volatile int *m, unsigned long val) " ldi %1, 1\n" " wr_f %1\n" " bis $31, %4, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %1, 0(%3)\n" " rd_f %1\n" " beq %1, 2f\n" @@ -113,9 +104,6 @@ ____xchg(_u64, volatile long *m, unsigned long val) " ldi %1, 1\n" " wr_f %1\n" " bis $31, %4, %1\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %1, 0(%3)\n" " rd_f %1\n" " beq %1, 2f\n" @@ -176,9 +164,6 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) " wr_f %3\n" " masklb %2, %5, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%4)\n" " rd_f %2\n" " beq %3, 2f\n" @@ -207,9 +192,6 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) " wr_f %3\n" " masklh %2, %5, %2\n" " or %1, %2, %2\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %2, 0(%4)\n" " rd_f %2\n" " beq %3, 2f\n" @@ -235,9 +217,6 @@ ____cmpxchg(_u32, volatile int *m, int old, int new) " cmpeq %0, %5, %1\n" " wr_f %1\n" " bis $31, %6, %4\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstw %4, 0(%3)\n" " rd_f %4\n" " beq %1, 2f\n" @@ -263,9 +242,6 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) " cmpeq %0, %5, %1\n" " wr_f %1\n" " bis $31, %6, %4\n" -#ifdef CONFIG_LOCK_FIXUP - " memb\n" -#endif " lstl %4, 0(%3)\n" " rd_f %4\n" " beq %1, 2f\n" -- Gitee From be52a0f884cb0cf8a2032218b17aa434c6029e08 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 14:58:56 +0800 Subject: [PATCH 067/150] sw64: remove unused file Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Remove arch/sw_64/kernel/pci_iommu.c. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/pci_iommu.c | 772 ---------------------------------- 1 file changed, 772 deletions(-) delete mode 100644 arch/sw_64/kernel/pci_iommu.c diff --git a/arch/sw_64/kernel/pci_iommu.c b/arch/sw_64/kernel/pci_iommu.c deleted file mode 100644 index 79760c4ac6fc..000000000000 --- a/arch/sw_64/kernel/pci_iommu.c +++ /dev/null @@ -1,772 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* iommu.c: Generic sw_64 IOMMU support for 3231 - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "proto.h" -#include "pci_impl.h" -#include "sw_pci_impl.h" - -#define DEBUG_ALLOC 0 -#if DEBUG_ALLOC > 0 -# define DBGA(args...) printk(KERN_DEBUG args) -#else -# define DBGA(args...) -#endif -#if DEBUG_ALLOC > 1 -# define DBGA2(args...) printk(KERN_DEBUG args) -#else -# define DBGA2(args...) -#endif - -unsigned long iommu_cmd; - -static void sw_iommu_create_new(struct pci_controller *hose, unsigned int error_bus_number, - unsigned int error_devfn, unsigned int error_da) -{ - unsigned long dtbr; - u64 *paddr; - u32 ofs; - unsigned long dtbbaseaddr, dtbbasecond; - - sw_read_piu_ior0(hose->node, hose->index, DTBASEADDR, &dtbr); - dtbr += PAGE_OFFSET; - ofs = error_da >> PAGE_SHIFT; - - dtbbaseaddr = dtbr + (error_bus_number << 3); - dtbbasecond = (*(u64 *)(dtbbaseaddr)) & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK; - dtbbasecond += (error_devfn << 3) + PAGE_OFFSET; - - paddr = (u64 *)get_zeroed_page(GFP_DMA); - sw_iommu_map(__pa(paddr), ofs, dtbbasecond, hose, NULL); -} - -irqreturn_t iommu_interrupt(int irq, void *dev) -{ - struct pci_controller *hose = (struct pci_controller *)dev; - unsigned long iommu_status; - unsigned int type, bus_number; - unsigned int devfn, error_da; - - sw_read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, &iommu_status); - if (!(iommu_status >> 63)) - return IRQ_NONE; - - type = (iommu_status >> 59) & 0x7; - bus_number = (iommu_status >> 45) & 0xff; - devfn = (iommu_status >> 37) & 0xff; - error_da = iommu_status & 0xffffffff; - - if (type == 0x3) { - iommu_status &= ~(1UL << 62); - iommu_status = iommu_status | (1UL << 63); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); - return IRQ_HANDLED; - } - - if (type == 0x2) - sw_iommu_create_new(hose, bus_number, devfn, error_da); - - udelay(100); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - - iommu_status = iommu_status | (3UL << 62); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS, iommu_status); - - return IRQ_HANDLED; -} - -struct irqaction iommu_irqaction = { - .handler = iommu_interrupt, - .flags = IRQF_SHARED | IRQF_NO_THREAD, - .name = "sw_iommu", -}; - -void sw_enable_iommu_func(struct pci_controller *hose) -{ - struct irqaction *action; - unsigned int iommu_irq; - unsigned long iommu_conf, iommu_ctrl; - - iommu_irq = hose->int_irq; - action = &iommu_irqaction; - action->dev_id = hose; - request_irq(iommu_irq, action.iommu_interrupt, action.flags, "sw_iommu", action->dev_id); - iommu_ctrl = (1UL << 63) | (0x100UL << 10); - sw_write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); - sw_read_piu_ior0(hose->node, hose->index, PIUCONFIG0, &iommu_conf); - iommu_conf = iommu_conf | (0x3 << 7); - sw_write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); - sw_write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); - sw_read_piu_ior0(hose->node, hose->index, PIUCONFIG0, &iommu_conf); - pr_info("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", - hose->node, hose->index, iommu_conf); -} - -struct sw_iommu_dev *pci_to_iommu(struct pci_dev *pdev) -{ - struct sw_iommu *iommu; - struct pci_controller *hose = (struct pci_controller *)pdev->sysdata; - struct sw_iommu_dev *sw_dev; - int busnumber, devid; - - iommu = hose->pci_iommu; - - list_for_each_entry(sw_dev, &iommu->dev_list, list) { - busnumber = sw_dev->dev_id >> 8; - devid = sw_dev->dev_id & 0xff; - if ((busnumber == pdev->bus->number) && (devid == pdev->devfn)) - return sw_dev; - } - - return NULL; -} - -struct sw_iommu_dev *create_sw_iommu_dev(struct pci_dev *dev, unsigned long *pte, struct sw_iommu *iommu) -{ - struct sw_iommu_dev *sw_dev = kzalloc(sizeof(struct sw_iommu_dev), GFP_KERNEL); - - sw_dev->dev_id = (dev->bus->number << 8) + dev->devfn; - sw_dev->io_page_base = pte; - sw_dev->iommu = iommu; - - list_add_tail(&sw_dev->list, &iommu->dev_list); - return sw_dev; -} - -void __sw_pci_iommu_dte_alloc(struct pci_bus *bus, struct sw_iommu *iommu) -{ - struct pci_dev *dev; - struct sw_iommu_dev *iommu_dev; - unsigned long *pte; - u64 *dte; - u64 dtebaseaddr; - u64 dtentry; - u64 dtebaseaddr2, ptentry; - - dtebaseaddr = (unsigned long)iommu->iommu_dtbr + (bus->number << 3); - dte = (u64 *)get_zeroed_page(GFP_KERNEL); - dtentry = (__pa(dte) & PAGE_MASK) | (1UL << 63); - *(u64 *)dtebaseaddr = dtentry; - - list_for_each_entry(dev, &bus->devices, bus_list) { - if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) { - pte = (unsigned long *)get_zeroed_page(GFP_KERNEL); - dtebaseaddr2 = ((unsigned long)dte & PAGE_MASK) + ((dev->devfn) << 3); - iommu_dev = create_sw_iommu_dev(dev, pte, iommu); - ptentry = (__pa(pte) & PAGE_MASK) | (1UL << 63); - iommu_dev->iommu_bypass = 0; - *(u64 *)dtebaseaddr2 = ptentry; - /* legacy VGA frame buffer has occupied 0xA0000-0xBFFFF memory segment */ - iommu_dev->iommu_area = sw_iommu_area_new(iommu_dev, 0x100000UL); - } else if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { - struct pci_bus *b = dev->subordinate; - - if (b) - __sw_pci_iommu_dte_alloc(b, iommu); - } - } -} - -static int iommu_cpu_suspend(void) -{ - return 0; -} - -static void iommu_cpu_resume(void) -{ -} - -struct syscore_ops iommu_cpu_syscore_ops = { - .suspend = iommu_cpu_suspend, - .resume = iommu_cpu_resume, -}; - -int sw_iommu_init(struct pci_controller *hose) -{ - struct sw_iommu *iommu; - unsigned long base; - unsigned long rc_mask = 0x1; - - rc_mask <<= (8 * hose->node + hose->index); - if (!(iommu_cmd & rc_mask)) - return 0; - sw_write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - hose->pci_iommu = kzalloc(sizeof(struct sw_iommu), GFP_KERNEL); - if (!hose->pci_iommu) { - printk("Can't alloc memory for pci_iommu!\n"); - return 0; - } - iommu = hose->pci_iommu; - spin_lock_init(&iommu->dt_lock); - iommu->index = hose->index; - iommu->enabled = true; - iommu->iommu_dtbr = (unsigned long *)get_zeroed_page(GFP_KERNEL); - base = __pa(iommu->iommu_dtbr) & PAGE_MASK; - sw_write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); - INIT_LIST_HEAD(&iommu->dev_list); - __sw_pci_iommu_dte_alloc(hose->bus, iommu); - sw_write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); - sw_enable_iommu_func(hose); - hose->iommu_enable = true; - - return 0; -} - -struct sw_pci_dev_iommu_area *sw_iommu_area_new(struct sw_iommu_dev *iommu_dev, dma_addr_t base) -{ - struct sw_pci_dev_iommu_area *iommu_area = kzalloc(sizeof(struct sw_pci_dev_iommu_area), GFP_KERNEL); - - if (!iommu_area) { - pr_err("SW arch could not allocate pci iommu dma_area.\n"); - return NULL; - } - - spin_lock_init(&iommu_area->lock); - iommu_area->iommu = iommu_dev->iommu; - iommu_area->dma_base = base; - iommu_area->bitmap = (void *)__get_free_pages(GFP_KERNEL, 3); - if (!iommu_area->bitmap) { - free_pages((unsigned long)iommu_area->bitmap, 3); - pr_err("SW arch could not allocate dma_area->bitmap.\n"); - return NULL; - } - memset(iommu_area->bitmap, 0, 8*PAGE_SIZE); - iommu_area->next_address = 0; - return iommu_area; -} - -/** - * sw_iommu_map - - * @paddr: buffer of the indicated size for PCI DMA - * @dma_ofs: virtual DMA buffer page frame number allocated from pdev private DMA zone - * @dtbaddr: Device Table Base Addr for Level 2 - * @index: PCIe host index - */ -int sw_iommu_map(unsigned long paddr, long dma_ofs, unsigned long dtbaddr, - struct pci_controller *hose, struct pci_dev *pdev) -{ - unsigned long pde, pte; /*pde means Page Table Base Addr for Level 2 pte means Page Table Entry*/ - unsigned long pdebaseaddr; - u64 *ptebasesecond, ptebaseaddr; /*ptebasesecond means Page Table Pointer for Level 2*/ - unsigned long pcache_flush_addr; - - pdebaseaddr = ((dma_ofs >> 10) & SW_IOMMU_LEVEL1_OFFSET) << 3; /* Offset of Page Table Entry for Level 1 */ - pdebaseaddr += ((*(volatile u64 *)dtbaddr) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - pte = (paddr & PAGE_MASK) | SW_IOMMU_ENTRY_VALID | SW_IOMMU_GRN | SW_IOMMU_ENABLE; - - /* If pde exists, no need to allocate a new page */ - if ((*(volatile u64 *)pdebaseaddr) & SW_IOMMU_ENTRY_VALID) { - ptebaseaddr = ((*(volatile u64 *)pdebaseaddr) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - ptebaseaddr += (dma_ofs & SW_IOMMU_LEVEL2_OFFSET) << 3; - - pcache_flush_addr = __pa(ptebaseaddr) & 0xffffffff80; - - *(volatile u64 *)ptebaseaddr = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } else { - ptebasesecond = (u64 *)get_zeroed_page(GFP_ATOMIC); - - if (!ptebasesecond) { - printk("allocating pages fails.\n"); - free_page((unsigned long)ptebasesecond); - return -1; - } - pde = (__pa(ptebasesecond) & PAGE_MASK) | SW_IOMMU_ENTRY_VALID; - - pcache_flush_addr = __pa(pdebaseaddr) & 0xffffffff80; - - *(volatile u64 *)pdebaseaddr = pde; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - - ptebaseaddr = (unsigned long)ptebasesecond + ((dma_ofs & SW_IOMMU_LEVEL2_OFFSET) << 3); - - pcache_flush_addr = __pa(ptebaseaddr) & 0xffffffff80; - - *(volatile u64 *)ptebaseaddr = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } - - return 0; -} - -static unsigned long -sw_iommu_area_alloc(struct sw_pci_dev_iommu_area *area, unsigned int pages, - unsigned long start) -{ - unsigned long next_bit = start >> PAGE_SHIFT; - unsigned long address = -1; - unsigned long boundary_size = ((4UL << 30)) >> PAGE_SHIFT; - unsigned long limit = boundary_size - (1UL << 17) - (area->dma_base >> PAGE_SHIFT); - - address = iommu_area_alloc(area->bitmap, limit, next_bit, pages, 0, boundary_size, 0); - if (address != -1) { - address = address << PAGE_SHIFT; - area->next_address = address + (pages << PAGE_SHIFT); - } - - return address; -} - -static long -sw_iommu_alloc_da(struct sw_pci_dev_iommu_area *area, long n) -{ - unsigned long address; - - address = sw_iommu_area_alloc(area, n, area->next_address); - if (address == -1) { - area->next_address = 0; - address = sw_iommu_area_alloc(area, n, area->next_address); - if (address == -1) - pr_err("SW arch failed to allocate device address.\n"); - } - - return address; -} - -static void sw_iommu_free_da(unsigned long *map, long dma_ofs, long n) -{ - bitmap_clear(map, dma_ofs, n); -} - -static void sw_iommu_unmap(struct pci_dev *pdev, long ofs) -{ - unsigned long dtbbaseaddr, dtbbasecond; /* dtbbaseaddr means Device Table Base Addr for Level 1 */ - /* dtbbasecond means Device Table Base Addr for Level 2 */ - unsigned long pde, pte; /* pde means Page Table Base Addr for Level 2 */ - /* pte means Page Table Entry */ - unsigned long tlb_flush_addr, pcache_flush_addr; - unsigned long addr; - unsigned long pdebaseaddr; /* ptebasefirst means Page Table Pointer for Level 1 */ - unsigned long ptebaseaddr; /* ptebasesecond means Page Table Pointer for Level 2 */ - unsigned long ptebaseaddr_full; /* ptebasesecond means Page Table Pointer for Level 2 */ - unsigned long ptebaseaddr_offset; - struct pci_controller *hose = (struct pci_controller *)pdev->sysdata; - int i; - u64 per_pte; - struct sw_iommu *sw_pci_iommu = hose->pci_iommu; - - addr = (unsigned long)sw_pci_iommu->iommu_dtbr; - dtbbaseaddr = addr + (pdev->bus->number << 3); - - dtbbasecond = (*(volatile u64 *)dtbbaseaddr) & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK; - dtbbasecond += (pdev->devfn << 3) + PAGE_OFFSET; - - pdebaseaddr = ((*(volatile u64 *)dtbbasecond) & (~(SW_IOMMU_ENTRY_VALID)) & (PAGE_MASK)) + PAGE_OFFSET; - pdebaseaddr += ((ofs >> 10) & SW_IOMMU_LEVEL1_OFFSET) << 3; - - pde = *(volatile u64 *)(pdebaseaddr); - ptebaseaddr = (pde & (~(SW_IOMMU_ENTRY_VALID)) & PAGE_MASK) + PAGE_OFFSET; - ptebaseaddr_offset = ptebaseaddr + ((ofs & SW_IOMMU_LEVEL2_OFFSET) << 3); - - tlb_flush_addr = (pdev->bus->number << 8) | pdev->devfn | (ofs << 16); - sw_write_piu_ior0(hose->node, hose->index, PTLB_FLUSHVADDR, tlb_flush_addr); /* TLB FLUSH*/ - - pte = *(volatile u64 *)(ptebaseaddr_offset); - pte &= ~(SW_IOMMU_ENTRY_VALID); /*disable Page Table Entry*/ - pcache_flush_addr = __pa(ptebaseaddr_offset) & 0xffffffff80; - - *(volatile u64 *)(ptebaseaddr_offset) = pte; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - - ptebaseaddr_full = ptebaseaddr + 0x1ff8; - if (ptebaseaddr_offset == ptebaseaddr_full) { - for (i = 0; i < 1024; i++) { - per_pte = *(volatile u64 *)(ptebaseaddr + i * 8); - if (per_pte & SW_IOMMU_ENTRY_VALID) - break; - } - if (i == 1024) { - free_page(ptebaseaddr); - pde &= ~(SW_IOMMU_ENTRY_VALID); - - pcache_flush_addr = __pa(pdebaseaddr) & 0xffffffff80; - - *(volatile u64 *)(pdebaseaddr) = pde; - mb(); - sw_write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHPADDR, pcache_flush_addr); - } - } -} - -static dma_addr_t __sw_map_single(struct pci_dev *pdev, unsigned long paddr, - struct sw_pci_dev_iommu_area *iommu_area, size_t size) -{ - long npages, dma_ofs, i, ofs; - unsigned long dtbbaseaddr; /* dtbbaseaddr means Device Table Base Addr for Level 1 */ - unsigned long dtbbasecond; /* dtbbasecond means Device Table Base Addr for Level 2 */ - unsigned long addr; - dma_addr_t ret = -1; - struct pci_controller *hose = pdev->sysdata; - struct sw_iommu *sw_pci_iommu = hose->pci_iommu; - unsigned long flags; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - addr = (unsigned long)sw_pci_iommu->iommu_dtbr; - dtbbaseaddr = addr + (pdev->bus->number << 3); - - dtbbasecond = (*(volatile u64 *)dtbbaseaddr) & ~(SW_IOMMU_ENTRY_VALID) & PAGE_MASK; - dtbbasecond += (pdev->devfn << 3) + PAGE_OFFSET; - npages = iommu_num_pages(paddr, size, PAGE_SIZE); - - if (hose->iommu_enable) { - spin_lock_irqsave(&iommu_area->lock, flags); - - dma_ofs = sw_iommu_alloc_da(iommu_area, npages); - if (dma_ofs == -1) { - pr_warn("%s %s failed: could not allocate dma page tables\n", - pci_name(pdev), __func__); - spin_unlock_irqrestore(&iommu_area->lock, flags); - return 0; - } - - ret = iommu_area->dma_base + dma_ofs; - - for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) { - ofs = (ret >> PAGE_SHIFT) + i; - sw_iommu_map(paddr, ofs, dtbbasecond, hose, pdev); - } - - spin_unlock_irqrestore(&iommu_area->lock, flags); - - ret += paddr & ~PAGE_MASK; - } - - return ret; -} - -/* - * Map a single buffer of the indicated size for PCI DMA in streaming - * mode. The 32-bit PCI bus mastering address to use is returned. - * Once the device is given the dma address, the device owns this memory - * until either pci_unmap_single or pci_dma_sync_single is performed. - */ - -static dma_addr_t -pci_iommu_map_single(struct pci_dev *pdev, void *cpu_addr, size_t size) -{ - struct pci_controller *hose = pdev->sysdata; - unsigned long paddr; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) { - unsigned long dma_offset; - - sw_read_piu_ior0(hose->node, hose->index, EPDMABAR, &dma_offset); - paddr = __pa(cpu_addr) + dma_offset; - } else { - struct sw_pci_dev_iommu_area *iommu_area; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - - paddr = __pa(cpu_addr); - iommu_area = sw_dev->iommu_area; - if (!iommu_area) { - pr_err("SW arch get iommu_area error!\n"); - return 0; - } - - paddr = __sw_map_single(pdev, paddr, iommu_area, size); - } - - return paddr; -} - -static dma_addr_t sw_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - - if (dir == PCI_DMA_NONE) - BUG(); - - return pci_iommu_map_single(pdev, (char *)page_address(page) + offset, size); -} - -/* - * Unmap a single streaming mode DMA translation. The DMA_ADDR and - * SIZE must match what was provided for in a previous pci_map_single - * call. All other usages are undefined. After this call, reads by - * the cpu to the buffer are guaranteed to see whatever the device - * wrote there. - */ - -static void sw_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - struct pci_controller *hose = pdev->sysdata; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - struct sw_pci_dev_iommu_area *iommu_area; - long dma_ofs, npages, ofs; - unsigned long flags; - int i; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) - return; - - iommu_area = sw_dev->iommu_area; - dma_ofs = (dma_addr - iommu_area->dma_base) >> PAGE_SHIFT; - npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - - spin_lock_irqsave(&iommu_area->lock, flags); - - for (i = 0; i < npages; ++i) { - ofs = (dma_addr >> PAGE_SHIFT) + i; - sw_iommu_unmap(pdev, ofs); - } - - sw_iommu_free_da(iommu_area->bitmap, dma_ofs, npages); - spin_unlock_irqrestore(&iommu_area->lock, flags); -} - -/* - * Allocate and map kernel buffer using consistent mode DMA for PCI - * device. Returns non-NULL cpu-view pointer to the buffer if - * successful and sets *DMA_ADDRP to the pci side dma address as well, - * else DMA_ADDRP is undefined. - */ -static void *sw_iommu_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - void *cpu_addr; - long order = get_order(size); - - gfp &= ~GFP_DMA; - -try_again: - cpu_addr = (void *)__get_free_pages(gfp | __GFP_ZERO, order); - if (!cpu_addr) { - pr_info("pci_alloc_consistent: get_free_pages failed from %ps\n", - __builtin_return_address(0)); - /* ??? Really atomic allocation? Otherwise we could play - * with vmalloc and sg if we can't find contiguous memory. - */ - return NULL; - } - memset(cpu_addr, 0, size); - - *dma_addrp = pci_iommu_map_single(pdev, cpu_addr, size); - if (*dma_addrp == 0) { - free_pages((unsigned long)cpu_addr, order); - if (gfp & GFP_DMA) - return NULL; - /* The address doesn't fit required mask and we - * do not have iommu. Try again with GFP_DMA. - */ - gfp |= GFP_DMA; - goto try_again; - } - - DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n", - size, cpu_addr, *dma_addrp, __builtin_return_address(0)); - - return cpu_addr; -} - -/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must - * be values that were returned from pci_alloc_consistent. SIZE must - * be the same as what as passed into pci_alloc_consistent. - * References to the memory and mappings associated with CPU_ADDR or - * DMA_ADDR past this call are illegal. - */ - -static void sw_iommu_free_coherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - - pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); - free_pages((unsigned long)cpu_addr, get_order(size)); - - DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n", - dma_addr, size, __builtin_return_address(0)); -} - -#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) -#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) - -static int sw_iommu_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - struct pci_dev *pdev = sw_gendev_to_pci(dev); - int out_nents = 0; - - if (dir == PCI_DMA_NONE) - BUG(); - - for_each_sg(sgl, sg, nents, i) { - BUG_ON(!sg_page(sg)); - - sg_dma_address(sg) = pci_iommu_map_single(pdev, SG_ENT_VIRT_ADDRESS(sg), sg->length); - if (sg_dma_address(sg) == 0) - goto error; - sg_dma_len(sg) = sg->length; - out_nents++; - } - - return nents; - -error: - pr_warn("pci_map_sg failed: could not allocate dma page tables\n"); - - /* Some allocation failed while mapping the scatterlist - * entries. Unmap them now. - */ - if (out_nents) - pci_unmap_sg(pdev, sgl, out_nents, dir); - return 0; -} - -/* - * Unmap a set of streaming mode DMA translations. Again, cpu read - * rules concerning calls here are the same as for pci_unmap_single() - * above. - */ -static void sw_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - struct pci_dev *pdev = sw_gendev_to_pci(dev); - struct pci_controller *hose = pdev->sysdata; - struct scatterlist *sg; - int i, j; - dma_addr_t dma_addr; - struct sw_pci_dev_iommu_area *iommu_area; - struct sw_iommu_dev *sw_dev = pci_to_iommu(pdev); - long dma_ofs, npages, ofs, size; - unsigned long flags; - - if (hose == NULL) { - pr_err("%s: hose does not exist!\n", __func__); - return 0; - } - - if (!hose->iommu_enable) - return; - - iommu_area = sw_dev->iommu_area; - for_each_sg(sgl, sg, nents, j) { - BUG_ON(!sg_page(sg)); - dma_addr = sg->dma_address; - size = sg->dma_length; - if (!size) - break; - npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); - dma_ofs = (dma_addr - iommu_area->dma_base) >> PAGE_SHIFT; - - spin_lock_irqsave(&iommu_area->lock, flags); - for (i = 0; i < npages; ++i) { - ofs = (dma_addr >> PAGE_SHIFT) + i; - sw_iommu_unmap(pdev, ofs); - } - - sw_iommu_free_da(iommu_area->bitmap, dma_ofs, npages); - - spin_unlock_irqrestore(&iommu_area->lock, flags); - } -} - -/* Return whether the given PCI device DMA address mask can be - * supported properly. - */ - -static int sw_iommu_supported(struct device *dev, u64 mask) -{ - /* As last resort try ZONE_DMA. */ - if (MAX_DMA_ADDRESS - PAGE_OFFSET - 1 <= mask) - return 1; - - return 0; -} - -static int sw_iommu_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr == 0; -} - -static int iommu_get_option(char **str, unsigned long *pint) -{ - char *cur = *str; - - if (!cur || !(*cur)) - return 0; - *pint = kstrtol(cur, str, 16); - - return 1; -} - -static int __init iommu_enable_setup(char *s) -{ - unsigned long rc_bitmap = 0; - - iommu_get_option(&s, &rc_bitmap); - iommu_cmd = rc_bitmap; - - return 1; -} -__setup("iommu_enable=", iommu_enable_setup); - -const struct dma_map_ops sw_iommu_dma_ops = { - .alloc = sw_iommu_alloc_coherent, - .free = sw_iommu_free_coherent, - .map_page = sw_iommu_map_page, - .unmap_page = sw_iommu_unmap_page, - .map_sg = sw_iommu_map_sg, - .unmap_sg = sw_iommu_unmap_sg, - .mapping_error = sw_iommu_mapping_error, - .dma_supported = sw_iommu_supported, -}; - -const struct dma_map_ops *dma_ops = &sw_iommu_dma_ops; -EXPORT_SYMBOL(dma_ops); -- Gitee From 8d2867f3fc131f10303cb2472457cdabe032d508 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 08:48:24 +0800 Subject: [PATCH 068/150] sw64: mm: rename _PAGE_PHU and _PAGE_PSE Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Rename _PAGE_PHU to _PAGE_CONT and _PAGE_PSE to _PAGE_LEAF. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 22 +++++++++++----------- arch/sw_64/mm/hugetlbpage.c | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 4b8358e7314d..846401b107ba 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -104,8 +104,8 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ #define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ #define _PAGE_ASM 0x0010 -#define _PAGE_PHU 0x0020 /* used for 256M page size bit */ -#define _PAGE_PSE 0x0040 /* used for 8M page size bit */ +#define _PAGE_CONT 0x0020 /* used for 256M page size bit */ +#define _PAGE_LEAF 0x0040 /* used for 8M page size bit */ #define _PAGE_PROTNONE 0x0080 /* used for numa page balancing */ #define _PAGE_SPECIAL 0x0100 #define _PAGE_KRE 0x0400 /* xxx - see below on the "accessed" bit */ @@ -144,7 +144,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) #define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL) -#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PHU) +#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_LEAF | _PAGE_CONT) /* * All the normal masks have the "page accessed" bits on, as any time they are used, @@ -272,7 +272,7 @@ static inline int pte_present(pte_t pte) static inline int pte_huge(pte_t pte) { - return pte_val(pte) & _PAGE_PSE; + return pte_val(pte) & _PAGE_LEAF; } static inline void pte_clear(struct mm_struct *mm, @@ -307,12 +307,12 @@ static inline int pmd_bad(pmd_t pmd) static inline int pmd_present(pmd_t pmd) { /* - * Checking for _PAGE_PSE is needed too because + * Checking for _PAGE_LEAF is needed too because * split_huge_page will temporarily clear the valid bit (but - * the _PAGE_PSE flag will remain set at all times while the + * the _PAGE_LEAF flag will remain set at all times while the * _PAGE_VALID bit is clear). */ - return pmd_val(pmd) & (_PAGE_VALID | _PAGE_PROTNONE | _PAGE_PSE); + return pmd_val(pmd) & (_PAGE_VALID | _PAGE_PROTNONE | _PAGE_LEAF); } static inline void pmd_clear(pmd_t *pmdp) @@ -388,7 +388,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) static inline pmd_t pmd_mkhuge(pmd_t pmd) { - pmd_val(pmd) |= _PAGE_PSE; + pmd_val(pmd) |= _PAGE_LEAF; return pmd; } @@ -495,7 +495,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkhuge(pte_t pte) { - pte_val(pte) |= _PAGE_PSE; + pte_val(pte) |= _PAGE_LEAF; return pte; } @@ -547,7 +547,7 @@ static inline int pmd_trans_splitting(pmd_t pmd) static inline int pmd_trans_huge(pmd_t pmd) { - return pmd_val(pmd) & _PAGE_PSE; + return pmd_val(pmd) & _PAGE_LEAF; } static inline int has_transparent_hugepage(void) @@ -630,7 +630,7 @@ extern pgd_t swapper_pg_dir[1024]; * * Format of swap PTE: * bit 0: _PAGE_VALID (must be zero) - * bit 6: _PAGE_PSE (must be zero) + * bit 6: _PAGE_LEAF (must be zero) * bit 7: _PAGE_PROTNONE (must be zero) * bits 8-15: swap type * bits 16-63: swap offset diff --git a/arch/sw_64/mm/hugetlbpage.c b/arch/sw_64/mm/hugetlbpage.c index 240d791b5c33..f385afcb09a3 100644 --- a/arch/sw_64/mm/hugetlbpage.c +++ b/arch/sw_64/mm/hugetlbpage.c @@ -19,7 +19,7 @@ int pmd_huge(pmd_t pmd) { return !pmd_none(pmd) && - (pmd_val(pmd) & (_PAGE_VALID | _PAGE_PSE)) != _PAGE_VALID; + (pmd_val(pmd) & (_PAGE_VALID | _PAGE_LEAF)) != _PAGE_VALID; } int pud_huge(pud_t pud) @@ -49,7 +49,7 @@ pte_t *sw64_256m_hugepte_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a return NULL; page = virt_to_page(pte); - pmd_val(*pmd) = pmd_val(*pmd) | _PAGE_PSE | _PAGE_PHU; + pmd_val(*pmd) = pmd_val(*pmd) | _PAGE_LEAF | _PAGE_CONT; for (i = 1; i < 32; i++) pmd_val(*(pmd+i)) = pmd_val(*pmd); return pte; @@ -101,7 +101,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, pmd = pmd_offset(pud, addr); if (!pmd_present(*pmd)) return NULL; - if (pmd_val(*pmd) & _PAGE_PHU) + if (pmd_val(*pmd) & _PAGE_CONT) pte = pte_offset_map(pmd, addr); else pte = (pte_t *) pmd; @@ -114,7 +114,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, static inline int sw64_huge_pmd_bad(pmd_t pmd) { return !(((pmd_val(pmd) & ~_PFN_MASK) == _PAGE_TABLE) || - ((pmd_val(pmd) & _PAGE_PHU) == _PAGE_PHU)); + ((pmd_val(pmd) & _PAGE_CONT) == _PAGE_CONT)); } static inline int sw64_huge_pmd_none_or_clear_bad(pmd_t *pmd) @@ -132,7 +132,7 @@ static void sw64_huge_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr) { if ((((unsigned long)pmd & 0xffUL) == 0) && - ((pmd_val(*pmd) & _PAGE_PHU) == _PAGE_PHU)) { + ((pmd_val(*pmd) & _PAGE_CONT) == _PAGE_CONT)) { pgtable_t token = pmd_pgtable(*pmd); pmd_clear(pmd); -- Gitee From 53c1442c124c2b0fb1e1b8e3ad6be24966f27e4c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 7 Sep 2023 17:36:59 +0800 Subject: [PATCH 069/150] sw64: pci: rename PCI_DEVICE_ID_CHIP3 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- This ID is not chip3 only. Rename it to PCI_DEVICE_ID_SW64_ROOT_BRIDGE. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pci.h | 2 +- arch/sw_64/kernel/chip_setup.c | 2 +- arch/sw_64/kernel/pci.c | 2 +- drivers/pci/controller/pci-sunway.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index c203390f4032..d826cec690c8 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -152,7 +152,7 @@ extern void __init reserve_mem_for_pci(void); extern int chip_pcie_configure(struct pci_controller *hose); #define PCI_VENDOR_ID_JN 0x5656 -#define PCI_DEVICE_ID_CHIP3 0x3231 +#define PCI_DEVICE_ID_SW64_ROOT_BRIDGE 0x3231 #define PCI_DEVICE_ID_JN_PCIESW 0x1000 #define PCI_DEVICE_ID_JN_PCIEUSIP 0x1200 #define PCI_DEVICE_ID_JN_PCIE2PCI 0x1314 diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 03f073ec9bca..d62a38cb5023 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -155,7 +155,7 @@ static void pcie_restore(void) write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); /* Fix up DEVICE_ID_VENDOR_ID register */ - value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; write_rc_conf(node, index, RC_VENDOR_ID, value); /* Set PCI-E root class code */ diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 6c6a64cca9ad..38ee0c5e191b 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -88,7 +88,7 @@ static void fixup_root_complex(struct pci_dev *dev) dev->no_msi = 1; } -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_CHIP3, fixup_root_complex); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_JN, PCI_DEVICE_ID_SW64_ROOT_BRIDGE, fixup_root_complex); /* Just declaring that the power-of-ten prefixes are actually the * power-of-two ones doesn't make it true :) diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 92adc5badbb9..6d933c660dda 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -195,7 +195,7 @@ static void set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl | 0x1); /* fix up DEVICE_ID_VENDOR_ID register */ - value = (PCI_DEVICE_ID_CHIP3 << 16) | PCI_VENDOR_ID_JN; + value = (PCI_DEVICE_ID_SW64_ROOT_BRIDGE << 16) | PCI_VENDOR_ID_JN; write_rc_conf(node, index, RC_VENDOR_ID, value); /* set PCI-E root class code */ -- Gitee From f07007ce3ca1e53fbe024f25bca6068e9bc9c25d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 08:57:49 +0800 Subject: [PATCH 070/150] sw64: mm: add _PAGE_CONT helpers Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add helpers that utilize _PAGE_CONT. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 846401b107ba..8daa0e709196 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -72,6 +72,11 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE - 1)) +#define CONT_PMD_SHIFT 6 +#define CONT_PMDS (1 << CONT_PMD_SHIFT) +#define CONT_PMD_SIZE (CONT_PMDS * PMD_SIZE) +#define CONT_PMD_MASK (~(CONT_PMD_SIZE - 1)) + /* * Entries per page directory level: the sw64 is three-level, with * all levels having a one-page page table. @@ -392,6 +397,12 @@ static inline pmd_t pmd_mkhuge(pmd_t pmd) return pmd; } +static inline pmd_t pmd_mkcont(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_CONT; + return pmd; +} + static inline int pud_none(pud_t pud) { return !pud_val(pud); @@ -456,6 +467,11 @@ static inline int pte_special(pte_t pte) return pte_val(pte) & _PAGE_SPECIAL; } +static inline int pte_cont(pte_t pte) +{ + return pte_val(pte) & _PAGE_CONT; +} + static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_FOW; @@ -545,6 +561,11 @@ static inline int pmd_trans_splitting(pmd_t pmd) return pmd_val(pmd) & _PAGE_SPLITTING; } +static inline int pmd_trans_cont(pmd_t pmd) +{ + return pmd_val(pmd) & _PAGE_CONT; +} + static inline int pmd_trans_huge(pmd_t pmd) { return pmd_val(pmd) & _PAGE_LEAF; -- Gitee From 6f3f88e77d23cec08a81ad2de8322d1083e1d7a2 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 08:59:56 +0800 Subject: [PATCH 071/150] sw64: mm: fix _PFN_MASK Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- _PFN_MASK does not need to cover all high bits. Set its range properly so we can utilize some high bits in the future. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 34 +++++++++++++++++--------------- arch/sw_64/mm/fault.c | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 8daa0e709196..ed8582802f47 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -15,6 +15,7 @@ #include #include +#include #include #include /* For TASK_SIZE */ #include @@ -143,9 +144,9 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define __DIRTY_BITS (_PAGE_DIRTY | _PAGE_KWE | _PAGE_UWE) #define __ACCESS_BITS (_PAGE_ACCESSED | _PAGE_KRE | _PAGE_URE) - #define _PFN_SHIFT 28 -#define _PFN_MASK ((-1UL) << _PFN_SHIFT) +#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) #define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) #define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL) @@ -233,36 +234,37 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) return pmd; } -static inline unsigned long pmd_page_vaddr(pmd_t pmd) -{ - return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PFN_SHIFT); -} - /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. */ #define page_to_pa(page) (page_to_pfn(page) << PAGE_SHIFT) -#define pud_pfn(pud) (pud_val(pud) >> _PFN_SHIFT) -#define pmd_pfn(pmd) (pmd_val(pmd) >> _PFN_SHIFT) -#define pte_pfn(pte) (pte_val(pte) >> _PFN_SHIFT) +#define p4d_pfn(p4d) ((p4d_val(p4d) & _PFN_MASK) >> _PFN_SHIFT) +#define pud_pfn(pud) ((pud_val(pud) & _PFN_MASK) >> _PFN_SHIFT) +#define pmd_pfn(pmd) ((pmd_val(pmd) & _PFN_MASK) >> _PFN_SHIFT) +#define pte_pfn(pte) ((pte_val(pte) & _PFN_MASK) >> _PFN_SHIFT) +#define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) +#define pud_page(pud) pfn_to_page(pud_pfn(pud)) +#define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) -#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) -#define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> _PFN_SHIFT)) -#define pud_page(pud) (pfn_to_page(pud_val(pud) >> _PFN_SHIFT)) -#define p4d_page(p4d) (pfn_to_page(p4d_val(p4d) >> _PFN_SHIFT)) +#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) static inline pud_t *p4d_pgtable(p4d_t p4d) { - return (pud_t *)pfn_to_virt(p4d_val(p4d) >> _PFN_SHIFT); + return (pud_t *)pfn_to_virt(p4d_pfn(p4d)); } static inline pmd_t *pud_pgtable(pud_t pud) { - return (pmd_t *)pfn_to_virt(pud_val(pud) >> _PFN_SHIFT); + return (pmd_t *)pfn_to_virt(pud_pfn(pud)); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pfn_to_virt(pmd_pfn(pmd)); } static inline int pte_none(pte_t pte) diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index 0f4ec9dac8d0..26e16bd7f9ee 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -121,7 +121,7 @@ unsigned long show_va_to_pa(struct mm_struct *mm, unsigned long addr) } pte = pte_offset_map(pmd, addr); if (pte_present(*pte)) { - ret = (unsigned long)pfn_to_virt(pte_val(*pte) >> _PFN_SHIFT); + ret = (unsigned long)pfn_to_virt(pte_pfn(*pte)); pr_debug("addr = %#lx, pgd = %#lx, pud = %#lx, pmd = %#lx, pte = %#lx, ret = %#lx\n", addr, *(unsigned long *)pgd, *(unsigned long *)pud, *(unsigned long *)pmd, *(unsigned long *)pte, ret); -- Gitee From 90827f895bf67fa3426805b487b2549869ed4d73 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 09:08:09 +0800 Subject: [PATCH 072/150] sw64: mm: add more helpers to pgtable.h Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add the following helpers to pgtable.h: pfn_pud() pmd_pte() pte_pmd() Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index ed8582802f47..3e31e7c50540 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -221,6 +221,13 @@ static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) pmd_val(pmd) = (pfn << _PFN_SHIFT) | pgprot_val(prot); return pmd; } +static inline pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot) +{ + pud_t pud; + + pud_val(pud) = (pfn << _PFN_SHIFT) | pgprot_val(pgprot); + return pud; +} static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { @@ -445,6 +452,16 @@ static inline void p4d_clear(p4d_t *p4dp) p4d_val(*p4dp) = 0; } +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +static inline pmd_t pte_pmd(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + /* * The following only work if pte_present() is true. * Undefined behaviour if not.. @@ -634,7 +651,6 @@ extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, extern int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); - #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH extern void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp); -- Gitee From a855a608015913a67940fafa050e95a95fa0571a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 5 Sep 2023 17:44:06 +0800 Subject: [PATCH 073/150] sw64: mm: unify _PAGE_CHG_MASK and _HPAGE_CHG_MASK Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Include _PAGE_LEAF and _PAGE_CONT in _PAGE_CHG_MASK. There is no need to have a separate _HPAGE_CHG_MASK for huge pages only. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 3e31e7c50540..ed11389b7b8b 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -149,8 +149,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) #define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) -#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL) -#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_LEAF | _PAGE_CONT) +#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) /* * All the normal masks have the "page accessed" bits on, as any time they are used, @@ -237,7 +236,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { - pmd_val(pmd) = (pmd_val(pmd) & _HPAGE_CHG_MASK) | pgprot_val(newprot); + pmd_val(pmd) = (pmd_val(pmd) & _PAGE_CHG_MASK) | pgprot_val(newprot); return pmd; } -- Gitee From f1ad75535800fb6cb21250773d98c083e0519754 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 11:08:01 +0800 Subject: [PATCH 074/150] sw64: mm: use static ZERO_PAGE Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Use statically defined ZERO_PAGE instead of dynamically allocated one. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 5 ++--- arch/sw_64/mm/init.c | 8 +------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index ed11389b7b8b..7c2e6069605f 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -201,9 +201,8 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ - -extern struct page *empty_zero_page; -#define ZERO_PAGE(vaddr) (empty_zero_page) +extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; +#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index a002e83bae34..c49973622c59 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -24,7 +24,7 @@ struct numa_node_desc_t numa_nodes_desc[1]; * empty_zero_page is a special page that is used for * zero-initialized data and COW. */ -struct page *empty_zero_page; +unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; EXPORT_SYMBOL(empty_zero_page); pg_data_t *node_data[MAX_NUMNODES] __read_mostly; EXPORT_SYMBOL(node_data); @@ -120,12 +120,6 @@ void __init zone_sizes_init(void) */ void __init paging_init(void) { - void *zero_page; - - zero_page = __va(memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE)); - pr_info("zero page start: %p\n", zero_page); - memset(zero_page, 0, PAGE_SIZE); - empty_zero_page = virt_to_page(zero_page); } void __init mem_detect(void) -- Gitee From 560f3930c8b63b97d3ee3f000ff5eba49eadbe57 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 16:03:56 +0800 Subject: [PATCH 075/150] sw64: rename some fields of struct vcpucb Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Rename the following fields to have a better show of their usage: whami -> soft_cid dtb_pcr -> dtb_vpcr tid -> soft_tid Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 6 +++--- arch/sw_64/kernel/hibernate.c | 4 ++-- arch/sw_64/kvm/kvm-sw64.c | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index c43ebe72e3a1..2039b1cf9f22 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -20,19 +20,19 @@ struct vcpucb { unsigned long new_a0; unsigned long new_a1; unsigned long new_a2; - unsigned long whami; + unsigned long soft_cid; unsigned long csr_save; unsigned long wakeup_magic; unsigned long host_vcpucb; unsigned long upcr; unsigned long vpcr; - unsigned long dtb_pcr; + unsigned long dtb_vpcr; unsigned long guest_ksp; unsigned long guest_usp; unsigned long vcpu_irq_disabled; unsigned long vcpu_irq; unsigned long ptbr; - unsigned long tid; + unsigned long soft_tid; unsigned long int_stat1; unsigned long int_stat2; unsigned long int_stat3; diff --git a/arch/sw_64/kernel/hibernate.c b/arch/sw_64/kernel/hibernate.c index 0e7e860c507e..644ea8504313 100644 --- a/arch/sw_64/kernel/hibernate.c +++ b/arch/sw_64/kernel/hibernate.c @@ -14,7 +14,7 @@ void save_processor_state(void) vcb->ksp = rdksp(); vcb->usp = rdusp(); - vcb->tid = rtid(); + vcb->soft_tid = rtid(); vcb->ptbr = rdptbr(); } @@ -24,7 +24,7 @@ void restore_processor_state(void) wrksp(vcb->ksp); wrusp(vcb->usp); - wrtp(vcb->tid); + wrtp(vcb->soft_tid); wrptbr(vcb->ptbr); sflush(); tbiv(); diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/kvm-sw64.c index c9182f6deadd..e6dc283cc9a3 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/kvm-sw64.c @@ -119,7 +119,7 @@ static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) /* Always update vpn */ /* Just setup vcb, hardware CSR will be changed later in HMcode */ vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(HARDWARE_VPN_MASK << 44))) | (vpn << 44); - vcpu->arch.vcb.dtb_pcr = ((vcpu->arch.vcb.dtb_pcr) & (~(HARDWARE_VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); + vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(HARDWARE_VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); /* * If vcpu migrate to a new physical cpu, the new physical cpu may keep @@ -514,7 +514,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.tsk = current; /* For guest kernel "sys_call HMC_whami", indicate virtual cpu id */ - vcpu->arch.vcb.whami = vcpu->vcpu_id; + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ @@ -526,7 +526,7 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) unsigned long addr = vcpu->kvm->arch.host_phys_addr; hrtimer_cancel(&vcpu->arch.hrt); - vcpu->arch.vcb.whami = vcpu->vcpu_id; + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ vcpu->arch.power_off = 0; @@ -816,7 +816,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); /* synchronize the longtime of source and destination */ - if (vcpu->arch.vcb.whami == 0) { + if (vcpu->arch.vcb.soft_cid == 0) { result = sw64_io_read(0, LONG_TIME); vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; longtime_offset = vcpu->arch.vcb.guest_longtime_offset; -- Gitee From 95f30af1f4780cd15b2932096aeabb2c0fcf6e3b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 17:19:07 +0800 Subject: [PATCH 076/150] sw64: topology: modify topology defines Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Use more standard BITS, SHIFT, MASK defines. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/core.h | 18 ++++++++++++++---- arch/sw_64/include/asm/topology.h | 18 ++++++++++++------ arch/sw_64/kernel/smp.c | 2 +- arch/sw_64/kernel/topology.c | 8 ++++---- arch/sw_64/mm/numa.c | 2 +- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h index e5e4cc138102..45b1772e8021 100644 --- a/arch/sw_64/include/asm/core.h +++ b/arch/sw_64/include/asm/core.h @@ -8,11 +8,21 @@ #define II_WAKE 3 #define II_NMII 6 -#ifdef CONFIG_SW64_CHIP3 #define II_RESET II_NMII -#define CORES_PER_NODE_SHIFT 5 -#endif -#define CORES_PER_NODE (1UL << CORES_PER_NODE_SHIFT) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 5 +#define DOMAIN_ID_MASK (GENMASK(DOMAIN_ID_BITS - 1, 0) << DOMAIN_ID_SHIFT) + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 31 +#define THREAD_ID_MASK (GENMASK(THREAD_ID_BITS - 1, 0) << THREAD_ID_SHIFT) + +#define CORE_ID_BITS 5 +#define CORE_ID_SHIFT 0 +#define CORE_ID_MASK (GENMASK(CORE_ID_BITS - 1, 0) << CORE_ID_SHIFT) + +#define MAX_CORES_PER_CPU (1 << CORE_ID_BITS) /* * 0x00 ~ 0xff for hardware mm fault diff --git a/arch/sw_64/include/asm/topology.h b/arch/sw_64/include/asm/topology.h index 74bd81524fe9..3ec906df276c 100644 --- a/arch/sw_64/include/asm/topology.h +++ b/arch/sw_64/include/asm/topology.h @@ -8,10 +8,6 @@ #include #include -#define THREAD_ID_SHIFT 5 /* thread_id is removed from rcid */ -#define THREAD_ID_MASK 0 /* set mask to 0 */ -#define CORE_ID_MASK ((1 << THREAD_ID_SHIFT) - 1) - extern struct cpu_topology cpu_topology[NR_CPUS]; #define topology_physical_package_id(cpu) (cpu_topology[cpu].package_id) @@ -25,9 +21,19 @@ void store_cpu_topology(int cpuid); void remove_cpu_topology(int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); -static inline int rcid_to_package(int rcid) +static inline int rcid_to_thread_id(int rcid) +{ + return (rcid & THREAD_ID_MASK) >> THREAD_ID_SHIFT; +} + +static inline int rcid_to_core_id(int rcid) +{ + return (rcid & CORE_ID_MASK) >> CORE_ID_SHIFT; +} + +static inline int rcid_to_domain_id(int rcid) { - return rcid >> CORES_PER_NODE_SHIFT; + return (rcid & DOMAIN_ID_MASK) >> DOMAIN_ID_SHIFT; } #ifdef CONFIG_NUMA diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 08c0c9c76328..5948074ca774 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -95,7 +95,7 @@ void smp_callin(void) notify_cpu_starting(cpuid); per_cpu(cpu_state, cpuid) = CPU_ONLINE; - per_cpu(hard_node_id, cpuid) = cpu_to_rcid(cpuid) >> CORES_PER_NODE_SHIFT; + per_cpu(hard_node_id, cpuid) = rcid_to_domain_id(cpu_to_rcid(cpuid)); /* Must have completely accurate bogos. */ local_irq_enable(); diff --git a/arch/sw_64/kernel/topology.c b/arch/sw_64/kernel/topology.c index 26200cf68cc8..2db9f5aec048 100644 --- a/arch/sw_64/kernel/topology.c +++ b/arch/sw_64/kernel/topology.c @@ -133,10 +133,10 @@ void store_cpu_topology(int cpu) goto topology_populated; } - cpu_topo->package_id = rcid_to_package(cpu_to_rcid(cpu)); - cpu_topo->core_id = cpu_to_rcid(cpu) & CORE_ID_MASK; - cpu_topo->thread_id = (cpu_to_rcid(cpu) >> THREAD_ID_SHIFT) & THREAD_ID_MASK; - cpu_topo->llc_id = rcid_to_package(cpu_to_rcid(cpu)); + cpu_topo->package_id = rcid_to_domain_id(cpu_to_rcid(cpu)); + cpu_topo->core_id = rcid_to_core_id(cpu_to_rcid(cpu)); + cpu_topo->thread_id = rcid_to_thread_id(cpu_to_rcid(cpu)); + cpu_topo->llc_id = rcid_to_domain_id(cpu_to_rcid(cpu)); pr_debug("CPU%u: socket %d core %d thread %d llc %d\n", cpu, cpu_topo->package_id, cpu_topo->core_id, diff --git a/arch/sw_64/mm/numa.c b/arch/sw_64/mm/numa.c index 7cb13587e465..86b52bb140af 100644 --- a/arch/sw_64/mm/numa.c +++ b/arch/sw_64/mm/numa.c @@ -378,7 +378,7 @@ void cpu_set_node(void) rr = first_node(node_online_map); for (i = 0; i < nr_cpu_ids; i++) { cid = cpu_to_rcid(i); - default_node = cid >> CORES_PER_NODE_SHIFT; + default_node = rcid_to_domain_id(cid); if (node_online(default_node)) { cpu_to_node_map[i] = default_node; } else { -- Gitee From 240995a32c8a46d9f3ec5a56ca7666c20f1e72ef Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 16:56:43 +0800 Subject: [PATCH 077/150] sw64: redesign __cpu_to_rcid initialization Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Setup __cpu_to_rcid in setup_arch(). Remove __rcid_to_cpu. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/smp.h | 27 ++++------------------ arch/sw_64/include/asm/sw64_init.h | 2 +- arch/sw_64/kernel/chip_setup.c | 37 +++++++++++++++++------------- arch/sw_64/kernel/head.S | 18 +++++++++++---- arch/sw_64/kernel/setup.c | 4 +++- arch/sw_64/kernel/smp.c | 22 +++--------------- 6 files changed, 46 insertions(+), 64 deletions(-) diff --git a/arch/sw_64/include/asm/smp.h b/arch/sw_64/include/asm/smp.h index fed9d682f1d7..3a2fcf62b30c 100644 --- a/arch/sw_64/include/asm/smp.h +++ b/arch/sw_64/include/asm/smp.h @@ -16,18 +16,6 @@ extern cpumask_t core_start; -static inline unsigned char -__hard_smp_processor_id(void) -{ - register unsigned char __r0 __asm__("$0"); - __asm__ __volatile__( - "sys_call %1 #whami" - : "=r"(__r0) - : "i" (HMC_whami) - : "$1", "$22", "$23", "$24", "$25"); - return __r0; -} - static inline unsigned long read_vpcr(void) { @@ -56,7 +44,6 @@ struct smp_rcb_struct { #define INIT_SMP_RCB ((struct smp_rcb_struct *) __va(0x820000UL)) -#define hard_smp_processor_id() __hard_smp_processor_id() #ifdef GENERATING_ASM_OFFSETS #define raw_smp_processor_id() (0) @@ -64,17 +51,11 @@ struct smp_rcb_struct { #include #define raw_smp_processor_id() (*((unsigned int *)((void *)current + TASK_CPU))) #endif +#define hard_smp_processor_id() cpu_to_rcid(raw_smp_processor_id()) /* The map from sequential logical cpu number to hard cid. */ extern int __cpu_to_rcid[NR_CPUS]; #define cpu_to_rcid(cpu) __cpu_to_rcid[cpu] - -/* - * Map from hard cid to sequential logical cpu number. This will only - * not be idempotent when cpus failed to come on-line. - */ -extern int __rcid_to_cpu[NR_CPUS]; -#define rcid_to_cpu(cpu) __rcid_to_cpu[cpu] #define cpu_physical_id(cpu) __cpu_to_rcid[cpu] extern unsigned long tidle_pcb[NR_CPUS]; @@ -89,8 +70,10 @@ void __cpu_die(unsigned int cpu); #else /* CONFIG_SMP */ #define hard_smp_processor_id() 0 #define smp_call_function_on_cpu(func, info, wait, cpu) ({ 0; }) -#define cpu_to_rcid(cpu) ((int)whami()) -#define rcid_to_cpu(rcid) 0 +/* The map from sequential logical cpu number to hard cid. */ +extern int __cpu_to_rcid[NR_CPUS]; +#define cpu_to_rcid(cpu) __cpu_to_rcid[0] +#define cpu_physical_id(cpu) __cpu_to_rcid[0] #endif /* CONFIG_SMP */ #define NO_PROC_ID (-1) diff --git a/arch/sw_64/include/asm/sw64_init.h b/arch/sw_64/include/asm/sw64_init.h index b7ca9cb89478..86ddd2cb65f8 100644 --- a/arch/sw_64/include/asm/sw64_init.h +++ b/arch/sw_64/include/asm/sw64_init.h @@ -8,7 +8,7 @@ #include struct sw64_early_init_ops { - void (*setup_core_start)(struct cpumask *cpumask); + void (*setup_core_map)(struct cpumask *cpumask); unsigned long (*get_node_mem)(int nodeid); void (*get_smp_info)(void); }; diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index d62a38cb5023..282f2f88b601 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -41,27 +41,32 @@ static unsigned long __init get_node_mem(int nodeid) return node_mem; } -static void __init setup_core_start(struct cpumask *cpumask) +static void __init setup_core_map(struct cpumask *cpumask) { - int i, j, cpus; + int i, j, cpu_num, cpuid, max_cores_per_cpu; unsigned long coreonline; - if (is_guest_or_emul()) { - coreonline = sw64_io_read(0, CORE_ONLINE); - for (i = 0; i < 64 ; i++) { - if (coreonline & (1UL << i)) - cpumask_set_cpu(i, cpumask); - } - } else { - cpus = get_cpu_nums(); - for (i = 0; i < cpus; i++) { - coreonline = sw64_io_read(i, CORE_ONLINE); - for (j = 0; j < 32 ; j++) { - if (coreonline & (1UL << j)) - cpumask_set_cpu(i * 32 + j, cpumask); + cpu_num = get_cpu_nums(); + cpuid = 0; + for (i = 0; i < cpu_num; i++) { + coreonline = sw64_io_read(i, CORE_ONLINE); + max_cores_per_cpu = MAX_CORES_PER_CPU; + + if (is_guest_or_emul()) + max_cores_per_cpu = 64; + + for (j = 0; j < max_cores_per_cpu; j++) { + if (coreonline & (1UL << j)) { + __cpu_to_rcid[cpuid] = (i << DOMAIN_ID_SHIFT) | (j << CORE_ID_SHIFT); + cpuid++; } } } + + while (cpuid < NR_CPUS) { + __cpu_to_rcid[cpuid] = -1; + cpuid++; + } } #ifdef CONFIG_PM @@ -218,7 +223,7 @@ static void io_resume(void) static struct sw64_chip_init_ops chip_init_ops = { .early_init = { - .setup_core_start = setup_core_start, + .setup_core_map = setup_core_map, .get_node_mem = get_node_mem, }, }; diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index c855d31de715..a791f875c6fe 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -67,11 +67,18 @@ __smp_callin: sys_call HMC_tbi sys_call HMC_whami # Get hard cid - - ldi $1, __rcid_to_cpu - s4addl $0, $1, $1 - ldw $0, 0($1) # Get logical cpu number - + ldi $1, __cpu_to_rcid + ldi $2, 0($31) + ldi $4, CONFIG_NR_CPUS +3: ldw $3, 0($1) + cmpeq $3, $0, $3 + bne $3, 4f + addl $1, 4, $1 + addl $2, 1, $2 + cmpeq $2, $4, $5 + bne $5, 5f + br $31, 3b +4: ldi $0, 0($2) ldi $2, idle_task_pointer s8addl $0, $2, $2 ldl $8, 0($2) # Get ksp of idle thread @@ -81,6 +88,7 @@ __smp_callin: ldi $30, ASM_THREAD_SIZE($30) call $26, smp_callin +5: sys_call HMC_halt .end __smp_callin #endif /* CONFIG_SMP */ diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 018feb25130b..39039ae377b4 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -44,6 +44,8 @@ #define DBGDCONT(args...) #endif +int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ +EXPORT_SYMBOL(__cpu_to_rcid); DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; @@ -764,7 +766,7 @@ setup_arch(char **cmdline_p) setup_chip_ops(); setup_socket_info(); show_socket_mem_layout(); - sw64_chip_init->early_init.setup_core_start(&core_start); + sw64_chip_init->early_init.setup_core_map(&core_start); if (is_guest_or_emul()) get_vt_smp_info(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 5948074ca774..4430e510dd2f 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -29,12 +29,6 @@ int smp_booted; #define DBGS(fmt, arg...) \ do { if (smp_debug) printk("SMP: " fmt, ## arg); } while (0) -int __cpu_to_rcid[NR_CPUS]; /* Map logical to physical */ -EXPORT_SYMBOL(__cpu_to_rcid); - -int __rcid_to_cpu[NR_CPUS]; /* Map physical to logical */ -EXPORT_SYMBOL(__rcid_to_cpu); - void *idle_task_pointer[NR_CPUS]; /* State of each CPU */ @@ -192,29 +186,19 @@ void __init smp_rcb_init(void) */ void __init setup_smp(void) { - int i = 0, num = 0; /* i: physical id, num: logical id */ + int i = 0, num = 0; init_cpu_possible(cpu_none_mask); /* For unified kernel, NR_CPUS is the maximum possible value */ for (; i < NR_CPUS; i++) { - if (cpumask_test_cpu(i, &core_start)) { - __cpu_to_rcid[num] = i; - __rcid_to_cpu[i] = num; + if (cpu_to_rcid(i) != -1) { set_cpu_possible(num, true); store_cpu_data(num); if (!cpumask_test_cpu(i, &cpu_offline)) set_cpu_present(num, true); num++; - } else - __rcid_to_cpu[i] = -1; - } - /* for sw64, the BSP must be logical core 0 */ - BUG_ON(cpu_to_rcid(0) != hard_smp_processor_id()); - - while (num < NR_CPUS) { - __cpu_to_rcid[num] = -1; - num++; + } } process_nr_cpu_ids(); -- Gitee From df6d4442e6a449529f88bb9da531c3813eca80b9 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 17:13:38 +0800 Subject: [PATCH 078/150] sw64: add C4 atomic operations support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support atomic operations on C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/atomic.h | 217 +++++++++++++++++++++++++++++++- arch/sw_64/include/asm/futex.h | 125 +++++++++++++----- 2 files changed, 306 insertions(+), 36 deletions(-) diff --git a/arch/sw_64/include/asm/atomic.h b/arch/sw_64/include/asm/atomic.h index cad03ba13076..1c662b077c31 100644 --- a/arch/sw_64/include/asm/atomic.h +++ b/arch/sw_64/include/asm/atomic.h @@ -35,6 +35,7 @@ #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) +#ifdef CONFIG_SUBARCH_C3B /** * atomic_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic_t @@ -68,7 +69,6 @@ static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) : "Ir" (u), "Ir" (a), "m" (v->counter)); return old; } -#define atomic_fetch_add_unless atomic_fetch_add_unless /** * atomic64_fetch_add_unless - add unless the number is a given value * @v: pointer of type atomic64_t @@ -102,7 +102,6 @@ static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) : "Ir" (u), "Ir" (a), "m" (v->counter)); return old; } -#define atomic64_fetch_add_unless atomic64_fetch_add_unless /* * atomic64_dec_if_positive - decrement by 1 if old value positive * @v: pointer of type atomic_t @@ -135,7 +134,6 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) } -#define atomic64_dec_if_positive atomic64_dec_if_positive #define ATOMIC_OP(op, asm_op) \ static inline void atomic_##op(int i, atomic_t *v) \ @@ -272,6 +270,219 @@ static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ return temp1; \ } \ +#else /* !CONFIG_SUBARCH_C3B */ + +/** + * atomic_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addw %0, %6, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/** + * atomic64_fetch_add_unless - add unless the number is a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as it was not @u. + * Returns the old value of @v. + */ +static inline long atomic64_fetch_add_unless(atomic64_t *v, long a, long u) +{ + long old, new, c; + unsigned long addr; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %4\n" + " bne %4, 2f\n" + " addl %0, %6, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (old), "=&r" (new), "=m" (v->counter), "=&r" (addr), "=&r" (c) + : "Ir" (u), "Ir" (a), "m" (v->counter)); + return old; +} + +/* + * atomic64_dec_if_positive - decrement by 1 if old value positive + * @v: pointer of type atomic_t + * + * The function returns the old value of *v minus 1, even if + * the atomic variable, v, was not decremented. + */ +static inline long atomic64_dec_if_positive(atomic64_t *v) +{ + unsigned long old, temp1, addr, temp2; + + __asm__ __volatile__( + " ldi %3, %2\n" + "1: lldl %4, 0(%3)\n" + " cmple %4, 0, %0\n" + " bne %0, 2f\n" + " subl %4, 1, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr), "=&r" (old) + : "m" (v->counter)); + return old - 1; +} + +#define ATOMIC_OP(op, asm_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldw %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstw %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC_OP_RETURN(op, asm_op) \ +static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#define ATOMIC_FETCH_OP(op, asm_op) \ +static inline int atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ +{ \ + int temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldw %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstw %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + + +#define ATOMIC64_OP(op, asm_op) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + unsigned long temp1, addr; \ + __asm__ __volatile__( \ + " ldi %2, %1\n" \ + "1: lldl %0, 0(%2)\n" \ + " " #asm_op " %0, %3, %0\n" \ + " lstl %0, 0(%2)\n" \ + " beq %0, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ +} \ + + +#define ATOMIC64_OP_RETURN(op, asm_op) \ +static inline long atomic64_##op##_return_relaxed(long i, atomic64_t *v)\ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " " #asm_op " %0, %4, %0\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} + +#define ATOMIC64_FETCH_OP(op, asm_op) \ +static inline long atomic64_fetch_##op##_relaxed(long i, atomic64_t *v) \ +{ \ + long temp1, temp2; \ + unsigned long addr; \ + __asm__ __volatile__( \ + " ldi %3, %2\n" \ + "1: lldl %0, 0(%3)\n" \ + " " #asm_op " %0, %4, %1\n" \ + " lstl %1, 0(%3)\n" \ + " beq %1, 2f\n" \ + ".subsection 2\n" \ + "2: lbr 1b\n" \ + ".previous" \ + : "=&r" (temp1), "=&r" (temp2), "=m" (v->counter), "=&r" (addr) \ + : "Ir" (i), "m" (v->counter)); \ + return temp1; \ +} \ + +#endif /* CONFIG_SUBARCH_C3B */ + +#define atomic_fetch_add_unless atomic_fetch_add_unless +#define atomic64_fetch_add_unless atomic64_fetch_add_unless +#define atomic64_dec_if_positive atomic64_dec_if_positive + #define ATOMIC_OPS(op) \ ATOMIC_OP(op, op##w) \ ATOMIC_OP_RETURN(op, op##w) \ diff --git a/arch/sw_64/include/asm/futex.h b/arch/sw_64/include/asm/futex.h index 2a814d57991a..783799813980 100644 --- a/arch/sw_64/include/asm/futex.h +++ b/arch/sw_64/include/asm/futex.h @@ -9,6 +9,8 @@ #include #include +#ifdef CONFIG_SUBARCH_C3B + #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ __asm__ __volatile__( \ "1: lldw %0, 0(%3)\n" \ @@ -32,41 +34,63 @@ : "r" (uaddr), "r"(oparg) \ : "memory") -static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, - u32 __user *uaddr) +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) { - int oldval = 0, ret; - unsigned long tmp; - - pagefault_disable(); - - switch (op) { - case FUTEX_OP_SET: - __futex_atomic_op("mov %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_ADD: - __futex_atomic_op("addw %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_OR: - __futex_atomic_op("or %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_ANDN: - __futex_atomic_op("andnot %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - case FUTEX_OP_XOR: - __futex_atomic_op("xor %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); - break; - default: - ret = -ENOSYS; - } + int ret = 0, cmp; + u32 prev, tmp; - pagefault_enable(); + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; - if (!ret) - *oval = oldval; + __asm__ __volatile__ ( + "1: lldw %1, 0(%4)\n" + " cmpeq %1, %5, %2\n" + " wr_f %2\n" + " bis $31, %6, %3\n" + "2: lstw %3, 0(%4)\n" + " rd_f %3\n" + " beq %2, 3f\n" + " beq %3, 4f\n" + "3: .subsection 2\n" + "4: br 1b\n" + " .previous\n" + " .section __ex_table, \"a\"\n" + " .long 1b-.\n" + " ldi $31, 3b-1b(%0)\n" + " .long 2b-.\n" + " ldi $31, 3b-2b(%0)\n" + " .previous\n" + : "+r"(ret), "=&r"(prev), "=&r"(cmp), "=&r"(tmp) + : "r"(uaddr), "r"((long)(int)oldval), "r"(newval) + : "memory"); + *uval = prev; return ret; } +#else /* !CONFIG_SUBARCH_C3B */ + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg, tmp) \ + __asm__ __volatile__( \ + "1: lldw %0, 0(%3)\n" \ + insn \ + "2: lstw %1, 0(%3)\n" \ + " beq %1, 4f\n" \ + " bis $31, $31, %1\n" \ + "3: .subsection 2\n" \ + "4: lbr 1b\n" \ + " .previous\n" \ + " .section __ex_table, \"a\"\n" \ + " .long 1b-.\n" \ + " ldi $31, 3b-1b(%1)\n" \ + " .long 2b-.\n" \ + " ldi $31, 3b-2b(%1)\n" \ + " .previous\n" \ + : "=&r" (oldval), "=&r"(ret), "=&r"(tmp) \ + : "r" (uaddr), "r"(oparg) \ + : "memory") + static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, @@ -81,14 +105,12 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, __asm__ __volatile__ ( "1: lldw %1, 0(%4)\n" " cmpeq %1, %5, %2\n" - " wr_f %2\n" + " beq %2, 3f\n" " bis $31, %6, %3\n" "2: lstw %3, 0(%4)\n" - " rd_f %3\n" - " beq %2, 3f\n" " beq %3, 4f\n" "3: .subsection 2\n" - "4: br 1b\n" + "4: lbr 1b\n" " .previous\n" " .section __ex_table, \"a\"\n" " .long 1b-.\n" @@ -103,6 +125,43 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, *uval = prev; return ret; } +#endif /* CONFIG_SUBARCH_C3B */ + +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) +{ + int oldval = 0, ret; + unsigned long tmp; + + pagefault_disable(); + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("addw %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("andnot %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %0, %4, %1\n", ret, oldval, uaddr, oparg, tmp); + break; + default: + ret = -ENOSYS; + } + + pagefault_enable(); + + if (!ret) + *oval = oldval; + + return ret; +} #endif /* __KERNEL__ */ -- Gitee From f07fd650c2dae688ce3d89499703f5fd79613a07 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 17:13:54 +0800 Subject: [PATCH 079/150] sw64: add C4 bit operations support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support bitops on C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/bitops.h | 301 +++++++++++++++++++++++--------- 1 file changed, 221 insertions(+), 80 deletions(-) diff --git a/arch/sw_64/include/asm/bitops.h b/arch/sw_64/include/asm/bitops.h index 8d2865ab7b74..46b242f4b4ba 100644 --- a/arch/sw_64/include/asm/bitops.h +++ b/arch/sw_64/include/asm/bitops.h @@ -9,10 +9,7 @@ #include #include -/* - * Copyright 1994, Linus Torvalds. - */ - +#ifdef CONFIG_SUBARCH_C3B /* * These have to be done with inline assembly: that way the bit-setting * is guaranteed to be atomic. All bit operations return 0 if the bit @@ -47,19 +44,6 @@ set_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -/* - * WARNING: non atomic version. - */ -static inline void -__set_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m |= 1 << (nr & 31); -} - -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() static inline void clear_bit(unsigned long nr, volatile void *addr) @@ -83,31 +67,6 @@ clear_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -static inline void -clear_bit_unlock(unsigned long nr, volatile void *addr) -{ - smp_mb(); - clear_bit(nr, addr); -} - -/* - * WARNING: non atomic version. - */ -static inline void -__clear_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m &= ~(1 << (nr & 31)); -} - -static inline void -__clear_bit_unlock(unsigned long nr, volatile void *addr) -{ - smp_mb(); - __clear_bit(nr, addr); -} - static inline void change_bit(unsigned long nr, volatile void *addr) { @@ -130,18 +89,6 @@ change_bit(unsigned long nr, volatile void *addr) : "Ir" (1UL << (nr & 31)), "m" (*m)); } -/* - * WARNING: non atomic version. - */ -static inline void -__change_bit(unsigned long nr, volatile void *addr) -{ - int *m = ((int *) addr) + (nr >> 5); - - *m ^= 1 << (nr & 31); -} - - static inline int test_and_set_bit(unsigned long nr, volatile void *addr) { @@ -198,20 +145,6 @@ test_and_set_bit_lock(unsigned long nr, volatile void *addr) return oldbit != 0; } -/* - * WARNING: non atomic version. - */ -static inline int -__test_and_set_bit(unsigned long nr, volatile void *addr) -{ - unsigned long mask = 1 << (nr & 0x1f); - int *m = ((int *) addr) + (nr >> 5); - int old = *m; - - *m = old | mask; - return (old & mask) != 0; -} - static inline int test_and_clear_bit(unsigned long nr, volatile void *addr) { @@ -240,18 +173,163 @@ test_and_clear_bit(unsigned long nr, volatile void *addr) return oldbit != 0; } -/* - * WARNING: non atomic version. - */ static inline int -__test_and_clear_bit(unsigned long nr, volatile void *addr) +test_and_change_bit(unsigned long nr, volatile void *addr) { - unsigned long mask = 1 << (nr & 0x1f); + unsigned long oldbit; + unsigned long temp, base; int *m = ((int *) addr) + (nr >> 5); - int old = *m; - *m = old & ~mask; - return (old & mask) != 0; + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " ldi %2, 1\n" + " wr_f %2\n" + " and %0, %4, %2\n" + " xor %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " rd_f %0\n" + " beq %0, 3f\n" + ".subsection 2\n" + "3: br 1b\n" + ".previous" + : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +#else /* !CONFIG_SUBARCH_C3B */ +static inline void +set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bis %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " bic %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline void +change_bit(unsigned long nr, volatile void *addr) +{ + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %2, %4\n" + "1: lldw %0, 0(%2)\n" + " xor %0, %3, %0\n" + " lstw %0, 0(%2)\n" + " beq %0, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m)); +} + +static inline int +test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_set_bit_lock(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " bne %2, 2f\n" // %2 is not zero, no need to set, return + " bis %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; +} + +static inline int +test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long oldbit; + unsigned long temp1, base; + int *m = ((int *) addr) + (nr >> 5); + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " and %0, %4, %2\n" + " beq %2, 2f\n" // %2 is zero, no need to set, return + " bic %0, %4, %0\n" + " lstw %0, 0(%3)\n" + " beq %0, 3f\n" // failed to set, try again. + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (temp1), "=m" (*m), "=&r" (oldbit), "=&r" (base) + : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); + + return oldbit != 0; } static inline int @@ -264,15 +342,12 @@ test_and_change_bit(unsigned long nr, volatile void *addr) __asm__ __volatile__( " ldi %3, %5\n" "1: lldw %0, 0(%3)\n" - " ldi %2, 1\n" - " wr_f %2\n" " and %0, %4, %2\n" " xor %0, %4, %0\n" " lstw %0, 0(%3)\n" - " rd_f %0\n" " beq %0, 3f\n" ".subsection 2\n" - "3: br 1b\n" + "3: lbr 1b\n" ".previous" : "=&r" (temp), "=m" (*m), "=&r" (oldbit), "=&r" (base) : "Ir" (1UL << (nr & 31)), "m" (*m) : "memory"); @@ -280,9 +355,75 @@ test_and_change_bit(unsigned long nr, volatile void *addr) return oldbit != 0; } + +#endif /* CONFIG_SUBARCH_C3B */ + /* * WARNING: non atomic version. */ +static inline void +__set_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m |= 1 << (nr & 31); +} + +#define smp_mb__before_clear_bit() smp_mb() +#define smp_mb__after_clear_bit() smp_mb() + +static inline void +clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + clear_bit(nr, addr); +} + +static inline void +__clear_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m &= ~(1 << (nr & 31)); +} + +static inline void +__clear_bit_unlock(unsigned long nr, volatile void *addr) +{ + smp_mb(); + __clear_bit(nr, addr); +} + +static inline void +__change_bit(unsigned long nr, volatile void *addr) +{ + int *m = ((int *) addr) + (nr >> 5); + + *m ^= 1 << (nr & 31); +} + +static inline int +__test_and_set_bit(unsigned long nr, volatile void *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old | mask; + return (old & mask) != 0; +} + +static inline int +__test_and_clear_bit(unsigned long nr, volatile void *addr) +{ + unsigned long mask = 1 << (nr & 0x1f); + int *m = ((int *) addr) + (nr >> 5); + int old = *m; + + *m = old & ~mask; + return (old & mask) != 0; +} + static inline int __test_and_change_bit(unsigned long nr, volatile void *addr) { -- Gitee From dc488ebc6ed534a8a1a777acb5b3924e3c0c5f3f Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 11:27:19 +0800 Subject: [PATCH 080/150] sw64: add C4 xchg operations support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support xchg operations on C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/xchg.h | 249 ++++++++++++++++++++++++++++++---- 1 file changed, 226 insertions(+), 23 deletions(-) diff --git a/arch/sw_64/include/asm/xchg.h b/arch/sw_64/include/asm/xchg.h index 154467ef30e4..16d3f3e4401c 100644 --- a/arch/sw_64/include/asm/xchg.h +++ b/arch/sw_64/include/asm/xchg.h @@ -11,6 +11,7 @@ * So this file is included twice from asm/cmpxchg.h. */ +#if defined(CONFIG_SUBARCH_C3B) /* * Atomic exchange. * Since it can be used to implement critical sections @@ -116,29 +117,6 @@ ____xchg(_u64, volatile long *m, unsigned long val) return val; } -/* - * This function doesn't exist, so you'll get a linker error - * if something tries to do an invalid xchg(). - */ -extern void __xchg_called_with_bad_pointer(void); - -static __always_inline unsigned long -____xchg(, volatile void *ptr, unsigned long x, int size) -{ - switch (size) { - case 1: - return ____xchg(_u8, ptr, x); - case 2: - return ____xchg(_u16, ptr, x); - case 4: - return ____xchg(_u32, ptr, x); - case 8: - return ____xchg(_u64, ptr, x); - } - __xchg_called_with_bad_pointer(); - return x; -} - /* * Atomic compare and exchange. Compare OLD with MEM, if identical, * store NEW in MEM. Return the initial value in MEM. Success is @@ -256,6 +234,231 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) return prev; } +#elif defined(CONFIG_SUBARCH_C4) +/* + * Atomic exchange. + * Since it can be used to implement critical sections + * it must clobber "memory" (also for interrupts in UP). + */ + +static inline unsigned long +____xchg(_u8, volatile char *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslb %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlb %2, %4, %0\n" + " masklb %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u16, volatile short *m, unsigned long val) +{ + unsigned long ret, tmp, addr64; + + __asm__ __volatile__( + " andnot %4, 7, %3\n" + " inslh %1, %4, %1\n" + "1: lldl %2, 0(%3)\n" + " extlh %2, %4, %0\n" + " masklh %2, %4, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%3)\n" + " beq %2, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64) + : "r" ((long)m), "1" (val) : "memory"); + + return ret; +} + +static inline unsigned long +____xchg(_u32, volatile int *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldw %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstw %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +static inline unsigned long +____xchg(_u64, volatile long *m, unsigned long val) +{ + unsigned long dummy, addr; + + __asm__ __volatile__( + " ldi %3, %5\n" + "1: lldl %0, 0(%3)\n" + " bis $31, %4, %1\n" + " lstl %1, 0(%3)\n" + " beq %1, 2f\n" + ".subsection 2\n" + "2: lbr 1b\n" + ".previous" + : "=&r" (val), "=&r" (dummy), "=m" (*m), "=&r"(addr) + : "rI" (val), "m" (*m) : "memory"); + + return val; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + * + * The memory barrier should be placed in SMP only when we actually + * make the change. If we don't change anything (so if the returned + * prev is equal to old) then we aren't acquiring anything new and + * we don't need any memory barrier as far I can tell. + */ +static inline unsigned long +____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslb %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlb %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklb %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new) +{ + unsigned long prev, tmp, cmp, addr64; + + __asm__ __volatile__( + " andnot %5, 7, %4\n" + " inslh %1, %5, %1\n" + "1: lldl %2, 0(%4)\n" + " extlh %2, %5, %0\n" + " cmpeq %0, %6, %3\n" + " beq %3, 2f\n" + " masklh %2, %5, %2\n" + " or %1, %2, %2\n" + " lstl %2, 0(%4)\n" + " beq %2, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64) + : "r" ((long)m), "Ir" (old), "1" (new) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u32, volatile int *m, int old, int new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldw %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstw %4, 0(%3)\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +static inline unsigned long +____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new) +{ + unsigned long prev, cmp, addr, tmp; + + __asm__ __volatile__( + " ldi %3, %7\n" + "1: lldl %0, 0(%3)\n" + " cmpeq %0, %5, %1\n" + " beq %1, 2f\n" + " bis $31, %6, %4\n" + " lstl %4, 0(%3)\n" + " beq %4, 3f\n" + "2:\n" + ".subsection 2\n" + "3: lbr 1b\n" + ".previous" + : "=&r"(prev), "=&r"(cmp), "=m"(*m), "=&r"(addr), "=&r"(tmp) + : "r"((long) old), "r"(new), "m"(*m) : "memory"); + + return prev; +} + +#endif + +/* This function doesn't exist, so you'll get a linker error + * if something tries to do an invalid xchg(). + */ +extern void __xchg_called_with_bad_pointer(void); + +static __always_inline unsigned long +____xchg(, volatile void *ptr, unsigned long x, int size) +{ + switch (size) { + case 1: + return ____xchg(_u8, ptr, x); + case 2: + return ____xchg(_u16, ptr, x); + case 4: + return ____xchg(_u32, ptr, x); + case 8: + return ____xchg(_u64, ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + /* * This function doesn't exist, so you'll get a linker error * if something tries to do an invalid cmpxchg(). -- Gitee From c65d17ce508f0697221b074535290f465c41ab28 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 17:14:07 +0800 Subject: [PATCH 081/150] sw64: add C4 barriers support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support new barriers on C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/barrier.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/sw_64/include/asm/barrier.h b/arch/sw_64/include/asm/barrier.h index 5f4a03d700c6..bff199126c9f 100644 --- a/arch/sw_64/include/asm/barrier.h +++ b/arch/sw_64/include/asm/barrier.h @@ -8,7 +8,13 @@ #define rmb() __asm__ __volatile__("memb" : : : "memory") +#if defined(CONFIG_SUBARCH_C3B) #define wmb() __asm__ __volatile__("memb" : : : "memory") +#elif defined(CONFIG_SUBARCH_C4) +#define wmb() __asm__ __volatile__("wmemb" : : : "memory") +#endif + +#define imemb() __asm__ __volatile__("imemb" : : : "memory") #ifdef CONFIG_SMP #define __ASM_SMP_MB "\tmemb\n" -- Gitee From 0a489bf3f07cfa7c1bf283349c172b10beebde62 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 10:54:35 +0800 Subject: [PATCH 082/150] sw64: add C4 CSRs Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add headers to allow reading or writing certain CSRs directly on C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 3 ++ arch/sw_64/include/asm/csr.h | 97 ++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 arch/sw_64/include/asm/csr.h diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 7fa38a675af1..86f3f85a5d74 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -185,6 +185,9 @@ config AUDIT_ARCH config SYS_HAS_EARLY_PRINTK bool +config HAVE_CSRRW + bool + menu "System setup" menu "Machine Configuration" diff --git a/arch/sw_64/include/asm/csr.h b/arch/sw_64/include/asm/csr.h new file mode 100644 index 000000000000..0610384208a4 --- /dev/null +++ b/arch/sw_64/include/asm/csr.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_CSR_H +#define _ASM_SW64_CSR_H + +#include + +#define CSR_EXC_SUM 0xd +#define CSR_INT_EN 0x1a +#define CSR_INT_STAT 0x1b +#define CSR_PCIE_MSI0_INT 0x1d +#define CSR_PCIE_MSI1_INT 0x1e +#define CSR_PCIE_MSI2_INT 0x1f +#define CSR_PCIE_MSI3_INT 0x20 +#define CSR_INT_VEC 0x2d +#define CSR_PCIE_MSI0_INTEN 0x35 +#define CSR_PCIE_MSI1_INTEN 0x36 +#define CSR_PCIE_MSI2_INTEN 0x37 +#define CSR_PCIE_MSI3_INTEN 0x38 +#define CSR_EXC_GPA 0x3b +#define CSR_EXC_PC 0xe +#define CSR_AS_INFO 0x3c +#define CSR_DS_STAT 0x48 +#define CSR_SOFTCID 0xc9 +#define CSR_DVA 0x54 +#define CSR_PTBR_SYS 0x68 +#define CSR_PTBR_USR 0x69 +#define CSR_APTP 0x6a +#define CSR_CID 0xc4 +#define CSR_WR_FREGS 0xc8 +#define CSR_SHTCLOCK 0xca +#define CSR_SHTCLOCK_OFFSET 0xcb + +#ifdef CONFIG_SUBARCH_C4 +#define CSR_IA_VPNMATCH 0xa +#define CSR_UPCR 0x15 +#define CSR_VPCR 0x16 +#define CSR_IA_MATCH 0x17 +#define CSR_IA_MASK 0x18 +#define CSR_IV_MATCH 0x19 +#define CSR_IA_UPNMATCH 0x3a +#define CSR_DC_CTLP 0x4e +#define CSR_DA_MATCH 0x51 +#define CSR_DA_MASK 0x52 +#define CSR_DA_MATCH_MODE 0x53 +#define CSR_DV_MATCH 0x56 +#define CSR_DV_MASK 0x57 +#define CSR_IDA_MATCH 0xc5 +#define CSR_IDA_MASK 0xc6 + +#define DA_MATCH_EN_S 4 +#define DV_MATCH_EN_S 6 +#define DAV_MATCH_EN_S 7 +#define DPM_MATCH 8 +#define DPM_MATCH_EN_S 10 +#define IDA_MATCH_EN_S 53 +#define IV_PM_EN_S 61 +#define IV_MATCH_EN_S 62 +#define IA_MATCH_EN_S 63 + +#endif + + +#ifdef CONFIG_HAVE_CSRRW +#define read_csr(x) \ + ({ unsigned long __val; \ + __asm__ __volatile__("csrr %0,%1" : "=r"(__val) : "i"(x)); \ + __val; }) + +#define write_csr(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1" ::"r"(x), "i"(y)); }) + +#define write_csr_imb(x, y) \ + ({ __asm__ __volatile__("csrw %0,%1; imemb" ::"r"(x), "i"(y)); }) + + +#ifndef __ASSEMBLY__ +#include +static inline void update_ptbr_sys(unsigned long ptbr) +{ + imemb(); + write_csr_imb(ptbr, CSR_PTBR_SYS); +} +#endif +#else +#define read_csr(x) (0) +#define write_csr(x, y) do { } while (0) +#define write_csr_imb(x, y) do { } while (0) + +#ifndef __ASSEMBLY__ +static inline void update_ptbr_sys(unsigned long ptbr) +{ + wrptbr(ptbr); +} +#endif + +#endif +#endif /* _ASM_SW64_CSR_H */ -- Gitee From 8a7e5515c8de5f487c0ba2fa67618997e89b9a1d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 10:43:16 +0800 Subject: [PATCH 083/150] sw64: topology: add C4 topology info Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add C4 topology info and setup code. Enable SCHED_SMT for C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 8 ++++++++ arch/sw_64/include/asm/core.h | 33 ++++++++++++++++++++++++++++++--- arch/sw_64/include/asm/hmcall.h | 2 ++ arch/sw_64/kernel/chip_setup.c | 7 +++++++ arch/sw_64/kernel/head.S | 5 +++++ 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 86f3f85a5d74..7da617075c8b 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -359,6 +359,14 @@ config TRACE_IRQFLAGS_SUPPORT config ARCH_SUPPORTS_UPROBES def_bool y +config SCHED_SMT + bool "SMT scheduler support" + depends on SMP && SUBARCH_C4 + help + Improves the CPU scheduler's decision making when dealing with + MultiThreading at a cost of slightly increased overhead in some + places. If unsure say N here. + config NR_CPUS int "Maximum number of CPUs (2-256)" range 2 256 diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h index 45b1772e8021..bf2e95cd8a3a 100644 --- a/arch/sw_64/include/asm/core.h +++ b/arch/sw_64/include/asm/core.h @@ -2,6 +2,8 @@ #ifndef _ASM_SW64_CORE_H #define _ASM_SW64_CORE_H +#include + #define II_II0 0 #define II_II1 1 #define II_SLEEP 2 @@ -10,18 +12,43 @@ #define II_RESET II_NMII +#if defined(CONFIG_SUBARCH_C3B) + #define DOMAIN_ID_BITS 2 #define DOMAIN_ID_SHIFT 5 -#define DOMAIN_ID_MASK (GENMASK(DOMAIN_ID_BITS - 1, 0) << DOMAIN_ID_SHIFT) #define THREAD_ID_BITS 1 #define THREAD_ID_SHIFT 31 -#define THREAD_ID_MASK (GENMASK(THREAD_ID_BITS - 1, 0) << THREAD_ID_SHIFT) #define CORE_ID_BITS 5 #define CORE_ID_SHIFT 0 -#define CORE_ID_MASK (GENMASK(CORE_ID_BITS - 1, 0) << CORE_ID_SHIFT) +static inline bool core_is_ht(void) +{ + return 0; +} + +#elif defined(CONFIG_SUBARCH_C4) + +#define DOMAIN_ID_BITS 2 +#define DOMAIN_ID_SHIFT 12 + +#define THREAD_ID_BITS 1 +#define THREAD_ID_SHIFT 8 + +#define CORE_ID_BITS 6 +#define CORE_ID_SHIFT 0 + +static inline bool core_is_ht(void) +{ + return rdhtctl() == 0x3; +} + +#endif + +#define DOMAIN_ID_MASK (GENMASK(DOMAIN_ID_BITS - 1, 0) << DOMAIN_ID_SHIFT) +#define THREAD_ID_MASK (GENMASK(THREAD_ID_BITS - 1, 0) << THREAD_ID_SHIFT) +#define CORE_ID_MASK (GENMASK(CORE_ID_BITS - 1, 0) << CORE_ID_SHIFT) #define MAX_CORES_PER_CPU (1 << CORE_ID_BITS) /* diff --git a/arch/sw_64/include/asm/hmcall.h b/arch/sw_64/include/asm/hmcall.h index 8bd5f5357bc0..65e644bd6a23 100644 --- a/arch/sw_64/include/asm/hmcall.h +++ b/arch/sw_64/include/asm/hmcall.h @@ -17,6 +17,7 @@ #define HMC_wrktp 0x0A #define HMC_rdptbr 0x0B #define HMC_wrptbr 0x0C +#define HMC_rdhtctl 0x0D #define HMC_wrksp 0x0E #define HMC_mtinten 0x0F #define HMC_load_mm 0x11 @@ -164,6 +165,7 @@ __CALL_HMC_W1(wrusp, unsigned long); __CALL_HMC_R0(rdksp, unsigned long); __CALL_HMC_W1(wrksp, unsigned long); +__CALL_HMC_R0(rdhtctl, unsigned long); /* * Load a mm context. This is needed when we change the page diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 282f2f88b601..18da63db812c 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -63,6 +63,13 @@ static void __init setup_core_map(struct cpumask *cpumask) } } + if (is_in_host() && core_is_ht()) { + for (i = 0; i < cpuid; i++) + __cpu_to_rcid[cpuid + i] = __cpu_to_rcid[i] | (1 << THREAD_ID_SHIFT); + + cpuid = cpuid + i; + } + while (cpuid < NR_CPUS) { __cpu_to_rcid[cpuid] = -1; cpuid++; diff --git a/arch/sw_64/kernel/head.S b/arch/sw_64/kernel/head.S index a791f875c6fe..fd0fbfbcf5b6 100644 --- a/arch/sw_64/kernel/head.S +++ b/arch/sw_64/kernel/head.S @@ -66,6 +66,7 @@ __smp_callin: bis $31, $31, $16 # invalidate all TLB with current VPN sys_call HMC_tbi +#if defined(CONFIG_SUBARCH_C3B) sys_call HMC_whami # Get hard cid ldi $1, __cpu_to_rcid ldi $2, 0($31) @@ -79,6 +80,10 @@ __smp_callin: bne $5, 5f br $31, 3b 4: ldi $0, 0($2) +#else + rcid $0 +#endif + ldi $2, idle_task_pointer s8addl $0, $2, $2 ldl $8, 0($2) # Get ksp of idle thread -- Gitee From e87ef8f2c4cb4049f0c9635883d99fa4ad0579af Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 15:02:50 +0800 Subject: [PATCH 084/150] sw64: mm: add C4 MMU support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support C4 MMU. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/mmu_context.h | 2 +- arch/sw_64/include/asm/pgtable.h | 118 ++++++++++++++++++++++----- arch/sw_64/kernel/smp.c | 2 +- arch/sw_64/mm/init.c | 14 +++- 4 files changed, 113 insertions(+), 23 deletions(-) diff --git a/arch/sw_64/include/asm/mmu_context.h b/arch/sw_64/include/asm/mmu_context.h index 5ae9d4616937..c69de5e7a2aa 100644 --- a/arch/sw_64/include/asm/mmu_context.h +++ b/arch/sw_64/include/asm/mmu_context.h @@ -11,7 +11,7 @@ * The maximum ASID's the processor supports. */ -#ifdef CONFIG_SUBARCH_C3B +#if defined(CONFIG_SUBARCH_C3B) || defined(CONFIG_SUBARCH_C4) #define ASID_BITS 10 #endif diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index 7c2e6069605f..dea7e16c60b6 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -23,21 +23,6 @@ struct mm_struct; struct vm_area_struct; -/* Certain architectures need to do special things when PTEs - * within a page table are directly modified. Thus, the following - * hook is made available. - */ -static inline void set_pte(pte_t *ptep, pte_t pteval) -{ - *ptep = pteval; -} - -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) -{ - set_pte(ptep, pteval); -} - static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; @@ -105,7 +90,10 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) /* * HMcode-imposed page table bits */ +#if defined(CONFIG_SUBARCH_C3B) + #define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID #define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ #define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ #define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ @@ -145,11 +133,6 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define __ACCESS_BITS (_PAGE_ACCESSED | _PAGE_KRE | _PAGE_URE) #define _PFN_SHIFT 28 -#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) -#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) - -#define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) -#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) /* * All the normal masks have the "page accessed" bits on, as any time they are used, @@ -162,6 +145,59 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_ASM | _PAGE_KRE | _PAGE_KWE) #define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | (x)) +#define page_valid_kern(x) (0) + +#elif defined(CONFIG_SUBARCH_C4) + +#define _PAGE_VALID 0x0001 +#define _PAGE_PRESENT _PAGE_VALID +#define _PAGE_FOR 0x0002 /* used for page protection (fault on read) */ +#define _PAGE_FOW 0x0004 /* used for page protection (fault on write) */ +#define _PAGE_FOE 0x0008 /* used for page protection (fault on exec) */ +#define _PAGE_FIXED 0x0010 +#define _PAGE_CONT 0x0020 /* used for 512M page size bit*/ +#define _PAGE_LEAF 0x0040 /* used for huge page bit */ +#define _PAGE_PCD 0x0080 /* used for page cache disabled */ + +/* and these are sw definition */ +#define _PAGE_WCD 0x0100 +#define _PAGE_ACCESSED 0x0200 +#define _PAGE_SPLITTING 0x0400 /* For Transparent Huge Page */ +#define _PAGE_SPECIAL 0x0800 +#define _PAGE_DEVMAP 0x1000 /* For ZONE DEVICE page */ +#define _PAGE_KERN 0x2000 +#define _PAGE_DIRTY _BITUL(62) +#define _PAGE_PROTNONE _BITUL(63) +#define _PAGE_BIT_FOW 2 /* bit of _PAGE_FOW */ +#define _PAGE_BIT_ACCESSED 9 /* bit of _PAGE_ACCESSED */ +#define _PAGE_BIT_SPLITTING 10 /* bit of _PAGE_SPLITTING */ +#define _PAGE_BIT_DEVMAP 12 /* bit of _PAGE_DEVMAP */ + +#define __DIRTY_BITS _PAGE_DIRTY +#define __ACCESS_BITS _PAGE_ACCESSED + +#define _PFN_SHIFT 24 + +/* + * All the normal masks have the "page accessed" bits on, as any time they are used, + * the page is accessed. They are cleared only by the page-out routines + */ +#define PAGE_NONE __pgprot(__ACCESS_BITS | _PAGE_FOR | _PAGE_FOW | _PAGE_FOE | _PAGE_LEAF | _PAGE_PROTNONE) +#define PAGE_SHARED __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF) +#define PAGE_COPY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_READONLY __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_FOW | _PAGE_LEAF) +#define PAGE_KERNEL __pgprot(_PAGE_VALID | _PAGE_KERN | _PAGE_LEAF) +#define _PAGE_NORMAL(x) __pgprot(_PAGE_VALID | __ACCESS_BITS | _PAGE_LEAF | (x)) + +#define page_valid_kern(x) ((x & (_PAGE_VALID | _PAGE_KERN)) == (_PAGE_VALID | _PAGE_KERN)) +#endif + +#define _PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#define _PFN_MASK (GENMASK(_PFN_BITS - 1, 0) << _PFN_SHIFT) + +#define _PAGE_TABLE (_PAGE_VALID | __DIRTY_BITS | __ACCESS_BITS) +#define _PAGE_CHG_MASK (_PFN_MASK | __DIRTY_BITS | __ACCESS_BITS | _PAGE_SPECIAL | _PAGE_LEAF | _PAGE_CONT) + #define _PAGE_P(x) _PAGE_NORMAL((x) | _PAGE_FOW) #define _PAGE_S(x) _PAGE_NORMAL(x) @@ -204,6 +240,23 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; + + if (page_valid_kern(pte_val(pteval))) { + mb(); + if ((pte_val(pteval) & _PAGE_FOE) == 0) + imemb(); + } +} + +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + set_pte(ptep, pteval); +} + static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { pte_t pte; @@ -430,6 +483,12 @@ static inline void pud_clear(pud_t *pudp) pud_val(*pudp) = 0; } +static inline pud_t pud_mkhuge(pud_t pud) +{ + pud_val(pud) |= _PAGE_LEAF; + return pud; +} + static inline int p4d_none(p4d_t p4d) { return !p4d_val(p4d); @@ -662,6 +721,8 @@ extern pgd_t swapper_pg_dir[1024]; #define update_mmu_cache(vma, address, ptep) do { } while (0) #define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) +#if defined(CONFIG_SUBARCH_C3B) + /* * Encode and decode a swap entry: * @@ -674,6 +735,23 @@ extern pgd_t swapper_pg_dir[1024]; */ #define __SWP_TYPE_SHIFT 8 #define __SWP_TYPE_BITS 8 + +#elif defined(CONFIG_SUBARCH_C4) + +/* + * Encode and decode a swap entry: + * + * Format of swap PTE: + * bit 0: _PAGE_VALID (must be zero) + * bits 6-10: swap type + * bits 11-58: swap offset + * bit 63: _PAGE_PROTNONE (must be zero) + */ +#define __SWP_TYPE_SHIFT 6 +#define __SWP_TYPE_BITS 5 + +#endif + #define __SWP_OFFSET_BITS 48 #define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 4430e510dd2f..cd5e4edec76c 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -83,7 +83,7 @@ void smp_callin(void) mmgrab(&init_mm); current->active_mm = &init_mm; /* update csr:ptbr */ - wrptbr(virt_to_phys(init_mm.pgd)); + update_ptbr_sys(virt_to_phys(init_mm.pgd)); /* inform the notifiers about the new cpu */ notify_cpu_starting(cpuid); diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index c49973622c59..73098db59f8d 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -65,6 +65,7 @@ static int __init setup_mem_size(char *p) } early_param("mem", setup_mem_size); +#if defined(CONFIG_SUBARCH_C3B) pgd_t * pgd_alloc(struct mm_struct *mm) { @@ -77,6 +78,17 @@ pgd_alloc(struct mm_struct *mm) return ret; } +#elif defined(CONFIG_SUBARCH_C4) +pgd_t * +pgd_alloc(struct mm_struct *mm) +{ + pgd_t *ret; + + ret = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + + return ret; +} +#endif /* Set up initial PCB, VPTB, and other such nicities. */ @@ -84,7 +96,7 @@ static inline void switch_to_system_map(void) { memset(swapper_pg_dir, 0, PAGE_SIZE); - wrptbr(virt_to_phys(swapper_pg_dir)); + update_ptbr_sys(virt_to_phys(swapper_pg_dir)); tbiv(); } -- Gitee From fc3c83da1e34696fc94f1b238ab374688c3f6fbe Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 16:53:40 +0800 Subject: [PATCH 085/150] sw64: mm: add C4 hugetlb support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add C4 hugetlb support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/hugetlb.h | 35 +++ arch/sw_64/mm/Makefile | 4 + arch/sw_64/mm/hugetlbpage_c4.c | 457 +++++++++++++++++++++++++++++++ 3 files changed, 496 insertions(+) create mode 100644 arch/sw_64/mm/hugetlbpage_c4.c diff --git a/arch/sw_64/include/asm/hugetlb.h b/arch/sw_64/include/asm/hugetlb.h index 246d12c2103d..0b6f16f913b2 100644 --- a/arch/sw_64/include/asm/hugetlb.h +++ b/arch/sw_64/include/asm/hugetlb.h @@ -3,6 +3,41 @@ #define _ASM_SW64_HUGETLB_H #include + +#ifdef CONFIG_SUBARCH_C4 +#define __HAVE_ARCH_HUGE_PTE_CLEAR +extern void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz); + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +extern void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +extern pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +extern void huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, + pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +extern void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep); + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +extern int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, pte_t pte, int dirty); + +#define arch_make_huge_pte arch_make_huge_pte +extern pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writable); + +#define set_huge_swap_pte_at set_huge_swap_pte_at +extern void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); +#endif + #include #endif /* _ASM_SW64_HUGETLB_H */ diff --git a/arch/sw_64/mm/Makefile b/arch/sw_64/mm/Makefile index 92be882cc82b..6f2ccec1d70c 100644 --- a/arch/sw_64/mm/Makefile +++ b/arch/sw_64/mm/Makefile @@ -8,5 +8,9 @@ obj-y := init.o fault.o physaddr.o mmap.o obj-$(CONFIG_NUMA) += numa.o +ifeq ($(CONFIG_SUBARCH_C4),y) +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage_c4.o +else obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +endif obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += thp.o diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c new file mode 100644 index 000000000000..0c763efc1d16 --- /dev/null +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -0,0 +1,457 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SW_64 Huge TLB Page Support for Kernel. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* + * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal + * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. + * Otherwise, returns 0. + */ +int pmd_huge(pmd_t pmd) +{ + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} + +int pud_huge(pud_t pud) +{ + return !pud_none(pud) && + (pud_val(pud) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; +} + +#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE +#define want_pmd_share() (1) +#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ +#define want_pmd_share() (0) +#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ + +/* + * Select all bits except the pfn + */ +static inline pgprot_t pte_pgprot(pte_t pte) +{ + unsigned long pfn = pte_pfn(pte); + + return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte)); +} + +static inline int num_contig_ptes(unsigned long size, size_t *pgsize) +{ + int contig_ptes = 0; + + *pgsize = size; + + switch (size) { + case PUD_SIZE: + case PMD_SIZE: + contig_ptes = 1; + break; + case CONT_PMD_SIZE: + *pgsize = PMD_SIZE; + contig_ptes = CONT_PMDS; + break; + default: + break; + } + + return contig_ptes; +} + +static pte_t get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, + unsigned long pgsize, unsigned long ncontig) +{ + pte_t orig_pte = huge_ptep_get(ptep); + unsigned long i; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) { + pte_t pte = ptep_get_and_clear(mm, addr, ptep); + + if (pte_dirty(pte)) + orig_pte = pte_mkdirty(orig_pte); + + if (pte_young(pte)) + orig_pte = pte_mkyoung(orig_pte); + } + + return orig_pte; +} + +static void clear_flush(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long pgsize, + unsigned long ncontig) +{ + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long i, saddr = addr; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); + + flush_tlb_range(&vma, saddr, addr); +} + +pte_t *huge_pte_alloc(struct mm_struct *mm, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pte_t *pte = NULL; + + pgd = pgd_offset(mm, addr); + p4d = p4d_alloc(mm, pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + + if (sz == PUD_SIZE) { + pte = (pte_t *)pud; + } else if (sz == PMD_SIZE) { + if (want_pmd_share() && pud_none(*pud)) + pte = huge_pmd_share(mm, addr, pud); + else + pte = (pte_t *)pmd_alloc(mm, pud, addr); + } else if (sz == (PMD_SIZE * CONT_PMDS)) { + pte = (pte_t *)pmd_alloc(mm, pud, addr); + WARN_ON(addr & (sz - 1)); + } + + WARN_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); + return pte; +} + +pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, + unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd = NULL; + + pgd = pgd_offset(mm, addr); + if (!pgd_present(*pgd)) + return NULL; + + p4d = p4d_offset(pgd, addr); + if (!p4d_present(*p4d)) + return NULL; + + pud = pud_offset(p4d, addr); + + if (sz != PUD_SIZE && pud_none(*pud)) + return NULL; + /* hugepage or swap? */ + if (pud_huge(*pud) || !pud_present(*pud)) + return (pte_t *)pud; + /* table; check the next level */ + + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + pmd = pmd_offset(pud, addr); + if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) && + pmd_none(*pmd)) + return NULL; + if (pmd_huge(*pmd) || !pmd_present(*pmd)) + return (pte_t *)pmd; + + return NULL; +} + +pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, + struct page *page, int writable) +{ + size_t pagesize = huge_page_size(hstate_vma(vma)); + + if (pagesize == CONT_PMD_SIZE) { + entry = pmd_pte(pmd_mkcont(pte_pmd(entry))); + } else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) { + pr_warn("%s: unrecognized huge page size 0x%lx\n", + __func__, pagesize); + } + return entry; +} + +void huge_pte_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + pte_clear(mm, addr, ptep); +} + +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + size_t pgsize; + int i; + int ncontig; + unsigned long pfn; + pgprot_t hugeprot; + + /* + * Code needs to be expanded to handle huge swap and migration + * entries. Needed for HUGETLB and MEMORY_FAILURE. + */ + WARN_ON(!pte_present(pte)); + + if (!pte_cont(pte)) { + set_pte_at(mm, addr, ptep, pte); + return; + } + + ncontig = CONT_PMDS; + pfn = pte_pfn(pte); + hugeprot = pte_pgprot(pte); + + get_and_clear(mm, addr, ptep, pgsize, ncontig); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz) +{ + int i, ncontig; + size_t pgsize; + + ncontig = num_contig_ptes(sz, &pgsize); + + for (i = 0; i < ncontig; i++, ptep++) + set_pte(ptep, pte); +} + +void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + unsigned long pfn; + pgprot_t hugeprot; + int ncontig, i; + size_t pgsize; + pte_t pte; + + if (!pte_cont(READ_ONCE(*ptep))) { + ptep_set_wrprotect(mm, addr, ptep); + return; + } + + ncontig = CONT_PMDS; + + pte = get_and_clear(mm, addr, ptep, pgsize, ncontig); + pte = pte_wrprotect(pte); + + hugeprot = pte_pgprot(pte); + pfn = pte_pfn(pte); + + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot)); +} + +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + int ncontig; + size_t pgsize; + pte_t orig_pte = huge_ptep_get(ptep); + + if (!pte_cont(orig_pte)) + return ptep_get_and_clear(mm, addr, ptep); + + ncontig = CONT_PMDS; + + return get_and_clear(mm, addr, ptep, pgsize, ncontig); +} + +void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + size_t pgsize; + int ncontig; + + if (!pte_cont(READ_ONCE(*ptep))) { + ptep_clear_flush(vma, addr, ptep); + return; + } + + ncontig = CONT_PMDS; + clear_flush(vma->vm_mm, addr, ptep, pgsize, ncontig); +} + +static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig) +{ + int i; + + if (pte_write(pte) != pte_write(huge_ptep_get(ptep))) + return 1; + + for (i = 0; i < ncontig; i++) { + pte_t orig_pte = huge_ptep_get(ptep + i); + + if (pte_dirty(pte) != pte_dirty(orig_pte)) + return 1; + + if (pte_young(pte) != pte_young(orig_pte)) + return 1; + } + + return 0; +} + +int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int ncontig, i; + size_t pgsize = 0; + unsigned long pfn = pte_pfn(pte); + pgprot_t hugeprot; + pte_t orig_pte; + + if (!pte_cont(pte)) + return ptep_set_access_flags(vma, addr, ptep, pte, dirty); + + ncontig = CONT_PMDS; + + if (!__cont_access_flags_changed(ptep, pte, ncontig)) + return 0; + + orig_pte = get_and_clear(vma->vm_mm, addr, ptep, pgsize, ncontig); + flush_tlb_fix_spurious_fault(vma, addr); + + /* Make sure we don't lose the dirty or young state */ + if (pte_dirty(orig_pte)) + pte = pte_mkdirty(pte); + + if (pte_young(orig_pte)) + pte = pte_mkyoung(pte); + + hugeprot = pte_pgprot(pte); + for (i = 0; i < ncontig; i++, ptep++, addr += pgsize) + set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot)); + + return 1; +} + +#ifdef CONFIG_HUGETLB_PAGE +static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + + info.flags = 0; + info.length = len; + info.low_limit = current->mm->mmap_legacy_base; + info.high_limit = TASK_SIZE; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + return vm_unmapped_area(&info); +} + +static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, + unsigned long addr0, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct vm_unmapped_area_info info; + unsigned long addr; + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = current->mm->mmap_base; + info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = TASK_SIZE; + addr = vm_unmapped_area(&info); + } + + return addr; +} + + unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, unsigned long flags) +{ + struct hstate *h = hstate_file(file); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if (len > TASK_SIZE) + return -ENOMEM; + + if (flags & MAP_FIXED) { + if (prepare_hugepage_range(file, addr, len)) + return -EINVAL; + return addr; + } + + if (addr) { + addr = ALIGN(addr, huge_page_size(h)); + vma = find_vma(mm, addr); + if (TASK_SIZE - len >= addr && + (!vma || addr + len <= vma->vm_start)) + return addr; + } + if (mm->get_unmapped_area == arch_get_unmapped_area) + return hugetlb_get_unmapped_area_bottomup(file, addr, len, + pgoff, flags); + else + return hugetlb_get_unmapped_area_topdown(file, addr, len, + pgoff, flags); +} +#endif /* CONFIG_HUGETLB_PAGE */ + +static __init int setup_hugepagesz(char *opt) +{ + unsigned long ps = memparse(opt, &opt); + + switch (ps) { + case PUD_SIZE: + case PMD_SIZE * CONT_PMDS: + case PMD_SIZE: + hugetlb_add_hstate(ilog2(ps) - PAGE_SHIFT); + return 1; + } + + pr_err("hugepagesz: Unsupported page size %lu M\n", + ps >> 20); + return 0; +} +__setup("hugepagesz=", setup_hugepagesz); -- Gitee From 719ea9c4fceab921dbb22baa2f1187f12e9cbcd4 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 09:05:42 +0800 Subject: [PATCH 086/150] sw64: add SW8A io addresses Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support IO of SW8A cpu. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 18 +- arch/sw_64/include/asm/msi.h | 4 +- arch/sw_64/include/asm/platform.h | 6 +- arch/sw_64/include/asm/sw64io.h | 8 +- arch/sw_64/include/asm/uncore_io_junzhang.h | 199 ++++++++++++++++++ .../asm/{chip3_io.h => uncore_io_xuelang.h} | 17 +- arch/sw_64/kernel/dup_print.c | 6 - drivers/cpufreq/Kconfig | 2 +- drivers/irqchip/Kconfig | 2 +- drivers/mfd/Kconfig | 2 +- drivers/spi/Kconfig | 2 +- 11 files changed, 236 insertions(+), 30 deletions(-) create mode 100644 arch/sw_64/include/asm/uncore_io_junzhang.h rename arch/sw_64/include/asm/{chip3_io.h => uncore_io_xuelang.h} (97%) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 7da617075c8b..32784c4c9e30 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -201,11 +201,19 @@ config SUBARCH_C3B endchoice choice - prompt "Chipset Family" + prompt "Uncore Configuration" -config SW64_CHIP3 - bool "Chip3" +config UNCORE_XUELANG + bool "Uncore for C3B" depends on SUBARCH_C3B + help + Sunway cpu uncore for C3B + +config UNCORE_JUNZHANG + bool "Uncore for C4" + depends on SUBARCH_C4 + help + Sunway cpu uncore for C4 endchoice choice @@ -213,7 +221,7 @@ choice config PLATFORM_XUELANG bool "Xuelang" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG select SPARSE_IRQ select SYS_HAS_EARLY_PRINTK select SW64_INTC_V2 @@ -371,7 +379,7 @@ config NR_CPUS int "Maximum number of CPUs (2-256)" range 2 256 depends on SMP - default "64" if SW64_CHIP3 + default "64" if UNCORE_XUELANG help SW6 support can handle a maximum of 256 CPUs. diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index 554ab7289fb1..e708ee191dab 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -2,6 +2,8 @@ #ifndef _ASM_SW64_MSI_H #define _ASM_SW64_MSI_H +#include + #define NR_VECTORS NR_IRQS #define NR_IRQ_VECTORS NR_IRQS @@ -15,8 +17,6 @@ #define PERCPU_MSI_IRQS 256 -#define MSIX_MSG_ADDR (0x91abc0UL) - #define VT_MSIX_MSG_ADDR (0x8000fee00000UL) #define VT_MSIX_ADDR_DEST_ID_SHIFT 12 #define VT_MSIX_ADDR_DEST_ID_MASK (0xff << VT_MSIX_ADDR_DEST_ID_SHIFT) diff --git a/arch/sw_64/include/asm/platform.h b/arch/sw_64/include/asm/platform.h index f3d1fc6774f8..ad54cdc772e1 100644 --- a/arch/sw_64/include/asm/platform.h +++ b/arch/sw_64/include/asm/platform.h @@ -3,7 +3,11 @@ #define _ASM_SW64_PLATFORM_H #include -#include +#if defined(CONFIG_UNCORE_XUELANG) +#include +#elif defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif #ifdef CONFIG_EFI #define BIOS_VERSION_GUID EFI_GUID(0xc47a23c3, 0xcebb, 0x4cc9, 0xa5, 0xe2, 0xde, 0xd0, 0x8f, 0xe4, 0x20, 0xb5) diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index 281b0b3da32c..efddcf6a2faa 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -5,8 +5,12 @@ #include #include -#if defined(CONFIG_SW64_CHIP3) -#include +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif + +#if defined(CONFIG_UNCORE_JUNZHANG) +#include #endif #define MK_RC_CFG(nid, idx) \ diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h new file mode 100644 index 000000000000..efceb4a14227 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_JUNZHANG_H + +#include + +#define IO_BASE (0x1UL << 47) +#define PCI_BASE (0x1UL << 43) +#define PCI_IOR0_BASE (0x2UL << 32) +#define PCI_IOR1_BASE (0x3UL << 32) + +#define PCI_RC_CFG (0x5UL << 32) + +#define PCI_EP_CFG (0x3UL << 33) +#define PCI_LEGACY_IO (0x1UL << 32) +#define PCI_LEGACY_IO_SIZE (0x100000000UL) +#define PCI_MEM_UNPRE 0x0UL +#define PCI_32BIT_VT_MEMIO (0xc0000000UL) +#define PCI_32BIT_MEMIO (0xe0000000UL) +#define PCI_32BIT_MEMIO_SIZE (0x20000000UL) +#define PCI_64BIT_MEMIO (0x1UL << 39) +#define PCI_64BIT_MEMIO_SIZE (0x8000000000UL) + +#define IO_RC_SHIFT 40 +#define IO_NODE_SHIFT 44 +#define IO_MARK_BIT 47 + +#define VT_MAX_CPUS_SHIFT 0 +#define VT_MAX_CPUS_MASK 0x3ff +#define VT_CORES_SHIFT 10 +#define VT_CORES_MASK 0x3ff +#define VT_THREADS_SHIFT 20 +#define VT_THREADS_MASK 0xfff + +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | SPBU_BASE | 0x40000UL) + +/* MSIConfig */ +#define MSICONFIG_VALID (0x1UL << 63) +#define MSICONFIG_EN (0x1UL << 62) +#define MSICONFIG_VECTOR_SHIFT 10 + +#define MSIX_MSG_ADDR (0xfff00000UL) + +#define SW64_PCI_IO_BASE(m, n) \ + (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) +#define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) + +#define SW64_PCI0_BUS 0 +#define PCI0_BUS SW64_PCI0_BUS + +#define MAX_NR_NODES 0x2 +#define MAX_NR_RCS 0x6 + +#define SPBU_BASE (0x3UL << 36) +#define INTPU_BASE (0x3aUL << 32) +#define IIC0_BASE (0x31UL << 32) +#define SPI_BASE (0x32UL << 32) +#define UART_BASE (0x33UL << 32) +#define IIC1_BASE (0x34UL << 32) +#define IIC2_BASE (0x35UL << 32) +#define GPIO_BASE (0x36UL << 32) +#define LPC_BASE (0x37UL << 32) +#define LPC_LEGACY_IO (0x1UL << 28 | IO_BASE | LPC_BASE) +#define LPC_MEM_IO (0x2UL << 28 | IO_BASE | LPC_BASE) +#define LPC_FIRMWARE_IO (0x3UL << 28 | IO_BASE | LPC_BASE) +#define PCI_VT_LEGACY_IO (IO_BASE | PCI_BASE | PCI_LEGACY_IO) + +#define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) + +/*-----------------------addr-----------------------*/ +/* INTPU REG */ +enum { + DEVINT_MISS = INTPU_BASE | 0x100UL, + MT_INT_CONFIG = INTPU_BASE | 0x300UL, + DEV_INT_CONFIG = INTPU_BASE | 0x480UL, + FMT_ERR = INTPU_BASE | 0x700UL, + FAULT_INT_CONFIG = INTPU_BASE | 0x780UL, + SERR_CNTTH = INTPU_BASE | 0x880UL, + SPBUSERR_CNT = INTPU_BASE | 0x900UL, + IRUSERR_CNT = INTPU_BASE | 0xa80UL, + ERRRPT_EN = INTPU_BASE | 0xb00UL, + IINT_MISS_VECTOR0 = INTPU_BASE | 0x1080UL, + IINT_MISS_VECTOR1 = INTPU_BASE | 0x1100UL, + IINT_MISS = INTPU_BASE | 0x1180UL, + IINT_MISS_RPTEN = INTPU_BASE | 0x1200UL, + DEVINT_MISS_RPTEN = INTPU_BASE | 0x1280UL, + ECCSERR = INTPU_BASE | 0x1300UL, + ECCSERR_RPTEN = INTPU_BASE | 0x1380UL, + ECCMERR = INTPU_BASE | 0x1400UL, + ECCMERR_RPTEN = INTPU_BASE | 0x1480UL, + DEVINT_WKEN = INTPU_BASE | 0x1500UL, + ADR_INT_CONFIG = INTPU_BASE | 0x1580UL, + DEVINTWK_INTEN = INTPU_BASE | 0x1600UL, +}; + +/* SPBU CSR */ +enum { + SMP_INFO = SPBU_BASE | 0x80UL, + INIT_CTL = SPBU_BASE | 0x680UL, + CORE_ONLINE = SPBU_BASE | 0x780UL, + DLI_RLTD_FAULT = SPBU_BASE | 0x980UL, + DLI_RLTD_FAULT_EN = SPBU_BASE | 0xa00UL, + DLI_RLTD_FAULT_INTEN = SPBU_BASE | 0xa80UL, + CFG_INFO = SPBU_BASE | 0x1100UL, + IO_START = SPBU_BASE | 0x1300UL, + I2C0_SRST_L = SPBU_BASE | 0x1900UL, + I2C1_SRST_L = SPBU_BASE | 0x1980UL, + I2C2_SRST_L = SPBU_BASE | 0x1a00UL, + MCU_DVC_INT = SPBU_BASE | 0x3000UL, + MCU_DVC_INT_EN = SPBU_BASE | 0x3080UL, + SI_FAULT_STAT = SPBU_BASE | 0x3100UL, + SI_FAULT_STAT_EN = SPBU_BASE | 0x3180UL, + SI_FAULT_INT_EN = SPBU_BASE | 0x3200UL, + ADR_CTL = SPBU_BASE | 0x3600UL, + MC_ONLINE = SPBU_BASE | 0x3780UL, + PIU_TOP0_CONFIG = SPBU_BASE | 0x4c80UL, + PIU_TOP1_CONFIG = SPBU_BASE | 0x4d00UL, + SOFT_INFO0 = SPBU_BASE | 0xa000UL, +}; + +/*--------------------------offset-----------------------------------*/ +/* PIU IOR0 */ +enum { + PIUCONFIG0 = 0x0UL, + EPDMABAR = 0x80UL, + IOMMUSEGITEM0 = 0x100UL, + IOMMUEXCPT_CTRL = 0x2100UL, + MSIADDR = 0x2180UL, + MSICONFIG0 = 0x2200UL, + INTACONFIG = 0xa200UL, + INTBCONFIG = 0xa280UL, + INTCCONFIG = 0xa300UL, + INTDCONFIG = 0xa380UL, + AERERRINTCONFIG = 0xa400UL, + AERERRMSICONFIG = 0xa480UL, + PMEINTCONFIG = 0xa500UL, + PMEMSICONFIG = 0xa580UL, + HPINTCONFIG = 0xa600UL, + HPMSICONFIG = 0xa680UL, + DTBASEADDR = 0xb000UL, + DTLB_FLUSHALL = 0xb080UL, + DTLB_FLUSHDEV = 0xb100UL, + PTLB_FLUSHALL = 0xb180UL, + PTLB_FLUSHDEV = 0xb200UL, + PTLB_FLUSHVADDR = 0xb280UL, + PCACHE_FLUSHALL = 0xb300UL, + PCACHE_FLUSHDEV = 0xb380UL, + PCACHE_FLUSHPADDR = 0xb400UL, + TIMEOUT_CONFIG = 0xb480UL, + IOMMUEXCPT_STATUS = 0xb500UL, + IOMMUPAGE_PADDR1 = 0xb580UL, + IOMMUPAGE_PADDR2 = 0xb600UL, + IOMMUPAGE_PADDR3 = 0xb680UL, + PTLB_ACCESS = 0xb700UL, + PTLB_ITEM_TAG = 0xb780UL, + PTLB_ITEM_DATA = 0xb800UL, + PCACHE_ACCESS = 0xb880UL, + PCACHE_ITEM_TAG = 0xb900UL, + PCACHE_ITEM_DATA0 = 0xb980UL, +}; + +/* PIU IOR1 */ +enum { + PIUCONFIG1 = 0x0UL, + ERRENABLE = 0x880UL, + RCDEBUGINF1 = 0xc80UL, + DCACONTROL = 0x1a00UL, + DEVICEID0 = 0x1a80UL, +}; + +/* RC */ +enum { + RC_VENDOR_ID = 0x0UL, + RC_COMMAND = 0x80UL, + RC_REVISION_ID = 0x100UL, + RC_PRIMARY_BUS = 0x300UL, + RC_MSI_CONTROL = 0xa00UL, + RC_EXP_DEVCAP = 0xe80UL, + RC_EXP_DEVCTL = 0xf00UL, + RC_SLOT_CTRL = 0x1100UL, + RC_LINK_STAT = 0x1000UL, + RC_CONTROL = 0X1180UL, + RC_STATUS = 0X1200UL, + RC_EXP_DEVCTL2 = 0x1300UL, + RC_PORT_LINK_CTL = 0xe200UL, + RC_ORDER_RULE_CTL = 0x11680UL, + RC_MISC_CONTROL_1 = 0x11780UL, + RC_PHY_INT_REG = 0x80000UL, + RC_PHY_EXT_GEN1 = 0x82400UL, + RC_PHY_EXT_GEN2 = 0x82480UL, +}; +/* GPIO */ +enum { + GPIO_SWPORTA_DR = GPIO_BASE | 0x0UL, + GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, +}; +/*--------------------------------------------------------------------------*/ +#endif /* _ASM_SW64_UNCORE_IO_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/chip3_io.h b/arch/sw_64/include/asm/uncore_io_xuelang.h similarity index 97% rename from arch/sw_64/include/asm/chip3_io.h rename to arch/sw_64/include/asm/uncore_io_xuelang.h index 6661e41057d1..00981eb92dc6 100644 --- a/arch/sw_64/include/asm/chip3_io.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_SW64_CHIP3_IO_H -#define _ASM_SW64_CHIP3_IO_H +#ifndef _ASM_SW64_UNCORE_IO_XUELANG_H +#define _ASM_SW64_UNCORE_IO_XUELANG_H #include @@ -31,11 +31,15 @@ #define VT_THREADS_SHIFT 20 #define VT_THREADS_MASK 0xfff +#define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) + /* MSIConfig */ #define MSICONFIG_VALID (0x1UL << 63) #define MSICONFIG_EN (0x1UL << 62) #define MSICONFIG_VECTOR_SHIFT 10 +#define MSIX_MSG_ADDR (0x91abc0UL) + #define SW64_PCI_IO_BASE(m, n) \ (IO_BASE | ((m) << IO_NODE_SHIFT) | PCI_BASE | ((n) << IO_RC_SHIFT)) #define SW64_IO_BASE(x) (IO_BASE | ((x) << IO_NODE_SHIFT)) @@ -46,13 +50,6 @@ #define MAX_NR_NODES 0x2 #define MAX_NR_RCS 0x6 -#define SW64_PCI_DEBUG 0 -#if SW64_PCI_DEBUG -#define PCIINFO(fmt, args...) printk(fmt, ##args) -#else -#define PCIINFO(fmt, args...) -#endif - #define MCU_BASE (0x3UL << 36) #define CAB0_BASE (0x10UL << 32) #define INTPU_BASE (0x2aUL << 32) @@ -321,4 +318,4 @@ enum { GPIO_SWPORTA_DDR = GPIO_BASE | 0x200UL, }; /*--------------------------------------------------------------------------*/ -#endif /* _ASM_SW64_CHIP3_IO_H */ +#endif /* _ASM_SW64_UNCORE_IO_XUELANG_H */ diff --git a/arch/sw_64/kernel/dup_print.c b/arch/sw_64/kernel/dup_print.c index 107363dc0513..439ac75feb01 100644 --- a/arch/sw_64/kernel/dup_print.c +++ b/arch/sw_64/kernel/dup_print.c @@ -15,12 +15,6 @@ static DEFINE_SPINLOCK(printk_lock); unsigned long sw64_printk_offset; #define PRINTK_SIZE 0x100000UL -/* - * For output the kernel message on the console - * with full-system emulator. - */ -#define QEMU_PRINTF_BUFF_BASE (IO_BASE | MCU_BASE | 0x40000UL) - int sw64_printk(const char *fmt, va_list args) { char *sw64_printk_buf; diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig index 43682f9b6e09..e85a4a54356b 100644 --- a/drivers/cpufreq/Kconfig +++ b/drivers/cpufreq/Kconfig @@ -357,7 +357,7 @@ endif if SW64 config SW64_CPUFREQ bool "SW64 CPU Frequency interface" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG default y help This adds the CPUFreq driver for SW64 processor which supports diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 52b476eda53b..8ddf6461b175 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -13,7 +13,7 @@ config ARM_GIC config SW64_INTC_V2 bool "SW64 Interrupt Controller V2" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG default y select GENERIC_IRQ_CHIP select IRQ_DOMAIN diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index b4a59e131f08..30eca221a3a1 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -570,7 +570,7 @@ config LPC_SCH config LPC_CHIP3 tristate "CHIP3 LPC" - depends on SW64_CHIP3 + depends on UNCORE_XUELANG select MFD_CORE help LPC bridge function of the chip3 provides support for diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index ff3443e6d2e4..47c9f9aab43b 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -983,7 +983,7 @@ config SPI_AMD # config SPI_CHIP3 tristate "Memory-mapped io interface driver for SUNWAY CHIP3 SPI core" - depends on SW64 + depends on UNCORE_XUELANG help general driver for SPI controller core from DesignWare -- Gitee From cf31815b3c529df7217badf3e59f5dc7fe0c4c4d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 09:06:32 +0800 Subject: [PATCH 087/150] sw64: unify io operations interface for C3B and C4 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Use a common interface for io operations to hide the differences between C3B and C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/sw64io.h | 9 ++++ .../include/asm/uncore_io_ops_junzhang.h | 33 ++++++++++++ .../sw_64/include/asm/uncore_io_ops_xuelang.h | 52 +++++++++++++++++++ arch/sw_64/kernel/chip_setup.c | 30 ++++------- drivers/clocksource/timer-sw64.c | 4 +- 5 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 arch/sw_64/include/asm/uncore_io_ops_junzhang.h create mode 100644 arch/sw_64/include/asm/uncore_io_ops_xuelang.h diff --git a/arch/sw_64/include/asm/sw64io.h b/arch/sw_64/include/asm/sw64io.h index efddcf6a2faa..d52cd8cc86bf 100644 --- a/arch/sw_64/include/asm/sw64io.h +++ b/arch/sw_64/include/asm/sw64io.h @@ -97,4 +97,13 @@ sw64_io_write(unsigned long node, unsigned long reg, unsigned long data) addr = __va(SW64_IO_BASE(node) | reg); writeq(data, addr); } + +#if defined(CONFIG_UNCORE_XUELANG) +#include +#endif + +#if defined(CONFIG_UNCORE_JUNZHANG) +#include +#endif + #endif /* _ASM_SW64_SW64IO_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h new file mode 100644 index 000000000000..4797ed2ad574 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H +#define _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long cfg_info; + + cfg_info = sw64_io_read(0, CFG_INFO); + cfg_info = (cfg_info >> 33) & 0x3; + cpus = 1 << cfg_info; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long total_mem; + + total_mem = sw64_io_read(node, CFG_INFO) >> 3; + total_mem = (total_mem & 0xffff) << 28; + node_mem = total_mem / __get_cpu_nums(); + + return node_mem; +} + +#define __io_read_longtime(node) (0UL) +#define __io_write_longtime(node, data) do { } while (0) +#define __io_write_longtime_start_en(node, data) do { } while (0) + +#endif /* _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h new file mode 100644 index 000000000000..1ae2473f15f6 --- /dev/null +++ b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_UNCORE_IO_OPS_XUELANG_H +#define _ASM_SW64_UNCORE_IO_OPS_XUELANG_H + +static inline int __get_cpu_nums(void) +{ + int cpus; + unsigned long trkmode; + + trkmode = sw64_io_read(0, TRKMODE); + trkmode = (trkmode >> 6) & 0x3; + cpus = 1 << trkmode; + + return cpus; +} + +static inline unsigned long __get_node_mem(int node) +{ + unsigned long node_mem; + unsigned long mc_config; + unsigned long mc_online; + unsigned long mc_cap; + unsigned long mc_num; + + mc_config = sw64_io_read(node, MC_CAP_CFG) & 0xf; + mc_cap = (1UL << mc_config) << 28; + mc_online = sw64_io_read(node, MC_ONLINE) & 0xff; + mc_num = __kernel_ctpop(mc_online); + node_mem = mc_cap * mc_num; + + return node_mem; +} + +static inline unsigned long +__io_read_longtime(int node) +{ + return sw64_io_read(node, LONG_TIME); +} + +static inline void +__io_write_longtime(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME, data); +} + +static inline void +__io_write_longtime_start_en(int node, unsigned long data) +{ + sw64_io_write(node, LONG_TIME_START_EN, data); +} + +#endif /* _ASM_SW64_UNCORE_IO_OPS_XUELANG_H */ diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 18da63db812c..68f912ab9f87 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -11,34 +11,19 @@ struct sw64_chip_init_ops *sw64_chip_init; static int get_cpu_nums(void) { - unsigned long trkmode; - int cpus; - if (is_guest_or_emul()) return 1; - trkmode = sw64_io_read(0, TRKMODE); - trkmode = (trkmode >> 6) & 0x3; - cpus = 1 << trkmode; - - return cpus; + return __get_cpu_nums(); } static unsigned long __init get_node_mem(int nodeid) { - unsigned long mc_config, mc_online, mc_cap, mc_num; - unsigned long node_mem; if (is_guest_or_emul()) return *(unsigned long *)MMSIZE & MMSIZE_MASK; - mc_config = sw64_io_read(nodeid, MC_CAP_CFG) & 0xf; - mc_cap = (1UL << mc_config) << 28; - mc_online = sw64_io_read(nodeid, MC_ONLINE) & 0xff; - mc_num = __kernel_ctpop(mc_online); - node_mem = mc_cap * mc_num; - - return node_mem; + return __get_node_mem(nodeid); } static void __init setup_core_map(struct cpumask *cpumask) @@ -184,15 +169,20 @@ static unsigned long saved_dvc_int, saved_long_time; static inline void intpu_save(void) { - saved_long_time = sw64_io_read(0, LONG_TIME); + switch (cpu_desc.model) { + case CPU_SW831: + saved_long_time = __io_read_longtime(0); + default: + break; + } } static inline void intpu_restore(void) { switch (cpu_desc.model) { case CPU_SW831: - sw64_io_write(0, LONG_TIME, saved_long_time); - sw64_io_write(0, LONG_TIME_START_EN, 0x1); + __io_write_longtime(0, saved_long_time); + __io_write_longtime_start_en(0, 0x1); break; default: pr_info("long time start is disable!"); diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 8f2ab450cf40..14f51b4e952a 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -150,7 +150,7 @@ static u64 read_longtime(struct clocksource *cs) unsigned long node; node = __this_cpu_read(hard_node_id); - result = sw64_io_read(node, LONG_TIME); + result = __io_read_longtime(node); return result; } @@ -163,7 +163,7 @@ static int longtime_enable(struct clocksource *cs) sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); break; case CPU_SW831: - sw64_io_write(0, LONG_TIME_START_EN, 0x1); + __io_write_longtime_start_en(0, 0x1); break; default: break; -- Gitee From 6d24c303da0b2b234a3fc8099c9db17c6f35424a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 16:03:00 +0800 Subject: [PATCH 088/150] sw64: timer: add C4 timer support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add timer setup and handling for basic C4 support. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 3 +- arch/sw_64/kernel/smp.c | 5 + arch/sw_64/kernel/vdso/vgettimeofday.c | 8 + drivers/clocksource/timer-sw64.c | 236 +++++++++++++++---------- 4 files changed, 153 insertions(+), 99 deletions(-) diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 0857acbf60ef..b0ff1ae82b54 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -15,10 +15,11 @@ endif obj-y := entry.o fpu.o traps.o process.o sys_sw64.o irq.o \ irq_sw64.o signal.o setup.o ptrace.o time.o \ - systbls.o dup_print.o tc.o chip_setup.o \ + systbls.o dup_print.o chip_setup.o \ insn.o early_init.o topology.o cacheinfo.o \ vdso.o vdso/ hmcall.o stacktrace.o idle.o reset.o +obj-$(CONFIG_SUBARCH_C3B) += tc.o obj-$(CONFIG_ACPI) += acpi.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_PCI) += pci.o pci-sysfs.o diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index cd5e4edec76c..30f58883c61e 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -258,7 +258,10 @@ int vt_cpu_up(unsigned int cpu, struct task_struct *tidle) return cpu_online(cpu) ? 0 : -ENOSYS; } +#ifdef CONFIG_SUBARCH_C3B DECLARE_STATIC_KEY_FALSE(use_tc_as_sched_clock); +#endif + int __cpu_up(unsigned int cpu, struct task_struct *tidle) { if (is_in_guest()) @@ -283,6 +286,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) } smp_boot_one_cpu(cpu, tidle); +#ifdef CONFIG_SUBARCH_C3B #ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE if (static_branch_likely(&use_tc_as_sched_clock)) { if (smp_booted) { @@ -291,6 +295,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) tc_sync_set(); } } +#endif #endif return cpu_online(cpu) ? 0 : -ENOSYS; diff --git a/arch/sw_64/kernel/vdso/vgettimeofday.c b/arch/sw_64/kernel/vdso/vgettimeofday.c index 49bb4e2e66ed..0aa16e988e88 100644 --- a/arch/sw_64/kernel/vdso/vgettimeofday.c +++ b/arch/sw_64/kernel/vdso/vgettimeofday.c @@ -16,6 +16,7 @@ #include #include +#include #include static __always_inline int syscall_fallback(clockid_t clkid, struct timespec64 *ts) @@ -74,6 +75,7 @@ static __always_inline int do_monotonic_coarse(struct timespec64 *ts, return 0; } +#if defined(CONFIG_SUBARCH_C3B) static __always_inline u64 read_longtime(void) { register unsigned long __r0 __asm__("$0"); @@ -83,6 +85,12 @@ static __always_inline u64 read_longtime(void) return __r0; } +#elif defined(CONFIG_SUBARCH_C4) +static __always_inline u64 read_longtime(void) +{ + return read_csr(CSR_SHTCLOCK); +} +#endif static __always_inline u64 get_ns(const struct vdso_data *data) { diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 14f51b4e952a..7b58c9450a7b 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -2,19 +2,143 @@ #include #include +#include +#include #include +#include +#include #include +#include #include #include -DECLARE_PER_CPU(u64, tc_offset); +#define SHTCLK_RATE 2500000 +#define SHTCLK_RATE_KHZ 2500 + +#if defined(CONFIG_SUBARCH_C4) +static u64 read_longtime(struct clocksource *cs) +{ + return read_csr(CSR_SHTCLOCK); +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 notrace read_sched_clock(void) +{ + return read_csr(CSR_SHTCLOCK); +} -static u64 sc_start; -static u64 sc_shift; -static u64 sc_multi; +void __init sw64_setup_clocksource(void) +{ + clocksource_register_khz(&clocksource_longtime, SHTCLK_RATE_KHZ); + sched_clock_register(read_sched_clock, BITS_PER_LONG, SHTCLK_RATE); +} +void __init setup_sched_clock(void) { } +#elif defined(CONFIG_SUBARCH_C3B) +#ifdef CONFIG_SMP +static u64 read_longtime(struct clocksource *cs) +{ + unsigned long node; + + node = __this_cpu_read(hard_node_id); + return __io_read_longtime(node); +} + +static int longtime_enable(struct clocksource *cs) +{ + switch (cpu_desc.model) { + case CPU_SW3231: + sw64_io_write(0, GPIO_SWPORTA_DR, 0); + sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); + break; + case CPU_SW831: + __io_write_longtime_start_en(0, 0x1); + break; + default: + break; + } + + return 0; +} + +static struct clocksource clocksource_longtime = { + .name = "longtime", + .rating = 100, + .enable = longtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_longtime, +}; + +static u64 read_vtime(struct clocksource *cs) +{ + unsigned long vtime_addr; + + vtime_addr = IO_BASE | LONG_TIME; + return rdio64(vtime_addr); +} + +static int vtime_enable(struct clocksource *cs) +{ + return 0; +} + +static struct clocksource clocksource_vtime = { + .name = "vtime", + .rating = 100, + .enable = vtime_enable, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 0, + .mult = 0, + .read = read_vtime, +}; +#else /* !SMP */ +static u64 read_tc(struct clocksource *cs) +{ + return rdtc(); +} + +static struct clocksource clocksource_tc = { + .name = "tc", + .rating = 300, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, + .mask = CLOCKSOURCE_MASK(64), + .shift = 22, + .mult = 0, /* To be filled in */ + .read = read_tc, +}; +#endif /* SMP */ + +void __init sw64_setup_clocksource(void) +{ +#ifdef CONFIG_SMP + if (is_in_host()) + clocksource_register_khz(&clocksource_longtime, 25000); + else + clocksource_register_khz(&clocksource_vtime, 25000); +#else + clocksource_register_hz(&clocksource_tc, get_cpu_freq()); + pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); +#endif +} + +DECLARE_PER_CPU(u64, tc_offset); +static u64 sc_start, sc_shift, sc_multi; DEFINE_STATIC_KEY_FALSE(use_tc_as_sched_clock); + static int __init sched_clock_setup(char *opt) { if (!opt) @@ -47,11 +171,19 @@ void __init setup_sched_clock(void) } #ifdef CONFIG_GENERIC_SCHED_CLOCK -static u64 notrace sched_clock_read(void) +static u64 notrace read_sched_clock(void) { return (rdtc() - sc_start) >> sc_shift; } + +void __init sw64_sched_clock_init(void) +{ + sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); +} #else /* !CONFIG_GENERIC_SCHED_CLOCK */ +/* + * scheduler clock - returns current time in nanoseconds. + */ unsigned long long notrace sched_clock(void) { if (static_branch_likely(&use_tc_as_sched_clock)) @@ -129,101 +261,9 @@ late_initcall(sched_clock_debug_init); #endif /* CONFIG_DEBUG_FS */ #endif /* CONFIG_GENERIC_SCHED_CLOCK */ -static u64 read_tc(struct clocksource *cs) -{ - return rdtc(); -} - -static struct clocksource clocksource_tc = { - .name = "tc", - .rating = 300, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 22, - .mult = 0, /* To be filled in */ - .read = read_tc, -}; - -static u64 read_longtime(struct clocksource *cs) -{ - u64 result; - unsigned long node; - - node = __this_cpu_read(hard_node_id); - result = __io_read_longtime(node); - - return result; -} - -static int longtime_enable(struct clocksource *cs) -{ - switch (cpu_desc.model) { - case CPU_SW3231: - sw64_io_write(0, GPIO_SWPORTA_DR, 0); - sw64_io_write(0, GPIO_SWPORTA_DDR, 0xff); - break; - case CPU_SW831: - __io_write_longtime_start_en(0, 0x1); - break; - default: - break; - } - - return 0; -} - -static struct clocksource clocksource_longtime = { - .name = "longtime", - .rating = 100, - .enable = longtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_longtime, -}; - -static u64 read_vtime(struct clocksource *cs) -{ - u64 result; - unsigned long vtime_addr = IO_BASE | LONG_TIME; - - result = rdio64(vtime_addr); - return result; -} - -static int vtime_enable(struct clocksource *cs) -{ - return 0; -} - -static struct clocksource clocksource_vtime = { - .name = "vtime", - .rating = 100, - .enable = vtime_enable, - .flags = CLOCK_SOURCE_IS_CONTINUOUS, - .mask = CLOCKSOURCE_MASK(64), - .shift = 0, - .mult = 0, - .read = read_vtime, -}; +#endif -void __init sw64_setup_clocksource(void) -{ - if (!IS_ENABLED(CONFIG_SMP)) { - clocksource_register_hz(&clocksource_tc, get_cpu_freq()); - pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); - } else { - if (is_in_host()) - clocksource_register_khz(&clocksource_longtime, 25000); - else - clocksource_register_khz(&clocksource_vtime, 25000); - } -#ifdef CONFIG_GENERIC_SCHED_CLOCK - sched_clock_register(sched_clock_read, BITS_PER_LONG, get_cpu_freq() >> sc_shift); -#endif -} static int timer_next_event(unsigned long delta, struct clock_event_device *evt); -- Gitee From 197d0e27bd1ed2207e36eaa08803bc6af2627d21 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 16:25:03 +0800 Subject: [PATCH 089/150] sw64: irq: add basic C4 irq support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add C4 irq handling mechanism. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_ops_junzhang.h | 6 ++++++ arch/sw_64/include/asm/uncore_io_ops_xuelang.h | 13 +++++++++++++ arch/sw_64/kernel/irq_sw64.c | 7 +++++++ arch/sw_64/kernel/smp.c | 6 ++++++ drivers/irqchip/irq-sunway-cpu.c | 16 ++++++++-------- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h index 4797ed2ad574..95a3b5c80531 100644 --- a/arch/sw_64/include/asm/uncore_io_ops_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_ops_junzhang.h @@ -30,4 +30,10 @@ static inline unsigned long __get_node_mem(int node) #define __io_write_longtime(node, data) do { } while (0) #define __io_write_longtime_start_en(node, data) do { } while (0) +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, FAULT_INT_CONFIG, data); +} + #endif /* _ASM_SW64_UNCORE_IO_OPS_JUNZHANG_H */ diff --git a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h index 1ae2473f15f6..9336e473211d 100644 --- a/arch/sw_64/include/asm/uncore_io_ops_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_ops_xuelang.h @@ -49,4 +49,17 @@ __io_write_longtime_start_en(int node, unsigned long data) sw64_io_write(node, LONG_TIME_START_EN, data); } +static inline void +__io_write_fault_int_en(int node, unsigned long data) +{ + sw64_io_write(node, DUAL_CG0_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG1_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG2_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG3_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG4_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG5_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG6_FAULT_INTEN, data); + sw64_io_write(node, DUAL_CG7_FAULT_INTEN, data); +} + #endif /* _ASM_SW64_UNCORE_IO_OPS_XUELANG_H */ diff --git a/arch/sw_64/kernel/irq_sw64.c b/arch/sw_64/kernel/irq_sw64.c index 88809fa531dd..989d55ee1b1b 100644 --- a/arch/sw_64/kernel/irq_sw64.c +++ b/arch/sw_64/kernel/irq_sw64.c @@ -16,6 +16,13 @@ init_IRQ(void) * Just in case the platform init_irq() causes interrupts/mchecks * (as is the case with RAWHIDE, at least). */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } + wrent(entInt, 0); sw64_init_irq(); diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 30f58883c61e..0878702b2895 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -74,6 +74,12 @@ void smp_callin(void) trap_init(); /* Set interrupt vector. */ + if (is_in_host()) { + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI0_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI1_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI2_INTEN); + write_csr(0xffffffffffffffffUL, CSR_PCIE_MSI3_INTEN); + } wrent(entInt, 0); /* Get our local ticker going. */ diff --git a/drivers/irqchip/irq-sunway-cpu.c b/drivers/irqchip/irq-sunway-cpu.c index 05ed5a045300..ca1b62b2d645 100644 --- a/drivers/irqchip/irq-sunway-cpu.c +++ b/drivers/irqchip/irq-sunway-cpu.c @@ -76,20 +76,20 @@ EXPORT_SYMBOL(perf_irq); static void handle_fault_int(void) { int node; + unsigned long value; node = __this_cpu_read(hard_node_id); pr_info("enter fault int, si_fault_stat = %#lx\n", sw64_io_read(node, SI_FAULT_STAT)); sw64_io_write(node, SI_FAULT_INT_EN, 0); sw64_io_write(node, DLI_RLTD_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG0_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG1_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG2_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG3_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG4_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG5_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG6_FAULT_INTEN, 0); - sw64_io_write(node, DUAL_CG7_FAULT_INTEN, 0); +#if defined(CONFIG_UNCORE_XUELANG) + value = 0; +#elif defined(CONFIG_UNCORE_JUNZHANG) + value = sw64_io_read(node, FAULT_INT_CONFIG); + value |= (1 << 8); +#endif + __io_write_fault_int_en(node, value); } static void handle_mt_int(void) -- Gitee From 3664c41211f26cac7a70cf455e92f73a49e84205 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 16:26:07 +0800 Subject: [PATCH 090/150] sw64: pci: add C4 PCI support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add pci setup for C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/uncore_io_junzhang.h | 2 ++ arch/sw_64/include/asm/uncore_io_xuelang.h | 2 ++ arch/sw_64/kernel/pci-noop.c | 1 + arch/sw_64/kernel/pci.c | 5 +++- drivers/pci/controller/pci-sunway.c | 29 +++++++++++++++++---- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/include/asm/uncore_io_junzhang.h b/arch/sw_64/include/asm/uncore_io_junzhang.h index efceb4a14227..37cfe1fd6807 100644 --- a/arch/sw_64/include/asm/uncore_io_junzhang.h +++ b/arch/sw_64/include/asm/uncore_io_junzhang.h @@ -68,6 +68,8 @@ #define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) #define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x8UL << 10) +#define PIUCONFIG0_INIT_VAL 0x38016 + /*-----------------------addr-----------------------*/ /* INTPU REG */ enum { diff --git a/arch/sw_64/include/asm/uncore_io_xuelang.h b/arch/sw_64/include/asm/uncore_io_xuelang.h index 00981eb92dc6..aeaadec5be16 100644 --- a/arch/sw_64/include/asm/uncore_io_xuelang.h +++ b/arch/sw_64/include/asm/uncore_io_xuelang.h @@ -72,6 +72,8 @@ #define PME_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) #define AER_ENABLE_INTD_CORE0 (0x1UL << 62 | 0x1UL << 10) +#define PIUCONFIG0_INIT_VAL 0x38056 + /*-----------------------addr-----------------------*/ /* CAB0 REG */ enum { diff --git a/arch/sw_64/kernel/pci-noop.c b/arch/sw_64/kernel/pci-noop.c index a0aa2e5bb675..abfba92fa6a9 100644 --- a/arch/sw_64/kernel/pci-noop.c +++ b/arch/sw_64/kernel/pci-noop.c @@ -8,6 +8,7 @@ #include #include #include +#include /* * The PCI controller list. diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 38ee0c5e191b..0c4dca96ccb4 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -634,6 +634,7 @@ sw64_init_host(unsigned long node, unsigned long index) } void __weak set_devint_wken(int node) {} +void __weak set_adr_int(int node) {} void __init sw64_init_arch(void) { @@ -647,8 +648,10 @@ void __init sw64_init_arch(void) cpu_num = sw64_chip->get_cpu_num(); for (node = 0; node < cpu_num; node++) { - if (is_in_host()) + if (is_in_host()) { set_devint_wken(node); + set_adr_int(node); + } rc_enable = sw64_chip_init->pci_init.get_rc_enable(node); if (rc_enable == 0) { printk("PCIe is disabled on node %ld\n", node); diff --git a/drivers/pci/controller/pci-sunway.c b/drivers/pci/controller/pci-sunway.c index 6d933c660dda..c57c2350ffef 100644 --- a/drivers/pci/controller/pci-sunway.c +++ b/drivers/pci/controller/pci-sunway.c @@ -12,6 +12,14 @@ void set_devint_wken(int node) sw64_io_write(node, DEVINTWK_INTEN, val); } +#ifdef CONFIG_UNCORE_JUNZHANG +void set_adr_int(int node) +{ + sw64_io_write(node, ADR_INT_CONFIG, (0x0 << 16 | 0x3f)); + sw64_io_write(node, ADR_CTL, 0xc); +} +#endif + void set_pcieport_service_irq(int node, int index) { if (IS_ENABLED(CONFIG_PCIE_PME)) @@ -172,12 +180,13 @@ static int check_pci_linkup(unsigned long node, unsigned long index) rc_debug = read_piu_ior1(node, index, RCDEBUGINF1); } - return !(rc_debug & 0x1); + return !(rc_debug == 0x111); } static void set_rc_piu(unsigned long node, unsigned long index) { - unsigned int i, value; + unsigned int i __maybe_unused; + unsigned int value; u32 rc_misc_ctrl; if (is_guest_or_emul()) @@ -206,7 +215,8 @@ static void set_rc_piu(unsigned long node, unsigned long index) write_rc_conf(node, index, RC_MISC_CONTROL_1, rc_misc_ctrl); write_rc_conf(node, index, RC_PRIMARY_BUS, 0xffffff); - write_piu_ior0(node, index, PIUCONFIG0, 0x38056); + write_piu_ior0(node, index, PIUCONFIG0, PIUCONFIG0_INIT_VAL); + write_piu_ior1(node, index, PIUCONFIG1, 0x2); write_piu_ior1(node, index, ERRENABLE, -1); @@ -214,8 +224,10 @@ static void set_rc_piu(unsigned long node, unsigned long index) write_piu_ior0(node, index, EPDMABAR, PCITODMA_OFFSET); if (IS_ENABLED(CONFIG_PCI_MSI)) { write_piu_ior0(node, index, MSIADDR, MSIX_MSG_ADDR); - for (i = 0; i < 256; i++) - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); +#ifdef CONFIG_UNCORE_XUELANG + for (i = 0; i < 256; i++) + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), 0); +#endif } } @@ -225,10 +237,17 @@ static void set_intx(unsigned long node, unsigned long index, if (is_guest_or_emul()) return; +#if defined(CONFIG_UNCORE_XUELANG) write_piu_ior0(node, index, INTACONFIG, int_conf | (0x8UL << 10)); write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x4UL << 10)); write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x2UL << 10)); write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x1UL << 10)); +#elif defined(CONFIG_UNCORE_JUNZHANG) + write_piu_ior0(node, index, INTACONFIG, int_conf | (0x1UL << 10)); + write_piu_ior0(node, index, INTBCONFIG, int_conf | (0x2UL << 10)); + write_piu_ior0(node, index, INTCCONFIG, int_conf | (0x4UL << 10)); + write_piu_ior0(node, index, INTDCONFIG, int_conf | (0x8UL << 10)); +#endif } static unsigned long get_rc_enable(unsigned long node) -- Gitee From f738c7e8766d11431189a92197fa8cb54729ee0d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 15:59:31 +0800 Subject: [PATCH 091/150] sw64: pci: add C4 PCI_MSI support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add msi setup and handling mechanism for C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/msi.h | 20 ++- arch/sw_64/kernel/chip_setup.c | 16 ++- arch/sw_64/kernel/msi.c | 1 + drivers/irqchip/irq-sunway-msi.c | 215 +++++++++++++++++++++++++------ 4 files changed, 209 insertions(+), 43 deletions(-) diff --git a/arch/sw_64/include/asm/msi.h b/arch/sw_64/include/asm/msi.h index e708ee191dab..e378326b6754 100644 --- a/arch/sw_64/include/asm/msi.h +++ b/arch/sw_64/include/asm/msi.h @@ -40,9 +40,15 @@ extern void handle_pci_msi_interrupt(unsigned long type, #define MSI_ADDR_BASE_HI 0 #define MSI_ADDR_BASE_LO 0x91abc0 +#define MSI_ADDR_SHIFT 20 +#define MSI_ADDR_DEST_ID_SHIFT 10 + struct sw64_msi_chip_data { spinlock_t cdata_lock; - unsigned long msi_config; + union { + unsigned long msi_config; + unsigned long msiaddr; + }; unsigned long rc_node; unsigned long rc_index; unsigned int msi_config_index; @@ -50,9 +56,21 @@ struct sw64_msi_chip_data { unsigned int vector; unsigned int prev_cpu; unsigned int prev_vector; + unsigned int multi_msi; bool move_in_progress; }; +static inline int rcid_to_msicid(int rcid) +{ + int msicid = 0; + + msicid |= (rcid_to_domain_id(rcid) << 7); + msicid |= (rcid_to_thread_id(rcid) << 6); + msicid |= (rcid_to_core_id(rcid) << 0); + + return msicid; +} + extern void arch_init_msi_domain(struct irq_domain *domain); enum irq_alloc_type { IRQ_ALLOC_TYPE_MSI, diff --git a/arch/sw_64/kernel/chip_setup.c b/arch/sw_64/kernel/chip_setup.c index 68f912ab9f87..b8c359db2ef6 100644 --- a/arch/sw_64/kernel/chip_setup.c +++ b/arch/sw_64/kernel/chip_setup.c @@ -93,9 +93,11 @@ static void pcie_save(void) piu_save->epdmabar = read_piu_ior0(node, index, EPDMABAR); piu_save->msiaddr = read_piu_ior0(node, index, MSIADDR); - for (i = 0; i < 256; i++) { - piu_save->msiconfig[i] = read_piu_ior0(node, index, - MSICONFIG0 + (i << 7)); + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + piu_save->msiconfig[i] = read_piu_ior0(node, index, + MSICONFIG0 + (i << 7)); + } } piu_save->iommuexcpt_ctrl = read_piu_ior0(node, index, IOMMUEXCPT_CTRL); @@ -131,9 +133,11 @@ static void pcie_restore(void) write_piu_ior0(node, index, EPDMABAR, piu_save->epdmabar); write_piu_ior0(node, index, MSIADDR, piu_save->msiaddr); - for (i = 0; i < 256; i++) { - write_piu_ior0(node, index, MSICONFIG0 + (i << 7), - piu_save->msiconfig[i]); + if (IS_ENABLED(CONFIG_UNCORE_XUELANG)) { + for (i = 0; i < 256; i++) { + write_piu_ior0(node, index, MSICONFIG0 + (i << 7), + piu_save->msiconfig[i]); + } } write_piu_ior0(node, index, IOMMUEXCPT_CTRL, piu_save->iommuexcpt_ctrl); diff --git a/arch/sw_64/kernel/msi.c b/arch/sw_64/kernel/msi.c index ee1bda3c6447..eeb01e63e636 100644 --- a/arch/sw_64/kernel/msi.c +++ b/arch/sw_64/kernel/msi.c @@ -2,6 +2,7 @@ #include #include #include +#include int msi_compose_msg(unsigned int irq, struct msi_msg *msg) { diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index a0ab4de8fa29..505c935132c6 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -29,11 +30,22 @@ static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_da static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { struct sw64_msi_chip_data *chip_data; + int rcid __maybe_unused; chip_data = irq_data_get_irq_chip_data(data->parent_data); +#if defined(CONFIG_UNCORE_XUELANG) msg->address_hi = MSI_ADDR_BASE_HI; msg->address_lo = MSI_ADDR_BASE_LO; msg->data = chip_data->msi_config_index; +#elif defined(CONFIG_UNCORE_JUNZHANG) + rcid = cpu_to_rcid(chip_data->dst_cpu); + msg->address_hi = MSI_ADDR_BASE_HI; + msg->address_lo = + (unsigned int)chip_data->msiaddr | + (rcid_to_msicid(rcid) << MSI_ADDR_DEST_ID_SHIFT); + msg->data = chip_data->vector; + printk("%s msg->address_lo:%#x msg->data:%#x\n", __func__, msg->address_lo, msg->data); +#endif } bool find_free_cpu_vector(const struct cpumask *search_mask, @@ -79,6 +91,45 @@ bool find_free_cpu_vector(const struct cpumask *search_mask, return true; } +static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_cpu, int *found_vector, unsigned int nr_irqs) +{ + int i, vector, cpu; + bool found = false, find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + for (vector = 0; vector < 256; vector++) { + for (i = 0; i < nr_irqs; i++) + if (per_cpu(vector_irq, cpu)[vector + i]) + break; + + if (i == nr_irqs) { + found = true; + *found_cpu = cpu; + *found_vector = vector; + return found; + } + + vector += i; + } + + cpu = cpumask_next(cpu, search_mask); + if (cpu < nr_cpu_ids) + goto try_again; + else { + if (find_once_global) { + printk("No global free vectors\n"); + return found; + } + printk("No local free vectors\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } +} + +#ifdef CONFIG_UNCORE_XUELANG static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, int msiconf_index, int vector) { @@ -96,16 +147,20 @@ static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, return msi_config; } +#endif static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) { struct sw64_msi_chip_data *cdata; - struct pci_controller *hose; struct irq_data *irqd; struct msi_desc *entry; struct cpumask searchmask; - unsigned long flags, msi_config; + unsigned long flags; int vector, cpu; + struct pci_controller *hose __maybe_unused; + unsigned long msi_config __maybe_unused; + int i __maybe_unused; + struct msi_msg msg __maybe_unused; /* Is this valid ? */ if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) @@ -130,22 +185,37 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, raw_spin_lock_irqsave(&vector_lock, flags); cpumask_and(&searchmask, cpumask, cpu_online_mask); - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { - raw_spin_unlock_irqrestore(&vector_lock, flags); - return -ENOSPC; + if (cdata->multi_msi > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, cdata->multi_msi)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } else { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } } /* update new setting */ entry = irq_get_msi_desc(irqd->irq); - hose = (struct pci_controller *)msi_desc_to_pci_sysdata(entry); spin_lock(&cdata->cdata_lock); +#if defined(CONFIG_UNCORE_XUELANG) + hose = (struct pci_controller *)msi_desc_to_pci_sysdata(entry); per_cpu(vector_irq, cpu)[vector] = irqd->irq; msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); + cdata->msi_config = msi_config; +#elif defined(CONFIG_UNCORE_JUNZHANG) + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; + BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); + __pci_write_msi_msg(entry, &msg); +#endif cdata->prev_vector = cdata->vector; cdata->prev_cpu = cdata->dst_cpu; cdata->dst_cpu = cpu; cdata->vector = vector; - cdata->msi_config = msi_config; cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); @@ -170,20 +240,24 @@ static struct irq_chip pci_msi_controller = { }; static int __assign_irq_vector(int virq, unsigned int nr_irqs, - struct irq_domain *domain, struct pci_controller *hose) + struct irq_domain *domain, struct pci_controller *hose, + enum irq_alloc_type type) { struct irq_data *irq_data; const struct cpumask *mask; struct cpumask searchmask; struct sw64_msi_chip_data *cdata; - int msiconf_index, node; + int node; int i, vector, cpu; - unsigned long msi_config; - int start_index; + unsigned long msi_config __maybe_unused; + unsigned long msiaddr __maybe_unused; + int msiconf_index __maybe_unused; + int start_index __maybe_unused; if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) nr_irqs = __roundup_pow_of_two(nr_irqs); +#if defined(CONFIG_UNCORE_XUELANG) msiconf_index = bitmap_find_next_zero_area(hose->piu_msiconfig, 256, 0, nr_irqs, nr_irqs - 1); @@ -193,6 +267,9 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, } start_index = msiconf_index; +#elif defined(CONFIG_UNCORE_JUNZHANG) + msiaddr = read_piu_ior0(hose->node, hose->index, MSIADDR); +#endif irq_data = irq_domain_get_irq_data(domain, virq); BUG_ON(!irq_data); irq_data->chip = &pci_msi_controller; @@ -208,48 +285,87 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, if (cpumask_first(&searchmask) >= nr_cpu_ids) cpumask_copy(&searchmask, cpu_online_mask); - for (i = 0; i < nr_irqs; i++) { - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + if (type == IRQ_ALLOC_TYPE_MSI && nr_irqs > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, nr_irqs)) return -ENOSPC; - per_cpu(vector_irq, cpu)[vector] = virq + i; - - if (i) { - irq_data = irq_domain_get_irq_data(domain, virq + i); - irq_data->chip = &pci_msi_controller; - } - cdata = alloc_sw_msi_chip_data(irq_data); if (!cdata) { printk("error alloc irq chip data\n"); return -ENOMEM; } - irq_data->chip_data = cdata; - msiconf_index = start_index + i; - msi_config = set_piu_msi_config(hose, cpu, msiconf_index, vector); + for (i = 0; i < nr_irqs; i++) { + per_cpu(vector_irq, cpu)[vector + i] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + irq_data->chip_data = cdata; + } cdata->dst_cpu = cpu; cdata->vector = vector; - cdata->rc_index = hose->index; - cdata->rc_node = hose->node; - cdata->msi_config = msi_config; - cdata->msi_config_index = msiconf_index; + cdata->msiaddr = msiaddr; cdata->prev_cpu = cpu; cdata->prev_vector = vector; + cdata->multi_msi = nr_irqs; cdata->move_in_progress = false; + printk("TYPE_MSI cpu:%#x vector:%#x msiaddr:%#lx virq:%d\n", + cpu, vector, msiaddr, virq); + } else { + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + printk("error alloc irq chip data\n"); + return -ENOMEM; + } + + irq_data->chip_data = cdata; + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->rc_index = hose->index; + cdata->rc_node = hose->node; +#if defined(CONFIG_UNCORE_XUELANG) + msiconf_index = start_index + i; + msi_config = set_piu_msi_config(hose, cpu, msiconf_index, vector); + cdata->msi_config = msi_config; + cdata->msi_config_index = msiconf_index; +#elif defined(CONFIG_UNCORE_JUNZHANG) + cdata->msiaddr = msiaddr; +#endif + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = 1; + cdata->move_in_progress = false; + } } return 0; } static int assign_irq_vector(int irq, unsigned int nr_irqs, - struct irq_domain *domain, struct pci_controller *hose) + struct irq_domain *domain, struct pci_controller *hose, + enum irq_alloc_type type) { int err; unsigned long flags; raw_spin_lock_irqsave(&vector_lock, flags); - err = __assign_irq_vector(irq, nr_irqs, domain, hose); + err = __assign_irq_vector(irq, nr_irqs, domain, hose, type); raw_spin_unlock_irqrestore(&vector_lock, flags); return err; } @@ -257,28 +373,33 @@ static int assign_irq_vector(int irq, unsigned int nr_irqs, static void sw64_vector_free_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { - int i; + int i, j; struct irq_data *irq_data; unsigned long flags; + struct msi_desc *entry __maybe_unused; + struct pci_controller *hose __maybe_unused; for (i = 0; i < nr_irqs; i++) { irq_data = irq_domain_get_irq_data(domain, virq + i); if (irq_data && irq_data->chip_data) { struct sw64_msi_chip_data *cdata; - struct msi_desc *entry; - struct pci_controller *hose; raw_spin_lock_irqsave(&vector_lock, flags); cdata = irq_data->chip_data; +#ifdef CONFIG_UNCORE_XUELANG entry = irq_get_msi_desc(virq + i); if (entry) { hose = (struct pci_controller *)msi_desc_to_pci_sysdata(entry); clear_bit(cdata->msi_config_index, hose->piu_msiconfig); } +#endif irq_domain_reset_irq_data(irq_data); - per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; + for (j = 0; j < cdata->multi_msi; j++) + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; kfree(cdata); raw_spin_unlock_irqrestore(&vector_lock, flags); + if (cdata->multi_msi > 1) + break; } } } @@ -313,11 +434,13 @@ static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, int err; struct irq_alloc_info *info = arg; struct pci_controller *hose; + enum irq_alloc_type msi_type; if (arg == NULL) return -ENODEV; hose = info->msi_dev->sysdata; - err = assign_irq_vector(virq, nr_irqs, domain, hose); + msi_type = info->type; + err = assign_irq_vector(virq, nr_irqs, domain, hose, msi_type); if (err) goto error; return 0; @@ -402,23 +525,39 @@ static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vec { if (likely(!cdata->move_in_progress)) return; +#if defined(CONFIG_UNCORE_XUELANG) if (vector == cdata->vector && cdata->dst_cpu == cpu) { raw_spin_lock(&vector_lock); cdata->move_in_progress = 0; per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; raw_spin_unlock(&vector_lock); } +#elif defined(UNCORE_JUNZHANG) + if (cdata->dst_cpu == cpu) { + if (vector >= cdata->vector && + vector < cdata->vector + cdata->multi_msi) { + int i; + + raw_spin_lock(&vector_lock); + cdata->move_in_progress = false; + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector + i] = 0; + raw_spin_unlock(&vector_lock); + } + } +#endif } void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) { - int i, irq, piu_index, msi_index = 0; + int i, irq, msi_index = 0; int cpu, vector_index = 0; - unsigned long value = 0; unsigned long int_pci_msi[3]; unsigned long *ptr; struct irq_data *irq_data; struct sw64_msi_chip_data *cdata; + int piu_index __maybe_unused; + unsigned long value __maybe_unused = 0; if (is_guest_or_emul()) { cpu = smp_processor_id(); @@ -437,6 +576,8 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned for (i = 0; i < 4; i++) { vector_index = i * 64; while (vector != 0) { + int irq = 0; + msi_index = find_next_bit(&vector, 64, msi_index); if (msi_index == 64) { msi_index = 0; @@ -448,9 +589,11 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned cdata = irq_data_get_irq_chip_data(irq_data); spin_lock(&cdata->cdata_lock); irq_move_complete(cdata, cpu, vector_index + msi_index); +#ifdef CONFIG_UNCORE_XUELANG piu_index = cdata->msi_config_index; value = cdata->msi_config | (1UL << 63); write_piu_ior0(cdata->rc_node, cdata->rc_index, MSICONFIG0 + (piu_index << 7), value); +#endif spin_unlock(&cdata->cdata_lock); handle_irq(irq); -- Gitee From 2fce866f4ffed63893e3dc0007247b1c4c81ca4b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 1 Sep 2023 16:47:43 +0800 Subject: [PATCH 092/150] sw64: iommu: add new iommu v2 support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add a new version of IOMMU. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/device.h | 2 +- arch/sw_64/include/asm/pci.h | 2 +- drivers/iommu/sw64/Kconfig | 11 + drivers/iommu/sw64/Makefile | 1 + drivers/iommu/sw64/sunway_iommu.c | 5 + drivers/iommu/sw64/sunway_iommu.h | 7 +- drivers/iommu/sw64/sunway_iommu_v2.c | 1816 ++++++++++++++++++++++++++ 7 files changed, 1840 insertions(+), 4 deletions(-) create mode 100644 drivers/iommu/sw64/sunway_iommu_v2.c diff --git a/arch/sw_64/include/asm/device.h b/arch/sw_64/include/asm/device.h index bc1408c47dd3..d999207e07d1 100644 --- a/arch/sw_64/include/asm/device.h +++ b/arch/sw_64/include/asm/device.h @@ -3,7 +3,7 @@ #define _ASM_SW64_DEVICE_H struct dev_archdata { -#if defined(CONFIG_SUNWAY_IOMMU) +#if defined(CONFIG_SUNWAY_IOMMU) || defined(CONFIG_SUNWAY_IOMMU_V2) void *iommu; #endif }; diff --git a/arch/sw_64/include/asm/pci.h b/arch/sw_64/include/asm/pci.h index d826cec690c8..37d82d84449d 100644 --- a/arch/sw_64/include/asm/pci.h +++ b/arch/sw_64/include/asm/pci.h @@ -95,7 +95,7 @@ extern void __init setup_chip_pci_ops(void); #define setup_chip_pci_ops() do { } while (0) #endif -#ifdef CONFIG_SUNWAY_IOMMU +#if defined(CONFIG_SUNWAY_IOMMU) || defined(CONFIG_SUNWAY_IOMMU_V2) extern struct syscore_ops iommu_cpu_syscore_ops; #endif diff --git a/drivers/iommu/sw64/Kconfig b/drivers/iommu/sw64/Kconfig index a313c6e2d11b..39b0417ae98a 100644 --- a/drivers/iommu/sw64/Kconfig +++ b/drivers/iommu/sw64/Kconfig @@ -7,3 +7,14 @@ config SUNWAY_IOMMU depends on SW64 && PCI help Support for IOMMU on SW64 platform. + +# SW64 IOMMU V2 SUPPORT +config SUNWAY_IOMMU_V2 + bool "Sunway IOMMU V2 Support" + select IOMMU_API + select IOMMU_IOVA + depends on SW64 && PCI + depends on !SUNWAY_IOMMU + help + Support for IOMMU V2 on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". diff --git a/drivers/iommu/sw64/Makefile b/drivers/iommu/sw64/Makefile index e23dbd40a74d..da7a604f253c 100644 --- a/drivers/iommu/sw64/Makefile +++ b/drivers/iommu/sw64/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_SUNWAY_IOMMU) += sunway_iommu.o +obj-$(CONFIG_SUNWAY_IOMMU_V2) += sunway_iommu_v2.o diff --git a/drivers/iommu/sw64/sunway_iommu.c b/drivers/iommu/sw64/sunway_iommu.c index 86920f88faac..89e56e4b9006 100644 --- a/drivers/iommu/sw64/sunway_iommu.c +++ b/drivers/iommu/sw64/sunway_iommu.c @@ -38,6 +38,11 @@ #define SW64_DMA_LIMIT (0xe0000000 - 1) #define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) +#define SW64_IOMMU_LEVEL1_OFFSET 0x1ff +#define SW64_IOMMU_LEVEL2_OFFSET 0x3ff + +#define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ +#define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ #define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) | ((1ULL) << PAGE_8M_SHIFT)) #define IDENTMAP_ALL ((1U) << 0) diff --git a/drivers/iommu/sw64/sunway_iommu.h b/drivers/iommu/sw64/sunway_iommu.h index 52d6452fa14c..b9cb68351cad 100644 --- a/drivers/iommu/sw64/sunway_iommu.h +++ b/drivers/iommu/sw64/sunway_iommu.h @@ -60,11 +60,12 @@ struct sunway_iommu_group { }; #define SW64_IOMMU_ENTRY_VALID ((1UL) << 63) +#define SW64_PTE_LAST_MASK ((1UL) << 8) /*last stage valid*/ #define SW64_DMA_START 0x1000000 -#define SW64_IOMMU_GRN_8K ((0UL) << 4) /* page size as 8KB */ -#define SW64_IOMMU_GRN_8M ((0x2UL) << 4) /* page size as 8MB */ #define SW64_PTE_GRN_MASK ((0x3UL) << 4) #define PAGE_8M_SHIFT 23 +#define PAGE_512M_SHIFT 29 +#define PAGE_8G_SHIFT 33 #define SW64_IOMMU_ENABLE 3 #define SW64_IOMMU_DISABLE 0 #define SW64_IOMMU_LEVEL1_OFFSET 0x1ff @@ -75,3 +76,5 @@ struct sunway_iommu_group { #define PAGE_SHIFT_IOMMU 18 #define PAGE_SIZE_IOMMU (_AC(1, UL) << PAGE_SHIFT_IOMMU) + +#define PCACHE_FLUSHPADDR_MASK 0xffffffffff80UL diff --git a/drivers/iommu/sw64/sunway_iommu_v2.c b/drivers/iommu/sw64/sunway_iommu_v2.c new file mode 100644 index 000000000000..1d31f42a2b6d --- /dev/null +++ b/drivers/iommu/sw64/sunway_iommu_v2.c @@ -0,0 +1,1816 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * iommu.c: Generic sw64 IOMMU support + * + * This is designed and tested for 3231. If there are no changes in hardware + * in later chips, then it should work just as well. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sunway_iommu.h" + +#define MAX_DOMAIN_NUM 65536 +#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) +#define SW64_32BIT_DMA_LIMIT (0xe0000000 - 1) +#define SW64_64BIT_DMA_LIMIT ((1UL << 41) - 1) +#define SW64_BAR_ADDRESS (IO_BASE | PCI_BASE) + +#define SW64_IOMMU_PGSIZES (((1ULL) << PAGE_SHIFT) \ + | ((1ULL) << PAGE_8M_SHIFT) \ + | ((1ULL) << PAGE_512M_SHIFT) \ + | ((1ULL) << PAGE_8G_SHIFT)) + +#define IDENTMAP_ALL ((1U) << 0) +#define DMA_MASK64 ((1U) << 1) + +#define PTE_VALID 0x8000000000000000UL +#define LAST_STAGE 0x100UL +#define PTE_GRN_8M 0x10UL +#define PTE_GRN_512M 0x20UL +#define PTE_GRN_8G 0x30UL +#define PTE_WRITEE 0x2UL +#define PTE_READE 0x1UL +#define PTE_RWE 0x3UL +#define PTE_FLAGS_MASK 0x8000000000000133UL +#define PAGE_8G_OFFSET_MASK ((1UL << PAGE_8G_SHIFT) - 1) +#define PAGE_512M_OFFSET_MASK ((1UL << PAGE_512M_SHIFT) - 1) +#define PAGE_8M_OFFSET_MASK ((1UL << PAGE_8M_SHIFT) - 1) + +/* IOMMU Exceptional Status */ +enum exceptype { + DTE_LEVEL1 = 0x0, + DTE_LEVEL2, + PTE_LEVEL1, + PTE_LEVEL2, + PTE_LEVEL3, + UNAUTHORIZED_ACCESS, + ILLEGAL_RESPONSE, + DTE_LEVEL1_VAL, + DTE_LEVEL2_VAL, + PTE_LEVEL1_VAL, + PTE_LEVEL2_VAL, + PTE_LEVEL3_VAL, +}; + +u64 iommu_enable_cmd; /* default IOMMU boot param: 0 */ + +unsigned long *sunway_iommu_domain_bitmap; + +static DEFINE_SPINLOCK(domain_bitmap_lock); +static DEFINE_SPINLOCK(sunway_iommu_device_table_lock); +spinlock_t sunway_domain_lock; + +static LLIST_HEAD(dev_data_list); +LIST_HEAD(sunway_domain_list); + +struct dma_domain { + struct sunway_iommu_domain sdomain; + struct iova_domain iovad; +}; +const struct iommu_ops sunway_iommu_ops; +static const struct dma_map_ops sunway_dma_ops; + +struct pci_controller *get_hose_from_domain(struct sunway_iommu_domain *sdomain) +{ + struct pci_controller *hose = NULL; + struct sunway_iommu *iommu; + + if (!sdomain) + return NULL; + + iommu = sdomain->iommu; + + if (!iommu) + return NULL; + + hose = iommu->hose_pt; + + return hose; +} + + +/* flush helpers */ +static void piu_flush_all(struct pci_controller *hose) +{ + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); +} + +struct sunway_iommu *get_iommu_from_device(struct device *dev) +{ + struct sunway_iommu *iommu; + struct pci_controller *hose; + + hose = to_pci_dev(dev)->sysdata; + iommu = hose->pci_iommu; + + return iommu; +} + +void domain_flush_all(struct sunway_iommu_domain *sdomain) +{ + struct pci_controller *hose; + + hose = get_hose_from_domain(sdomain); + if (!hose) + return; + + piu_flush_all(hose); +} + +void domain_flush_ptlb(struct sunway_iommu_domain *sdomain) +{ + struct pci_controller *hose; + + hose = get_hose_from_domain(sdomain); + if (!hose) + return; + + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); +} + +void dev_flush_dtlb(struct sunway_iommu_domain *sdomain, struct sunway_iommu_dev *sdev_data) +{ + struct pci_controller *hose; + u16 devid; + + hose = get_hose_from_domain(sdomain); + if (!hose) + return; + + devid = sdev_data->devid; + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, devid); +} + +/* + * This function is designed to support IOMMU code only, + * as it only provides 2 specific types of flush ops + */ +void +flush_device_tlb(struct sunway_iommu_domain *sdomain, + unsigned long flush_addr, unsigned long hflush_addr) +{ + struct pci_controller *hose; + struct pci_dev *pdev; + struct sunway_iommu_dev *sdev_data; + + hose = get_hose_from_domain(sdomain); + if (!hose) + return; + + switch (hflush_addr) { + case PCACHE_FLUSHPADDR: + flush_addr = __pa(flush_addr) & PCACHE_FLUSHPADDR_MASK; + /* Set memory bar here */ + mb(); + write_piu_ior0(hose->node, hose->index, + hflush_addr, flush_addr); + break; + + case PTLB_FLUSHVADDR: + list_for_each_entry(sdev_data, &sdomain->dev_list, list) { + pdev = sdev_data->pdev; + flush_addr = (pdev->bus->number << 8) + | pdev->devfn | (flush_addr << 16); + write_piu_ior0(hose->node, hose->index, + hflush_addr, flush_addr); + } + break; + + default: + break; + } +} + +/* domain helpers */ +static struct sunway_iommu_domain *to_sunway_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct sunway_iommu_domain, domain); +} + +static struct dma_domain *to_dma_domain(struct sunway_iommu_domain *sdomain) +{ + return container_of(sdomain, struct dma_domain, sdomain); +} + +static void add_domain_to_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_add(&sdomain->list, &sunway_domain_list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void del_domain_from_list(struct sunway_iommu_domain *sdomain) +{ + unsigned long flags; + + spin_lock_irqsave(&sunway_domain_lock, flags); + list_del(&sdomain->list); + spin_unlock_irqrestore(&sunway_domain_lock, flags); +} + +static void free_pagetable(struct sunway_iommu_domain *sdomain) +{ + unsigned long *l2_pte, *l3_pte; + unsigned long l2_pte_val, l3_pte_val; + int l2_index, l3_index, ptes_one_page; + + l2_pte = sdomain->pt_root; + if (!l2_pte) + return; + + ptes_one_page = PAGE_SIZE/sizeof(unsigned long); + for (l2_index = 0; l2_index < ptes_one_page; l2_index++, l2_pte++) { + l2_pte_val = *l2_pte; + if ((l2_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l2_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l2_pte_val |= PAGE_OFFSET; + l3_pte = (unsigned long *)l2_pte_val; + for (l3_index = 0; l3_index < ptes_one_page; l3_index++, l3_pte++) { + l3_pte_val = *l3_pte; + if ((l3_pte_val & SW64_IOMMU_ENTRY_VALID) == 0) + continue; + + l3_pte_val &= ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK; + l3_pte_val |= PAGE_OFFSET; + free_page(l3_pte_val); + } + free_page(l2_pte_val); + } + + free_page((unsigned long)sdomain->pt_root); +} + +static void domain_id_free(int id) +{ + spin_lock(&domain_bitmap_lock); + if (id > 0) + __clear_bit(id, sunway_iommu_domain_bitmap); + spin_unlock(&domain_bitmap_lock); +} + +static void dma_domain_free(struct dma_domain *dma_dom) +{ + if (!dma_dom) + return; + + del_domain_from_list(&dma_dom->sdomain); + put_iova_domain(&dma_dom->iovad); + free_pagetable(&dma_dom->sdomain); + if (dma_dom->sdomain.id) + domain_id_free(dma_dom->sdomain.id); + + kfree(dma_dom); +} + +static void sunway_domain_free(struct sunway_iommu_domain *sdomain) +{ + if (!sdomain) + return; + + del_domain_from_list(sdomain); + if (sdomain->id) + domain_id_free(sdomain->id); + + kfree(sdomain); +} + +static u16 sunway_domain_id_alloc(void) +{ + int id; + + spin_lock(&domain_bitmap_lock); + id = find_first_zero_bit(sunway_iommu_domain_bitmap, MAX_DOMAIN_NUM); + if (id > 0 && id < MAX_DOMAIN_NUM) + __set_bit(id, sunway_iommu_domain_bitmap); + else + id = 0; + spin_unlock(&domain_bitmap_lock); + + return id; +} + +static int sunway_domain_init(struct sunway_iommu_domain *sdomain) +{ + spin_lock_init(&sdomain->lock); + mutex_init(&sdomain->api_lock); + sdomain->id = sunway_domain_id_alloc(); + if (!sdomain->id) + return -ENOMEM; + INIT_LIST_HEAD(&sdomain->dev_list); + + return 1; +} + +static struct sunway_iommu_domain *sunway_domain_alloc(void) +{ + struct sunway_iommu_domain *sdomain; + + sdomain = kzalloc(sizeof(struct sunway_iommu_domain), GFP_KERNEL); + if (!sdomain) + return NULL; + + if (!sunway_domain_init(sdomain)) { + kfree(sdomain); + return NULL; + } + + add_domain_to_list(sdomain); + return sdomain; +} + +static struct dma_domain *dma_domain_alloc(void) +{ + struct dma_domain *dma_dom; + struct page; + + dma_dom = kzalloc(sizeof(struct dma_domain), GFP_KERNEL); + if (!dma_dom) + return NULL; + + sunway_domain_init(&dma_dom->sdomain); + dma_dom->sdomain.type = IOMMU_DOMAIN_DMA; + init_iova_domain(&dma_dom->iovad, PAGE_SIZE, IOVA_PFN(SW64_DMA_START)); + reserve_iova(&dma_dom->iovad, (0xe0000000UL >> PAGE_SHIFT), (0x100000000UL >> PAGE_SHIFT)); + + add_domain_to_list(&dma_dom->sdomain); + + return dma_dom; +} + +static void device_flush_all(struct sunway_iommu_dev *sdata) +{ + struct pci_controller *hose = sdata->pdev->sysdata; + + if (hose == NULL) + return; + + write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PTLB_FLUSHDEV, sdata->devid); + write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHDEV, sdata->devid); +} + +/* iommu_ops device attach/unattach helpers */ +static void +set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct page *dt_page, *pt_page; + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_base, dte_l2_val; + + pdev = sdev->pdev; + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return; + + sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + iommu = sdomain->iommu; + sdev->iommu = iommu; + dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); + dte_l1_val = *dte_l1; + + if (!dte_l1_val) { + /* Alloc a new level-2 device table page */ + dt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, + get_order(PAGE_SIZE)); + + WARN_ON(!dt_page); + dte_l2_base = (unsigned long)page_address(dt_page); + dte_l1_val = (__pa(dte_l2_base) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + *dte_l1 = dte_l1_val; + } + + if (!sdomain->pt_root) { + pt_page = alloc_pages_node(iommu->node, GFP_KERNEL | __GFP_ZERO, 0); + WARN_ON(!pt_page); + sdomain->pt_root = page_address(pt_page); + } + + dte_l2 = __va(dte_l1_val & ~(SW64_IOMMU_ENTRY_VALID) & PAGE_MASK) + (pdev->devfn << 3); + dte_l2_val = (__pa(sdomain->pt_root) & PAGE_MASK) | SW64_IOMMU_ENTRY_VALID; + if (sdomain->type == IOMMU_DOMAIN_IDENTITY) { + dte_l2_val |= 0x1; + sdev->passthrough = IDENTMAP_ALL; + } + *dte_l2 = dte_l2_val; + device_flush_all(sdev); +} + +static void +do_attach(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + sdev_data->domain = sdomain; + list_add(&sdev_data->list, &sdomain->dev_list); + + sdomain->dev_cnt++; + set_dte_entry(sdev_data, sdomain); + + pr_debug("iommu: device %d add to domain: %d\n", + sdev_data->devid, sdomain->id); +} + +static void do_detach(struct sunway_iommu_dev *sdev_data) +{ + struct sunway_iommu_domain *sdomain = sdev_data->domain; + + sdev_data->domain = NULL; + list_del(&sdev_data->list); + device_flush_all(sdev_data); + + sdomain->dev_cnt--; + pr_debug("iommu: device %d detached from domain %d\n", + sdev_data->devid, sdomain->id); +} + +static int +__attach_device(struct sunway_iommu_dev *sdev_data, struct sunway_iommu_domain *sdomain) +{ + int ret; + + spin_lock(&sdomain->lock); + ret = -EBUSY; + if (sdev_data->domain != NULL) + goto out_unlock; + + do_attach(sdev_data, sdomain); + ret = 0; + +out_unlock: + spin_unlock(&sdomain->lock); + return ret; +} + +static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) +{ + struct sunway_iommu_domain *domain; + + domain = sunway_dev_data->domain; + + spin_lock(&domain->lock); + do_detach(sunway_dev_data); + spin_unlock(&domain->lock); +} + +static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *sdev_data; + unsigned long flags; + int ret; + + sdev_data = dev->archdata.iommu; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + ret = __attach_device(sdev_data, sdomain); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + return ret; +} + +static void detach_device(struct device *dev) +{ + struct sunway_iommu_domain *sunway_domain; + struct sunway_iommu_dev *sdev_data; + unsigned long flags; + + sdev_data = dev->archdata.iommu; + sunway_domain = sdev_data->domain; + + if (WARN_ON(!sdev_data->domain)) + return; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + __detach_device(sdev_data); + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); + + if (!dev_is_pci(dev)) + return; +} + +static struct sunway_iommu_dev *search_dev_data(u16 devid) +{ + struct sunway_iommu_dev *sdev_data; + struct llist_node *node; + + if (llist_empty(&dev_data_list)) + return NULL; + + node = dev_data_list.first; + llist_for_each_entry(sdev_data, node, dev_data_list) { + if (sdev_data->devid == devid) + return sdev_data; + } + + return NULL; +} + +/* iommu device init/uninit ops */ +static int iommu_init_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev_data; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct pci_controller *hose; + + if (dev->archdata.iommu) + return 0; + + sdev_data = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); + if (!sdev_data) + return -ENOMEM; + + pdev = to_pci_dev(dev); + hose = pdev->sysdata; + llist_add(&sdev_data->dev_data_list, &dev_data_list); + sdev_data->pdev = pdev; + dev->archdata.iommu = sdev_data; + iommu = hose->pci_iommu; + iommu_device_link(&iommu->iommu, dev); + + return 0; +} + +static void init_iommu_group(struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return; + + iommu_group_put(group); +} + +static void iommu_uninit_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; + + sdev = dev->archdata.iommu; + if (!sdev) + return; + + if (sdev->domain) + detach_device(dev); + + iommu = sdev->iommu; + iommu_device_unlink(&iommu->iommu, dev); + iommu_group_remove_device(dev); + dev->dma_ops = NULL; +} + +/* dma_ops helpers*/ +static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) +{ + struct sunway_iommu_domain *sdomain; + struct iommu_domain *domain; + struct pci_dev *pdev; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return ERR_PTR(-ENODEV); + + sdev = dev->archdata.iommu; + sdomain = sdev->domain; + if (sdomain == NULL) { + domain = iommu_get_domain_for_dev(dev); + sdomain = to_sunway_domain(domain); + attach_device(dev, sdomain); + } + + if (sdomain == NULL) + return ERR_PTR(-EBUSY); + + return sdomain; +} + +/********************************************************************** + * + * Following functions describe IOMMU init ops + * + **********************************************************************/ + +static struct sunway_iommu *sunway_iommu_early_init(struct pci_controller *hose) +{ + struct sunway_iommu *iommu; + struct page *page; + unsigned long base; + + hose->pci_iommu = kzalloc(sizeof(struct sunway_iommu), GFP_KERNEL); + if (!hose->pci_iommu) + return 0; + + iommu = hose->pci_iommu; + spin_lock_init(&iommu->dt_lock); + + iommu->node = hose->node; + if (!node_online(hose->node)) + iommu->node = -1; + + page = alloc_pages_node(iommu->node, __GFP_ZERO, get_order(PAGE_SIZE)); + iommu->iommu_dtbr = page_address(page); + + iommu->hose_pt = hose; + iommu->index = hose->index; + + iommu->enabled = true; + + base = __pa(iommu->iommu_dtbr) & PAGE_MASK; + write_piu_ior0(hose->node, hose->index, DTBASEADDR, base); + + return iommu; +} + +unsigned long fetch_dte(struct sunway_iommu *iommu, unsigned long devid, + enum exceptype type) +{ + unsigned long *dte_l1, *dte_l2; + unsigned long dte_l1_val, dte_l2_val; + + if (!iommu) + return 0; + dte_l1 = iommu->iommu_dtbr + (devid >> 8); + if (type == DTE_LEVEL1) + return (unsigned long)dte_l1; + + dte_l1_val = *dte_l1; + if (type == DTE_LEVEL1_VAL) + return dte_l1_val; + + dte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + dte_l1_val |= PAGE_OFFSET; + dte_l2 = (unsigned long *)(dte_l1_val + ((devid & 0xff) << 3)); + if (type == DTE_LEVEL2) + return (unsigned long)dte_l2; + + dte_l2_val = *dte_l2; + if (type == DTE_LEVEL2_VAL) + return dte_l2_val; + + return dte_l2_val; +} + +unsigned long fetch_pte(struct sunway_iommu_domain *sdomain, dma_addr_t iova, + enum exceptype type) +{ + unsigned long iova_pfn; + unsigned long pte_l1_val, pte_l2_val, pte_l3_val; + unsigned long *pte_l1, *pte_l2, *pte_l3; + unsigned long pte_root; + unsigned long offset; + + if (!sdomain) + return -EINVAL; + + pte_root = __pa(sdomain->pt_root) & PAGE_MASK; + iova_pfn = iova >> PAGE_SHIFT; + pte_root = ((pte_root) & (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK)); + pte_root |= PAGE_OFFSET; + offset = ((iova_pfn >> 20) & SW64_IOMMU_LEVEL1_OFFSET) << 3; + pte_l1 = (unsigned long *)(pte_root + offset); + if (type == PTE_LEVEL1) + return (unsigned long)pte_l1; + + pte_l1_val = *pte_l1; + if (type == PTE_LEVEL1_VAL) + return pte_l1_val; + + pte_l1_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l1_val |= PAGE_OFFSET; + offset = ((iova_pfn >> 10) & SW64_IOMMU_LEVEL2_OFFSET) << 3; + pte_l2 = (unsigned long *)(pte_l1_val + offset); + + if (type == PTE_LEVEL2) + return (unsigned long)pte_l2; + + pte_l2_val = *pte_l2; + if (type == PTE_LEVEL2_VAL) + return pte_l2_val; + + pte_l2_val &= (~(SW64_IOMMU_ENTRY_VALID)) & (PAGE_MASK); + pte_l2_val |= PAGE_OFFSET; + offset = (iova_pfn & SW64_IOMMU_LEVEL3_OFFSET) << 3; + pte_l3 = (unsigned long *)(pte_l2_val + offset); + if (type == PTE_LEVEL3) + return (unsigned long)pte_l3; + + pte_l3_val = *pte_l3; + if (type == PTE_LEVEL3_VAL) + return pte_l3_val; + + return pte_l3_val; +} + +/* IOMMU Interrupt handle */ +irqreturn_t iommu_interrupt(int irq, void *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev; + struct sunway_iommu_domain *sdomain; + struct sunway_iommu_dev *sdev; + unsigned long iommu_status; + unsigned long type; + unsigned long devid, dva; + + iommu_status = read_piu_ior0(hose->node, hose->index, IOMMUEXCPT_STATUS); + if (!(iommu_status >> 63)) + return IRQ_NONE; + + type = (iommu_status >> 58) & 0xf; + devid = (iommu_status >> 36) & 0xffff; + dva = ((iommu_status & 0xffffffff) >> 3) << 13; + pr_info("%s, iommu_status = %#lx, devid %#lx, dva %#lx, ", + __func__, iommu_status, devid, dva); + + sdev = search_dev_data(devid); + if (sdev == NULL) { + pr_info("no such dev!!!\n"); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + + return IRQ_HANDLED; + } + + sdomain = sdev->domain; + switch (type) { + case DTE_LEVEL1: + pr_info("invalid level1 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL1_VAL)); + break; + case DTE_LEVEL2: + pr_info("invalid level2 dte, addr:%#lx, val:%#lx\n", + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2), + fetch_dte(hose->pci_iommu, devid, DTE_LEVEL2_VAL)); + break; + case PTE_LEVEL1: + pr_info("invalid level1 pte, addr: %#lx, val:%#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL1), + fetch_pte(sdomain, dva, PTE_LEVEL1_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + case PTE_LEVEL2: + pr_info("invalid level2 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL2), + fetch_pte(sdomain, dva, PTE_LEVEL2_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + + case PTE_LEVEL3: + pr_info("invalid level3 pte, addr: %#lx, val: %#lx\n", + fetch_pte(sdomain, dva, PTE_LEVEL3), + fetch_pte(sdomain, dva, PTE_LEVEL3_VAL)); + + iommu_status &= ~(1UL << 62); + write_piu_ior0(hose->node, hose->index, + IOMMUEXCPT_STATUS, iommu_status); + break; + default: + pr_info("iommu exception type %ld\n", type); + break; + } + + return IRQ_HANDLED; +} + +struct irqaction iommu_irqaction = { + .handler = iommu_interrupt, + .flags = IRQF_SHARED | IRQF_NO_THREAD, + .name = "sunway_iommu", +}; + +void sunway_enable_iommu_func(struct pci_controller *hose) +{ + unsigned int iommu_irq, err; + unsigned long iommu_conf, iommu_ctrl; + + iommu_irq = hose->int_irq; + pr_debug("%s node %ld rc %ld iommu_irq %d\n", + __func__, hose->node, hose->index, iommu_irq); + err = request_irq(iommu_irq, iommu_interrupt, + IRQF_SHARED, "sunway_iommu", hose); + if (err < 0) + pr_info("sw iommu request irq failed!\n"); + + iommu_ctrl = (1UL << 63) | (0x100UL << 10); + write_piu_ior0(hose->node, hose->index, IOMMUEXCPT_CTRL, iommu_ctrl); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + iommu_conf = iommu_conf | (0x3 << 7); + write_piu_ior0(hose->node, hose->index, PIUCONFIG0, iommu_conf); + write_piu_ior0(hose->node, hose->index, TIMEOUT_CONFIG, 0xf); + iommu_conf = read_piu_ior0(hose->node, hose->index, PIUCONFIG0); + pr_debug("SW arch configure node %ld hose-%ld iommu_conf = %#lx\n", + hose->node, hose->index, iommu_conf); +} + +static bool is_iommu_enable(struct pci_controller *hose) +{ + u64 rc_mask = 0x1; + + rc_mask <<= (8 * hose->node + hose->index); + if (iommu_enable_cmd & rc_mask) + return true; + + return false; +} + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); + +int sunway_iommu_init(struct pci_controller *hose_head) +{ + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; + int iommu_index = 0; + + sunway_iommu_domain_bitmap = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(MAX_DOMAIN_NUM / 8)); + if (sunway_iommu_domain_bitmap == NULL) + return 0; + __set_bit(0, sunway_iommu_domain_bitmap); + + /* Do the loop */ + for (hose = hose_head; hose; hose = hose->next) { + if (!is_iommu_enable(hose)) { + hose->iommu_enable = false; + continue; + } + + iommu = sunway_iommu_early_init(hose); + iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "%d", + iommu_index); + iommu_device_set_ops(&iommu->iommu, &sunway_iommu_ops); + iommu_device_register(&iommu->iommu); + iommu_index++; + sunway_enable_iommu_func(hose); + hose->iommu_enable = true; + } + + ret = iova_cache_get(); + if (ret) + return ret; + + ret = bus_set_iommu(&pci_bus_type, &sunway_iommu_ops); + if (ret) + return ret; + + for (hose = hose_head; hose; hose = hose->next) + if (hose->iommu_enable) + piu_flush_all(hose); + + return 1; +} +EXPORT_SYMBOL(sunway_iommu_init); + +/* iommu cpu syscore ops */ +static int iommu_cpu_suspend(void) +{ + return 0; +} + +static void iommu_cpu_resume(void) +{ + +} + +struct syscore_ops iommu_cpu_syscore_ops = { + .suspend = iommu_cpu_suspend, + .resume = iommu_cpu_resume, +}; + +/******************************************************************************* + * + * DMA OPS Functions + * + ******************************************************************************/ + +static unsigned long +sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, + unsigned long iova, unsigned long page_size) +{ + unsigned long offset, iova_pfn; + unsigned long *pte_base, *pte; + unsigned long grn; + int level, current_level; + int tmp = 1; + + pr_debug("%s iova %#lx, page_size %#lx\n", __func__, iova, page_size); + BUG_ON(!is_power_of_2(page_size)); + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + level = 2; + grn = PTE_GRN_8M; + break; + default: + level = 3; + break; + } + + pte_base = sunway_domain->pt_root; + iova_pfn = iova >> PAGE_SHIFT; + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + if (current_level == level) { + if (grn == PTE_GRN_512M) { + int i; + + for (i = 0; i < 64; i++) { + *(pte + i) = 0; + flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + } + + } else { + *pte = 0; + flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + } + flush_device_tlb(sunway_domain, (iova >> PAGE_SHIFT), PTLB_FLUSHVADDR); + break; + } + + pte_base = (unsigned long *)((*pte & (~PTE_FLAGS_MASK)) | PAGE_OFFSET); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return page_size; +} + +int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, + unsigned long bus_addr, unsigned long paddr, + size_t page_size) +{ + struct page *page; + struct sunway_iommu *iommu; + unsigned long iova_pfn, pte_val; + unsigned long *pte_base, *pte; + unsigned long offset, grn = 0; + int level = 0, current_level; + int tmp = 1; + + iommu = sunway_domain->iommu; + if (!iommu) + return -1; + iova_pfn = bus_addr >> PAGE_SHIFT; + pte_base = sunway_domain->pt_root; + + switch (page_size) { + case (1UL << 33): + level = 1; + grn = PTE_GRN_8G; + break; + case (1UL << 29): + level = 2; + grn = PTE_GRN_512M; + break; + case (1UL << 23): + grn = PTE_GRN_8M; + level = 2; + break; + default: + level = 3; + break; + } + + offset = (iova_pfn >> 20) & 0x1ff; + current_level = 1; + while (current_level <= level) { + pte = &pte_base[offset]; + + if (!(*pte) || (current_level == level)) { + pte_val = PTE_VALID | PTE_RWE | grn; + if (current_level == level) { + *(volatile u64 *)(pte) = 0; + pte_val |= ((paddr & PAGE_MASK) | LAST_STAGE); + } else { + page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); + if (!page) { + pr_err("Allocating level%d page table pages failed.\n", (level + 1)); + return -ENOMEM; + } + + pte_val |= (page_to_phys(page) & PAGE_MASK); + } + + if ((grn == PTE_GRN_512M) && (current_level == 2)) { + int i; + + for (i = 0; i < 64; i++) { + cmpxchg64((volatile u64 *)(pte + i), 0UL, pte_val); + flush_device_tlb(sunway_domain, (unsigned long)(pte + i), PCACHE_FLUSHPADDR); + } + } else { + if (cmpxchg64((volatile u64 *)pte, 0UL, pte_val)) + free_page((unsigned long)page_address(page)); + else + flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + } + } + + pte_base = (unsigned long *)__va((*pte) & (~PTE_FLAGS_MASK)); + offset = (iova_pfn >> (tmp--) * 10) & 0x3ff; + current_level++; + } + + return 0; +} + +static unsigned long +sunway_alloc_iova(struct dma_domain *dma_dom, unsigned long pages, struct pci_dev *pdev) +{ + struct device *dev; + unsigned long pfn = 0; + + pages = __roundup_pow_of_two(pages); + dev = &(pdev->dev); + if (min(dev->coherent_dma_mask, *dev->dma_mask) == DMA_BIT_MASK(32)) { + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_32BIT_DMA_LIMIT), true); + } else { + /* IOVA boundary should be 16M ~ 3.5G */ + pfn = alloc_iova_fast(&dma_dom->iovad, pages, + IOVA_PFN(SW64_64BIT_DMA_LIMIT), true); + } + + return (pfn << PAGE_SHIFT); +} + +static void sunway_free_iova(struct dma_domain *dma_dom, + unsigned long address, unsigned long pages) +{ + pages = __roundup_pow_of_two(pages); + address >>= PAGE_SHIFT; + + free_iova_fast(&dma_dom->iovad, address, pages); +} + +static dma_addr_t +__sunway_map_single(struct dma_domain *dma_dom, + struct pci_dev *pdev, phys_addr_t paddr, size_t size) +{ + dma_addr_t ret, address, start; + unsigned long npages, i; + + npages = iommu_num_pages(paddr, size, PAGE_SIZE); + + address = sunway_alloc_iova(dma_dom, npages, pdev); + if (!address) + return 0; + + start = address; + for (i = 0; i < npages; ++i) { + ret = sunway_iommu_map_page(&dma_dom->sdomain, start, + paddr, PAGE_SIZE); + if (ret) { + pr_info("error when map page.\n"); + goto out_unmap; + } + + start += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + address += paddr & ~PAGE_MASK; + return address; + +out_unmap: + for (--i; i >= 0; --i) { + start -= PAGE_SIZE; + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + } + + sunway_free_iova(dma_dom, address, npages); + return 0; +} + +static dma_addr_t +pci_iommu_map_single(struct pci_dev *pdev, + struct dma_domain *dma_dom, void *cpu_addr, size_t size) +{ + struct pci_controller *hose = pdev->sysdata; + unsigned long paddr; + + if (hose == NULL) { + pr_err("%s: hose does not exist!\n", __func__); + return 0; + } + + paddr = __sunway_map_single(dma_dom, pdev, __pa(cpu_addr), size); + + pr_debug("pci_alloc_consistent: %zx -> [%px,%lx] from %ps\n", + size, cpu_addr, paddr, __builtin_return_address(0)); + + return paddr; +} + +static void *sunway_alloc_coherent(struct device *dev, + size_t size, + dma_addr_t *dma_addr, gfp_t gfp, + unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct sunway_iommu_dev *sdev; + struct page *page; + void *cpu_addr; + + if (!pdev) + return NULL; + + hose = pdev->sysdata; + if (!hose) + return NULL; + + gfp &= ~GFP_DMA; + +try_again: + page = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, get_order(size)); + cpu_addr = page_address(page); + if (!cpu_addr) { + pr_info + ("pci_alloc_consistent: get_free_pages failed from %ps\n", + __builtin_return_address(0)); + + return NULL; + } + + *dma_addr = __pa(cpu_addr); + if (!(hose->iommu_enable)) + return cpu_addr; + + sdev = dev->archdata.iommu; + if (sdev->passthrough & DMA_MASK64) + return cpu_addr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return cpu_addr; + } + + __free_pages(page, get_order(size)); + set_dma_ops(dev, &swiotlb_dma_ops); + return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + *dma_addr = pci_iommu_map_single(pdev, dma_dom, cpu_addr, size); + if (*dma_addr == 0) { + free_pages((unsigned long)cpu_addr, get_order(size)); + if (gfp & GFP_DMA) + return NULL; + + gfp |= GFP_DMA; + goto try_again; + } + + return cpu_addr; +} + +static void +__sunway_unmap_single(struct dma_domain *dma_dom, dma_addr_t dma_addr, size_t size) +{ + dma_addr_t start; + unsigned long npages; + int i; + + npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); + dma_addr &= PAGE_MASK; + start = dma_addr; + + for (i = 0; i < npages; i++) { + sunway_iommu_unmap_page(&dma_dom->sdomain, start, PAGE_SIZE); + start += PAGE_SIZE; + } + + sunway_free_iova(dma_dom, dma_addr, npages); + pr_debug("pci_free_consistent: %zx -> [%llx] from %ps\n", + size, dma_addr, __builtin_return_address(0)); + +} + +static void +sunway_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_addr, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + if (!pdev) + goto out_unmap; + + hose = pdev->sysdata; + if (!hose || !(hose->iommu_enable)) + goto out_unmap; + + sdev = dev->archdata.iommu; + if (sdev->passthrough) + goto out_unmap; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); + goto out_free; + +out_unmap: + pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL); + +out_free: + pr_debug("sunway_free_consistent: [%llx,%zx] from %ps\n", + dma_addr, size, __builtin_return_address(0)); + + free_pages((unsigned long)vaddr, get_order(size)); +} + +static dma_addr_t +sunway_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + phys_addr_t paddr = page_to_phys(page) + offset; + + if (dir == PCI_DMA_NONE) + BUG(); + + if (!pdev) + return 0; + + hose = pdev->sysdata; + if (!hose || !(hose->iommu_enable)) + return paddr; + + sdev = dev->archdata.iommu; + if (sdev->passthrough & DMA_MASK64) + return paddr; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + return paddr; + } + + set_dma_ops(dev, &swiotlb_dma_ops); + return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); + } + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + return pci_iommu_map_single(pdev, dma_dom, + (char *)page_address(page) + offset, size); +} + +static void +sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pdev->sysdata; + if (hose == NULL) + return; + + if (!hose->iommu_enable) + return; + + sdev = dev->archdata.iommu; + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + __sunway_unmap_single(dma_dom, dma_addr, size); +} + +#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) +static int +sunway_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom = NULL; + struct scatterlist *sg; + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + int i, out_nents = 0; + + if (dir == PCI_DMA_NONE) + BUG(); + + if (!pdev) + return 0; + + hose = pdev->sysdata; + if (!hose) + return 0; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, i) { + BUG_ON(!sg_page(sg)); + + sg_dma_address(sg) = __pa(SG_ENT_VIRT_ADDRESS(sg)); + if (!(hose->iommu_enable)) + goto check; + + sdev = dev->archdata.iommu; + if (sdev->passthrough & DMA_MASK64) + goto check; + else if (sdev->passthrough) { + if (min(dev->coherent_dma_mask, *dev->dma_mask) > DMA_BIT_MASK(32)) { + sdev->passthrough |= DMA_MASK64; + goto check; + } + + set_dma_ops(dev, &swiotlb_dma_ops); + return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); + } + + sg_dma_address(sg) = + pci_iommu_map_single(pdev, dma_dom, + SG_ENT_VIRT_ADDRESS(sg), sg->length); +check: + if (sg_dma_address(sg) == 0) + goto error; + + sg_dma_len(sg) = sg->length; + out_nents++; + } + + return nents; + +error: + pr_warn("pci_map_sg failed:"); + pr_warn("could not allocate dma page tables\n"); + + if (out_nents) + pci_unmap_sg(pdev, sgl, out_nents, dir); + return 0; +} + +static void +sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + struct scatterlist *sg; + struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu_dev *sdev; + dma_addr_t dma_addr; + long size; + int j; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + hose = pdev->sysdata; + if (!hose->iommu_enable) + return; + + sdev = dev->archdata.iommu; + if (sdev->passthrough) + return; + + sdomain = get_sunway_domain(dev); + dma_dom = to_dma_domain(sdomain); + + for_each_sg(sgl, sg, nents, j) { + dma_addr = sg->dma_address; + size = sg->dma_length; + if (!size) + break; + + __sunway_unmap_single(dma_dom, dma_addr, size); + } +} + +static int sunway_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_addr == 0; +} + +static int sunway_supported(struct device *dev, u64 mask) +{ + if (mask < DMA_BIT_MASK(32)) + return 0; + /* + * Upstream PCI/PCIe bridges or SoC interconnects may not carry + * as many DMA address bits as the device itself supports. + */ + if (dev->bus_dma_mask && mask > dev->bus_dma_mask) + return 0; + return 1; +} + +static const struct dma_map_ops sunway_dma_ops = { + .alloc = sunway_alloc_coherent, + .free = sunway_free_coherent, + .map_sg = sunway_map_sg, + .unmap_sg = sunway_unmap_sg, + .map_page = sunway_map_page, + .unmap_page = sunway_unmap_page, + .mapping_error = sunway_mapping_error, + .dma_supported = sunway_supported, +}; + +/********************************************************************** + * + * IOMMU OPS Functions + * + **********************************************************************/ + +static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + sdomain = sunway_domain_alloc(); + if (!sdomain) { + pr_err("Allocating sunway_domain failed!\n"); + return NULL; + } + + sdomain->domain.geometry.aperture_start = 0UL; + sdomain->domain.geometry.aperture_end = ~0ULL; + sdomain->domain.geometry.force_aperture = true; + sdomain->type = IOMMU_DOMAIN_UNMANAGED; + break; + + case IOMMU_DOMAIN_DMA: + dma_dom = dma_domain_alloc(); + if (!dma_dom) { + pr_err("Failed to alloc dma domain!\n"); + return NULL; + } + + sdomain = &dma_dom->sdomain; + break; + + case IOMMU_DOMAIN_IDENTITY: + sdomain = sunway_domain_alloc(); + if (!sdomain) + return NULL; + + sdomain->type = IOMMU_DOMAIN_IDENTITY; + break; + + default: + return NULL; + } + + return &sdomain->domain; +} + +static void clean_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu_dev *entry; + unsigned long flags; + + spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); + + while (!list_empty(&sdomain->dev_list)) { + entry = list_first_entry(&sdomain->dev_list, + struct sunway_iommu_dev, list); + + BUG_ON(!entry->domain); + __detach_device(entry); + } + + spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); +} + +static void sunway_iommu_domain_free(struct iommu_domain *dom) +{ + struct sunway_iommu_domain *sdomain; + struct dma_domain *dma_dom; + + sdomain = to_sunway_domain(dom); + + if (sdomain->dev_cnt > 0) + clean_domain(sdomain); + + BUG_ON(sdomain->dev_cnt != 0); + + if (!dom) + return; + + switch (dom->type) { + case IOMMU_DOMAIN_DMA: + dma_dom = to_dma_domain(sdomain); + dma_domain_free(dma_dom); + break; + + default: + free_pagetable(sdomain); + sunway_domain_free(sdomain); + break; + } + +} + +static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + struct sunway_iommu_dev *sdev_data; + struct pci_dev *pdev; + struct pci_controller *hose; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return -EINVAL; + + hose = pdev->sysdata; + if (!hose) + return -EINVAL; + + if (!hose->iommu_enable) + return -EINVAL; + + if (!sdomain->iommu) + sdomain->iommu = hose->pci_iommu; + + sdev_data = dev->archdata.iommu; + if (!sdev_data) + return -EINVAL; + + if (sdev_data->domain) + detach_device(dev); + + ret = attach_device(dev, sdomain); + + return ret; +} + +static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) +{ + struct sunway_iommu_dev *sunway_dev_data = dev->archdata.iommu; + struct pci_dev *pdev = to_pci_dev(dev); + + if (!pdev) + return; + + if (sunway_dev_data->domain != NULL) + detach_device(dev); +} + +static phys_addr_t +sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + unsigned long paddr, grn; + unsigned long is_last; + + if (iova > SW64_BAR_ADDRESS) + return iova; + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL1_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_8G) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8G_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + is_last = paddr & SW64_PTE_LAST_MASK; + grn = paddr & SW64_PTE_GRN_MASK; + if (is_last) { + if (grn == PTE_GRN_512M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_512M_OFFSET_MASK; + return paddr; + } + + if (grn == PTE_GRN_8M) { + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_8M_OFFSET_MASK; + return paddr; + } + + return 0; + } + + paddr = fetch_pte(sdomain, iova, PTE_LEVEL3_VAL); + if ((paddr & SW64_IOMMU_ENTRY_VALID) == 0) + return 0; + + grn = paddr & SW64_PTE_GRN_MASK; + if (grn != 0) + return 0; + + paddr &= ~PTE_FLAGS_MASK; + paddr += iova & PAGE_MASK; + return paddr; +} + +static int +sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t page_size, int iommu_prot) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + int ret; + + /* + * As VFIO cannot distinguish between normal DMA request + * and pci device BAR, check should be introduced manually + * to avoid VFIO trying to map pci config space. + */ + if (iova > SW64_BAR_ADDRESS) + return 0; + + mutex_lock(&sdomain->api_lock); + ret = sunway_iommu_map_page(sdomain, iova, paddr, page_size); + mutex_unlock(&sdomain->api_lock); + + return ret; +} + +static size_t +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t page_size) +{ + struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); + size_t unmap_size; + + if (iova > SW64_BAR_ADDRESS) + return page_size; + + mutex_lock(&sdomain->api_lock); + unmap_size = sunway_iommu_unmap_page(sdomain, iova, page_size); + mutex_unlock(&sdomain->api_lock); + + return unmap_size; +} + +static int sunway_iommu_add_device(struct device *dev) +{ + struct pci_dev *pdev; + struct pci_controller *hose; + int ret; + + pdev = to_pci_dev(dev); + if (!pdev) + return 0; + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + return 0; + + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return 0; + + hose = pdev->sysdata; + if (!hose || !hose->iommu_enable) + return 0; + + ret = iommu_init_device(dev); + if (ret) { + pr_warn("Attention: %s %s failed, device may not work properly!\n", + dev_name(dev), __func__); + return ret; + } + + init_iommu_group(dev); + + dev->dma_ops = &sunway_dma_ops; + + return 0; +} + +static void sunway_iommu_remove_device(struct device *dev) +{ + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + if (!pdev) + return; + + iommu_uninit_device(dev); +} + +static bool sunway_iommu_capable(enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_INTR_REMAP: + return true; + default: + return false; + } +} + +const struct iommu_ops sunway_iommu_ops = { + .capable = sunway_iommu_capable, + .domain_alloc = sunway_iommu_domain_alloc, + .domain_free = sunway_iommu_domain_free, + .attach_dev = sunway_iommu_attach_device, + .detach_dev = sunway_iommu_detach_device, + .map = sunway_iommu_map, + .unmap = sunway_iommu_unmap, + .iova_to_phys = sunway_iommu_iova_to_phys, + .add_device = sunway_iommu_add_device, + .remove_device = sunway_iommu_remove_device, + .device_group = pci_device_group, + .pgsize_bitmap = SW64_IOMMU_PGSIZES, +}; + +/***************************************************************************** + * + * Boot param handle + * + *****************************************************************************/ +static int __init iommu_enable_setup(char *str) +{ + int ret; + unsigned long rc_bitmap = 0xffffffffUL; + + ret = kstrtoul(str, 16, &rc_bitmap); + iommu_enable_cmd = rc_bitmap; + + return ret; +} +__setup("iommu_enable=", iommu_enable_setup); -- Gitee From b3e64100d4667c61cd986cf2a5bbd93669fecdf1 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 14:50:10 +0800 Subject: [PATCH 093/150] sw64: kvm: add C4 vcpu support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Modify headers to support C4 vcpu structure. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/vcpu.h | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/arch/sw_64/include/asm/vcpu.h b/arch/sw_64/include/asm/vcpu.h index 2039b1cf9f22..c4e3caacbc70 100644 --- a/arch/sw_64/include/asm/vcpu.h +++ b/arch/sw_64/include/asm/vcpu.h @@ -4,6 +4,8 @@ #ifndef __ASSEMBLY__ +#ifdef CONFIG_SUBARCH_C3B + struct vcpucb { unsigned long go_flag; unsigned long pcbb; @@ -53,5 +55,52 @@ struct vcpucb { unsigned long reserved[3]; }; +#else + +struct vcpucb { + unsigned long ktp; + unsigned long pcbb; + unsigned long ksp; + unsigned long usp; + unsigned long kgp; + unsigned long ent_arith; + unsigned long ent_if; + unsigned long ent_int; + unsigned long ent_mm; + unsigned long ent_sys; + unsigned long ent_una; + unsigned long stack_pc; + unsigned long new_a0; + unsigned long new_a1; + unsigned long new_a2; + unsigned long soft_cid; + unsigned long csr_save; + unsigned long wakeup_magic; + unsigned long host_vcpucb; + unsigned long upcr; + unsigned long vpcr; + unsigned long dtb_vpcr; + unsigned long dtb_upcr; + unsigned long guest_ksp; + unsigned long guest_usp; + unsigned long vcpu_irq_disabled; + unsigned long vcpu_irq; + unsigned long ptbr_usr; + unsigned long ptbr_sys; + unsigned long soft_tid; + unsigned long int_stat0; + unsigned long int_stat1; + unsigned long int_stat2; + unsigned long int_stat3; + unsigned long reset_entry; + unsigned long pvcpu; + unsigned long exit_reason; + unsigned long ipaddr; + unsigned long vcpu_pc_save; + unsigned long shtclock_offset; + unsigned long reserved[8]; +}; +#endif + #endif /* __ASSEMBLY__ */ #endif /* _ASM_SW64_VCPU_H */ -- Gitee From 6f26b55a8bb8742649d5db4447e87f4a454103cf Mon Sep 17 00:00:00 2001 From: Hang Yiyi Date: Thu, 7 Sep 2023 17:43:59 +0800 Subject: [PATCH 094/150] sw64: kvm: add config guard for C3B implementation Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Signed-off-by: Hang Yiyi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 2 +- arch/sw_64/kernel/setup.c | 8 ++++++++ arch/sw_64/kvm/vmem.c | 6 ++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index b0ff1ae82b54..c35063a30616 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -44,7 +44,7 @@ obj-y += pci-noop.o endif ifdef CONFIG_KVM -obj-y += kvm_cma.o +obj-$(CONFIG_SUBARCH_C3B) += kvm_cma.o endif # Core logic support diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 39039ae377b4..b1d9d70a79c2 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -49,6 +49,7 @@ EXPORT_SYMBOL(__cpu_to_rcid); DEFINE_PER_CPU(unsigned long, hard_node_id) = { 0 }; +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) struct cma *sw64_kvm_cma; EXPORT_SYMBOL(sw64_kvm_cma); @@ -59,6 +60,7 @@ static phys_addr_t kvm_mem_base; struct gen_pool *sw64_kvm_pool; EXPORT_SYMBOL(sw64_kvm_pool); #endif +#endif static inline int phys_addr_valid(unsigned long addr) { @@ -734,6 +736,7 @@ static void __init reserve_mem_for_initrd(void) } #endif /* CONFIG_BLK_DEV_INITRD */ +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) static int __init early_kvm_reserved_mem(char *p) { @@ -756,6 +759,7 @@ void __init sw64_kvm_reserve(void) PAGE_SIZE, 0, "sw64_kvm_cma", &sw64_kvm_cma); } #endif +#endif void __init setup_arch(char **cmdline_p) @@ -823,8 +827,10 @@ setup_arch(char **cmdline_p) reserve_crashkernel(); /* Reserve large chunks of memory for use by CMA for KVM. */ +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) sw64_kvm_reserve(); +#endif #endif sw64_numa_init(); @@ -1003,6 +1009,7 @@ static int __init sw64_of_init(void) core_initcall(sw64_of_init); #endif +#ifdef CONFIG_SUBARCH_C3B #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) static int __init sw64_kvm_pool_init(void) { @@ -1047,3 +1054,4 @@ static int __init sw64_kvm_pool_init(void) } core_initcall_sync(sw64_kvm_pool_init); #endif +#endif diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index efac24ce5ede..f1afc3a0d602 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -70,6 +70,7 @@ static void vmem_vm_close(struct vm_area_struct *vma) size_t size; struct vmem_info *info; +#ifdef CONFIG_SUBARCH_C3B info = vma->vm_private_data; addr = info->start; size = round_up(info->size, 8 << 20); @@ -82,6 +83,7 @@ static void vmem_vm_close(struct vm_area_struct *vma) } kfree(info); } +#endif } const struct vm_operations_struct vmem_vm_ops = { @@ -165,7 +167,7 @@ static struct miscdevice vmem_dev = { .fops = &vmem_fops, }; -static int __init vmem_init(void) +int __init vmem_init(void) { int err; @@ -177,7 +179,7 @@ static int __init vmem_init(void) return 0; } -static void vmem_exit(void) +void vmem_exit(void) { misc_deregister(&vmem_dev); } -- Gitee From 8b918b3d87e1b3a0357f7c0181ead65b007889e4 Mon Sep 17 00:00:00 2001 From: Hang Yiyi Date: Thu, 7 Sep 2023 17:46:35 +0800 Subject: [PATCH 095/150] sw64: kvm: add support for Core4 memory virtualization Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Signed-off-by: Hang Yiyi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_asm.h | 5 + arch/sw_64/include/asm/kvm_mmu.h | 131 +++ arch/sw_64/kvm/Kconfig | 2 + arch/sw_64/kvm/handle_exit.c | 8 +- arch/sw_64/kvm/mmu.c | 1561 ++++++++++++++++++++++++++++++ arch/sw_64/mm/hugetlbpage_c4.c | 1 + 6 files changed, 1707 insertions(+), 1 deletion(-) create mode 100644 arch/sw_64/include/asm/kvm_mmu.h create mode 100644 arch/sw_64/kvm/mmu.c diff --git a/arch/sw_64/include/asm/kvm_asm.h b/arch/sw_64/include/asm/kvm_asm.h index 67b4ff594074..2e86c3745c9f 100644 --- a/arch/sw_64/include/asm/kvm_asm.h +++ b/arch/sw_64/include/asm/kvm_asm.h @@ -12,6 +12,7 @@ #define SW64_KVM_EXIT_IPI 14 #define SW64_KVM_EXIT_STOP 16 #define SW64_KVM_EXIT_RESTART 17 +#define SW64_KVM_EXIT_APT_FAULT 18 #define SW64_KVM_EXIT_FATAL_ERROR 22 #define SW64_KVM_EXIT_DEBUG 24 @@ -28,8 +29,12 @@ {14, "IPI" }, \ {16, "STOP" }, \ {17, "RESTART" }, \ + {18, "APT_FAULT" }, \ {22, "FATAL_ERROR" }, \ {23, "MEMHOTPLUG" }, \ {24, "DEBUG" } + +#include + #endif /* _ASM_SW64_KVM_ASM_H */ diff --git a/arch/sw_64/include/asm/kvm_mmu.h b/arch/sw_64/include/asm/kvm_mmu.h new file mode 100644 index 000000000000..f4493de934ba --- /dev/null +++ b/arch/sw_64/include/asm/kvm_mmu.h @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SW64_KVM_MMU_H +#define _ASM_SW64_KVM_MMU_H + +#define AF_ACCESS_TYPE_SHIFT 55 +#define AF_INV_LEVEL_SHIFT 53 +#define AF_FAULT_STATUS_SHIFT 48 + +#define AF_ACCESS_TYPE_MASK 0x3 +#define AF_INV_LEVEL_MASK 0x3 +#define AF_FAULT_STATUS_MASK 0x1f +#define AF_ENTRY_ADDR_MASK ((0x1UL << AF_FAULT_STATUS_SHIFT) - 1) + +/* access type defination */ +#define AF_READ_ACCESS_TYPE 0x1 +#define AF_WRITE_ACCESS_TYPE 0x2 +#define AF_EXEC_ACCESS_TYPE 0x3 + +/* invalid page level */ +#define AF_INV_LEVEL_1 0 +#define AF_INV_LEVEL_2 1 +#define AF_INV_LEVEL_3 2 +#define AF_INV_LEVEL_4 3 + +/* fault status */ +#define AF_STATUS_MISCONFIG 0x1 +#define AF_STATUS_FOR 0x2 +#define AF_STATUS_FOW 0x4 +#define AF_STATUS_FOE 0x8 +#define AF_STATUS_INV 0x10 + +#define KVM_MMU_CACHE_MIN_PAGES 2 + +static inline void kvm_set_aptpte_readonly(pte_t *pte) +{ + pte_val(*pte) |= _PAGE_FOW; +} + +static inline bool kvm_aptpte_readonly(pte_t *pte) +{ + return (pte_val(*pte) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpmd_readonly(pmd_t *pmd) +{ + pmd_val(*pmd) |= _PAGE_FOW; +} + +static inline bool kvm_aptpmd_readonly(pmd_t *pmd) +{ + return (pmd_val(*pmd) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline void kvm_set_aptpud_readonly(pud_t *pud) +{ + pud_val(*pud) |= _PAGE_FOW; +} + +static inline bool kvm_aptpud_readonly(pud_t *pud) +{ + return (pud_val(*pud) & _PAGE_FOW) == _PAGE_FOW; +} + +static inline pte_t kvm_pte_mkwrite(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOW; + return pte; +} + +static inline pte_t kvm_pte_mkexec(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_FOE; + return pte; +} + +static inline bool kvm_pte_exec(pte_t *pte) +{ + return !(pte_val(*pte) & _PAGE_FOE); +} + +static inline pmd_t kvm_pmd_mkwrite(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOW; + return pmd; +} + +static inline pmd_t kvm_pmd_mkexec(pmd_t pmd) +{ + pmd_val(pmd) &= ~_PAGE_FOE; + return pmd; +} + +static inline bool kvm_pmd_exec(pmd_t *pmd) +{ + return !(pmd_val(*pmd) & _PAGE_FOE); +} + +static inline pud_t kvm_pud_mkwrite(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOW; + return pud; +} + +static inline pud_t kvm_pud_mkexec(pud_t pud) +{ + pud_val(pud) &= ~_PAGE_FOE; + return pud; +} + +static inline bool kvm_pud_exec(pud_t *pud) +{ + return !(pud_val(*pud) & _PAGE_FOE); +} + +void kvm_core4_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change); +void kvm_core4_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +void kvm_core4_flush_shadow_all(struct kvm *kvm); +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +void kvm_handle_apt_fault(struct kvm_vcpu *vcpu); +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm); +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot); +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run); +void apt_unmap_vm(struct kvm *kvm); +#endif /* _ASM_SW64_KVM_MMU_H */ diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index 8077ea452765..9d75c8733ee0 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -27,6 +27,8 @@ config KVM select HAVE_KVM_IRQFD select HAVE_KVM_MSI select KVM_VFIO + select MMU_NOTIFIER + select KVM_GENERIC_DIRTYLOG_READ_PROTECT select TUN select GENERIC_ALLOCATOR select KVM_GENERIC_DIRTYLOG_READ_PROTECT diff --git a/arch/sw_64/kvm/handle_exit.c b/arch/sw_64/kvm/handle_exit.c index 7ff826d0ed09..08c7f283c14e 100644 --- a/arch/sw_64/kvm/handle_exit.c +++ b/arch/sw_64/kvm/handle_exit.c @@ -8,12 +8,14 @@ #include #include #include +#include #include +#include int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index, struct hcall_args *hargs) { - gfn_t gfn; + gfn_t gfn __maybe_unused; switch (exception_index) { case SW64_KVM_EXIT_IO: @@ -66,6 +68,10 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, vcpu->stat.memhotplug_exits++; vcpu_mem_hotplug(vcpu, hargs->arg0); return 1; +#endif +#ifdef CONFIG_SUBARCH_C4 + case SW64_KVM_EXIT_APT_FAULT: + return kvm_handle_guest_abort(vcpu, run); #endif case SW64_KVM_EXIT_FATAL_ERROR: vcpu->stat.fatal_error_exits++; diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c new file mode 100644 index 000000000000..b0d15a1e549f --- /dev/null +++ b/arch/sw_64/kvm/mmu.c @@ -0,0 +1,1561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 - os kernal + * Author: lff + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#define KVM_APT_FLAG_LOGGING_ACTIVE (1UL << 1) + +static bool memslot_is_logging(struct kvm_memory_slot *memslot) +{ + return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); +} + +/* + * Return values of kvm_handle_mmio_page_fault and mmu.page_fault: + * RET_AF_RETRY: let CPU fault again on the address. + * RET_AF_EMULATE: mmio page fault, emulate the instruction directly. + * + * For kvm_handle_mmio_page_fault only: + * RET_AF_INVALID: the spte is invalid, let the real page fault path update it. + */ +enum { + RET_AF_RETRY = 0, + RET_AF_EMULATE = 1, + RET_AF_INVALID = 2, +}; + +/** + * apt_dissolve_pmd() - clear and flush huge PMD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pmd: pmd pointer for IPA + * + * Function clears a PMD entry, flushes TLBs. + */ +static void apt_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) +{ + int i; + + if (!pmd_trans_huge(*pmd)) + return; + + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); +} + +/** + * apt_dissolve_pud() - clear and flush huge PUD entry + * @kvm: pointer to kvm structure. + * @addr: IPA + * @pud: pud pointer for IPA + * + * Function clears a PUD entry, flushes TLBs. + */ +static void apt_dissolve_pud(struct kvm *kvm, phys_addr_t addr, pud_t *pudp) +{ + if (!pud_huge(*pudp)) + return; + + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pudp)); +} + +static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, + int min, int max) +{ + void *page; + + BUG_ON(max > KVM_NR_MEM_OBJS); + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < max) { + page = (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!page) + return -ENOMEM; + cache->objects[cache->nobjs++] = page; + } + return 0; +} + +static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + free_page((unsigned long)mc->objects[--mc->nobjs]); +} + +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); +} + +static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc) +{ + void *p; + + BUG_ON(!mc || !mc->nobjs); + p = mc->objects[--mc->nobjs]; + return p; +} + +static void unmap_apt_ptes(struct kvm *kvm, pmd_t *pmd, + phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte, *start_pte; + struct page *ptr_page; + + start_pte = pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + put_page(virt_to_page(pte)); + } + } while (pte++, addr += PAGE_SIZE, addr != end); + + ptr_page = virt_to_page(start_pte); + if (page_count(ptr_page) == 1) { + pte_t *pte_table = pte_offset_kernel(pmd, 0); + + pmd_clear(pmd); + free_page((unsigned long)pte_table); + put_page(virt_to_page(pmd)); + } +} + +static void unmap_apt_pmds(struct kvm *kvm, pud_t *pud, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pmd_t *pmd, *start_pmd; + struct page *ptr_page; + int i; + + start_pmd = pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (pmd_trans_cont(*pmd)) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pmd)); + } else { + unmap_apt_ptes(kvm, pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pmd); + if (page_count(ptr_page) == 1) { + pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0UL); + pud_clear(pud); + free_page((unsigned long)pmd_table); + put_page(virt_to_page(pud)); + } +} + +static void unmap_apt_puds(struct kvm *kvm, p4d_t *p4d, + phys_addr_t addr, phys_addr_t end) +{ + phys_addr_t next; + pud_t *pud, *start_pud; + struct page *ptr_page; + + start_pud = pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + pud_clear(pud); + /* Do we need flush tlb???? edited by lff */ + kvm_flush_remote_tlbs(kvm); + put_page(virt_to_page(pud)); + } else { + unmap_apt_pmds(kvm, pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); + + ptr_page = virt_to_page(start_pud); + if (page_count(ptr_page) == 1) { + pud_t *pud_table __maybe_unused = pud_offset(p4d, 0UL); + p4d_clear(p4d); + kvm_flush_remote_tlbs(kvm); + free_page((unsigned long)pud_table); + put_page(virt_to_page(p4d)); + } +} + +/** + * unmap_apt_range -- Clear addtional page table entries to unmap a range + * @kvm: The VM pointer + * @start: The intermediate physical base address of the range to unmap + * @size: The size of the area to unmap + * + * Clear a range of apt mappings, lowering the various ref-counts. Must + * be called while holding mmu_lock (unless for freeing the apt pgd before + * destroying the VM), otherwise another faulting VCPU may come in and mess + * with things behind our backs. + */ +static void unmap_apt_range(struct kvm *kvm, phys_addr_t start, u64 size) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t addr = start, end = start + size; + phys_addr_t next; + + assert_spin_locked(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + do { + /* + * Make sure the page table is still active, as another thread + * could have possibly freed the page table, while we released + * the lock. + */ + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (!p4d_none(*p4d)) + unmap_apt_puds(kvm, p4d, addr, next); + /* + * If the range is too large, release the kvm->mmu_lock + * to prevent starvation and lockup detector warnings. + */ + if (next != end) + cond_resched_lock(&kvm->mmu_lock); + } while (pgd++, addr = next, addr != end); +} + +static void apt_unmap_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + hva_t hva = memslot->userspace_addr; + phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t size = PAGE_SIZE * memslot->npages; + hva_t reg_end = hva + size; + + /* + * A memory region could potentially cover multiple VMAs, and any holes + * between them, so iterate over all of them to find out if we should + * unmap any of them. + * + * +--------------------------------------------+ + * +---------------+----------------+ +----------------+ + * | : VMA 1 | VMA 2 | | VMA 3 : | + * +---------------+----------------+ +----------------+ + * | memory region | + * +--------------------------------------------+ + */ + do { + struct vm_area_struct *vma = find_vma(current->mm, hva); + hva_t vm_start, vm_end; + + if (!vma || vma->vm_start >= reg_end) + break; + + /* + * Take the intersection of this VMA with the memory region + */ + vm_start = max(hva, vma->vm_start); + vm_end = min(reg_end, vma->vm_end); + + if (!(vma->vm_flags & VM_PFNMAP)) { + gpa_t gpa = addr + (vm_start - memslot->userspace_addr); + + unmap_apt_range(kvm, gpa, vm_end - vm_start); + } + hva = vm_end; + } while (hva < reg_end); +} + +/** + * apt_unmap_vm - Unmap Additional Stage RAM mappings + * @kvm: The struct kvm pointer + * + * Go through the memregions and unmap any reguler RAM + * backing memory already mapped to the VM. + */ +void apt_unmap_vm(struct kvm *kvm) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int idx; + + idx = srcu_read_lock(&kvm->srcu); + down_read(¤t->mm->mmap_lock); + spin_lock(&kvm->mmu_lock); + + slots = kvm_memslots(kvm); + kvm_for_each_memslot(memslot, slots) + apt_unmap_memslot(kvm, memslot); + spin_unlock(&kvm->mmu_lock); + up_read(¤t->mm->mmap_lock); + srcu_read_unlock(&kvm->srcu, idx); +} + +static pud_t *apt_get_pud(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr) +{ + p4d_t *p4d; + pud_t *pud; + + pgd += pgd_index(addr); + if (pgd_none(*pgd)) { + /* Not used on SW64 yet */ + VM_BUG_ON(pgd); + return NULL; + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + if (!cache) + return NULL; + pud = mmu_memory_cache_alloc(cache); + p4d_populate(NULL, p4d, pud); + get_page(virt_to_page(p4d)); + } + return pud_offset(p4d, addr); +} + +static pmd_t *apt_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, unsigned long sz) +{ + pud_t *pud; + pmd_t *pmd; + + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud || pud_huge(*pud)) + return NULL; + + if (pud_none(*pud)) { + if (!cache) + return NULL; + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + if (sz == CONT_PMD_SIZE) + addr &= CONT_PMD_MASK; + + return pmd_offset(pud, addr); +} + +static bool kvm_is_write_fault(unsigned long access_type) +{ + if (access_type == AF_WRITE_ACCESS_TYPE) + return true; + + return false; +} + +static bool kvm_is_exec_fault(unsigned long access_type) +{ + if (access_type == AF_EXEC_ACCESS_TYPE) + return true; + + return false; +} +/** + * apt_wp_ptes - write protect PMD range + * @pmd: pointer to pmd entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) +{ + pte_t *pte; + + pte = pte_offset_kernel(pmd, addr); + do { + if (!pte_none(*pte)) { + if (!kvm_aptpte_readonly(pte)) + kvm_set_aptpte_readonly(pte); + } + } while (pte++, addr += PAGE_SIZE, addr != end); +} + +/** + * apt_wp_pmds - write protect PUD range + * @pud: pointer to pud entry + * @addr: range start address + * @end: range end address + */ +static void apt_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) +{ + pmd_t *pmd; + phys_addr_t next; + + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + if (!pmd_none(*pmd)) { + if (pmd_trans_huge(*pmd)) { + if (!kvm_aptpmd_readonly(pmd)) + kvm_set_aptpmd_readonly(pmd); + } else { + apt_wp_ptes(pmd, addr, next); + } + } + } while (pmd++, addr = next, addr != end); +} + +/** + * apt_wp_puds - write protect PGD range + * @pgd: pointer to pgd entry + * @addr: range start address + * @end: range end address + * + * Process PUD entries, for a huge PUD we cause a panic. + */ +static void apt_wp_puds(p4d_t *p4d, phys_addr_t addr, phys_addr_t end) +{ + pud_t *pud; + phys_addr_t next; + + pud = pud_offset(p4d, addr); + do { + next = pud_addr_end(addr, end); + if (!pud_none(*pud)) { + if (pud_huge(*pud)) { + if (!kvm_aptpud_readonly(pud)) + kvm_set_aptpud_readonly(pud); + } else { + /* TODO:PUD not supported, revisit later if supported */ +// BUG_ON(pud_trans_huge(*pud)); + apt_wp_pmds(pud, addr, next); + } + } + } while (pud++, addr = next, addr != end); +} + +/** + * apt_wp_range() - write protect apt memory region range + * @kvm: The KVM pointer + * @addr: Start address of range + * @end: End address of range + */ +static void apt_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) +{ + pgd_t *pgd; + p4d_t *p4d; + phys_addr_t next; + + pgd = kvm->arch.pgd + pgd_index(addr); + p4d = p4d_offset(pgd, addr); + + do { + /* + * Release kvm_mmu_lock periodically if the memory region is + * large. Otherwise, we may see kernel panics with + * CONFIG_DETECT_HUNG_TASK, CONFIG_LOCKUP_DETECTOR, + * CONFIG_LOCKDEP. Additionally, holding the lock too long + * will also starve other vCPUs. We have to also make sure + * that the page tables are not freed while we released + * the lock. + */ + cond_resched_lock(&kvm->mmu_lock); + if (!READ_ONCE(kvm->arch.pgd)) + break; + next = p4d_addr_end(addr, end); + if (p4d_present(*p4d)) + apt_wp_puds(p4d, addr, next); + } while (p4d++, addr = next, addr != end); +} + +/** + * kvm_mmu_wp_memory_region() - write protect apt entries for memory slot + * @kvm: The KVM pointer + * @slot: The memory slot to write protect + * + * Called to start logging dirty pages after memory region + * KVM_MEM_LOG_DIRTY_PAGES operation is called. After this function returns + * all present PMD and PTEs are write protected in the memory region. + * Afterwards read of dirty page log can be called. + * + * Acquires kvm_mmu_lock. Called with kvm->slots_lock mutex acquired, + * serializing operations for VM memory regions. + */ +void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot) +{ + struct kvm_memslots *slots = kvm_memslots(kvm); + struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); + phys_addr_t start = memslot->base_gfn << PAGE_SHIFT; + phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); + apt_wp_range(kvm, start, end); + spin_unlock(&kvm->mmu_lock); + kvm_flush_remote_tlbs(kvm); // 需要通知其他vcpu进行tlb刷新，利用request机制 +} + +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + int cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.migration_mark = mark; +} + +void kvm_core4_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * There is no need to do this in any of the following cases: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + */ + if (change == KVM_MR_FLAGS_ONLY && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + new->flags & KVM_MEM_LOG_DIRTY_PAGES)) { + kvm_mark_migration(kvm, 1); + kvm_mmu_wp_memory_region(kvm, mem->slot); + } + /* If dirty logging has been stopped, do nothing for now. */ + if ((change != KVM_MR_DELETE) + && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } +} + +void kvm_core4_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ + gpa_t gpa = slot->base_gfn << PAGE_SHIFT; + phys_addr_t size = slot->npages << PAGE_SHIFT; + + spin_lock(&kvm->mmu_lock); +// flush_apt_tlbs(kvm); + unmap_apt_range(kvm, gpa, size); + spin_unlock(&kvm->mmu_lock); +} + +/** + * kvm_alloc_addtional_stage_pgd - allocate level-1 table for addtional stage translation. + * @kvm: The KVM struct pointer for the VM. + * + * Allocates only the addtional stage HW PGD level table(s) (can support full + * 48-bit input addresses). Clears the allocated pages. + * + * Note we don't need locking here as this is only called when the VM is + * created, which can only be done once. + */ +int kvm_alloc_addtional_stage_pgd(struct kvm *kvm) +{ + pgd_t *pgd; + + if (kvm->arch.pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + /* Allocate the HW PGD, making sure that each page gets its own refcount */ + pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!pgd) + return -ENOMEM; + + kvm->arch.pgd = pgd; + return 0; +} + +/** + * kvm_free_apt_pgd - free all apt tables + * @kvm: The KVM struct pointer for the VM. + * + * Walks the level-1 page table pointed to by kvm->arch.pgd and frees all + * underlying level-2 and level-3 tables before freeing the actual level-1 table + * and setting the struct pointer to NULL. + */ +void kvm_free_apt_pgd(struct kvm *kvm) +{ + void *pgd = NULL; + + spin_lock(&kvm->mmu_lock); + if (kvm->arch.pgd) { + unmap_apt_range(kvm, 0, KVM_PHYS_SIZE); + pgd = READ_ONCE(kvm->arch.pgd); + kvm->arch.pgd = NULL; + } + spin_unlock(&kvm->mmu_lock); + + /* Free the HW pgd, one page at a time */ + if (pgd) + free_pages_exact(pgd, PAGE_SIZE); +} + +void kvm_core4_flush_shadow_all(struct kvm *kvm) +{ + kvm_free_apt_pgd(kvm); +} + +static void kvm_send_hwpoison_signal(unsigned long address, + struct vm_area_struct *vma) +{ + kernel_siginfo_t info; + + clear_siginfo(&info); + info.si_signo = SIGBUS; + info.si_errno = 0; + info.si_code = BUS_MCEERR_AR; + info.si_addr = (void __user *)address; + + if (is_vm_hugetlb_page(vma)) + info.si_addr_lsb = huge_page_shift(hstate_vma(vma)); + else + info.si_addr_lsb = PAGE_SHIFT; + + send_sig_info(SIGBUS, &info, current); +} + +static bool fault_supports_apt_huge_mapping(struct kvm_memory_slot *memslot, + unsigned long hva, + unsigned long map_size) +{ + gpa_t gpa_start; + hva_t uaddr_start, uaddr_end; + size_t size; + + /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ + if (map_size == PAGE_SIZE) + return true; + + size = memslot->npages * PAGE_SIZE; + + gpa_start = memslot->base_gfn << PAGE_SHIFT; + + uaddr_start = memslot->userspace_addr; + uaddr_end = uaddr_start + size; + + /* + * Pages belonging to memslots that don't have the same alignment + * within a PMD/PUD for userspace and IPA cannot be mapped with stage-2 + * PMD/PUD entries, because we'll end up mapping the wrong pages. + * + * Consider a layout like the following: + * + * memslot->userspace_addr: + * +-----+--------------------+--------------------+---+ + * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| + * +-----+--------------------+--------------------+---+ + * + * memslot->base_gfn << PAGE_SHIFT: + * +---+--------------------+--------------------+-----+ + * |abc|def Stage-2 block | Stage-2 block |tvxyz| + * +---+--------------------+--------------------+-----+ + * + * If we create those stage-2 blocks, we'll end up with this incorrect + * mapping: + * d -> f + * e -> g + * f -> h + */ + if ((gpa_start & (map_size - 1)) != (uaddr_start & (map_size - 1))) + return false; + + /* + * Next, let's make sure we're not trying to map anything not covered + * by the memslot. This means we have to prohibit block size mappings + * for the beginning and end of a non-block aligned and non-block sized + * memory slot (illustrated by the head and tail parts of the + * userspace view above containing pages 'abcde' and 'xyz', + * respectively). + * + * Note that it doesn't matter if we do the check using the + * userspace_addr or the base_gfn, as both are equally aligned (per + * the check above) and equally sized. + */ + return (hva & ~(map_size - 1)) >= uaddr_start && + (hva & ~(map_size - 1)) + map_size <= uaddr_end; +} + +/* + * apt_get_leaf_entry - walk the stage2 VM page tables and return + * true if a valid and present leaf-entry is found. A pointer to the + * leaf-entry is returned in the appropriate level variable - pudpp, + * pmdpp, ptepp. + */ +static bool apt_get_leaf_entry(struct kvm *kvm, phys_addr_t addr, + pud_t **pudpp, pmd_t **pmdpp, pte_t **ptepp) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + *pudpp = NULL; + *pmdpp = NULL; + *ptepp = NULL; + + pudp = apt_get_pud(kvm->arch.pgd, NULL, addr); + if (!pudp || pud_none(*pudp) || !pud_present(*pudp)) + return false; + + if (pud_huge(*pudp)) { + *pudpp = pudp; + return true; + } + + pmdp = pmd_offset(pudp, addr); + if (!pmdp || pmd_none(*pmdp) || !pmd_present(*pmdp)) + return false; + + if (pmd_trans_huge(*pmdp)) { + *pmdpp = pmdp; + return true; + } + + ptep = pte_offset_kernel(pmdp, addr); + if (!ptep || pte_none(*ptep) || !pte_present(*ptep)) + return false; + + *ptepp = ptep; + return true; +} + +static bool apt_is_exec(struct kvm *kvm, phys_addr_t addr) +{ + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + bool found; + + found = apt_get_leaf_entry(kvm, addr, &pudp, &pmdp, &ptep); + if (!found) + return false; + + if (pudp) + return kvm_pud_exec(pudp); + else if (pmdp) + return kvm_pmd_exec(pmdp); + else + return kvm_pte_exec(ptep); +} + +static int apt_set_pte_fast(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + int inv_level = ((read_csr(CSR_AS_INFO)) >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + unsigned long inv_hpa = read_csr(CSR_AS_INFO) & AF_ENTRY_ADDR_MASK; + + VM_BUG_ON(logging_active && !cache); + + if (inv_level == 1) { + pud = (pud_t *)(inv_hpa | PAGE_OFFSET); + goto find_pud; + } else if (inv_level == 2) { + pmd = (pmd_t *)(inv_hpa | PAGE_OFFSET); + goto find_pmd; + } else if (inv_level == 3) { + pte = (pte_t *)(inv_hpa | PAGE_OFFSET); + goto find_pte; + } + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + +find_pud: + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + +find_pmd: + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + +find_pte: + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + +static int apt_set_pte(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pte_t *new_pte, + unsigned long flags) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte, old_pte; + bool logging_active = flags & KVM_APT_FLAG_LOGGING_ACTIVE; + + VM_BUG_ON(logging_active && !cache); + + /* Create addtional page table mapping - Levels 0 and 1 */ + pud = apt_get_pud(kvm->arch.pgd, cache, addr); + if (!pud) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PUD, then continue + * on to allocate page. + */ + if (logging_active) + apt_dissolve_pud(kvm, addr, pud); + + if (pud_none(*pud)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pmd = mmu_memory_cache_alloc(cache); + pud_populate(NULL, pud, pmd); + get_page(virt_to_page(pud)); + } + + pmd = pmd_offset(pud, addr); + if (!pmd) { + /* + * Ignore calls from kvm_set_spte_hva for unallocated + * address ranges. + */ + return 0; + } + + /* + * While dirty page logging - dissolve huge PMD, then continue on to + * allocate page. + */ + if (logging_active) + apt_dissolve_pmd(kvm, addr, pmd); + + /* Create stage-2 page mappings - Level 2 */ + if (pmd_none(*pmd)) { + if (!cache) + return 0; /* ignore calls from kvm_set_spte_hva */ + pte = mmu_memory_cache_alloc(cache); + pmd_populate_kernel(NULL, pmd, pte); + get_page(virt_to_page(pmd)); + } + + pte = pte_offset_kernel(pmd, addr); + + /* Create 2nd stage page table mapping - Level 3 */ + old_pte = *pte; + + /* new pte should be readonly? */ +// *new_pte = pte_wrprotect(*new_pte); + + if (pte_present(old_pte)) { + /* Skip page table update if there is no change */ + if (pte_val(old_pte) == pte_val(*new_pte)) + return 0; + + /* Do we need WRITE_ONCE(pte, 0)? */ + set_pte(pte, __pte(0)); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pte)); + } + + /* Do we need WRITE_ONCE(pte, new_pte)? */ + set_pte(pte, *new_pte); + return 0; +} + + + +static int apt_set_pmd_huge(struct kvm *kvm, struct kvm_mmu_memory_cache + *cache, phys_addr_t addr, const pmd_t *new_pmd, unsigned long sz) +{ + pmd_t *pmd, old_pmd, *ori_pmd; + int i; +retry: + pmd = apt_get_pmd(kvm, cache, addr, sz); + VM_BUG_ON(!pmd); + ori_pmd = pmd; + old_pmd = *pmd; + if (pmd_present(old_pmd)) { + /* + * If we already have PTE level mapping for this block, + * we must unmap it to avoid inconsistent TLB state and + * leaking the table page. We could end up in this situation + * if the memory slot was marked for dirty logging and was + * reverted, leaving PTE level mappings for the pages accessed + * during the period. So, unmap the PTE level mapping for this + * block and retry, as we could have released the upper level + * table in the process. + * + * Normal THP split/merge follows mmu_notifier callbacks and do + * get handled accordingly. + */ + if (!pmd_trans_huge(old_pmd)) { + unmap_apt_range(kvm, addr & PMD_MASK, PMD_SIZE); + goto retry; + } + /* + * Multiple vcpus faulting on the same PMD entry, can + * lead to them sequentially updating the PMD with the + * same value. Following the break-before-make + * (pmd_clear() followed by tlb_flush()) process can + * hinder forward progress due to refaults generated + * on missing translations. + * + * Skip updating the page table if the entry is + * unchanged. + */ + if (pmd_val(old_pmd) == pmd_val(*new_pmd)) + return 0; + + /* + * Mapping in huge pages should only happen through a + * fault. If a page is merged into a transparent huge + * page, the individual subpages of that huge page + * should be unmapped through MMU notifiers before we + * get here. + * + * Merging of CompoundPages is not supported; they + * should become splitting first, unmapped, merged, + * and mapped back in on-demand. + */ + VM_BUG_ON(pmd_pfn(old_pmd) != pmd_pfn(*new_pmd)); + + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, pmd++) + pmd_clear(pmd); + } else + pmd_clear(pmd); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pmd)); + } + + /* Do we need WRITE_ONCE(pmd, new_pmd)? */ + if (sz == CONT_PMD_SIZE) { + for (i = 0; i < CONT_PMDS; i++, ori_pmd++) + set_pmd(ori_pmd, *new_pmd); + } else + set_pmd(pmd, *new_pmd); + return 0; +} + +static int apt_set_pud_huge(struct kvm *kvm, struct kvm_mmu_memory_cache *cache, + phys_addr_t addr, const pud_t *new_pudp) +{ + pud_t *pudp, old_pud; + +retry: + pudp = apt_get_pud(kvm->arch.pgd, cache, addr); + VM_BUG_ON(!pudp); + + old_pud = *pudp; + + /* + * A large number of vcpus faulting on the same stage 2 entry, + * can lead to a refault due to the stage2_pud_clear()/tlb_flush(). + * Skip updating the page tables if there is no change. + */ + if (pud_val(old_pud) == pud_val(*new_pudp)) + return 0; + + if (pud_present(old_pud)) { + /* + * If we already have table level mapping for this block, unmap + * the range for this block and retry. + */ + if (!pud_huge(old_pud)) { + unmap_apt_range(kvm, addr & PUD_MASK, PUD_SIZE); + goto retry; + } + +// WARN_ON_ONCE(kvm_pud_pfn(old_pud) != kvm_pud_pfn(*new_pudp)); + pud_clear(pudp); + kvm_flush_remote_tlbs(kvm); + } else { + get_page(virt_to_page(pudp)); + } + + set_pud(pudp, *new_pudp); + return 0; +} + +static unsigned long +transparent_hugepage_adjust(struct kvm_memory_slot *memslot, + unsigned long hva, kvm_pfn_t *pfnp, + phys_addr_t *gpap) +{ + kvm_pfn_t pfn = *pfnp; + struct page *page = pfn_to_page(pfn); + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and IPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (!PageHuge(page) && PageTransCompoundMap(page) && + fault_supports_apt_huge_mapping(memslot, hva, PMD_SIZE)) { + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + *gpap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~(PTRS_PER_PMD - 1); + kvm_get_pfn(pfn); + *pfnp = pfn; + return PMD_SIZE; + } + + return PAGE_SIZE; +} + +static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_gpa, + struct kvm_memory_slot *memslot, unsigned long hva, + unsigned long fault_status) +{ + int ret; + bool write_fault, exec_fault, writable, force_pte = false; + unsigned long mmu_seq; + gfn_t gfn = fault_gpa >> PAGE_SHIFT; + struct kvm *kvm = vcpu->kvm; + struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; + struct vm_area_struct *vma; + kvm_pfn_t pfn; + pgprot_t mem_type = PAGE_READONLY; + bool logging_active = memslot_is_logging(memslot); + unsigned long vma_pagesize, flags = 0; + unsigned long as_info, access_type; + unsigned int vma_shift; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + write_fault = kvm_is_write_fault(access_type); + exec_fault = kvm_is_exec_fault(access_type); + VM_BUG_ON(write_fault && exec_fault); + + if (fault_status == AF_STATUS_FOR) { + kvm_err("Unexpected APT read permission error\n"); + return -EFAULT; + } + + /* Let's check if we will get back a huge page backed by hugetlbfs */ + down_read(¤t->mm->mmap_lock); + vma = find_vma_intersection(current->mm, hva, hva + 1); + if (unlikely(!vma)) { + kvm_err("Failed to find VMA for hva 0x%lx\n", hva); + up_read(¤t->mm->mmap_lock); + return -EFAULT; + } + + if (is_vm_hugetlb_page(vma)) + vma_shift = huge_page_shift(hstate_vma(vma)); + else + vma_shift = PAGE_SHIFT; + + vma_pagesize = 1ULL << vma_shift; + if (logging_active || (vma->vm_flags & VM_PFNMAP) || + !fault_supports_apt_huge_mapping(memslot, hva, vma_pagesize)) { + force_pte = true; + vma_pagesize = PAGE_SIZE; + } + + if (vma_pagesize == PMD_SIZE || vma_pagesize == CONT_PMD_SIZE || vma_pagesize == PUD_SIZE) + gfn = (fault_gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT; + up_read(¤t->mm->mmap_lock); + /* We need minimum second+third level pages */ + ret = mmu_topup_memory_cache(memcache, KVM_MMU_CACHE_MIN_PAGES, + KVM_NR_MEM_OBJS); + if (ret) + return ret; + + mmu_seq = vcpu->kvm->mmu_notifier_seq; + /* + * Ensure the read of mmu_notifier_seq happens before we call + * gfn_to_pfn_prot (which calls get_user_pages), so that we don't risk + * the page we just got a reference to gets unmapped before we have a + * chance to grab the mmu_lock, which ensure that if the page gets + * unmapped afterwards, the call to kvm_unmap_hva will take it away + * from us again properly. This smp_rmb() interacts with the smp_wmb() + * in kvm_mmu_notifier_invalidate_. + */ + smp_rmb(); + + pfn = gfn_to_pfn_prot(kvm, gfn, write_fault, &writable); + if (pfn == KVM_PFN_ERR_HWPOISON) { + kvm_send_hwpoison_signal(hva, vma); + return 0; + } + if (is_error_noslot_pfn(pfn)) + return -EFAULT; + + if (logging_active) { + /* + * Faults on pages in a memslot with logging enabled + * should not be mapped with huge pages (it introduces churn + * and performance degradation), so force a pte mapping. + */ + flags |= KVM_APT_FLAG_LOGGING_ACTIVE; + + /* + * Only actually map the page as writable if this was a write + * fault. + */ + if (!write_fault) + writable = false; + } + + spin_lock(&kvm->mmu_lock); + if (mmu_notifier_retry(kvm, mmu_seq)) + goto out_unlock; + + /* + * If we are not forced to use page mapping, check if we are + * backed by a THP and thus use block mapping if possible. + */ + if (vma_pagesize == PAGE_SIZE && !force_pte) { + vma_pagesize = transparent_hugepage_adjust(memslot, hva, + &pfn, &fault_gpa); + } + + if (vma_pagesize == PUD_SIZE) { + pud_t new_pud = pfn_pud(pfn, mem_type); + + new_pud = pud_mkhuge(new_pud); + + if (writable) { + new_pud = kvm_pud_mkwrite(new_pud); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pud = kvm_pud_mkexec(new_pud); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pud = kvm_pud_mkexec(new_pud); + } + + ret = apt_set_pud_huge(kvm, memcache, fault_gpa, &new_pud); + } else if (vma_pagesize == CONT_PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + new_pmd = pmd_mkcont(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else if (vma_pagesize == PMD_SIZE) { + pmd_t new_pmd = pfn_pmd(pfn, mem_type); + + new_pmd = pmd_mkhuge(new_pmd); + + if (writable) { + new_pmd = kvm_pmd_mkwrite(new_pmd); + kvm_set_pfn_dirty(pfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pmd = kvm_pmd_mkexec(new_pmd); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pmd = kvm_pmd_mkexec(new_pmd); + } + + ret = apt_set_pmd_huge(kvm, memcache, fault_gpa, &new_pmd, vma_pagesize); + } else { + pte_t new_pte = pfn_pte(pfn, mem_type); + + if (writable) { + new_pte = kvm_pte_mkwrite(new_pte); + kvm_set_pfn_dirty(pfn); + mark_page_dirty(kvm, gfn); + } + + if (exec_fault && fault_status == AF_STATUS_INV) { + new_pte = kvm_pte_mkexec(new_pte); + } else if (fault_status == AF_STATUS_FOE) { + /* Preserve execute if FOE was already cleared */ + if (apt_is_exec(kvm, fault_gpa)) + new_pte = kvm_pte_mkexec(new_pte); + } + + ret = apt_set_pte_fast(kvm, memcache, fault_gpa, &new_pte, flags); + if (!ret) + goto out_unlock; + } + +out_unlock: + spin_unlock(&kvm->mmu_lock); + kvm_set_pfn_accessed(pfn); + kvm_release_pfn_clean(pfn); + return ret; +} + +/** + * kvm_handle_guest_abort - handles all 2nd stage aborts + * @vcpu: the VCPU pointer + * @run: the kvm_run structure + * + * Any abort that gets to the host is almost guaranteed to be caused by a + * missing second stage translation table entry, which can mean that either the + * guest simply needs more memory and we must allocate an appropriate page or it + * can mean that the guest tried to access I/O memory, which is emulated by user + * space. The distinction is based on the IPA causing the fault and whether this + * memory region has been registered as standard RAM by user space. + */ +#ifdef CONFIG_SUBARCH_C4 +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + unsigned long as_info; /* the value of CSR: AS_INFO */ + unsigned int access_type, inv_level; + unsigned int fault_status; + unsigned long fault_entry_addr; + phys_addr_t fault_gpa; + struct kvm_memory_slot *memslot; + unsigned long hva; + bool write_fault, writable; + gfn_t gfn; + + int ret, idx; + + as_info = read_csr(CSR_AS_INFO); + access_type = (as_info >> AF_ACCESS_TYPE_SHIFT) & AF_ACCESS_TYPE_MASK; + inv_level = (as_info >> AF_INV_LEVEL_SHIFT) & AF_INV_LEVEL_MASK; + fault_status = (as_info >> AF_FAULT_STATUS_SHIFT) & AF_FAULT_STATUS_MASK; + fault_entry_addr = (as_info & AF_ENTRY_ADDR_MASK) >> 3; + + fault_gpa = read_csr(CSR_EXC_GPA); + idx = srcu_read_lock(&vcpu->kvm->srcu); + + gfn = fault_gpa >> PAGE_SHIFT; + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + + write_fault = kvm_is_write_fault(access_type); + + /* The memory slot for IO doesn't register in memory region + * with kvm, if hva == KVM_HVA_ERR_BAD, the gpa used for MMIO + * needs emulation. + */ + + if (hva == KVM_HVA_ERR_BAD) { + ret = io_mem_abort(vcpu, run, NULL); + goto out_unlock; + } + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_gpa >= KVM_PHYS_SIZE); + + ret = user_mem_abort(vcpu, fault_gpa, memslot, hva, fault_status); + if (ret == 0) + ret = 1; +out_unlock: + srcu_read_unlock(&vcpu->kvm->srcu, idx); + return ret; +} +#endif +static int handle_hva_to_gpa(struct kvm *kvm, unsigned long start, unsigned long end, + int (*handler)(struct kvm *kvm, gpa_t gpa, u64 size, void *data), + void *data) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *memslot; + int ret = 0; + + slots = kvm_memslots(kvm); + + /* we only care about the pages that the guest sees */ + kvm_for_each_memslot(memslot, slots) { + unsigned long hva_start, hva_end; + gfn_t gpa; + + hva_start = max(start, memslot->userspace_addr); + hva_end = min(end, memslot->userspace_addr + + (memslot->npages << PAGE_SHIFT)); + if (hva_start >= hva_end) + continue; + + gpa = hva_to_gfn_memslot(hva_start, memslot) << PAGE_SHIFT; + ret |= handler(kvm, gpa, (u64)(hva_end - hva_start), data); + } + + return ret; +} + +static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + unmap_apt_range(kvm, gpa, size); + return 0; +} + +int kvm_unmap_hva_range(struct kvm *kvm, + unsigned long start, unsigned long end, bool blockable) +{ + if (!kvm->arch.pgd) + return 0; + + handle_hva_to_gpa(kvm, start, end, &kvm_unmap_hva_handler, NULL); + return 1; +} + +static int apt_ptep_test_and_clear_young(pte_t *pte) +{ + if (pte_young(*pte)) { + *pte = pte_mkold(*pte); + return 1; + } + return 0; +} + +static int apt_pmdp_test_and_clear_young(pmd_t *pmd) +{ + return apt_ptep_test_and_clear_young((pte_t *)pmd); +} + +static int apt_pudp_test_and_clear_young(pud_t *pud) +{ + return apt_ptep_test_and_clear_young((pte_t *)pud); +} + +static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE); + if (!apt_get_leaf_entry(kvm, gpa, &pud, &pmd, &pte)) + return 0; + + if (pud) + return apt_pudp_test_and_clear_young(pud); + else if (pmd) + return apt_pmdp_test_and_clear_young(pmd); + else + return apt_ptep_test_and_clear_young(pte); +} + +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end) +{ + if (!kvm->arch.pgd) + return 0; + + return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL); +} + +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) +{ + if (!kvm->arch.pgd) + return 0; + return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL); +} + +static int kvm_set_apte_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *data) +{ + pte_t *pte = (pte_t *)data; + + WARN_ON(size != PAGE_SIZE); + + apt_set_pte(kvm, NULL, gpa, pte, 0); + return 0; +} + +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) +{ + unsigned long end = hva + PAGE_SIZE; + pte_t apt_pte; + + if (!kvm->arch.pgd) + return 0; + + apt_pte = pte_wrprotect(pte); + handle_hva_to_gpa(kvm, hva, end, &kvm_set_apte_handler, &apt_pte); + return 0; +} + +/** + * kvm_mmu_write_protect_pt_masked() - write protect dirty pages + * @kvm: The KVM pointer + * @slot: The memory slot associated with mask + * @gfn_offset: The gfn offset in memory slot + * @mask: The mask of dirty pages at offset 'gfn_offset' in this memory + * slot to be write protected + * + * Walks bits set in mask write protects the associated pte's. Caller must + * acquire kvm_mmu_lock. + */ +static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + phys_addr_t base_gfn = slot->base_gfn + gfn_offset; + phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; + phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; + + apt_wp_range(kvm, start, end); +} + +/* + * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected + * dirty pages. + * + * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to + * enable dirty logging for them. + */ +void kvm_core4_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) +{ + kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); +} diff --git a/arch/sw_64/mm/hugetlbpage_c4.c b/arch/sw_64/mm/hugetlbpage_c4.c index 0c763efc1d16..8c33643e6b87 100644 --- a/arch/sw_64/mm/hugetlbpage_c4.c +++ b/arch/sw_64/mm/hugetlbpage_c4.c @@ -34,6 +34,7 @@ int pud_huge(pud_t pud) return !pud_none(pud) && (pud_val(pud) & (_PAGE_PRESENT|_PAGE_LEAF)) != _PAGE_PRESENT; } +EXPORT_SYMBOL(pud_huge); #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE #define want_pmd_share() (1) -- Gitee From 26fcc30f63209fd4b37dd56cdbd798880d616166 Mon Sep 17 00:00:00 2001 From: Hang Yiyi Date: Thu, 7 Sep 2023 17:47:52 +0800 Subject: [PATCH 096/150] sw64: kvm: rebuild Core3 and Core4 virtualization Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Signed-off-by: Hang Yiyi Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_host.h | 89 ++++- arch/sw_64/include/uapi/asm/kvm.h | 11 + arch/sw_64/kvm/Kconfig | 8 - arch/sw_64/kvm/Makefile | 9 +- arch/sw_64/kvm/emulate.c | 11 + arch/sw_64/kvm/kvm_core3.c | 505 +++++++++++++++++++++++ arch/sw_64/kvm/kvm_core4.c | 252 ++++++++++++ arch/sw_64/kvm/mmio.c | 6 + arch/sw_64/kvm/{kvm-sw64.c => sw64.c} | 555 +++++--------------------- 9 files changed, 978 insertions(+), 468 deletions(-) create mode 100644 arch/sw_64/kvm/kvm_core3.c create mode 100644 arch/sw_64/kvm/kvm_core4.c rename arch/sw_64/kvm/{kvm-sw64.c => sw64.c} (53%) diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 56e7ef56c82b..9b6614773e54 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -28,8 +28,22 @@ #include +#define last_vpn(cpu) (cpu_data[cpu].last_vpn) + +#ifdef CONFIG_SUBARCH_C3B +#define VPN_BITS 8 +#endif + +#ifdef CONFIG_SUBARCH_C4 +#define VPN_BITS 10 +#endif + +#define VPN_FIRST_VERSION (1UL << VPN_BITS) +#define VPN_MASK ((1UL << VPN_BITS) - 1) +#define VPN_SHIFT (64 - VPN_BITS) + #define KVM_MAX_VCPUS 64 -#define KVM_USER_MEM_SLOTS 64 +#define KVM_USER_MEM_SLOTS 512 #define KVM_HALT_POLL_NS_DEFAULT 0 #define KVM_IRQCHIP_NUM_PINS 256 @@ -41,6 +55,13 @@ #define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) #define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) +/* + * The architecture supports 48-bit GPA as input to the addtional stage translations. + */ +#define KVM_PHYS_SHIFT (48) +#define KVM_PHYS_SIZE (_AC(1, ULL) << KVM_PHYS_SHIFT) +#define KVM_PHYS_MASK (KVM_PHYS_SIZE - _AC(1, ULL)) + struct kvm_arch_memory_slot { unsigned long host_phys_addr; bool valid; @@ -52,8 +73,43 @@ struct kvm_arch { /* segment table */ unsigned long *seg_pgd; + + struct swvm_mem mem; + /* Addtional stage page table*/ + pgd_t *pgd; +}; + +struct kvm_sw64_ops { + unsigned long (*get_new_vpn_context)(struct kvm_vcpu *vcpu, long cpu); + void (*update_vpn)(struct kvm_vcpu *vcpu, unsigned long vpn); + int (*init_vm)(struct kvm *kvm); + void (*destroy_vm)(struct kvm *kvm); + int (*prepare_memory_region)(struct kvm *kvm, struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change); + void (*mmu_enable_log_dirty_pt_masked)(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask); + void (*commit_memory_region)(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, const struct kvm_memory_slot *new, + enum kvm_mr_change change); + void (*flush_shadow_memslot)(struct kvm *kvm, struct kvm_memory_slot *slot); + void (*flush_shadow_all)(struct kvm *kvm); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); + int (*vcpu_run)(struct kvm_vcpu *vcpu, struct kvm_run *run); + void (*vcpu_free)(struct kvm_vcpu *vcpu); + long (*get_vcb)(struct file *filp, unsigned long arg); + long (*set_vcb)(struct file *filp, unsigned long arg); }; +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; struct kvm_vcpu_arch { struct kvm_regs regs __attribute__((__aligned__(32))); @@ -65,6 +121,7 @@ struct kvm_vcpu_arch { struct hrtimer hrt; unsigned long timer_next_event; unsigned long vtimer_freq; + int first_run; int halted; int stopped; @@ -74,6 +131,9 @@ struct kvm_vcpu_arch { DECLARE_BITMAP(irqs_pending, SWVM_IRQS); unsigned long vpnc[NR_CPUS]; + /* Detect first run of a vcpu */ + bool has_run_once; + /* WAIT executed */ int wait; @@ -84,6 +144,13 @@ struct kvm_vcpu_arch { bool pause; struct kvm_decode mmio_decode; + + /* Cache some mmu pages needed inside spinlock regions */ + struct kvm_mmu_memory_cache mmu_page_cache; + + /* guest live migration */ + unsigned long migration_mark; + unsigned long shtclock; }; struct vmem_info { @@ -130,6 +197,21 @@ struct kvm_vcpu_stat { #ifdef CONFIG_KVM_MEMHOTPLUG void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr); #endif +#ifdef CONFIG_SUBARCH_C4 +#define KVM_ARCH_WANT_MMU_NOTIFIER +#endif +int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); + +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat); +void check_vcpu_requests(struct kvm_vcpu *vcpu); +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu); +int vmem_init(void); +void vmem_exit(void); +int __sw64_vcpu_run(unsigned long vcb_pa, struct kvm_regs *regs, + struct hcall_args *args); int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index, struct hcall_args *hargs); void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type); @@ -140,13 +222,12 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} -static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} -static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); + int kvm_sw64_perf_init(void); int kvm_sw64_perf_teardown(void); diff --git a/arch/sw_64/include/uapi/asm/kvm.h b/arch/sw_64/include/uapi/asm/kvm.h index 0ca8c10b8550..2253475deaa5 100644 --- a/arch/sw_64/include/uapi/asm/kvm.h +++ b/arch/sw_64/include/uapi/asm/kvm.h @@ -117,4 +117,15 @@ struct kvm_sync_regs { struct kvm_sregs { }; +struct swvm_mem_bank { + unsigned long guest_phys_addr; + unsigned long host_phys_addr; + unsigned long host_addr; + unsigned long size; +}; + +struct swvm_mem { + struct swvm_mem_bank membank[SWVM_NUM_NUMA_MEMBANKS]; +}; + #endif /* _UAPI_ASM_SW64_KVM_H */ diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index 9d75c8733ee0..94f1729fad1c 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -17,7 +17,6 @@ if VIRTUALIZATION config KVM tristate "Kernel-based Virtual Machine (KVM) support" - select KVM_SW64_HOST select PREEMPT_NOTIFIERS select CMA depends on NET @@ -38,13 +37,6 @@ config KVM If unsure, say N. -config KVM_SW64_HOST - tristate "KVM for SW64 processors support" - depends on KVM - help - Provides host support for SW64 processors. - To compile this as a module, choose M here. - config KVM_MEMHOTPLUG bool "Memory hotplug support for guest" depends on KVM && MEMORY_HOTPLUG diff --git a/arch/sw_64/kvm/Makefile b/arch/sw_64/kvm/Makefile index 43cea19215ff..5e24b82ce5b8 100644 --- a/arch/sw_64/kvm/Makefile +++ b/arch/sw_64/kvm/Makefile @@ -7,7 +7,10 @@ KVM := ../../../virt/kvm ccflags-y += -Ivirt/kvm -Iarch/sw_64/kvm -kvm-$(CONFIG_KVM_SW64_HOST) += $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o -kvm-$(CONFIG_KVM_SW64_HOST) += kvm-sw64.o entry.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o +obj-$(CONFIG_KVM) += kvm.o -obj-$(CONFIG_KVM_SW64_HOST) += kvm.o +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ + sw64.o entry.o emulate.o mmio.o kvm_timer.o handle_exit.o perf.o + +kvm-$(CONFIG_SUBARCH_C3B) += kvm_core3.o +kvm-$(CONFIG_SUBARCH_C4) += kvm_core4.o mmu.o diff --git a/arch/sw_64/kvm/emulate.c b/arch/sw_64/kvm/emulate.c index bcc06c0dd618..160c5c29c4d0 100644 --- a/arch/sw_64/kvm/emulate.c +++ b/arch/sw_64/kvm/emulate.c @@ -5,6 +5,7 @@ * linhn */ #include +#include #include #include @@ -12,8 +13,18 @@ void sw64_decode(struct kvm_vcpu *vcpu, unsigned int insn, struct kvm_run *run) { int opc, ra; +#ifdef CONFIG_SUBARCH_C3B opc = (insn >> 26) & 0x3f; ra = (insn >> 21) & 0x1f; +#elif defined(CONFIG_SUBARCH_C4) + unsigned long ds_stat, exc_sum; + + ds_stat = read_csr(CSR_DS_STAT); + exc_sum = read_csr(CSR_EXC_SUM); + + opc = (ds_stat >> 4) & 0x3f; + ra = (exc_sum >> 8) & 0x1f; +#endif switch (opc) { case 0x20: /* LDBU */ diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c new file mode 100644 index 000000000000..529a04a2bed0 --- /dev/null +++ b/arch/sw_64/kvm/kvm_core3.c @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "trace.h" + +#include "../kernel/pci_impl.h" +#include "vmem.c" + +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) +extern bool bind_vcpu_enabled; +#endif + +#define GUEST_RESET_PC 0xffffffff80011100 + +static unsigned long longtime_offset; + +#ifdef CONFIG_KVM_MEMHOTPLUG +static u64 get_vpcr_memhp(u64 seg_base, u64 vpn) +{ + return seg_base | ((vpn & VPN_MASK) << 44); +} +#else +static u64 get_vpcr(u64 hpa_base, u64 mem_size, u64 vpn) +{ + return (hpa_base >> 23) | ((mem_size >> 23) << 16) + | ((vpn & VPN_MASK) << 44); +} +#endif + +static unsigned long core3_get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +{ + unsigned long vpn = last_vpn(cpu); + unsigned long next = vpn + 1; + + if ((vpn & VPN_MASK) >= VPN_MASK) { + tbia(); + next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + } + last_vpn(cpu) = next; + return next; +} + +static void core3_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(VPN_MASK << 44))) | (vpn << 44); + vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); +} + +int kvm_core3_init_vm(struct kvm *kvm) +{ +#ifdef CONFIG_KVM_MEMHOTPLUG + unsigned long *seg_pgd; + + if (kvm->arch.seg_pgd != NULL) { + kvm_err("kvm_arch already initialized?\n"); + return -EINVAL; + } + + seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); + if (!seg_pgd) + return -ENOMEM; + + kvm->arch.seg_pgd = seg_pgd; + #endif + return 0; +} + +void kvm_core3_destroy_vm(struct kvm *kvm) +{ + int i; + #ifdef CONFIG_KVM_MEMHOTPLUG + void *seg_pgd = NULL; + + if (kvm->arch.seg_pgd) { + seg_pgd = READ_ONCE(kvm->arch.seg_pgd); + kvm->arch.seg_pgd = NULL; + } + + if (seg_pgd) + free_pages_exact(seg_pgd, PAGE_SIZE); + #endif + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) + kvm_vcpu_destroy(kvm->vcpus[i]); + } + atomic_set(&kvm->online_vcpus, 0); +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +static void setup_segment_table(struct kvm *kvm, + struct kvm_memory_slot *memslot, unsigned long addr, size_t size) +{ + unsigned long *seg_pgd = kvm->arch.seg_pgd; + unsigned long num_of_entry; + unsigned long base_hpa = addr; + unsigned long i; + + num_of_entry = round_up(size, 1 << 30) >> 30; + + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } +} +#endif + +int kvm_core3_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + unsigned long addr; + struct file *vm_file; + struct vm_area_struct *vma; + struct vmem_info *info; + unsigned long ret; + size_t size; + + if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) + return 0; + + if (test_bit(IO_MARK_BIT, &(mem->guest_phys_addr))) + return 0; + + if (test_bit(IO_MARK_BIT + 1, &(mem->guest_phys_addr))) + return 0; + +#ifndef CONFIG_KVM_MEMHOTPLUG + if (mem->guest_phys_addr) { + pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); + return 0; + } +#endif + if (!sw64_kvm_pool) + return -ENOMEM; + + pr_info("%s: %#llx %#llx, user addr: %#llx\n", __func__, + mem->guest_phys_addr, mem->memory_size, mem->userspace_addr); + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + vm_file = vma->vm_file; + + if (!vm_file) { + info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); + + size = round_up(mem->memory_size, 8<<20); + addr = gen_pool_alloc(sw64_kvm_pool, size); + if (!addr) + return -ENOMEM; + vm_munmap(mem->userspace_addr, mem->memory_size); + ret = vm_mmap(vm_file, mem->userspace_addr, mem->memory_size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, 0); + if ((long)ret < 0) + return ret; + + vma = find_vma(current->mm, mem->userspace_addr); + if (!vma) + return -ENOMEM; + +#ifdef CONFIG_KVM_MEMHOTPLUG + if (memslot->base_gfn == 0x0UL) { + setup_segment_table(kvm, memslot, addr, size); + kvm->arch.host_phys_addr = (u64)addr; + memslot->arch.host_phys_addr = addr; + } else { + /* used for memory hotplug */ + memslot->arch.host_phys_addr = addr; + memslot->arch.valid = false; + } +#endif + info->start = addr; + info->size = size; + vma->vm_private_data = (void *) info; + + vma->vm_ops = &vmem_vm_ops; + vma->vm_ops->open(vma); + + ret = vmem_vm_insert_page(vma); + if ((int)ret < 0) + return ret; + } else { + info = vm_file->private_data; + addr = info->start; + } + + pr_info("guest phys addr = %#lx, size = %#lx\n", + addr, vma->vm_end - vma->vm_start); +#ifndef CONFIG_KVM_MEMHOTPLUG + kvm->arch.host_phys_addr = (u64)addr; + kvm->arch.size = round_up(mem->memory_size, 8<<20); +#endif + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +/* + * kvm_mark_migration write the mark on every vcpucbs of the kvm, which tells + * the system to do migration while the mark is on, and flush all vcpu's tlbs + * at the beginning of the migration. + */ +void kvm_mark_migration(struct kvm *kvm, int mark) +{ + struct kvm_vcpu *vcpu; + int cpu; + + kvm_for_each_vcpu(cpu, vcpu, kvm) + vcpu->arch.vcb.migration_mark = mark << 2; + + kvm_flush_remote_tlbs(kvm); +} + +void kvm_core3_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + const struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + /* + * At this point memslot has been committed and there is an + * allocated dirty_bitmap[], dirty pages will be tracked while the + * memory slot is write protected. + */ + + /* If dirty logging has been stopped, do nothing for now. */ + if ((change != KVM_MR_DELETE) && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) + && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { + kvm_mark_migration(kvm, 0); + return; + } + + /* If it's the first time dirty logging, flush all vcpu tlbs. */ + if ((change == KVM_MR_FLAGS_ONLY) && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES)) + && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + kvm_mark_migration(kvm, 1); +} + +int kvm_core3_vcpu_reset(struct kvm_vcpu *vcpu) +{ + unsigned long addr = vcpu->kvm->arch.host_phys_addr; + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + if (vcpu->vcpu_id == 0) + memset(__va(addr), 0, 0x2000000); + + return 0; +} + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int kvm_core3_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + int ret; + struct vcpucb *vcb = &(vcpu->arch.vcb); + struct hcall_args hargs; + int irq; + bool more; + sigset_t sigsaved; + + /* Set guest vcb */ + /* vpn will update later when vcpu is running */ + if (vcpu->arch.vcb.vpcr == 0) { +#ifndef CONFIG_KVM_MEMHOTPLUG + vcpu->arch.vcb.vpcr + = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); +#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) + if (unlikely(bind_vcpu_enabled)) { + int nid; + unsigned long end; + + end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; + nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); + if (pfn_to_nid(PHYS_PFN(end)) == nid) + set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); + } +#endif +#else /* !CONFIG_KVM_MEMHOTPLUG */ + unsigned long seg_base = virt_to_phys(vcpu->kvm->arch.seg_pgd); + + vcpu->arch.vcb.vpcr = get_vpcr_memhp(seg_base, 0); +#endif /* CONFIG_KVM_MEMHOTPLUG */ + vcpu->arch.vcb.upcr = 0x7; + } + +#ifdef CONFIG_PERF_EVENTS + vcpu_load(vcpu); +#endif + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + if (run->exit_reason == KVM_EXIT_MMIO) + kvm_handle_mmio_return(vcpu, run); + + run->exit_reason = KVM_EXIT_UNKNOWN; + ret = 1; + while (ret > 0) { + /* Check conditions before entering the guest */ + cond_resched(); + + preempt_disable(); + local_irq_disable(); + + if (signal_pending(current)) { + ret = -EINTR; + run->exit_reason = KVM_EXIT_INTR; + vcpu->stat.signal_exits++; + } + + if (ret <= 0) { + local_irq_enable(); + preempt_enable(); + continue; + } + + memset(&hargs, 0, sizeof(hargs)); + + clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + + irq = interrupt_pending(vcpu, &more); + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); + + vcpu->arch.halted = 0; + + sw64_kvm_switch_vpn(vcpu); + check_vcpu_requests(vcpu); + guest_enter_irqoff(); + + /* Enter the guest */ + trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); + vcpu->mode = IN_GUEST_MODE; + + ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); + + /* Back from guest */ + vcpu->mode = OUTSIDE_GUEST_MODE; + + vcpu->stat.exits++; + local_irq_enable(); + guest_exit_irqoff(); + + trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); + + preempt_enable(); + + /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ + ret = handle_exit(vcpu, run, ret, &hargs); + update_vcpu_stat_time(&vcpu->stat); + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + +#ifdef CONFIG_PERF_EVENTS + vcpu_put(vcpu); +#endif + return ret; +} + +static long kvm_core3_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.vcb.migration_mark) { + unsigned long result = sw64_io_read(0, LONG_TIME) + + vcpu->arch.vcb.guest_longtime_offset; + vcpu->arch.vcb.guest_longtime = result; + vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; + } + + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +static long kvm_core3_set_vcb(struct file *filp, unsigned long arg) +{ + unsigned long result; + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.vcb.migration_mark) { + /* updated vpcr needed by destination vm */ + vcpu->arch.vcb.vpcr + = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); + + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) { + result = sw64_io_read(0, LONG_TIME); + vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; + longtime_offset = vcpu->arch.vcb.guest_longtime_offset; + } else + vcpu->arch.vcb.guest_longtime_offset = longtime_offset; + + set_timer(vcpu, 200000000); + vcpu->arch.vcb.migration_mark = 0; + } + + return 0; +} + +#ifdef CONFIG_KVM_MEMHOTPLUG +void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_memory_slot *slot; + unsigned long start_pfn = start_addr >> PAGE_SHIFT; + + kvm_for_each_memslot(slot, kvm_memslots(kvm)) { + if (start_pfn == slot->base_gfn) { + unsigned long *seg_pgd; + unsigned long num_of_entry = slot->npages >> 17; + unsigned long base_hpa = slot->arch.host_phys_addr; + unsigned long i; + + seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); + for (i = 0; i < num_of_entry; i++) { + *seg_pgd = base_hpa + (i << 30); + seg_pgd++; + } + } + } +} +#endif + +static struct kvm_sw64_ops core3_sw64_ops __ro_after_init = { + .get_new_vpn_context = core3_get_new_vpn_context, + .update_vpn = core3_update_vpn, + .init_vm = kvm_core3_init_vm, + .destroy_vm = kvm_core3_destroy_vm, + .prepare_memory_region = kvm_core3_prepare_memory_region, + .commit_memory_region = kvm_core3_commit_memory_region, + .vcpu_reset = kvm_core3_vcpu_reset, + .vcpu_run = kvm_core3_vcpu_ioctl_run, + .get_vcb = kvm_core3_get_vcb, + .set_vcb = kvm_core3_set_vcb, +}; + +static int __init kvm_core3_init(void) +{ + int i, ret; + + ret = vmem_init(); + if (ret) + goto out; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(&core3_sw64_ops, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) { + vmem_exit(); + goto out; + } + return 0; +out: + return ret; +} + +static void __exit kvm_core3_exit(void) +{ + kvm_exit(); + vmem_exit(); +} + +module_init(kvm_core3_init); +module_exit(kvm_core3_exit); diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c new file mode 100644 index 000000000000..5301abadcdc0 --- /dev/null +++ b/arch/sw_64/kvm/kvm_core4.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 - os kernal + * Author: fire3 yangzh + * linhn + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include "trace.h" + +#include "../kernel/pci_impl.h" + +#define GUEST_RESET_PC 0xfff0000000011002 +static unsigned long shtclock_offset; + +static unsigned long core4_get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +{ + unsigned long vpn = last_vpn(cpu); + unsigned long next = vpn + 1; + + if ((vpn & VPN_MASK) >= VPN_MASK) { + tbivpn(-1, 0, 0); + next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + } + last_vpn(cpu) = next; + return next; +} + +static void core4_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +{ + vcpu->arch.vcb.vpcr = vpn << 44; + vcpu->arch.vcb.dtb_vpcr = vpn; +} + +int kvm_core4_init_vm(struct kvm *kvm) +{ + return kvm_alloc_addtional_stage_pgd(kvm); +} + +void kvm_core4_destroy_vm(struct kvm *kvm) +{ + int i; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) { + kvm_arch_vcpu_free(kvm->vcpus[i]); + kvm->vcpus[i] = NULL; + } + } + atomic_set(&kvm->online_vcpus, 0); +} + +int kvm_core4_vcpu_reset(struct kvm_vcpu *vcpu) +{ + if (vcpu->arch.has_run_once) + apt_unmap_vm(vcpu->kvm); + + hrtimer_cancel(&vcpu->arch.hrt); + vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; + vcpu->arch.vcb.vcpu_irq_disabled = 1; + vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ + vcpu->arch.power_off = 0; + memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + + return 0; +} + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int kvm_core4_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + int ret; + struct vcpucb *vcb = &(vcpu->arch.vcb); + struct hcall_args hargs; + int irq; + bool more; + sigset_t sigsaved; + +#ifdef CONFIG_PERF_EVENTS + vcpu_load(vcpu); +#endif + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + if (run->exit_reason == KVM_EXIT_MMIO) + kvm_handle_mmio_return(vcpu, run); + + run->exit_reason = KVM_EXIT_UNKNOWN; + ret = 1; + while (ret > 0) { + /* + * Check conditions before entering the guest + */ + cond_resched(); + + preempt_disable(); + local_irq_disable(); + + if (signal_pending(current)) { + ret = -EINTR; + run->exit_reason = KVM_EXIT_INTR; + vcpu->stat.signal_exits++; + } + + if (ret <= 0) { + local_irq_enable(); + preempt_enable(); + continue; + } + + memset(&hargs, 0, sizeof(hargs)); + + clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + + irq = interrupt_pending(vcpu, &more); + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); + + vcpu->arch.halted = 0; + + sw64_kvm_switch_vpn(vcpu); + check_vcpu_requests(vcpu); + guest_enter_irqoff(); + + /* update aptp before the guest runs */ + imemb(); + write_csr_imb((unsigned long)vcpu->kvm->arch.pgd, CSR_APTP); + + /* Enter the guest */ + trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); + vcpu->mode = IN_GUEST_MODE; + + ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); + + /* Back from guest */ + vcpu->mode = OUTSIDE_GUEST_MODE; + + vcpu->stat.exits++; + local_irq_enable(); + guest_exit_irqoff(); + + trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); + + preempt_enable(); + + /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ + ret = handle_exit(vcpu, run, ret, &hargs); + update_vcpu_stat_time(&vcpu->stat); + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + +#ifdef CONFIG_PERF_EVENTS + vcpu_put(vcpu); +#endif + return ret; +} + +static void kvm_core4_vcpu_free(struct kvm_vcpu *vcpu) +{ + kvm_mmu_free_memory_caches(vcpu); +} + +static long kvm_core4_get_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + if (vcpu->arch.migration_mark) + vcpu->arch.shtclock = read_csr(CSR_SHTCLOCK) + + vcpu->arch.vcb.shtclock_offset; + if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) + return -EINVAL; + + return 0; +} + +static long kvm_core4_set_vcb(struct file *filp, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + struct vcpucb *kvm_vcb; + + kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); + memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); + + if (vcpu->arch.migration_mark) { + /* synchronize the longtime of source and destination */ + if (vcpu->arch.vcb.soft_cid == 0) + shtclock_offset = vcpu->arch.shtclock - read_csr(CSR_SHTCLOCK); + vcpu->arch.vcb.shtclock_offset = shtclock_offset; + set_timer(vcpu, 200000000); + vcpu->arch.migration_mark = 0; + } + return 0; +} + +static struct kvm_sw64_ops core4_sw64_ops __ro_after_init = { + .get_new_vpn_context = core4_get_new_vpn_context, + .update_vpn = core4_update_vpn, + .init_vm = kvm_core4_init_vm, + .destroy_vm = kvm_core4_destroy_vm, + .commit_memory_region = kvm_core4_commit_memory_region, + .flush_shadow_memslot = kvm_core4_flush_shadow_memslot, + .flush_shadow_all = kvm_core4_flush_shadow_all, + .vcpu_reset = kvm_core4_vcpu_reset, + .vcpu_run = kvm_core4_vcpu_ioctl_run, + .vcpu_free = kvm_core4_vcpu_free, + .get_vcb = kvm_core4_get_vcb, + .set_vcb = kvm_core4_set_vcb, +}; + +static int __init kvm_core4_init(void) +{ + int i, ret; + + for (i = 0; i < NR_CPUS; i++) + last_vpn(i) = VPN_FIRST_VERSION; + + ret = kvm_init(&core4_sw64_ops, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + + if (ret) + return ret; + + return 0; +} + +static void __exit kvm_core4_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_core4_init); +module_exit(kvm_core4_exit); diff --git a/arch/sw_64/kvm/mmio.c b/arch/sw_64/kvm/mmio.c index fe6ae6f5ed5c..21ad89722f9a 100644 --- a/arch/sw_64/kvm/mmio.c +++ b/arch/sw_64/kvm/mmio.c @@ -7,6 +7,7 @@ #include #include #include +#include static unsigned long mmio_read_buf(char *buf, unsigned int len) { @@ -63,8 +64,13 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, { int ret; +#ifdef CONFIG_SUBARCH_C3B run->mmio.phys_addr = hargs->arg1 & 0xfffffffffffffUL; sw64_decode(vcpu, hargs->arg2, run); +#elif defined(CONFIG_SUBARCH_C4) + run->mmio.phys_addr = read_csr(CSR_DVA) & 0xfffffffffffffUL; + sw64_decode(vcpu, 0, run); +#endif if (run->mmio.is_write) ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr, run->mmio.len, run->mmio.data); diff --git a/arch/sw_64/kvm/kvm-sw64.c b/arch/sw_64/kvm/sw64.c similarity index 53% rename from arch/sw_64/kvm/kvm-sw64.c rename to arch/sw_64/kvm/sw64.c index e6dc283cc9a3..86788d548b19 100644 --- a/arch/sw_64/kvm/kvm-sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -1,9 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2018 - os kernal - * Author: fire3 yangzh - * linhn - */ #include #include @@ -16,35 +11,23 @@ #include #include #include +#include +#include #include #define CREATE_TRACE_POINTS #include "trace.h" #include "../kernel/pci_impl.h" -#include "vmem.c" bool set_msi_flag; -static unsigned long longtime_offset; - -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) -extern bool bind_vcpu_enabled; -#endif -#define last_vpn(cpu) (cpu_data[cpu].last_vpn) - -#ifdef CONFIG_SUBARCH_C3B -#define WIDTH_HARDWARE_VPN 8 -#endif - -#define VPN_FIRST_VERSION (1UL << WIDTH_HARDWARE_VPN) -#define HARDWARE_VPN_MASK ((1UL << WIDTH_HARDWARE_VPN) - 1) -#define VPN_SHIFT (64 - WIDTH_HARDWARE_VPN) - -#define GUEST_RESET_PC 0xffffffff80011100 #define DFX_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu_stat, x), DFX_STAT_U64, ## __VA_ARGS__ } +struct kvm_sw64_ops *kvm_sw64_ops __read_mostly; +EXPORT_SYMBOL_GPL(kvm_sw64_ops); + int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) { set_bit(number, (vcpu->arch.irqs_pending)); @@ -52,6 +35,11 @@ int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) return 0; } +int kvm_arch_check_processor_compat(void *opaque) +{ + return 0; +} + int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { @@ -70,36 +58,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq return vcpu_interrupt_line(vcpu, irq, true); } -extern int __sw64_vcpu_run(unsigned long vcb_pa, - struct kvm_regs *regs, struct hcall_args *args); - -#ifdef CONFIG_KVM_MEMHOTPLUG -static u64 get_vpcr_memhp(u64 seg_base, u64 vpn) -{ - return seg_base | ((vpn & HARDWARE_VPN_MASK) << 44); -} -#else -static u64 get_vpcr(u64 hpa_base, u64 mem_size, u64 vpn) -{ - return (hpa_base >> 23) | ((mem_size >> 23) << 16) - | ((vpn & HARDWARE_VPN_MASK) << 44); -} -#endif - -static unsigned long __get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) -{ - unsigned long vpn = last_vpn(cpu); - unsigned long next = vpn + 1; - - if ((vpn & HARDWARE_VPN_MASK) >= HARDWARE_VPN_MASK) { - tbia(); - next = (vpn & ~HARDWARE_VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ - } - last_vpn(cpu) = next; - return next; -} - -static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) +void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) { unsigned long vpn; unsigned long vpnc; @@ -108,18 +67,17 @@ static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) vpn = last_vpn(cpu); vpnc = vcpu->arch.vpnc[cpu]; - if ((vpnc ^ vpn) & ~HARDWARE_VPN_MASK) { + if ((vpnc ^ vpn) & ~VPN_MASK) { /* vpnc and cpu vpn not in the same version, get new vpnc and vpn */ - vpnc = __get_new_vpn_context(vcpu, cpu); + vpnc = kvm_sw64_ops->get_new_vpn_context(vcpu, cpu); vcpu->arch.vpnc[cpu] = vpnc; } - vpn = vpnc & HARDWARE_VPN_MASK; + vpn = vpnc & VPN_MASK; /* Always update vpn */ /* Just setup vcb, hardware CSR will be changed later in HMcode */ - vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(HARDWARE_VPN_MASK << 44))) | (vpn << 44); - vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(HARDWARE_VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); + kvm_sw64_ops->update_vpn(vcpu, vpn); /* * If vcpu migrate to a new physical cpu, the new physical cpu may keep @@ -137,14 +95,14 @@ static void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) } } -static void check_vcpu_requests(struct kvm_vcpu *vcpu) +void check_vcpu_requests(struct kvm_vcpu *vcpu) { unsigned long vpn; long cpu = smp_processor_id(); if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { - vpn = vcpu->arch.vpnc[cpu] & HARDWARE_VPN_MASK; + vpn = vcpu->arch.vpnc[cpu] & VPN_MASK; tbivpn(0, 0, vpn); } } @@ -206,11 +164,6 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) && !vcpu->arch.power_off); } -int kvm_arch_check_processor_compat(void *opaque) -{ - return 0; -} - int kvm_arch_hardware_enable(void) { return 0; @@ -240,49 +193,6 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; } -/* - * kvm_mark_migration write the mark on every vcpucbs of the kvm, which tells - * the system to do migration while the mark is on, and flush all vcpu's tlbs - * at the beginning of the migration. - */ -void kvm_mark_migration(struct kvm *kvm, int mark) -{ - struct kvm_vcpu *vcpu; - int cpu; - - kvm_for_each_vcpu(cpu, vcpu, kvm) - vcpu->arch.vcb.migration_mark = mark << 2; - - kvm_flush_remote_tlbs(kvm); -} - -void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) -{ - /* - * At this point memslot has been committed and there is an - * allocated dirty_bitmap[], dirty pages will be tracked while the - * memory slot is write protected. - */ - - /* If dirty logging has been stopped, do nothing for now. */ - if ((change != KVM_MR_DELETE) - && (old->flags & KVM_MEM_LOG_DIRTY_PAGES) - && (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))) { - kvm_mark_migration(kvm, 0); - return; - } - - /* If it's the first time dirty logging, flush all vcpu tlbs. */ - if ((change == KVM_MR_FLAGS_ONLY) - && (!(old->flags & KVM_MEM_LOG_DIRTY_PAGES)) - && (new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - kvm_mark_migration(kvm, 1); -} - int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) { int r = 0; @@ -291,7 +201,6 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_IRQCHIP: case KVM_CAP_IOEVENTFD: case KVM_CAP_SYNC_MMU: - case KVM_CAP_IMMEDIATE_EXIT: r = 1; break; case KVM_CAP_NR_VCPUS: @@ -305,10 +214,16 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) return r; } +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ +} + void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { + if (kvm_sw64_ops->mmu_enable_log_dirty_pt_masked) + kvm_sw64_ops->mmu_enable_log_dirty_pt_masked(kvm, slot, gfn_offset, mask); } int kvm_sw64_pending_timer(struct kvm_vcpu *vcpu) @@ -326,54 +241,21 @@ int kvm_arch_hardware_setup(void *opaque) return 0; } -void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) -{ - hrtimer_cancel(&vcpu->arch.hrt); -} - int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { -#ifdef CONFIG_KVM_MEMHOTPLUG - unsigned long *seg_pgd; - - if (kvm->arch.seg_pgd != NULL) { - kvm_err("kvm_arch already initialized?\n"); + if (type) return -EINVAL; - } - seg_pgd = alloc_pages_exact(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO); - if (!seg_pgd) - return -ENOMEM; - - kvm->arch.seg_pgd = seg_pgd; -#endif + if (kvm_sw64_ops->init_vm) + return kvm_sw64_ops->init_vm(kvm); return 0; } void kvm_arch_destroy_vm(struct kvm *kvm) { - int i; -#ifdef CONFIG_KVM_MEMHOTPLUG - void *seg_pgd = NULL; - - if (kvm->arch.seg_pgd) { - seg_pgd = READ_ONCE(kvm->arch.seg_pgd); - kvm->arch.seg_pgd = NULL; - } - - if (seg_pgd) - free_pages_exact(seg_pgd, PAGE_SIZE); -#endif - - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_vcpu_destroy(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - - atomic_set(&kvm->online_vcpus, 0); + if (kvm_sw64_ops->destroy_vm) + return kvm_sw64_ops->destroy_vm(kvm); } long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) @@ -411,96 +293,26 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change) { - unsigned long addr; - struct file *vm_file; - struct vm_area_struct *vma; - struct vmem_info *info; - unsigned long ret; - size_t size; - - if (change == KVM_MR_FLAGS_ONLY || change == KVM_MR_DELETE) - return 0; - - if (test_bit(IO_MARK_BIT, &(mem->guest_phys_addr))) - return 0; + if (kvm_sw64_ops->prepare_memory_region) + return kvm_sw64_ops->prepare_memory_region(kvm, memslot, + mem, change); - if (test_bit(IO_MARK_BIT + 1, &(mem->guest_phys_addr))) - return 0; - -#ifndef CONFIG_KVM_MEMHOTPLUG - if (mem->guest_phys_addr) { - pr_info("%s, No KVM MEMHOTPLUG support!\n", __func__); - return 0; - } -#endif - - if (!sw64_kvm_pool) - return -ENOMEM; - - pr_info("%s: %#llx %#llx, user addr: %#llx\n", __func__, - mem->guest_phys_addr, mem->memory_size, mem->userspace_addr); - - vma = find_vma(current->mm, mem->userspace_addr); - if (!vma) - return -ENOMEM; - vm_file = vma->vm_file; - - if (!vm_file) { - info = kzalloc(sizeof(struct vmem_info), GFP_KERNEL); - - size = round_up(mem->memory_size, 8 << 20); - addr = gen_pool_alloc(sw64_kvm_pool, size); - if (!addr) - return -ENOMEM; - vm_munmap(mem->userspace_addr, mem->memory_size); - ret = vm_mmap(vm_file, mem->userspace_addr, mem->memory_size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, 0); - if ((long)ret < 0) - return ret; - - vma = find_vma(current->mm, mem->userspace_addr); - if (!vma) - return -ENOMEM; - -#ifdef CONFIG_KVM_MEMHOTPLUG - if (memslot->base_gfn == 0x0UL) { - setup_segment_table(kvm, memslot, addr, size); - kvm->arch.host_phys_addr = (u64)addr; - memslot->arch.host_phys_addr = addr; - } else { - /* used for memory hotplug */ - memslot->arch.host_phys_addr = addr; - memslot->arch.valid = false; - } -#endif - - info->start = addr; - info->size = mem->memory_size; - vma->vm_private_data = (void *) info; - - vma->vm_ops = &vmem_vm_ops; - vma->vm_ops->open(vma); - - ret = vmem_vm_insert_page(vma); - if ((int)ret < 0) - return ret; - } else { - info = vm_file->private_data; - addr = info->start; - } + return 0; +} - pr_info("guest phys addr = %#lx, size = %#lx\n", - addr, vma->vm_end - vma->vm_start); +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +{ + hrtimer_cancel(&vcpu->arch.hrt); -#ifndef CONFIG_KVM_MEMHOTPLUG - kvm->arch.host_phys_addr = (u64)addr; - kvm->arch.size = round_up(mem->memory_size, 8 << 20); -#endif + if (kvm_sw64_ops->vcpu_free) + kvm_sw64_ops->vcpu_free(vcpu); - memset(__va(addr), 0, 0x2000000); + kfree(vcpu); +} - return 0; +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + kvm_arch_vcpu_free(vcpu); } int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) @@ -513,7 +325,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.hrt.function = clockdev_fn; vcpu->arch.tsk = current; - /* For guest kernel "sys_call HMC_whami", indicate virtual cpu id */ vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; vcpu->arch.vcb.vcpu_irq_disabled = 1; vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ @@ -521,19 +332,15 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) return 0; } -int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) { - unsigned long addr = vcpu->kvm->arch.host_phys_addr; - - hrtimer_cancel(&vcpu->arch.hrt); - vcpu->arch.vcb.soft_cid = vcpu->vcpu_id; - vcpu->arch.vcb.vcpu_irq_disabled = 1; - vcpu->arch.pcpu_id = -1; /* force flush tlb for the first time */ - vcpu->arch.power_off = 0; - memset(&vcpu->arch.irqs_pending, 0, sizeof(vcpu->arch.irqs_pending)); + return 0; +} - if (vcpu->vcpu_id == 0) - memset(__va(addr), 0, 0x2000000); +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +{ + if (kvm_sw64_ops->vcpu_reset) + return kvm_sw64_ops->vcpu_reset(vcpu); return 0; } @@ -635,25 +442,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return 0; } -void _debug_printk_vcpu(struct kvm_vcpu *vcpu) -{ - unsigned long pc = vcpu->arch.regs.pc; - unsigned long offset = vcpu->kvm->arch.host_phys_addr; - unsigned int *pc_phys = __va((pc & 0x7fffffffUL) + offset); - unsigned int insn; - int opc, ra, disp16; - - insn = *pc_phys; - opc = (insn >> 26) & 0x3f; - ra = (insn >> 21) & 0x1f; - disp16 = insn & 0xffff; - - if (opc == 0x06 && disp16 == 0x1000) /* RD_F */ - pr_info("vcpu exit: pc = %#lx (%p), insn[%x] : rd_f r%d [%#lx]\n", - pc, pc_phys, insn, ra, vcpu_get_reg(vcpu, ra)); -} - -static void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) +void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) { vcpu_stat->utime = current->utime; vcpu_stat->stime = current->stime; @@ -666,171 +455,31 @@ static void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) */ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { - int ret; struct kvm_run *run = vcpu->run; - struct vcpucb *vcb = &(vcpu->arch.vcb); - struct hcall_args hargs; - int irq; - bool more; - sigset_t sigsaved; - - if (run->immediate_exit) - return -EINTR; - - /* Set guest vcb */ - /* vpn will update later when vcpu is running */ - if (vcpu->arch.vcb.vpcr == 0) { -#ifndef CONFIG_KVM_MEMHOTPLUG - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) - if (unlikely(bind_vcpu_enabled)) { - int nid; - unsigned long end; - - end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; - nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); - if (pfn_to_nid(PHYS_PFN(end)) == nid) - set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); - } -#endif -#else /* !CONFIG_KVM_MEMHOTPLUG */ - unsigned long seg_base = virt_to_phys(vcpu->kvm->arch.seg_pgd); - - vcpu->arch.vcb.vpcr = get_vpcr_memhp(seg_base, 0); -#endif /* CONFIG_KVM_MEMHOTPLUG */ - vcpu->arch.vcb.upcr = 0x7; - } - -#ifdef CONFIG_PERF_EVENTS - vcpu_load(vcpu); -#endif - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - if (run->exit_reason == KVM_EXIT_MMIO) - kvm_handle_mmio_return(vcpu, run); - - run->exit_reason = KVM_EXIT_UNKNOWN; - ret = 1; - - while (ret > 0) { - /* Check conditions before entering the guest */ - cond_resched(); - - preempt_disable(); - local_irq_disable(); - - if (signal_pending(current)) { - ret = -EINTR; - run->exit_reason = KVM_EXIT_INTR; - vcpu->stat.signal_exits++; - } - - if (ret <= 0) { - local_irq_enable(); - preempt_enable(); - continue; - } - - memset(&hargs, 0, sizeof(hargs)); - - clear_vcpu_irq(vcpu); - - if (vcpu->arch.restart == 1) { - /* handle reset vCPU */ - vcpu->arch.regs.pc = GUEST_RESET_PC; - vcpu->arch.restart = 0; - } - - irq = interrupt_pending(vcpu, &more); - if (irq < SWVM_IRQS) - try_deliver_interrupt(vcpu, irq, more); - - vcpu->arch.halted = 0; - - sw64_kvm_switch_vpn(vcpu); - check_vcpu_requests(vcpu); - guest_enter_irqoff(); - - /* Enter the guest */ - trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); - vcpu->mode = IN_GUEST_MODE; - - ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); - - /* Back from guest */ - vcpu->mode = OUTSIDE_GUEST_MODE; - - vcpu->stat.exits++; - local_irq_enable(); - guest_exit_irqoff(); - - trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); - - preempt_enable(); - - /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ - ret = handle_exit(vcpu, run, ret, &hargs); - update_vcpu_stat_time(&vcpu->stat); - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - -#ifdef CONFIG_PERF_EVENTS - vcpu_put(vcpu); -#endif - return ret; + return kvm_sw64_ops->vcpu_run(vcpu, run); } long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { - unsigned long result; struct kvm_vcpu *vcpu = filp->private_data; - struct vcpucb *kvm_vcb; + int r; switch (ioctl) { case KVM_SW64_VCPU_INIT: - return kvm_arch_vcpu_reset(vcpu); + r = kvm_arch_vcpu_reset(vcpu); + break; case KVM_SW64_GET_VCB: - if (vcpu->arch.vcb.migration_mark) { - result = sw64_io_read(0, LONG_TIME) - + vcpu->arch.vcb.guest_longtime_offset; - vcpu->arch.vcb.guest_longtime = result; - vcpu->arch.vcb.guest_irqs_pending = vcpu->arch.irqs_pending[0]; - } - - if (copy_to_user((void __user *)arg, &(vcpu->arch.vcb), sizeof(struct vcpucb))) - return -EINVAL; + r = kvm_sw64_ops->get_vcb(filp, arg); break; case KVM_SW64_SET_VCB: - kvm_vcb = memdup_user((void __user *)arg, sizeof(*kvm_vcb)); - memcpy(&(vcpu->arch.vcb), kvm_vcb, sizeof(struct vcpucb)); - - if (vcpu->arch.vcb.migration_mark) { - /* updated vpcr needed by destination vm */ - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - - /* synchronize the longtime of source and destination */ - if (vcpu->arch.vcb.soft_cid == 0) { - result = sw64_io_read(0, LONG_TIME); - vcpu->arch.vcb.guest_longtime_offset = vcpu->arch.vcb.guest_longtime - result; - longtime_offset = vcpu->arch.vcb.guest_longtime_offset; - } else - vcpu->arch.vcb.guest_longtime_offset = longtime_offset; - - set_timer(vcpu, 200000000); - vcpu->arch.vcb.migration_mark = 0; - } + r = kvm_sw64_ops->set_vcb(filp, arg); break; default: - return -EINVAL; + r = -EINVAL; } - return 0; + + return r; } long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) @@ -855,30 +504,29 @@ long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) int kvm_arch_init(void *opaque) { + int r; + struct kvm_sw64_ops *ops = opaque; + + if (kvm_sw64_ops) { + printk(KERN_ERR "kvm: already loaded the other module\n"); + r = -EEXIST; + goto out; + } + + kvm_sw64_ops = ops; kvm_sw64_perf_init(); + return 0; +out: + return r; } void kvm_arch_exit(void) { + kvm_sw64_ops = NULL; kvm_sw64_perf_teardown(); } -void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) -{ -} - -void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - kvm_flush_remote_tlbs(kvm); -} - -int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) -{ - return 0; -} - int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { @@ -910,6 +558,34 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) +{ + if (kvm_sw64_ops->flush_shadow_memslot) + kvm_sw64_ops->flush_shadow_memslot(kvm, slot); +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ + if (kvm_sw64_ops->flush_shadow_all) + kvm_sw64_ops->flush_shadow_all(kvm); +} + +void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, + struct kvm_memory_slot *memslot) +{ + /* Let implementation handle TLB/GVA invalidation */ + kvm_arch_flush_shadow_memslot(kvm, memslot); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, + const struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + kvm_sw64_ops->commit_memory_region(kvm, mem, old, new, change); +} + int kvm_dev_ioctl_check_extension(long ext) { int r; @@ -981,30 +657,3 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return vcpu_interrupt_line(vcpu, irq_num, level); } -static int __init kvm_sw64_init(void) -{ - int i, ret; - - ret = vmem_init(); - if (ret) - return ret; - - for (i = 0; i < NR_CPUS; i++) - last_vpn(i) = VPN_FIRST_VERSION; - - ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); - if (ret) { - vmem_exit(); - return ret; - } - return 0; -} - -static void __exit kvm_sw64_exit(void) -{ - kvm_exit(); - vmem_exit(); -} - -module_init(kvm_sw64_init); -module_exit(kvm_sw64_exit); -- Gitee From 9686340989064ad14708acf70da53f73c09da401 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 11 Sep 2023 16:13:32 +0800 Subject: [PATCH 097/150] sw64: kconfig: add SW8A platform support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add a new platform which is for SW8A. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 32784c4c9e30..ef22f65e8c5c 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -227,7 +227,15 @@ config PLATFORM_XUELANG select SW64_INTC_V2 select I2C_SUNWAY if I2C help - Sunway chip3 board chipset + Sunway board chipset for C3B + +config PLATFORM_JUNZHANG + bool "JunZhang" + depends on UNCORE_JUNZHANG + select SPARSE_IRQ + select SYS_HAS_EARLY_PRINTK + help + Sunway board chipset for C4 endchoice -- Gitee From 226e5d9db5396dc17ea21986bf6b148ec2c128cd Mon Sep 17 00:00:00 2001 From: He Sheng Date: Thu, 31 Aug 2023 08:48:49 +0800 Subject: [PATCH 098/150] sw64: dts: add an empty dts to support SW8A temporarily Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- There is no SW8A machine yet. Use empty dts for now. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/boot/dts/Makefile | 6 ++++++ arch/sw_64/boot/dts/empty.dts | 15 +++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 arch/sw_64/boot/dts/empty.dts diff --git a/arch/sw_64/boot/dts/Makefile b/arch/sw_64/boot/dts/Makefile index b9834c70be22..7a9615b79124 100644 --- a/arch/sw_64/boot/dts/Makefile +++ b/arch/sw_64/boot/dts/Makefile @@ -1,7 +1,13 @@ # SPDX-License-Identifier: GPL-2.0 # Built-in dtb +ifeq ($(CONFIG_PLATFORM_XUELANG),y) builtindtb-y := chip3 +endif + +ifeq ($(CONFIG_PLATFORM_JUNZHANG),y) +builtindtb-y := empty +endif ifeq ($(CONFIG_SW64_BUILTIN_DTB), y) ifneq ($(CONFIG_SW64_BUILTIN_DTB_NAME),"") diff --git a/arch/sw_64/boot/dts/empty.dts b/arch/sw_64/boot/dts/empty.dts new file mode 100644 index 000000000000..f8fe34e29641 --- /dev/null +++ b/arch/sw_64/boot/dts/empty.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Default device tree; + */ + +/dts-v1/; +/ { + compatible = "sunway,chip3"; + model = "chip3"; + #address-cells = <2>; + #size-cells = <2>; + + soc { + }; +}; -- Gitee From 7e4aebc564f410e00fae661c5119557c16130483 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 5 Sep 2023 17:13:41 +0800 Subject: [PATCH 099/150] sw64: use PAGE_OFFSET as ktext address head Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Always use PAGE_OFFSET for ktext address. Keep the old address for C3B during init to be compatible with older grub, and these code will be deprecated in future. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/page.h | 14 ++++++++---- arch/sw_64/mm/init.c | 2 +- arch/sw_64/mm/physaddr.c | 40 +++++++++++++++++++---------------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/arch/sw_64/include/asm/page.h b/arch/sw_64/include/asm/page.h index 1ba34f4f5077..5a32d1298c3b 100644 --- a/arch/sw_64/include/asm/page.h +++ b/arch/sw_64/include/asm/page.h @@ -33,20 +33,26 @@ extern void copy_page(void *_to, void *_from); typedef struct page *pgtable_t; extern unsigned long __phys_addr(unsigned long); +#ifdef CONFIG_SUBARCH_C3B +extern unsigned long __boot_phys_addr(unsigned long); +#else +#define __boot_phys_addr(x) __phys_addr(x) +#endif + #endif /* !__ASSEMBLY__ */ #define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) #include -#if defined(CONFIG_SW64_LEGACY_KTEXT_ADDRESS) #define __START_KERNEL_map PAGE_OFFSET -#else -#define __START_KERNEL_map 0xffffffff80000000 -#endif #define __pa(x) __phys_addr((unsigned long)(x)) #define __va(x) ((void *)((unsigned long) (x) | PAGE_OFFSET)) + +#define __boot_pa(x) __boot_phys_addr((unsigned long)(x)) +#define __boot_va(x) __va(x) + #define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) diff --git a/arch/sw_64/mm/init.c b/arch/sw_64/mm/init.c index 73098db59f8d..f16bb4e1d206 100644 --- a/arch/sw_64/mm/init.c +++ b/arch/sw_64/mm/init.c @@ -175,7 +175,7 @@ void __init sw64_memblock_init(void) /* Make sure initrd is in memory range. */ if (sunway_boot_params->initrd_start) { - phys_addr_t base = __pa(sunway_boot_params->initrd_start); + phys_addr_t base = __boot_pa(sunway_boot_params->initrd_start); phys_addr_t size = sunway_boot_params->initrd_size; memblock_add(base, size); diff --git a/arch/sw_64/mm/physaddr.c b/arch/sw_64/mm/physaddr.c index 17840f4ef40b..c123dcad6aa6 100644 --- a/arch/sw_64/mm/physaddr.c +++ b/arch/sw_64/mm/physaddr.c @@ -6,30 +6,34 @@ unsigned long __phys_addr(unsigned long x) { - if (x >= __START_KERNEL_map) { - x -= __START_KERNEL_map; - VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE); - } else { - VIRTUAL_BUG_ON(x < PAGE_OFFSET); - x -= PAGE_OFFSET; - VIRTUAL_BUG_ON(!phys_addr_valid(x)); - } + VIRTUAL_BUG_ON(x < PAGE_OFFSET); + x &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(x)); return x; } EXPORT_SYMBOL(__phys_addr); bool __virt_addr_valid(unsigned long x) { - if (x >= __START_KERNEL_map) { - x -= __START_KERNEL_map; - if (x >= KERNEL_IMAGE_SIZE) - return false; - } else { - if (x < PAGE_OFFSET) - return false; - x -= PAGE_OFFSET; - } - + if (x < PAGE_OFFSET) + return false; + x &= ~PAGE_OFFSET; return pfn_valid(x >> PAGE_SHIFT); } EXPORT_SYMBOL(__virt_addr_valid); + +#ifdef CONFIG_SUBARCH_C3B +#define LEGACY_BOOT_VA 0xffffffff80000000 +unsigned long __boot_phys_addr(unsigned long x) +{ + if (x >= LEGACY_BOOT_VA) { + x &= ~LEGACY_BOOT_VA; + VIRTUAL_BUG_ON(x >= KERNEL_IMAGE_SIZE); + } else { + VIRTUAL_BUG_ON(x < PAGE_OFFSET); + x &= ~PAGE_OFFSET; + VIRTUAL_BUG_ON(!phys_addr_valid(x)); + } + return x; +} +#endif -- Gitee From c653c0ef9d9b46cd4dbd1c424068fc19315ccd31 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 18 Aug 2023 17:26:59 +0800 Subject: [PATCH 100/150] sw64: add basic support for C4 ISA Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- This patch introduces baisc support for new C4 ISA and SW8A CPU based on it. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 4 ++++ arch/sw_64/include/asm/hw_init.h | 1 + 2 files changed, 5 insertions(+) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index ef22f65e8c5c..221b5e8243c9 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -198,6 +198,10 @@ choice config SUBARCH_C3B bool "C3B" +config SUBARCH_C4 + bool "C4" + select HAVE_CSRRW + select GENERIC_SCHED_CLOCK endchoice choice diff --git a/arch/sw_64/include/asm/hw_init.h b/arch/sw_64/include/asm/hw_init.h index 612ef83b9f52..84dd2ee01887 100644 --- a/arch/sw_64/include/asm/hw_init.h +++ b/arch/sw_64/include/asm/hw_init.h @@ -102,6 +102,7 @@ DECLARE_STATIC_KEY_FALSE(run_mode_emul_key); #define CPU_SW3231 0x31 #define CPU_SW831 0x32 +#define CPU_SW8A 0x41 #define GET_TABLE_ENTRY 1 #define GET_VENDOR_ID 2 -- Gitee From 93d2b0e64b028b5192c99b0f79b3b05486aa9a5b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 30 Aug 2023 17:27:02 +0800 Subject: [PATCH 101/150] sw64: defconfig: add junzhang_defconfig for C4 serials Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add a new defconfig for C4 cpus. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/configs/junzhang_defconfig | 697 ++++++++++++++++++++++++++ 1 file changed, 697 insertions(+) create mode 100644 arch/sw_64/configs/junzhang_defconfig diff --git a/arch/sw_64/configs/junzhang_defconfig b/arch/sw_64/configs/junzhang_defconfig new file mode 100644 index 000000000000..2a9a7bb9d70f --- /dev/null +++ b/arch/sw_64/configs/junzhang_defconfig @@ -0,0 +1,697 @@ +CONFIG_LOCALVERSION="-junzhang" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +# CONFIG_CROSS_MEMORY_ATTACH is not set +CONFIG_USELIB=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKHEADERS=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_MEMCG=y +CONFIG_BLK_CGROUP=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_NAMESPACES=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_RELAY=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y +CONFIG_PERF_EVENTS=y +CONFIG_DEBUG_PERF_USE_VMALLOC=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SUBARCH_C4=y +CONFIG_SMP=y +CONFIG_SCHED_SMT=y +CONFIG_NR_CPUS=64 +CONFIG_ARCH_SPARSEMEM_ENABLE=y +CONFIG_NUMA=y +CONFIG_HZ=100 +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_USE_OF=y +CONFIG_FIRMWARE_MEMMAP=y +CONFIG_DMI_SYSFS=m +CONFIG_ACPI_TAD=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE=y +CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE=y +# CONFIG_CPU_IDLE is not set +CONFIG_VIRTUALIZATION=y +CONFIG_KVM=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_SCSI=m +CONFIG_VHOST_VSOCK=m +CONFIG_VHOST_CROSS_ENDIAN_LEGACY=y +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_OSF_PARTITION=y +CONFIG_BSD_DISKLABEL=y +CONFIG_MINIX_SUBPARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y +CONFIG_LDM_PARTITION=y +CONFIG_SGI_PARTITION=y +CONFIG_ULTRIX_PARTITION=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_CMA_AREAS=7 +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y +CONFIG_XFRM_USER=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_STATISTICS=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_IP_MROUTE=y +CONFIG_NET_IPVTI=m +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_UDP_DIAG=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6=m +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=m +CONFIG_IPV6_ILA=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=m +CONFIG_NFT_COUNTER=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_OSF=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_NF_TABLES_IPV4=y +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_FLOW_TABLE_IPV4=m +CONFIG_NF_LOG_ARP=m +CONFIG_IP_NF_IPTABLES=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE=m +CONFIG_VLAN_8021Q=m +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_DEFAULT=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +# CONFIG_WIRELESS is not set +CONFIG_PCI=y +CONFIG_PCIEPORTBUS=y +CONFIG_PCIEAER=y +# CONFIG_PCIEASPM is not set +CONFIG_PCI_MSI=y +CONFIG_PCI_IOV=y +CONFIG_UEVENT_HELPER=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_STANDALONE is not set +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_JEDECPROBE=y +CONFIG_MTD_CFI_ADV_OPTIONS=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_STAA=y +CONFIG_MTD_ROM=y +CONFIG_MTD_ABSENT=y +CONFIG_MTD_COMPLEX_MAPPINGS=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=y +CONFIG_BLK_DEV_RAM_SIZE=5000000 +CONFIG_VIRTIO_BLK=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_MULTIPATH=y +CONFIG_NVME_RDMA=m +CONFIG_NVME_FC=y +CONFIG_NVME_TARGET=y +CONFIG_NVME_TARGET_LOOP=y +CONFIG_NVME_TARGET_RDMA=m +CONFIG_NVME_TARGET_FC=y +CONFIG_NVME_TARGET_FCLOOP=y +CONFIG_RAID_ATTRS=y +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=y +CONFIG_BLK_DEV_SR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=y +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=y +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=y +CONFIG_SCSI_CXGB3_ISCSI=m +CONFIG_SCSI_CXGB4_ISCSI=m +CONFIG_SCSI_BNX2_ISCSI=m +CONFIG_MEGARAID_NEWGEN=y +CONFIG_MEGARAID_MM=m +CONFIG_MEGARAID_MAILBOX=m +CONFIG_MEGARAID_LEGACY=m +CONFIG_MEGARAID_SAS=m +CONFIG_SCSI_MPT3SAS=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=y +CONFIG_SCSI_DH_HP_SW=y +CONFIG_SCSI_DH_EMC=y +CONFIG_SCSI_DH_ALUA=y +CONFIG_ATA=y +CONFIG_SATA_AHCI=y +# CONFIG_ATA_SFF is not set +CONFIG_MD=y +CONFIG_MD_LINEAR=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BCACHE_DEBUG=y +CONFIG_BCACHE_CLOSURES_DEBUG=y +CONFIG_BLK_DEV_DM=m +CONFIG_DM_DEBUG=y +CONFIG_DM_DEBUG_BLOCK_MANAGER_LOCKING=y +CONFIG_DM_DEBUG_BLOCK_STACK_TRACING=y +CONFIG_DM_UNSTRIPED=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_WRITECACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_RAID=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_ISCSI_TARGET=m +CONFIG_NET_FC=y +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_VIRTIO_NET=y +# CONFIG_NET_VENDOR_3COM is not set +# CONFIG_NET_VENDOR_AMD is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +CONFIG_CAVIUM_PTP=y +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +CONFIG_E100=y +CONFIG_E1000=y +CONFIG_E1000E=y +CONFIG_IGB=y +CONFIG_IGBVF=m +CONFIG_IXGB=m +CONFIG_IXGBE=m +CONFIG_IXGBEVF=m +CONFIG_I40E=y +CONFIG_I40EVF=y +# CONFIG_NET_VENDOR_MARVELL is not set +CONFIG_MLX4_EN=y +CONFIG_MLX5_CORE=m +CONFIG_MLX5_FPGA=y +CONFIG_MLX5_CORE_EN=y +CONFIG_MLXSW_CORE=y +CONFIG_MLXSW_PCI=y +CONFIG_MLXSW_I2C=y +CONFIG_MLXSW_MINIMAL=y +# CONFIG_NET_VENDOR_MICREL is not set +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SMSC is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +# CONFIG_NET_VENDOR_VIA is not set +# CONFIG_NET_VENDOR_WIZNET is not set +# CONFIG_WLAN is not set +CONFIG_INPUT_FF_MEMLESS=y +CONFIG_INPUT_POLLDEV=y +CONFIG_INPUT_MOUSEDEV=y +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_LIBPS2=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_PCI is not set +CONFIG_SERIAL_8250_SUNWAY=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_VIRTIO_CONSOLE=y +# CONFIG_HW_RANDOM is not set +# CONFIG_DEVPORT is not set +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_MUX=y +CONFIG_SPI=y +CONFIG_SPI_SPIDEV=y +CONFIG_SENSORS_PVT=y +CONFIG_SENSORS_LM75=y +CONFIG_SSB=y +CONFIG_DRM=y +CONFIG_DRM_RADEON=y +CONFIG_DRM_AST=y +CONFIG_DRM_VIRTIO_GPU=y +CONFIG_FIRMWARE_EDID=y +CONFIG_LCD_CLASS_DEVICE=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_MTHCA=m +# CONFIG_INFINIBAND_MTHCA_DEBUG is not set +CONFIG_MLX4_INFINIBAND=m +CONFIG_MLX5_INFINIBAND=m +CONFIG_INFINIBAND_IPOIB=m +CONFIG_INFINIBAND_IPOIB_CM=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +# CONFIG_RTC_INTF_PROC is not set +CONFIG_RTC_DRV_PCF8523=y +CONFIG_UIO=y +CONFIG_UIO_PCI_GENERIC=m +CONFIG_VIRTIO_PCI=y +# CONFIG_VIRTIO_PCI_LEGACY is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_STAGING=y +CONFIG_FB_SM750=y +CONFIG_IOMMU_DEFAULT_PASSTHROUGH=y +CONFIG_SUNWAY_IOMMU=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_DEBUG=y +CONFIG_XFS_FS=y +CONFIG_GFS2_FS=y +CONFIG_FANOTIFY=y +CONFIG_QUOTA=y +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_FSCACHE=y +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=y +CONFIG_NTFS_RW=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_HUGETLBFS=y +CONFIG_CONFIGFS_FS=y +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_NFS_V4_1_MIGRATION=y +CONFIG_ROOT_NFS=y +CONFIG_NFS_FSCACHE=y +CONFIG_NFS_USE_LEGACY_DNS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_SCSILAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=y +CONFIG_NLS_CODEPAGE_950=y +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK=y +CONFIG_SECURITY_INFINIBAND=y +CONFIG_SECURITY_PATH=y +CONFIG_CRYPTO_AUTHENC=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_CRYPTO_CBC=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_DEFLATE=y +CONFIG_CRYPTO_LZO=y +# CONFIG_CRYPTO_HW is not set +CONFIG_CONSOLE_LOGLEVEL_QUIET=7 +# CONFIG_ENABLE_MUST_CHECK is not set +# CONFIG_FRAME_POINTER is not set +CONFIG_SCHEDSTATS=y +# CONFIG_RCU_TRACE is not set -- Gitee From 7ffa4715e2d62e690a0dc8376bcfbefef5f3cf6d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 14:08:08 +0800 Subject: [PATCH 102/150] sw64: update default FPCR value Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Enable hardware denormal support on C4 by default. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/uapi/asm/fpu.h | 10 ++++++++++ arch/sw_64/kernel/process.c | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/include/uapi/asm/fpu.h b/arch/sw_64/include/uapi/asm/fpu.h index 035ca65b1ba3..ca95ecbb6765 100644 --- a/arch/sw_64/include/uapi/asm/fpu.h +++ b/arch/sw_64/include/uapi/asm/fpu.h @@ -6,7 +6,11 @@ * SW-64 floating-point control register defines: */ #define FPCR_DNOD (1UL << 47) /* denorm INV trap disable */ +#ifdef CONFIG_SUBARCH_C3B #define FPCR_DNZ (1UL << 48) /* denorms to zero */ +#else +#define FPCR_DNOE (1UL << 48) /* hardware denormal support */ +#endif #define FPCR_INVD (1UL << 49) /* invalid op disable (opt.) */ #define FPCR_DZED (1UL << 50) /* division by zero disable (opt.) */ #define FPCR_OVFD (1UL << 51) /* overflow disable (optional) */ @@ -30,6 +34,12 @@ #define FPCR_MASK 0xffff800000000000L +#ifdef CONFIG_SUBARCH_C3B +#define FPCR_INIT FPCR_DYN_NORMAL +#else +#define FPCR_INIT (FPCR_DYN_NORMAL | FPCR_DNOE) +#endif + /* status bit coming from hardware fpcr . definde by fire3 */ #define FPCR_STATUS_INV0 (1UL << 52) #define FPCR_STATUS_DZE0 (1UL << 53) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 4ffa724897c3..3dadf450debe 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -34,7 +34,7 @@ flush_thread(void) * with respect to the FPU. This is all exceptions disabled. */ current_thread_info()->ieee_state = 0; - wrfpcr(FPCR_DYN_NORMAL | ieee_swcr_to_fpcr(0)); + wrfpcr(FPCR_INIT | ieee_swcr_to_fpcr(0)); /* Clean slate for TLS. */ current_thread_info()->pcb.tp = 0; -- Gitee From e15658e1738ec7e2d37b9bbd633ffa73882030a8 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 11 Sep 2023 17:35:12 +0800 Subject: [PATCH 103/150] sw64: add C4 FPE_INTDIV support Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- C4 supports hardware div and mod operations. Setup si_code accordingly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 934559d0d125..30dd2feeb3ec 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -164,6 +164,10 @@ do_entArith(unsigned long summary, unsigned long write_mask, if (!user_mode(regs)) die("Arithmetic fault", regs, 0); + /*summary<39> means integer divide by zero in C4.*/ + if ((summary >> 39) & 1) + si_code = FPE_INTDIV; + force_sig_fault(SIGFPE, si_code, (void __user *)regs->pc); } -- Gitee From 77214fb34d8f9cbcb17f471723dbc8a6a40f504a Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 16:57:36 +0800 Subject: [PATCH 104/150] sw64: add new match interface for C4 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add a new match interface which supports per process match and a user interface for C4. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig.debug | 7 + arch/sw_64/include/asm/core.h | 1 + arch/sw_64/include/asm/thread_info.h | 8 +- arch/sw_64/include/uapi/asm/ptrace.h | 6 + arch/sw_64/kernel/Makefile | 1 + arch/sw_64/kernel/match.c | 551 +++++++++++++++++++++++++++ arch/sw_64/kernel/ptrace.c | 251 +++++++++++- 7 files changed, 822 insertions(+), 3 deletions(-) create mode 100644 arch/sw_64/kernel/match.c diff --git a/arch/sw_64/Kconfig.debug b/arch/sw_64/Kconfig.debug index 2cd2036e0996..6cb3c2488b36 100644 --- a/arch/sw_64/Kconfig.debug +++ b/arch/sw_64/Kconfig.debug @@ -44,3 +44,10 @@ config SW64_RRK help Duplicate kernel log to specific space. Do not enable it in a production kernel. + +config DEBUG_MATCH + bool "instruction-flow and data-flow match debugfs interface" + depends on DEBUG_FS + default n + help + Turns on the DebugFS interface for instruction-flow and data-flow match. diff --git a/arch/sw_64/include/asm/core.h b/arch/sw_64/include/asm/core.h index bf2e95cd8a3a..3f4df6d6c92d 100644 --- a/arch/sw_64/include/asm/core.h +++ b/arch/sw_64/include/asm/core.h @@ -72,6 +72,7 @@ static inline bool core_is_ht(void) #define MMCSR__DAV_MATCH 0x102 #define MMCSR__IA_MATCH 0x103 #define MMCSR__IDA_MATCH 0x104 +#define MMCSR__IV_MATCH 0x105 /* entry.S */ extern void entArith(void); diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index c111dc88bb22..2523347c2976 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -18,7 +18,13 @@ struct pcb_struct { unsigned long tp; unsigned long da_match, da_mask; unsigned long dv_match, dv_mask; - unsigned long dc_ctl; + union { + unsigned long dc_ctl; + unsigned long match_ctl; + }; + unsigned long ia_match, ia_mask; + unsigned long iv_match; + unsigned long ida_match, ida_mask; }; struct thread_info { diff --git a/arch/sw_64/include/uapi/asm/ptrace.h b/arch/sw_64/include/uapi/asm/ptrace.h index e48dd68c5b4d..e2c76d58fa82 100644 --- a/arch/sw_64/include/uapi/asm/ptrace.h +++ b/arch/sw_64/include/uapi/asm/ptrace.h @@ -47,5 +47,11 @@ struct user_fpsimd_state { #define PT_DV_MATCH 165 #define PT_DV_MASK 166 #define PT_DC_CTL 167 +#define PT_MATCH_CTL 167 +#define PT_IA_MATCH 168 +#define PT_IA_MASK 169 +#define PT_IV_MATCH 170 +#define PT_IDA_MATCH 171 +#define PT_IDA_MASK 172 #endif /* _UAPI_ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index c35063a30616..3aee5b7f5fdb 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -34,6 +34,7 @@ obj-$(CONFIG_PCI) += pci_common.o obj-$(CONFIG_RELOCATABLE) += relocate.o obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o +obj-$(CONFIG_DEBUG_MATCH) += match.o ifeq ($(CONFIG_DEBUG_FS)$(CONFIG_NUMA),yy) obj-y += bindvcpu.o diff --git a/arch/sw_64/kernel/match.c b/arch/sw_64/kernel/match.c new file mode 100644 index 000000000000..3926391270da --- /dev/null +++ b/arch/sw_64/kernel/match.c @@ -0,0 +1,551 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include +#include + + +char da_match_buf[1024], dv_match_buf[1024], dav_match_buf[1024]; +char ia_match_buf[1024], iv_match_buf[1024], ida_match_buf[1024]; + +unsigned long da_match_cf1, da_match_cf2, da_match_cf3; +unsigned long dv_match_cf1, dv_match_cf2, dv_match_cf3; +unsigned long dav_match_cf1, dav_match_cf2, dav_match_cf3, + dav_match_cf4, dav_match_cf5; +unsigned long ia_match_cf1, ia_match_cf2, ia_match_cf3, ia_match_cf4; +unsigned long iv_match_cf1, iv_match_cf2; +unsigned long ida_match_cf1, ida_match_cf2; + +static int da_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", da_match_buf); + return 0; +} + +static int dv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dv_match_buf); + return 0; +} + +static int dav_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", dav_match_buf); + return 0; +} + +static int ia_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ia_match_buf); + return 0; +} + +static int iv_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", iv_match_buf); + return 0; +} + +static int ida_match_show(struct seq_file *m, void *v) +{ + + seq_printf(m, "%s", ida_match_buf); + return 0; +} + +static int da_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, da_match_show, NULL); +} + +static int dv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dv_match_show, NULL); +} + +static int dav_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, dav_match_show, NULL); +} + +static int ia_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ia_match_show, NULL); +} + +static int iv_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, iv_match_show, NULL); +} + +static int ida_match_open(struct inode *inode, struct file *file) +{ + return single_open(file, ida_match_show, NULL); +} + +static void +write_da_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(da_match_cf1, CSR_DA_MATCH); + write_csr(da_match_cf2, CSR_DA_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= da_match_cf3; + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dv_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dv_match_cf1, CSR_DV_MATCH); + write_csr(dv_match_cf2, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | dv_match_cf3); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_dav_match(void *i) +{ + unsigned long dc_ctl; + + write_csr(dav_match_cf1, CSR_DA_MATCH); + write_csr(dav_match_cf2, CSR_DA_MASK); + write_csr(dav_match_cf3, CSR_DV_MATCH); + write_csr(dav_match_cf4, CSR_DV_MASK); + dc_ctl = read_csr(CSR_DC_CTLP); + dc_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) + | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH)); + dc_ctl |= ((0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S) + | dav_match_cf5); + write_csr(dc_ctl, CSR_DC_CTLP); +} + +static void +write_ia_match(void *i) +{ + ia_match_cf1 |= (0x1UL << IA_MATCH_EN_S); + write_csr_imb(ia_match_cf1, CSR_IA_MATCH); + write_csr_imb(ia_match_cf2, CSR_IA_MASK); + write_csr(((0x3ffUL << 18) | ia_match_cf3), CSR_IA_VPNMATCH); + write_csr(((0x3ffUL << 18) | ia_match_cf4), CSR_IA_UPNMATCH); +} + +static void +write_iv_match(void *i) +{ + unsigned long ia_match_tmp; + + ia_match_tmp = read_csr(CSR_IA_MATCH); + ia_match_tmp &= ~(0x1UL << IV_PM_EN_S); + ia_match_tmp |= ((((iv_match_cf2 >> IV_PM_EN_S) & 0x1) << IV_PM_EN_S) + | (iv_match_cf2 & 0x3) | (0x1UL << IV_MATCH_EN_S)); + write_csr_imb(iv_match_cf1, CSR_IV_MATCH); + write_csr_imb(ia_match_tmp, CSR_IA_MATCH); +} + +static void +write_ida_match(void *i) +{ + + ida_match_cf1 |= (0x1UL << IDA_MATCH_EN_S); + write_csr(ida_match_cf1, CSR_IDA_MATCH); + write_csr(ida_match_cf2, CSR_IDA_MASK); +} + +static ssize_t da_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(da_match_buf) - 1, len); + if (copy_from_user(da_match_buf, user_buf, size)) + return -EFAULT; + + da_match_buf[size] = '\0'; + strcpy(tmp, da_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &da_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &da_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &da_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_da_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(dv_match_buf) - 1, len); + if (copy_from_user(dv_match_buf, user_buf, size)) + return -EFAULT; + + dv_match_buf[size] = '\0'; + strcpy(tmp, dv_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dv_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dv_match_cf3); + if (err) + return err; + + if (on_each_cpu(write_dv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static ssize_t dav_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[500]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[500]; + int err; + char *ret = NULL; + + size = min(sizeof(dav_match_buf) - 1, len); + if (copy_from_user(dav_match_buf, user_buf, size)) + return -EFAULT; + + dav_match_buf[size] = '\0'; + strcpy(tmp, dav_match_buf); + p = tmp; + + for (i = 0 ; i < 5; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[500] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &dav_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &dav_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &dav_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &dav_match_cf4); + if (err) + return err; + + err = kstrtoul(&tmp1[400], 0, &dav_match_cf5); + if (err) + return err; + + + if (on_each_cpu(write_dav_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t ia_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ia_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ia_match_cf2); + if (err) + return err; + + err = kstrtoul(&tmp1[200], 0, &ia_match_cf3); + if (err) + return err; + + err = kstrtoul(&tmp1[300], 0, &ia_match_cf4); + if (err) + return err; + + if (on_each_cpu(write_ia_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + +static ssize_t iv_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ia_match_buf) - 1, len); + if (copy_from_user(ia_match_buf, user_buf, size)) + return -EFAULT; + + ia_match_buf[size] = '\0'; + strcpy(tmp, ia_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &iv_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &iv_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_iv_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + return len; +} + + +static ssize_t ida_match_set(struct file *file, const char __user *user_buf, + size_t len, loff_t *ppos) +{ + size_t size; + char tmp[400]; + char *p; + int i, m; + const char *sep = " "; + char tmp1[400]; + int err; + char *ret = NULL; + + size = min(sizeof(ida_match_buf) - 1, len); + if (copy_from_user(ida_match_buf, user_buf, size)) + return -EFAULT; + + ida_match_buf[size] = '\0'; + strcpy(tmp, ida_match_buf); + p = tmp; + + for (i = 0 ; i < 4; i++) { + m = i*100; + ret = strsep(&p, sep); + if (ret != NULL) + strcpy(&tmp1[m], ret); + } + tmp1[400] = '\0'; + + err = kstrtoul(&tmp1[0], 0, &ida_match_cf1); + if (err) + return err; + + err = kstrtoul(&tmp1[100], 0, &ida_match_cf2); + if (err) + return err; + + if (on_each_cpu(write_ida_match, NULL, 1)) + pr_crit("%s: timed out\n", __func__); + + return len; +} + +static const struct file_operations set_da_match_fops = { + .open = da_match_open, + .read = seq_read, + .write = da_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dv_match_fops = { + .open = dv_match_open, + .read = seq_read, + .write = dv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_dav_match_fops = { + .open = dav_match_open, + .read = seq_read, + .write = dav_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_ia_match_fops = { + .open = ia_match_open, + .read = seq_read, + .write = ia_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static const struct file_operations set_iv_match_fops = { + .open = iv_match_open, + .read = seq_read, + .write = iv_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + + +static const struct file_operations set_ida_match_fops = { + .open = ida_match_open, + .read = seq_read, + .write = ida_match_set, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init match_debugfs_init(void) +{ + struct dentry *match_entry; + + if (!sw64_debugfs_dir) + return -ENODEV; + + match_entry = debugfs_create_file("da_match", 0600, + sw64_debugfs_dir, NULL, + &set_da_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dv_match", 0600, + sw64_debugfs_dir, NULL, + &set_dv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("dav_match", 0600, + sw64_debugfs_dir, NULL, + &set_dav_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ia_match", 0600, + sw64_debugfs_dir, NULL, + &set_ia_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("iv_match", 0600, + sw64_debugfs_dir, NULL, + &set_iv_match_fops); + if (!match_entry) + return -ENOMEM; + + match_entry = debugfs_create_file("ida_match", 0600, + sw64_debugfs_dir, NULL, + &set_ida_match_fops); + if (!match_entry) + return -ENOMEM; + + return 0; +} +late_initcall(match_debugfs_init); diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 01fdfc2babc9..25cb19340dde 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -15,6 +15,7 @@ #include #include "proto.h" +#include #define CREATE_TRACE_POINTS #include @@ -67,7 +68,13 @@ static int pcboff[] = { [PT_DA_MASK] = PCB_OFF(da_mask), [PT_DV_MATCH] = PCB_OFF(dv_match), [PT_DV_MASK] = PCB_OFF(dv_mask), - [PT_DC_CTL] = PCB_OFF(dc_ctl) + [PT_DC_CTL] = PCB_OFF(dc_ctl), + [PT_MATCH_CTL] = PCB_OFF(match_ctl), + [PT_IA_MATCH] = PCB_OFF(ia_match), + [PT_IA_MASK] = PCB_OFF(ia_mask), + [PT_IV_MATCH] = PCB_OFF(iv_match), + [PT_IDA_MATCH] = PCB_OFF(ida_match), + [PT_IDA_MASK] = PCB_OFF(ida_mask) }; static unsigned long zero; @@ -89,7 +96,12 @@ get_reg_addr(struct task_struct *task, unsigned long regno) case PT_DA_MASK: case PT_DV_MATCH: case PT_DV_MASK: - case PT_DC_CTL: + case PT_MATCH_CTL: + case PT_IA_MATCH: + case PT_IA_MASK: + case PT_IV_MATCH: + case PT_IDA_MATCH: + case PT_IDA_MASK: addr = (void *)task_thread_info(task) + pcboff[regno]; break; case PT_REG_BASE ... PT_REG_END: @@ -433,6 +445,7 @@ syscall_trace_leave(void) trace_sys_exit(regs, regs_return_value(regs)); } +#ifdef CONFIG_SUBARCH_C3B static long rwcsr(int rw, unsigned long csr, unsigned long value) { register unsigned long __r0 __asm__("$0"); @@ -556,6 +569,240 @@ void restore_da_match_after_sched(void) } } +#elif defined(CONFIG_SUBARCH_C4) +int do_match(unsigned long address, unsigned long mmcsr, long cause, struct pt_regs *regs) +{ + kernel_siginfo_t info; + unsigned long match_ctl, ia_match; + sigval_t sw64_value; + + printk("%s: pid %d, name = %s, cause = %#lx, mmcsr = %#lx, address = %#lx, pc %#lx\n", + __func__, current->pid, current->comm, cause, mmcsr, address, regs->pc); + + switch (mmcsr) { + case MMCSR__DA_MATCH: + case MMCSR__DV_MATCH: + case MMCSR__DAV_MATCH: + case MMCSR__IA_MATCH: + case MMCSR__IDA_MATCH: + case MMCSR__IV_MATCH: + show_regs(regs); + + if (!(current->ptrace & PT_PTRACED)) { + printk(" pid %d %s not be ptraced, return\n", current->pid, current->comm); + if (mmcsr == MMCSR__DA_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + return 1; + } + + info.si_signo = SIGTRAP; + info.si_addr = (void *) address; + sw64_value.sival_ptr = (void *)(regs->pc); + info.si_value = sw64_value; + info.si_code = TRAP_HWBKPT; + + if (mmcsr == MMCSR__DA_MATCH) { + info.si_errno = 1; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x3UL << DA_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + task_thread_info(current)->pcb.match_ctl &= ~0x1; + task_thread_info(current)->pcb.da_match = 0; + } + if (mmcsr == MMCSR__DV_MATCH) { + info.si_errno = 2; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~(0x1UL << DV_MATCH_EN_S); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 1); + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__DAV_MATCH) { + info.si_errno = 3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + write_csr(match_ctl, CSR_DC_CTLP); + write_csr(0, CSR_DA_MATCH); // clear da_match + write_csr(0, CSR_DV_MATCH); // clear dv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 | (0x1 << 1) | (0x1 << 2)); + task_thread_info(current)->pcb.da_match = 0; + task_thread_info(current)->pcb.dv_match = 0; + } + if (mmcsr == MMCSR__IA_MATCH) { + info.si_errno = 4; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IA_MATCH_EN_S) | (0x7ffffffffffffUL << 2)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 3); + task_thread_info(current)->pcb.ia_match = 0; + } + if (mmcsr == MMCSR__IV_MATCH) { + info.si_errno = 5; + ia_match = read_csr(CSR_IA_MATCH); + ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + write_csr(ia_match, CSR_IA_MATCH); // clear ia_match + write_csr(0, CSR_IV_MATCH); // clear iv_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 4); + task_thread_info(current)->pcb.ia_match &= ~((0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S)); + task_thread_info(current)->pcb.iv_match = 0; + } + if (mmcsr == MMCSR__IDA_MATCH) { + info.si_errno = 6; + write_csr(0, CSR_IDA_MATCH); // clear ida_match + task_thread_info(current)->pcb.match_ctl &= ~(0x1 << 5); + task_thread_info(current)->pcb.ida_match = 0; + } + printk("do_page_fault: want to send SIGTRAP, pid = %d\n", current->pid); + force_sig_info(&info); + return 1; + } + + return 0; +} + +/* + *pcb->match_ctl: + * [0] DA_MATCH + * [1] DV_MATCH + * [2] DAV_MATCH + * [3] IA_MATCH + * [4] IV_MATCH + * [5] IDA_MATCH + * [8:9] match_ctl_mode + * + */ +#define DA_MATCH 0x1 +#define DV_MATCH 0x2 +#define DAV_MATCH 0x4 +#define IA_MATCH 0x8 +#define IV_MATCH 0x10 +#define IDA_MATCH 0x20 + +void restore_da_match_after_sched(void) +{ + unsigned long match_ctl_mode; + unsigned long match_ctl; + struct pcb_struct *pcb = &task_thread_info(current)->pcb; + unsigned long vpn, upn; + + if (!pcb->match_ctl) + return; + pr_info("Restroe MATCH status, pid: %d\n", current->pid); + + if (pcb->match_ctl & DA_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx match_ctl:%#lx\n", pcb->da_match, pcb->da_mask, match_ctl); + } + + if (pcb->match_ctl & DV_MATCH) { + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x1UL << 3) | (0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (0x1UL << DV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & DAV_MATCH) { + write_csr(pcb->da_match, CSR_DA_MATCH); + write_csr(pcb->da_mask, CSR_DA_MASK); + write_csr(pcb->dv_match, CSR_DV_MATCH); + write_csr(pcb->dv_mask, CSR_DV_MASK); + write_csr(0xfffffffff, CSR_DA_MATCH_MODE); + match_ctl_mode = (pcb->match_ctl >> 8) & 0x3; + match_ctl = read_csr(CSR_DC_CTLP); + match_ctl &= ~((0x3UL << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) | (0x1UL << DAV_MATCH_EN_S)); + match_ctl |= (match_ctl_mode << DA_MATCH_EN_S) | (0x1UL << DV_MATCH_EN_S) + | (0x1UL << DAV_MATCH_EN_S) | (0x1UL << DPM_MATCH_EN_S) + | (0x3UL << DPM_MATCH); + write_csr(match_ctl, CSR_DC_CTLP); + pr_info("da_match:%#lx da_mask:%#lx dv_match:%#lx dv_mask:%#lx match_ctl:%#lx\n", + pcb->da_match, pcb->da_mask, pcb->dv_match, pcb->dv_mask, match_ctl); + } + + if (pcb->match_ctl & IA_MATCH) { + pcb->ia_match |= (0x1UL << IA_MATCH_EN_S) | 0x3; + pcb->ia_mask |= 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->ia_mask, CSR_IA_MASK); + vpn = read_csr(CSR_VPCR) >> 44; + vpn &= 0x3ff; + upn = read_csr(CSR_UPCR); + upn &= 0x3ff; + write_csr(((0x3ff << 18) | vpn), CSR_IA_VPNMATCH); + write_csr(((0x3ff << 18) | upn), CSR_IA_UPNMATCH); + pr_info("ia_match:%#lx ia_mask:%#lx\n", pcb->ia_match, pcb->ia_mask); + } + if (pcb->match_ctl & IV_MATCH) { + pcb->ia_match |= (0x1UL << IV_MATCH_EN_S) | (0x1UL << IV_PM_EN_S) | 0x3; + write_csr(pcb->ia_match, CSR_IA_MATCH); + write_csr(pcb->iv_match, CSR_IV_MATCH); + pr_info("ia_match:%#lx iv_match:%#lx\n", pcb->ia_match, pcb->iv_match); + } + if (pcb->match_ctl & IDA_MATCH) { + pcb->ida_match |= (0x1UL << IDA_MATCH_EN_S) | 0x3; + pcb->ida_mask |= 0x3; + write_csr(pcb->ida_match, CSR_IDA_MATCH); + write_csr(pcb->ida_mask, CSR_IDA_MASK); + pr_info("ida_match:%#lx ida_mask:%#lx\n", pcb->ida_match, pcb->ida_mask); + } +} +#endif + struct pt_regs_offset { const char *name; int offset; -- Gitee From 734c1436b9e23a6a82c2ff85ddfd1719d2eea5e5 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 8 Sep 2023 14:24:19 +0800 Subject: [PATCH 105/150] sw64: optimize __fpstate_save on C4 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Do not save fp regs if they are not modified. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/fpu.S | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/sw_64/kernel/fpu.S b/arch/sw_64/kernel/fpu.S index 3cb3bfab08e8..ddc988681fdd 100644 --- a/arch/sw_64/kernel/fpu.S +++ b/arch/sw_64/kernel/fpu.S @@ -3,11 +3,16 @@ #include #include #include +#include .text .set noat ENTRY(__fpstate_save) /* a0: prev task */ +#ifdef CONFIG_SUBARCH_C4 + csrr $1, CSR_WR_FREGS + beq $1, out +#endif vstd $f0, TASK_THREAD_F0(a0) vstd $f1, TASK_THREAD_F1(a0) vstd $f2, TASK_THREAD_F2(a0) @@ -42,6 +47,7 @@ ENTRY(__fpstate_save) vstd $f30, TASK_THREAD_F30(a0) fstd $f0, TASK_THREAD_FPCR(a0) vldd $f0, TASK_THREAD_F0(a0) +out: ret END(__fpstate_save) @@ -98,5 +104,8 @@ $setfpec_over: vldd $f28, TASK_THREAD_F28(a0) vldd $f29, TASK_THREAD_F29(a0) vldd $f30, TASK_THREAD_F30(a0) +#ifdef CONFIG_SUBARCH_C4 + csrw $31, CSR_WR_FREGS +#endif ret END(__fpstate_restore) -- Gitee From 7e704f66682cb221b045d325964f696c287c65e0 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 6 Sep 2023 13:44:01 +0800 Subject: [PATCH 106/150] sw64: lib: optimize lib functions for C4 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- C4 supports hardware unalign handling. Optimize memory operation libraries accordingly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/deep-clear_user.S | 4 + arch/sw_64/lib/deep-copy_page.S | 7 ++ arch/sw_64/lib/deep-copy_template_c4.S | 108 +++++++++++++++++++++++++ arch/sw_64/lib/deep-copy_user.S | 11 ++- arch/sw_64/lib/deep-memcpy.S | 4 + arch/sw_64/lib/deep-memset.S | 11 +++ arch/sw_64/lib/deep-set_template_c4.S | 93 +++++++++++++++++++++ 7 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 arch/sw_64/lib/deep-copy_template_c4.S create mode 100644 arch/sw_64/lib/deep-set_template_c4.S diff --git a/arch/sw_64/lib/deep-clear_user.S b/arch/sw_64/lib/deep-clear_user.S index 521586a7189f..c81418ed99a2 100644 --- a/arch/sw_64/lib/deep-clear_user.S +++ b/arch/sw_64/lib/deep-clear_user.S @@ -34,7 +34,11 @@ __clear_user: bis $31, $31, $7 mov $17, $18 bis $31, $31, $17 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif $out: bis $31, $18, $0 beq $7, $return diff --git a/arch/sw_64/lib/deep-copy_page.S b/arch/sw_64/lib/deep-copy_page.S index 5061837e537b..a9b9d97f318a 100644 --- a/arch/sw_64/lib/deep-copy_page.S +++ b/arch/sw_64/lib/deep-copy_page.S @@ -5,6 +5,7 @@ * Copy an entire page. */ #include +#include .text .align 4 @@ -19,6 +20,9 @@ copy_page: stl $4, 0($sp) bic $4, 0x1f, $4 vstd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrr $5, CSR_WR_FREGS +#endif /* Optimize by GUOY from SOC 2013-06-04 */ 1: @@ -46,6 +50,9 @@ copy_page: ldi $4, 0x40($sp) bic $4, 0x1f, $4 vldd $f16, 0($4) +#ifdef CONFIG_SUBARCH_C4 + csrw $5, CSR_WR_FREGS +#endif addl $sp, 0x60, $sp ret diff --git a/arch/sw_64/lib/deep-copy_template_c4.S b/arch/sw_64/lib/deep-copy_template_c4.S new file mode 100644 index 000000000000..e0740874dfa3 --- /dev/null +++ b/arch/sw_64/lib/deep-copy_template_c4.S @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memcpy and copy_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd and simd_u loop + * $16: latest dest, clobbered + * $17: latest src, clobbered + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x60($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + vstd $f2, 0x20($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vldd $f1, 0($23); \ + vldd $f2, 0x20($23); \ + ldi $sp, 0x60($sp); \ + bis $31, $31, $7 + + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 4 +$simd_loop: + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vldd $f2, 32($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f2, 32($16) ) + subl $18, 64, $18 + addl $17, 64, $17 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vldd $f1, 0($17) ) + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $17, 32, $17 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 4 +$quad_loop_tail: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( ldl $3, 8($17) ) + FIXUP_LDST( stl $2, 0($16) ) + FIXUP_LDST( stl $3, 8($16) ) + subl $18, 16, $18 + addl $17, 16, $17 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( ldl $2, 0($17) ) + FIXUP_LDST( stl $2, 0($16) ) + subl $18, 8, $18 + addl $17, 8, $17 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( ldbu $2, 0($17) ) + FIXUP_LDST( stb $2, 0($16) ) + subl $18, 1, $18 + addl $17, 1, $17 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out diff --git a/arch/sw_64/lib/deep-copy_user.S b/arch/sw_64/lib/deep-copy_user.S index 327cab322765..8f4ef09ab90e 100644 --- a/arch/sw_64/lib/deep-copy_user.S +++ b/arch/sw_64/lib/deep-copy_user.S @@ -11,10 +11,13 @@ .previous /* - * $7: SIMD status + * $7: SIMD status for C3B * 0: not in simd loop * 1: in simd loop * 2: in simd_u loop + * $7: SIMD status for C4 + * 0: not in simd loop + * 1: in simd and simd_u loop * $18: bytes left to copy * */ @@ -23,16 +26,22 @@ __copy_user: .prologue 0 bis $31, $31, $7 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif $out: bis $31, $18, $0 beq $7, $return subl $7, 1, $7 beq $7, $restore_simd +#if defined(CONFIG_SUBARCH_C3B) $restore_simd_u: RESTORE_SIMD_U_REGS br $31, $return +#endif $restore_simd: RESTORE_SIMD_REGS diff --git a/arch/sw_64/lib/deep-memcpy.S b/arch/sw_64/lib/deep-memcpy.S index c4b5bf3d26df..78a6bd85cf01 100644 --- a/arch/sw_64/lib/deep-memcpy.S +++ b/arch/sw_64/lib/deep-memcpy.S @@ -11,7 +11,11 @@ memcpy: .frame $30, 0, $26, 0 .prologue 0 mov $16, $0 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-copy_template_c4.S" +#endif $out: ret .end memcpy diff --git a/arch/sw_64/lib/deep-memset.S b/arch/sw_64/lib/deep-memset.S index 5d9beb1e2f53..c6b5355beec6 100644 --- a/arch/sw_64/lib/deep-memset.S +++ b/arch/sw_64/lib/deep-memset.S @@ -26,6 +26,7 @@ */ #include +#include #define FIXUP_LDST(x, y) \ x, y @@ -44,6 +45,9 @@ ___memset: .frame $30, 0, $26, 0 .prologue 0 +#ifdef CONFIG_SUBARCH_C4 + csrr $6, CSR_WR_FREGS +#endif /* expand 1 byte data to 8 bytes */ and $17, 0xff, $17 sll $17, 8, $4 @@ -56,8 +60,15 @@ ___memset: __constant_c_memset: bis $31, $31, $7 bis $31, $16, $0 +#if defined(CONFIG_SUBARCH_C3B) #include "deep-set_template.S" +#elif defined(CONFIG_SUBARCH_C4) +#include "deep-set_template_c4.S" +#endif $out: +#ifdef CONFIG_SUBARCH_C4 + csrw $6, CSR_WR_FREGS +#endif ret .end ___memset diff --git a/arch/sw_64/lib/deep-set_template_c4.S b/arch/sw_64/lib/deep-set_template_c4.S new file mode 100644 index 000000000000..2b1bcab8fec9 --- /dev/null +++ b/arch/sw_64/lib/deep-set_template_c4.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * template for memset and clear_user with SIMD + * + * $7: SIMD status + * 0: not in simd loop + * 1: in simd loop + * $16: latest dest, clobbered + * $17: 8-byte data to set + * $18: bytes left to copy + * + */ + +#define SAVE_SIMD_REGS \ + ldi $sp, -0x40($sp); \ + addl $sp, 0x1f, $23; \ + bic $23, 0x1f, $23; \ + vstd $f1, 0($23); \ + ldi $7, 1 + +#define RESTORE_SIMD_REGS \ + vldd $f1, 0($23); \ + ldi $sp, 0x40($sp); \ + bis $31, $31, $7 + + ble $18, $out + + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + cmplt $18, 16, $1 + bne $1, $quad_loop_end + cmplt $18, 32, $1 + bne $1, $simd_end + +$prep_simd_loop: + SAVE_SIMD_REGS + ifmovd $17, $f1 + vcpyf $f1, $f1 + cmplt $18, 64, $1 + bne $1, $simd_loop_end + + .align 3 +$simd_loop: + FIXUP_LDST( vstd $f1, 0($16) ) + FIXUP_LDST( vstd $f1, 32($16) ) + subl $18, 64, $18 + addl $16, 64, $16 + cmplt $18, 64, $1 + beq $1, $simd_loop + +$simd_loop_end: + cmplt $18, 32, $1 + bne $1, $no_more_simd + FIXUP_LDST( vstd $f1, 0($16) ) + subl $18, 32, $18 + addl $16, 32, $16 + +$no_more_simd: + RESTORE_SIMD_REGS + +$simd_end: + ble $18, $out + cmplt $18, 16, $1 + bne $1, $quad_loop_end + + .align 3 +$quad_loop_tail: + FIXUP_LDST( stl $17, 0($16) ) + FIXUP_LDST( stl $17, 8($16) ) + subl $18, 16, $18 + addl $16, 16, $16 + cmplt $18, 16, $1 + beq $1, $quad_loop_tail + +$quad_loop_end: + ble $18, $out + cmplt $18, 8, $1 + bne $1, $byte_loop_tail + +$move_one_quad: + FIXUP_LDST( stl $17, 0($16) ) + subl $18, 8, $18 + addl $16, 8, $16 + ble $18, $out + + .align 3 +$byte_loop_tail: + FIXUP_LDST( stb $17, 0($16) ) + subl $18, 1, $18 + addl $16, 1, $16 + bgt $18, $byte_loop_tail + br $31, $out -- Gitee From 5d3fc4182bb54411272aff002a818a77053a55a6 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 11 Sep 2023 17:33:26 +0800 Subject: [PATCH 107/150] sw64: kprobe: allow kprobe in atomic sequence on C4 Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Locking instructions on C4 can be interrupted, so probe between atomic is allowed. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/kprobes/decode-insn.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/sw_64/kernel/kprobes/decode-insn.c b/arch/sw_64/kernel/kprobes/decode-insn.c index d376a7e2bee4..411dcc89fc4f 100644 --- a/arch/sw_64/kernel/kprobes/decode-insn.c +++ b/arch/sw_64/kernel/kprobes/decode-insn.c @@ -90,10 +90,12 @@ bool __kprobes sw64_insn_can_kprobe(kprobe_opcode_t *addr) printk("addr can't steppable\n"); return false; } +#ifdef CONFIG_SUBARCH_C3B if (!is_probed_between_atomic(addr)) { printk("addr between atomic cant probe\n"); return false; } +#endif return true; } #endif -- Gitee From aa95ca334c23dd925b2c04793b0eef0d28bfcfda Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 31 Aug 2023 16:21:49 +0800 Subject: [PATCH 108/150] sw64: optimize C4 compilation Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCII -------------------------------- Add new cflags to generate C4 new instructions to optimize performance. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 678880cee7fd..81d530afef33 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -26,6 +26,9 @@ endif CHECKFLAGS += -D__sw__ cflags-y := -pipe -ffixed-8 -mno-fp-regs #-msmall-data +ifeq ($(CONFIG_SUBARCH_C4),y) + cflags-y += -fsw-rev +endif cflags-y += $(call cc-option, -fno-jump-tables) cflags-y += $(cpuflags-y) -- Gitee From 0b3860f0c0df1f6f40e9eb3991ded66851ae2bd8 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Mon, 25 Sep 2023 16:19:50 +0800 Subject: [PATCH 109/150] sw64: Kconfig: rename BUILTIN_DTB option Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 6 +++--- arch/sw_64/Makefile | 2 +- arch/sw_64/boot/dts/Makefile | 6 +++--- arch/sw_64/kernel/setup.c | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 221b5e8243c9..75b541dc742b 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -504,16 +504,16 @@ config USE_OF help Include support for flattened device tree machine descriptions. -config SW64_BUILTIN_DTB +config BUILTIN_DTB bool "Embed DTB in kernel image" depends on OF default n help Embeds a device tree binary in the kernel image. -config SW64_BUILTIN_DTB_NAME +config BUILTIN_DTB_NAME string "Built in DTB" - depends on SW64_BUILTIN_DTB + depends on BUILTIN_DTB help Set the name of the DTB to embed, leave blank to pick one automatically based on kernel configuration. diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 81d530afef33..9e54b91df7d0 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -44,7 +44,7 @@ core-$(CONFIG_MATHEMU) += arch/sw_64/math-emu/ drivers-$(CONFIG_OPROFILE) += arch/sw_64/oprofile/ libs-y += arch/sw_64/lib/ core-$(CONFIG_KVM) += arch/sw_64/kvm/ -core-$(CONFIG_SW64_BUILTIN_DTB) += arch/sw_64/boot/dts/ +core-$(CONFIG_BUILTIN_DTB) += arch/sw_64/boot/dts/ core-$(CONFIG_NET) += arch/sw_64/net/ # export what is needed by arch/sw_64/boot/Makefile diff --git a/arch/sw_64/boot/dts/Makefile b/arch/sw_64/boot/dts/Makefile index 7a9615b79124..e32c159cab64 100644 --- a/arch/sw_64/boot/dts/Makefile +++ b/arch/sw_64/boot/dts/Makefile @@ -9,9 +9,9 @@ ifeq ($(CONFIG_PLATFORM_JUNZHANG),y) builtindtb-y := empty endif -ifeq ($(CONFIG_SW64_BUILTIN_DTB), y) -ifneq ($(CONFIG_SW64_BUILTIN_DTB_NAME),"") - builtindtb-y := $(patsubst "%",%,$(CONFIG_SW64_BUILTIN_DTB_NAME)) +ifeq ($(CONFIG_BUILTIN_DTB), y) +ifneq ($(CONFIG_BUILTIN_DTB_NAME),"") + builtindtb-y := $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_NAME)) endif obj-y += $(builtindtb-y).dtb.o diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index b1d9d70a79c2..894291647af3 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -582,7 +582,7 @@ static void __init setup_machine_fdt(void) const char *name; /* Give a chance to select kernel builtin DTB firstly */ - if (IS_ENABLED(CONFIG_SW64_BUILTIN_DTB)) + if (IS_ENABLED(CONFIG_BUILTIN_DTB)) dt_virt = (void *)__dtb_start; else { dt_virt = (void *)sunway_boot_params->dtb_start; -- Gitee From da5c778a16aca3dfb7cae4d290bf1ada93cb456a Mon Sep 17 00:00:00 2001 From: He Sheng Date: Mon, 25 Sep 2023 16:39:54 +0800 Subject: [PATCH 110/150] sw64: Kconfig: clean up out-of-date deep sleep options Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/Kconfig | 10 ---------- arch/sw_64/kernel/smp.c | 13 ------------- arch/sw_64/kernel/suspend.c | 8 -------- 3 files changed, 31 deletions(-) diff --git a/arch/sw_64/Kconfig b/arch/sw_64/Kconfig index 75b541dc742b..a28344f33160 100644 --- a/arch/sw_64/Kconfig +++ b/arch/sw_64/Kconfig @@ -627,16 +627,6 @@ config ARCH_HIBERNATION_POSSIBLE depends on SW64 def_bool y -config SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE - depends on SUSPEND - bool "SW64 non bootcore suspend into deep sleep mode" - default n - -config SW64_SUSPEND_DEEPSLEEP_BOOTCORE - depends on SUSPEND - bool "SW64 bootcore suspend into deep sleep mode" - default n - source "drivers/cpuidle/Kconfig" source "drivers/idle/Kconfig" diff --git a/arch/sw_64/kernel/smp.c b/arch/sw_64/kernel/smp.c index 0878702b2895..5b71852ab872 100644 --- a/arch/sw_64/kernel/smp.c +++ b/arch/sw_64/kernel/smp.c @@ -276,10 +276,8 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) wmb(); smp_rcb->ready = 0; -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE /* send wake up signal */ send_wakeup_interrupt(cpu); -#endif /* send reset signal */ if (smp_booted) { if (is_in_host()) { @@ -293,7 +291,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) smp_boot_one_cpu(cpu, tidle); #ifdef CONFIG_SUBARCH_C3B -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE if (static_branch_likely(&use_tc_as_sched_clock)) { if (smp_booted) { tc_sync_clear(); @@ -301,7 +298,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) tc_sync_set(); } } -#endif #endif return cpu_online(cpu) ? 0 : -ENOSYS; @@ -581,19 +577,10 @@ void arch_cpu_idle_dead(void) } #ifdef CONFIG_SUSPEND - -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_NONBOOT_CORE sleepen(); send_sleep_interrupt(smp_processor_id()); while (1) asm("nop"); -#else - asm volatile("halt"); - while (1) - asm("nop"); -#endif /* SW64_SUSPEND_DEEPSLEEP */ - - #else asm volatile("memb"); asm volatile("halt"); diff --git a/arch/sw_64/kernel/suspend.c b/arch/sw_64/kernel/suspend.c index 94fb35fc9b47..27a240e66149 100644 --- a/arch/sw_64/kernel/suspend.c +++ b/arch/sw_64/kernel/suspend.c @@ -33,20 +33,12 @@ void sw64_suspend_enter(void) /* boot processor will go to deep sleep mode from here * After wake up boot processor, pc will go here */ - - disable_local_timer(); current_thread_info()->pcb.tp = rtid(); -#ifdef CONFIG_SW64_SUSPEND_DEEPSLEEP_BOOTCORE sw64_suspend_deep_sleep(&suspend_state); -#else - mtinten(); - asm("halt"); -#endif wrtp(current_thread_info()->pcb.tp); - disable_local_timer(); } -- Gitee From 4e1707be837a26b2c3a4a38c5cca46c4d9e7f507 Mon Sep 17 00:00:00 2001 From: He Sheng Date: Mon, 25 Sep 2023 16:41:52 +0800 Subject: [PATCH 111/150] sw64: add arch/sw_64/Kbuild Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Use the standard obj-y form to specify the sub-directories under arch/sw_64/. No functional change intended. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/Kbuild | 7 +++++++ arch/sw_64/Makefile | 7 +------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 arch/sw_64/Kbuild diff --git a/arch/sw_64/Kbuild b/arch/sw_64/Kbuild new file mode 100644 index 000000000000..aa0bf0507406 --- /dev/null +++ b/arch/sw_64/Kbuild @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-y += kernel/ mm/ platform/ +obj-$(CONFIG_NET) += net/ +obj-$(CONFIG_KVM) += kvm/ +obj-$(CONFIG_MATHEMU) += math-emu/ + +obj-$(CONFIG_BUILTIN_DTB) += boot/dts/ diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 9e54b91df7d0..57b899929b83 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -38,14 +38,9 @@ KBUILD_DEFCONFIG = openeuler_defconfig head-y := arch/sw_64/kernel/head.o -core-y += arch/sw_64/kernel/ arch/sw_64/mm/ -core-y += arch/sw_64/platform/ -core-$(CONFIG_MATHEMU) += arch/sw_64/math-emu/ +core-y += arch/sw_64/ drivers-$(CONFIG_OPROFILE) += arch/sw_64/oprofile/ libs-y += arch/sw_64/lib/ -core-$(CONFIG_KVM) += arch/sw_64/kvm/ -core-$(CONFIG_BUILTIN_DTB) += arch/sw_64/boot/dts/ -core-$(CONFIG_NET) += arch/sw_64/net/ # export what is needed by arch/sw_64/boot/Makefile LIBS_Y := $(patsubst %/, %/lib.a, $(libs-y)) -- Gitee From 6572b6067c31dde618cc5082c72dff55bb07be8f Mon Sep 17 00:00:00 2001 From: He Sheng Date: Tue, 26 Sep 2023 09:48:00 +0800 Subject: [PATCH 112/150] sw64: kvm: move bind vcpu init code into kvm_core3.c Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- The bind vcpu codes is only used by C3B, so move the init code into kvm_core3.c to make them clear. Signed-off-by: He Sheng Reviewed-by: Cui Wei Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 4 --- arch/sw_64/kernel/bindvcpu.c | 30 ---------------------- arch/sw_64/kernel/setup.c | 2 ++ arch/sw_64/kvm/kvm_core3.c | 49 +++++++++++++++++++++++++++++------- 4 files changed, 42 insertions(+), 43 deletions(-) delete mode 100644 arch/sw_64/kernel/bindvcpu.c diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 3aee5b7f5fdb..25a299daab1f 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -36,10 +36,6 @@ obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_DEBUG_MATCH) += match.o -ifeq ($(CONFIG_DEBUG_FS)$(CONFIG_NUMA),yy) -obj-y += bindvcpu.o -endif - ifndef CONFIG_PCI obj-y += pci-noop.o endif diff --git a/arch/sw_64/kernel/bindvcpu.c b/arch/sw_64/kernel/bindvcpu.c deleted file mode 100644 index 46617eb68b7a..000000000000 --- a/arch/sw_64/kernel/bindvcpu.c +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2022 Wang Yuanheng - * Author: Wang Yuanheng - * - */ - -#include -#include -#include -#include -#include - -__read_mostly bool bind_vcpu_enabled; -EXPORT_SYMBOL(bind_vcpu_enabled); - -static int __init bind_vcpu_init(void) -{ - struct dentry *bindvcpu; - - if (!sw64_debugfs_dir) - return -ENODEV; - - bindvcpu = debugfs_create_bool("bind_vcpu", 0644, - sw64_debugfs_dir, &bind_vcpu_enabled); - if (!bindvcpu) - return -ENOMEM; - return 0; -} -late_initcall(bind_vcpu_init); diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index 894291647af3..ec8980e3c0df 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -987,6 +987,8 @@ device_initcall(add_pcspkr); #ifdef CONFIG_DEBUG_FS struct dentry *sw64_debugfs_dir; +EXPORT_SYMBOL(sw64_debugfs_dir); + static int __init debugfs_sw64(void) { struct dentry *d; diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index 529a04a2bed0..f18d1f7a0336 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -5,6 +5,7 @@ * linhn */ +#include #include #include #include @@ -13,6 +14,7 @@ #include #include +#include #include #include #include @@ -22,8 +24,36 @@ #include "../kernel/pci_impl.h" #include "vmem.c" + +__read_mostly bool bind_vcpu_enabled; + #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) -extern bool bind_vcpu_enabled; +struct dentry *bindvcpu; + +static int __init bind_vcpu_init(void) +{ + if (!sw64_debugfs_dir) + return -ENODEV; + bindvcpu = debugfs_create_bool("bind_vcpu", 0644, + sw64_debugfs_dir, &bind_vcpu_enabled); + if (IS_ERR(bindvcpu)) + return PTR_ERR(bindvcpu); + return 0; +} + +static void bind_vcpu_exit(void) +{ + bind_vcpu_enabled = false; + debugfs_remove(bindvcpu); +} +#else +static int __init bind_vcpu_init(void) +{ + return 0; +} + +static void bind_vcpu_exit(void) { } + #endif #define GUEST_RESET_PC 0xffffffff80011100 @@ -290,7 +320,6 @@ int kvm_core3_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) #ifndef CONFIG_KVM_MEMHOTPLUG vcpu->arch.vcb.vpcr = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); -#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_NUMA) if (unlikely(bind_vcpu_enabled)) { int nid; unsigned long end; @@ -300,7 +329,6 @@ int kvm_core3_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) if (pfn_to_nid(PHYS_PFN(end)) == nid) set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); } -#endif #else /* !CONFIG_KVM_MEMHOTPLUG */ unsigned long seg_base = virt_to_phys(vcpu->kvm->arch.seg_pgd); @@ -477,8 +505,10 @@ static int __init kvm_core3_init(void) { int i, ret; + bind_vcpu_init(); + ret = vmem_init(); - if (ret) + if (unlikely(ret)) goto out; for (i = 0; i < NR_CPUS; i++) @@ -486,12 +516,12 @@ static int __init kvm_core3_init(void) ret = kvm_init(&core3_sw64_ops, sizeof(struct kvm_vcpu), 0, THIS_MODULE); - if (ret) { - vmem_exit(); - goto out; - } - return 0; + if (likely(!ret)) + return 0; + + vmem_exit(); out: + bind_vcpu_exit(); return ret; } @@ -499,6 +529,7 @@ static void __exit kvm_core3_exit(void) { kvm_exit(); vmem_exit(); + bind_vcpu_exit(); } module_init(kvm_core3_init); -- Gitee From 2d4a6614b0d64f2e8043796e226e9d091c44a260 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:44 +0800 Subject: [PATCH 113/150] sw64: use sp in pt_regs Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Since we have sp in pt_regs, there's no need to have another one in pcb. Rmove the extra one. There's no need to call rdusp() because we have saved it in pt_regs, unless we are not sure whether the one in pt_regs is ksp or usp. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/processor.h | 3 +-- arch/sw_64/include/asm/ptrace.h | 3 +-- arch/sw_64/include/asm/switch_to.h | 3 +-- arch/sw_64/include/asm/thread_info.h | 1 - arch/sw_64/include/uapi/asm/ptrace.h | 3 +-- arch/sw_64/kernel/kgdb.c | 2 +- arch/sw_64/kernel/perf_event.c | 2 +- arch/sw_64/kernel/process.c | 9 ++++----- arch/sw_64/kernel/ptrace.c | 10 +++------- arch/sw_64/kernel/signal.c | 19 ++++++++----------- arch/sw_64/kernel/traps.c | 10 +++------- 11 files changed, 24 insertions(+), 41 deletions(-) diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h index 4c1065b61af2..c0b7b683492e 100644 --- a/arch/sw_64/include/asm/processor.h +++ b/arch/sw_64/include/asm/processor.h @@ -65,8 +65,7 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) -#define KSTK_ESP(tsk) \ - ((tsk) == current ? rdusp() : task_thread_info(tsk)->pcb.usp) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->sp) #define cpu_relax() barrier() diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 95a26be29667..3eedfcc4dad0 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -63,8 +63,7 @@ struct pt_regs { #define user_mode(regs) (((regs)->ps & 8) != 0) #define instruction_pointer(regs) ((regs)->pc) #define profile_pc(regs) instruction_pointer(regs) -#define current_user_stack_pointer() rdusp() -#define user_stack_pointer(regs) rdusp() +#define user_stack_pointer(regs) (regs->sp) #define kernel_stack_pointer(regs) ((unsigned long)((regs) + 1)) #define instruction_pointer_set(regs, val) ((regs)->pc = val) diff --git a/arch/sw_64/include/asm/switch_to.h b/arch/sw_64/include/asm/switch_to.h index c81b004cfdd8..5c67fd421b80 100644 --- a/arch/sw_64/include/asm/switch_to.h +++ b/arch/sw_64/include/asm/switch_to.h @@ -16,7 +16,6 @@ static inline void aux_save(struct task_struct *task) if (likely(!(task->flags & PF_KTHREAD))) { pcb = &task_thread_info(task)->pcb; - pcb->usp = rdusp(); pcb->tp = rtid(); __fpstate_save(task); } @@ -28,7 +27,7 @@ static inline void aux_restore(struct task_struct *task) if (likely(!(task->flags & PF_KTHREAD))) { pcb = &task_thread_info(task)->pcb; - wrusp(pcb->usp); + wrusp(task_pt_regs(task)->sp); wrtp(pcb->tp); __fpstate_restore(task); } diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index 2523347c2976..1fd97ce5fab2 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -14,7 +14,6 @@ typedef struct { struct pcb_struct { - unsigned long usp; unsigned long tp; unsigned long da_match, da_mask; unsigned long dv_match, dv_mask; diff --git a/arch/sw_64/include/uapi/asm/ptrace.h b/arch/sw_64/include/uapi/asm/ptrace.h index e2c76d58fa82..3fd53450e418 100644 --- a/arch/sw_64/include/uapi/asm/ptrace.h +++ b/arch/sw_64/include/uapi/asm/ptrace.h @@ -30,8 +30,7 @@ struct user_fpsimd_state { /* PTRACE_DETACH is 17 */ #define PT_REG_BASE 0 -#define PT_REG_END 29 -#define PT_USP 30 +#define PT_REG_END 30 #define PT_FPREG_BASE 32 #define PT_FPREG_END 62 #define PT_FPCR 63 diff --git a/arch/sw_64/kernel/kgdb.c b/arch/sw_64/kernel/kgdb.c index 95970b293de0..4142b2ada817 100644 --- a/arch/sw_64/kernel/kgdb.c +++ b/arch/sw_64/kernel/kgdb.c @@ -57,7 +57,7 @@ struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { { "r27", 8, offsetof(struct pt_regs, r27)}, { "at", 8, offsetof(struct pt_regs, r28)}, { "gp", 8, offsetof(struct pt_regs, gp)}, - { "sp", 8, -1 }, + { "sp", 8, offsetof(struct pt_regs, sp)}, { "zero", 8, -1 }, { "f0", 8, -1 }, diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 72c5e506aa8b..336fab76d6f7 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -686,7 +686,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { - unsigned long usp = current_user_stack_pointer(); + unsigned long usp = rdusp(); unsigned long user_addr; int err; diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 3dadf450debe..29273f897dd3 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -22,6 +22,7 @@ start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { regs->pc = pc; regs->ps = 8; + regs->sp = sp; wrusp(sp); } EXPORT_SYMBOL(start_thread); @@ -80,7 +81,6 @@ copy_thread(unsigned long clone_flags, unsigned long usp, p->thread.ra = (unsigned long) ret_from_kernel_thread; p->thread.s[0] = usp; /* function */ p->thread.s[1] = kthread_arg; - childti->pcb.usp = 0; return 0; } @@ -95,9 +95,9 @@ copy_thread(unsigned long clone_flags, unsigned long usp, childti->pcb.tp = regs->r20; else regs->r20 = 0; - if (usp) - childti->pcb.usp = usp; *childregs = *regs; + if (usp) + childregs->sp = usp; childregs->r0 = 0; childregs->r19 = 0; p->thread.ra = (unsigned long) ret_from_fork; @@ -115,9 +115,8 @@ void sw64_elf_core_copy_regs(elf_greg_t *dest, struct pt_regs *regs) ti = (void *)((__u64)regs & ~(THREAD_SIZE - 1)); - for (i = 0; i < 30; i++) + for (i = 0; i < 31; i++) dest[i] = *(__u64 *)((void *)regs + regoffsets[i]); - dest[30] = ti == current_thread_info() ? rdusp() : ti->pcb.usp; dest[31] = regs->pc; dest[32] = ti->pcb.tp; } diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 25cb19340dde..2116f4c90c0f 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -54,7 +54,7 @@ short regoffsets[32] = { R(r9), R(r10), R(r11), R(r12), R(r13), R(r14), R(r15), R(r16), R(r17), R(r18), R(r19), R(r20), R(r21), R(r22), R(r23), R(r24), R(r25), R(r26), - R(r27), R(r28), R(gp), 0, 0 + R(r27), R(r28), R(gp), R(sp), 0 }; #undef R @@ -62,7 +62,6 @@ short regoffsets[32] = { #define PCB_OFF(var) offsetof(struct pcb_struct, var) static int pcboff[] = { - [PT_USP] = PCB_OFF(usp), [PT_TP] = PCB_OFF(tp), [PT_DA_MATCH] = PCB_OFF(da_match), [PT_DA_MASK] = PCB_OFF(da_mask), @@ -90,7 +89,6 @@ get_reg_addr(struct task_struct *task, unsigned long regno) int fno, vno; switch (regno) { - case PT_USP: case PT_UNIQUE: case PT_DA_MATCH: case PT_DA_MASK: @@ -280,10 +278,9 @@ static int gpr_get(struct task_struct *target, int i, ret; regs = task_pt_regs(target); - for (i = 0; i < 30; i++) + for (i = 0; i < 31; i++) uregs.regs[i] = *(__u64 *)((void *)regs + regoffsets[i]); - uregs.regs[30] = task_thread_info(target)->pcb.usp; uregs.pc = regs->pc; uregs.pstate = regs->ps; @@ -307,10 +304,9 @@ static int gpr_set(struct task_struct *target, return ret; regs = task_pt_regs(target); - for (i = 0; i < 30; i++) + for (i = 0; i < 31; i++) *(__u64 *)((void *)regs + regoffsets[i]) = uregs.regs[i]; - task_thread_info(target)->pcb.usp = uregs.regs[30]; regs->pc = uregs.pc; regs->ps = uregs.pstate; diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 037162e4e844..404f00a88413 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -94,7 +94,6 @@ extern char compile_time_assert static long restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) { - unsigned long usp; long err = __get_user(regs->pc, &sc->sc_pc); current->restart_block.fn = do_no_restart_syscall; @@ -129,8 +128,8 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) err |= __get_user(regs->r27, sc->sc_regs+27); err |= __get_user(regs->r28, sc->sc_regs+28); err |= __get_user(regs->gp, sc->sc_regs+29); - err |= __get_user(usp, sc->sc_regs+30); - wrusp(usp); + err |= __get_user(regs->sp, sc->sc_regs+30); + wrusp(regs->sp); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, offsetof(struct user_fpsimd_state, fpcr)); @@ -220,7 +219,7 @@ get_sigframe(struct ksignal *ksig, unsigned long sp, size_t frame_size) static long setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, - unsigned long mask, unsigned long sp) + unsigned long mask) { long err = 0; @@ -259,7 +258,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, err |= __put_user(regs->r27, sc->sc_regs+27); err |= __put_user(regs->r28, sc->sc_regs+28); err |= __put_user(regs->gp, sc->sc_regs+29); - err |= __put_user(sp, sc->sc_regs+30); + err |= __put_user(regs->sp, sc->sc_regs+30); err |= __put_user(0, sc->sc_regs+31); /* simd-fp */ __fpstate_save(current); @@ -273,11 +272,10 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { - unsigned long oldsp, err = 0; + unsigned long err = 0; struct rt_sigframe __user *frame; - oldsp = rdusp(); - frame = get_sigframe(ksig, oldsp, sizeof(*frame)); + frame = get_sigframe(ksig, regs->sp, sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) return -EFAULT; @@ -288,9 +286,8 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) err |= __put_user(0, &frame->uc.uc_flags); err |= __put_user(0, &frame->uc.uc_link); err |= __put_user(set->sig[0], &frame->uc.uc_old_sigmask); - err |= __save_altstack(&frame->uc.uc_stack, oldsp); - err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, - set->sig[0], oldsp); + err |= __save_altstack(&frame->uc.uc_stack, regs->sp); + err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]); err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); if (err) return -EFAULT; diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 30dd2feeb3ec..b0551086edce 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -72,8 +72,7 @@ void show_regs(struct pt_regs *regs) regs->r22, regs->r23, regs->r24); printk("t11= %016lx pv = %016lx at = %016lx\n", regs->r25, regs->r27, regs->r28); - printk("gp = %016lx sp = %px\n", regs->gp, - user_mode(regs) ? (void *)rdusp() : (regs + 1)); + printk("gp = %016lx sp = %016lx\n", regs->gp, regs->sp); } static void show_code(unsigned int *pc) @@ -626,12 +625,9 @@ do_entUnaUser(void __user *va, unsigned long opcode, if ((1L << opcode) & OP_INT_MASK) { /* it's an integer load/store */ - if (reg < 30) { + if (reg < 31) { reg_addr = (unsigned long *) ((char *)regs + regoffsets[reg]); - } else if (reg == 30) { - /* usp in HMCODE regs */ - fake_reg = rdusp(); } else { /* zero "register" */ fake_reg = 0; @@ -1466,7 +1462,7 @@ do_entUnaUser(void __user *va, unsigned long opcode, /* Only integer loads should get here; everyone else returns early. */ if (reg == 30) - wrusp(fake_reg); + wrusp(regs->sp); return; give_sigsegv: -- Gitee From c72b3a92f3a3b83c7bf7bc456b71b5857a94f123 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:45 +0800 Subject: [PATCH 114/150] sw64: avoid using wrusp() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Call wrusp() before returning to user space instead of calling it immediately when sp is modified. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/switch_to.h | 1 - arch/sw_64/kernel/entry.S | 6 +++++- arch/sw_64/kernel/process.c | 1 - arch/sw_64/kernel/signal.c | 3 +-- arch/sw_64/kernel/traps.c | 3 --- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/sw_64/include/asm/switch_to.h b/arch/sw_64/include/asm/switch_to.h index 5c67fd421b80..f4af18c5710c 100644 --- a/arch/sw_64/include/asm/switch_to.h +++ b/arch/sw_64/include/asm/switch_to.h @@ -27,7 +27,6 @@ static inline void aux_restore(struct task_struct *task) if (likely(!(task->flags & PF_KTHREAD))) { pcb = &task_thread_info(task)->pcb; - wrusp(task_pt_regs(task)->sp); wrtp(pcb->tp); __fpstate_restore(task); } diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index d3c961ba53e2..689cc58c30be 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -70,7 +70,11 @@ .endm .macro RESTORE_ALL - ldl $1, PT_REGS_R16($sp) + ldl $16, PT_REGS_SP($sp) + /* skip wrusp if returning to kernel */ + blt $16, 1f + sys_call HMC_wrusp +1: ldl $1, PT_REGS_R16($sp) ldl $2, PT_REGS_R17($sp) ldl $3, PT_REGS_R18($sp) ldl $4, PT_REGS_GP($sp) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index 29273f897dd3..ad4b71d4037a 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -23,7 +23,6 @@ start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) regs->pc = pc; regs->ps = 8; regs->sp = sp; - wrusp(sp); } EXPORT_SYMBOL(start_thread); diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 404f00a88413..281086e177a3 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -129,7 +129,6 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) err |= __get_user(regs->r28, sc->sc_regs+28); err |= __get_user(regs->gp, sc->sc_regs+29); err |= __get_user(regs->sp, sc->sc_regs+30); - wrusp(regs->sp); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, offsetof(struct user_fpsimd_state, fpcr)); @@ -305,7 +304,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) regs->r17 = 0; regs->r18 = (unsigned long) &frame->uc.uc_mcontext; } - wrusp((unsigned long) frame); + regs->sp = (unsigned long) frame; #if DEBUG_SIG printk("SIG deliver (%s:%d): sp=%p pc=%p ra=%p\n", diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index b0551086edce..c74b4b541192 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -1460,9 +1460,6 @@ do_entUnaUser(void __user *va, unsigned long opcode, goto give_sigbus; } - /* Only integer loads should get here; everyone else returns early. */ - if (reg == 30) - wrusp(regs->sp); return; give_sigsegv: -- Gitee From 06e7f5d157361e4aa1d069204e5b30439bb9d29b Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:45 +0800 Subject: [PATCH 115/150] sw64: fix pt_regs related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Struct user_pt_regs are now prefix of struct pt_regs, fix related code accordingly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/perf_regs.c | 19 +------------------ arch/sw_64/kernel/ptrace.c | 11 ++++++----- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/arch/sw_64/kernel/perf_regs.c b/arch/sw_64/kernel/perf_regs.c index 8fc3597d9e41..b036f213936b 100644 --- a/arch/sw_64/kernel/perf_regs.c +++ b/arch/sw_64/kernel/perf_regs.c @@ -8,24 +8,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) if (WARN_ON_ONCE((u32)idx >= PERF_REG_SW64_MAX)) return 0; - switch (idx) { - case PERF_REG_SW64_R16: - return regs->r16; - case PERF_REG_SW64_R17: - return regs->r17; - case PERF_REG_SW64_R18: - return regs->r18; - case PERF_REG_SW64_R19 ... PERF_REG_SW64_R28: - return ((unsigned long *)regs)[idx - 3]; - case PERF_REG_SW64_GP: - return regs->gp; - case PERF_REG_SW64_SP: - return (user_mode(regs) ? rdusp() : (u64)(regs + 1)); - case PERF_REG_SW64_PC: - return regs->pc; - default: - return ((unsigned long *)regs)[idx]; - } + return ((unsigned long *)regs)[idx]; } #define REG_RESERVED (~((1ULL << PERF_REG_SW64_MAX) - 1)) diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 2116f4c90c0f..4e8518f954cd 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -831,6 +831,9 @@ static const struct pt_regs_offset regoffset_table[] = { REG_OFFSET_NAME(r13), REG_OFFSET_NAME(r14), REG_OFFSET_NAME(r15), + REG_OFFSET_NAME(r16), + REG_OFFSET_NAME(r17), + REG_OFFSET_NAME(r18), REG_OFFSET_NAME(r19), REG_OFFSET_NAME(r20), REG_OFFSET_NAME(r21), @@ -841,12 +844,10 @@ static const struct pt_regs_offset regoffset_table[] = { REG_OFFSET_NAME(r26), REG_OFFSET_NAME(r27), REG_OFFSET_NAME(r28), - REG_OFFSET_NAME(ps), - REG_OFFSET_NAME(pc), REG_OFFSET_NAME(gp), - REG_OFFSET_NAME(r16), - REG_OFFSET_NAME(r17), - REG_OFFSET_NAME(r18), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(ps), REG_OFFSET_END, }; -- Gitee From 007511ba69dd7a9758448714a2dbebdc917f182c Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:46 +0800 Subject: [PATCH 116/150] sw64: simplify pt_regs related code Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Struct user_pt_regs are now prefix of struct pt_regs, simplify related code accordingly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/ptrace.c | 34 ++------------------ arch/sw_64/kernel/signal.c | 64 ++------------------------------------ 2 files changed, 5 insertions(+), 93 deletions(-) diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 4e8518f954cd..38116b893b5c 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -273,20 +273,7 @@ static int gpr_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { - struct pt_regs *regs; - struct user_pt_regs uregs; - int i, ret; - - regs = task_pt_regs(target); - for (i = 0; i < 31; i++) - uregs.regs[i] = *(__u64 *)((void *)regs + regoffsets[i]); - - uregs.pc = regs->pc; - uregs.pstate = regs->ps; - - ret = membuf_write(&to, &uregs, sizeof(uregs)); - - return ret; + return membuf_write(&to, task_pt_regs(target), sizeof(struct user_pt_regs)); } static int gpr_set(struct task_struct *target, @@ -294,23 +281,8 @@ static int gpr_set(struct task_struct *target, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) { - struct pt_regs *regs; - struct user_pt_regs uregs; - int i, ret; - - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, - &uregs, 0, sizeof(uregs)); - if (ret) - return ret; - - regs = task_pt_regs(target); - for (i = 0; i < 31; i++) - *(__u64 *)((void *)regs + regoffsets[i]) = uregs.regs[i]; - - regs->pc = uregs.pc; - regs->ps = uregs.pstate; - - return 0; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + task_pt_regs(target), 0, sizeof(struct user_pt_regs)); } static int fpr_get(struct task_struct *target, diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 281086e177a3..cbf916a3da9d 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -98,37 +98,7 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) current->restart_block.fn = do_no_restart_syscall; - err |= __get_user(regs->r0, sc->sc_regs+0); - err |= __get_user(regs->r1, sc->sc_regs+1); - err |= __get_user(regs->r2, sc->sc_regs+2); - err |= __get_user(regs->r3, sc->sc_regs+3); - err |= __get_user(regs->r4, sc->sc_regs+4); - err |= __get_user(regs->r5, sc->sc_regs+5); - err |= __get_user(regs->r6, sc->sc_regs+6); - err |= __get_user(regs->r7, sc->sc_regs+7); - err |= __get_user(regs->r8, sc->sc_regs+8); - err |= __get_user(regs->r9, sc->sc_regs+9); - err |= __get_user(regs->r10, sc->sc_regs+10); - err |= __get_user(regs->r11, sc->sc_regs+11); - err |= __get_user(regs->r12, sc->sc_regs+12); - err |= __get_user(regs->r13, sc->sc_regs+13); - err |= __get_user(regs->r14, sc->sc_regs+14); - err |= __get_user(regs->r15, sc->sc_regs+15); - err |= __get_user(regs->r16, sc->sc_regs+16); - err |= __get_user(regs->r17, sc->sc_regs+17); - err |= __get_user(regs->r18, sc->sc_regs+18); - err |= __get_user(regs->r19, sc->sc_regs+19); - err |= __get_user(regs->r20, sc->sc_regs+20); - err |= __get_user(regs->r21, sc->sc_regs+21); - err |= __get_user(regs->r22, sc->sc_regs+22); - err |= __get_user(regs->r23, sc->sc_regs+23); - err |= __get_user(regs->r24, sc->sc_regs+24); - err |= __get_user(regs->r25, sc->sc_regs+25); - err |= __get_user(regs->r26, sc->sc_regs+26); - err |= __get_user(regs->r27, sc->sc_regs+27); - err |= __get_user(regs->r28, sc->sc_regs+28); - err |= __get_user(regs->gp, sc->sc_regs+29); - err |= __get_user(regs->sp, sc->sc_regs+30); + err |= __copy_from_user(regs, sc->sc_regs, sizeof(unsigned long) * 31); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, offsetof(struct user_fpsimd_state, fpcr)); @@ -227,37 +197,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, err |= __put_user(regs->pc, &sc->sc_pc); err |= __put_user(8, &sc->sc_ps); - err |= __put_user(regs->r0, sc->sc_regs+0); - err |= __put_user(regs->r1, sc->sc_regs+1); - err |= __put_user(regs->r2, sc->sc_regs+2); - err |= __put_user(regs->r3, sc->sc_regs+3); - err |= __put_user(regs->r4, sc->sc_regs+4); - err |= __put_user(regs->r5, sc->sc_regs+5); - err |= __put_user(regs->r6, sc->sc_regs+6); - err |= __put_user(regs->r7, sc->sc_regs+7); - err |= __put_user(regs->r8, sc->sc_regs+8); - err |= __put_user(regs->r9, sc->sc_regs+9); - err |= __put_user(regs->r10, sc->sc_regs+10); - err |= __put_user(regs->r11, sc->sc_regs+11); - err |= __put_user(regs->r12, sc->sc_regs+12); - err |= __put_user(regs->r13, sc->sc_regs+13); - err |= __put_user(regs->r14, sc->sc_regs+14); - err |= __put_user(regs->r15, sc->sc_regs+15); - err |= __put_user(regs->r16, sc->sc_regs+16); - err |= __put_user(regs->r17, sc->sc_regs+17); - err |= __put_user(regs->r18, sc->sc_regs+18); - err |= __put_user(regs->r19, sc->sc_regs+19); - err |= __put_user(regs->r20, sc->sc_regs+20); - err |= __put_user(regs->r21, sc->sc_regs+21); - err |= __put_user(regs->r22, sc->sc_regs+22); - err |= __put_user(regs->r23, sc->sc_regs+23); - err |= __put_user(regs->r24, sc->sc_regs+24); - err |= __put_user(regs->r25, sc->sc_regs+25); - err |= __put_user(regs->r26, sc->sc_regs+26); - err |= __put_user(regs->r27, sc->sc_regs+27); - err |= __put_user(regs->r28, sc->sc_regs+28); - err |= __put_user(regs->gp, sc->sc_regs+29); - err |= __put_user(regs->sp, sc->sc_regs+30); + err |= __copy_to_user(sc->sc_regs, regs, sizeof(unsigned long) * 31); err |= __put_user(0, sc->sc_regs+31); /* simd-fp */ __fpstate_save(current); -- Gitee From fb3078548a9c2e49f650a1a73c79eee9d693d7e2 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:46 +0800 Subject: [PATCH 117/150] sw64: rewrite struct pt_regs Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Use an array to store all registers instead of having different members for each register in struct pt_regs. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/elf.h | 2 +- arch/sw_64/include/asm/kexec.h | 50 ++++++++----- arch/sw_64/include/asm/livepatch.h | 4 +- arch/sw_64/include/asm/processor.h | 2 +- arch/sw_64/include/asm/ptrace.h | 42 ++--------- arch/sw_64/include/asm/syscall.h | 38 +++++----- arch/sw_64/kernel/asm-offsets.c | 62 +++++++-------- arch/sw_64/kernel/kgdb.c | 68 ++++++++--------- arch/sw_64/kernel/kprobes/kprobes-ftrace.c | 4 +- arch/sw_64/kernel/kprobes/kprobes.c | 6 +- arch/sw_64/kernel/perf_event.c | 2 +- arch/sw_64/kernel/process.c | 14 ++-- arch/sw_64/kernel/ptrace.c | 87 ++++++++++++---------- arch/sw_64/kernel/signal.c | 46 ++++++------ arch/sw_64/kernel/stacktrace.c | 4 +- arch/sw_64/kernel/sys_sw64.c | 8 +- arch/sw_64/kernel/traps.c | 43 ++++++----- arch/sw_64/kernel/uprobes.c | 6 +- 18 files changed, 237 insertions(+), 251 deletions(-) diff --git a/arch/sw_64/include/asm/elf.h b/arch/sw_64/include/asm/elf.h index 8c858cff5573..2e29943b54f3 100644 --- a/arch/sw_64/include/asm/elf.h +++ b/arch/sw_64/include/asm/elf.h @@ -102,7 +102,7 @@ typedef struct user_fpsimd_state elf_fpregset_t; * such function. */ -#define ELF_PLAT_INIT(_r, load_addr) (_r->r0 = 0) +#define ELF_PLAT_INIT(_r, load_addr) (_r->regs[0] = 0) /* * The registers are laid out in pt_regs for HMCODE and syscall diff --git a/arch/sw_64/include/asm/kexec.h b/arch/sw_64/include/asm/kexec.h index a99aba9638e6..25e0d8da84f8 100644 --- a/arch/sw_64/include/asm/kexec.h +++ b/arch/sw_64/include/asm/kexec.h @@ -34,25 +34,37 @@ static inline void crash_setup_regs(struct pt_regs *newregs, if (oldregs) { memcpy(newregs, oldregs, sizeof(*newregs)); } else { - __asm__ __volatile__ ("stl $0, %0" : "=m" (newregs->r0)); - __asm__ __volatile__ ("stl $1, %0" : "=m" (newregs->r1)); - __asm__ __volatile__ ("stl $2, %0" : "=m" (newregs->r2)); - __asm__ __volatile__ ("stl $3, %0" : "=m" (newregs->r3)); - __asm__ __volatile__ ("stl $4, %0" : "=m" (newregs->r4)); - __asm__ __volatile__ ("stl $5, %0" : "=m" (newregs->r5)); - __asm__ __volatile__ ("stl $6, %0" : "=m" (newregs->r6)); - __asm__ __volatile__ ("stl $7, %0" : "=m" (newregs->r7)); - __asm__ __volatile__ ("stl $8, %0" : "=m" (newregs->r8)); - __asm__ __volatile__ ("stl $19, %0" : "=m" (newregs->r19)); - __asm__ __volatile__ ("stl $20, %0" : "=m" (newregs->r20)); - __asm__ __volatile__ ("stl $21, %0" : "=m" (newregs->r21)); - __asm__ __volatile__ ("stl $22, %0" : "=m" (newregs->r22)); - __asm__ __volatile__ ("stl $23, %0" : "=m" (newregs->r23)); - __asm__ __volatile__ ("stl $24, %0" : "=m" (newregs->r24)); - __asm__ __volatile__ ("stl $25, %0" : "=m" (newregs->r25)); - __asm__ __volatile__ ("stl $26, %0" : "=m" (newregs->r26)); - __asm__ __volatile__ ("stl $27, %0" : "=m" (newregs->r27)); - __asm__ __volatile__ ("stl $28, %0" : "=m" (newregs->r28)); + __asm__ __volatile__ ("stl $0, %0" : "=m" (newregs->regs[0])); + __asm__ __volatile__ ("stl $1, %0" : "=m" (newregs->regs[1])); + __asm__ __volatile__ ("stl $2, %0" : "=m" (newregs->regs[2])); + __asm__ __volatile__ ("stl $3, %0" : "=m" (newregs->regs[3])); + __asm__ __volatile__ ("stl $4, %0" : "=m" (newregs->regs[4])); + __asm__ __volatile__ ("stl $5, %0" : "=m" (newregs->regs[5])); + __asm__ __volatile__ ("stl $6, %0" : "=m" (newregs->regs[6])); + __asm__ __volatile__ ("stl $7, %0" : "=m" (newregs->regs[7])); + __asm__ __volatile__ ("stl $8, %0" : "=m" (newregs->regs[8])); + __asm__ __volatile__ ("stl $9, %0" : "=m" (newregs->regs[9])); + __asm__ __volatile__ ("stl $10, %0" : "=m" (newregs->regs[10])); + __asm__ __volatile__ ("stl $11, %0" : "=m" (newregs->regs[11])); + __asm__ __volatile__ ("stl $12, %0" : "=m" (newregs->regs[12])); + __asm__ __volatile__ ("stl $13, %0" : "=m" (newregs->regs[13])); + __asm__ __volatile__ ("stl $14, %0" : "=m" (newregs->regs[14])); + __asm__ __volatile__ ("stl $15, %0" : "=m" (newregs->regs[15])); + __asm__ __volatile__ ("stl $16, %0" : "=m" (newregs->regs[16])); + __asm__ __volatile__ ("stl $17, %0" : "=m" (newregs->regs[17])); + __asm__ __volatile__ ("stl $18, %0" : "=m" (newregs->regs[18])); + __asm__ __volatile__ ("stl $19, %0" : "=m" (newregs->regs[19])); + __asm__ __volatile__ ("stl $20, %0" : "=m" (newregs->regs[20])); + __asm__ __volatile__ ("stl $21, %0" : "=m" (newregs->regs[21])); + __asm__ __volatile__ ("stl $22, %0" : "=m" (newregs->regs[22])); + __asm__ __volatile__ ("stl $23, %0" : "=m" (newregs->regs[23])); + __asm__ __volatile__ ("stl $24, %0" : "=m" (newregs->regs[24])); + __asm__ __volatile__ ("stl $25, %0" : "=m" (newregs->regs[25])); + __asm__ __volatile__ ("stl $26, %0" : "=m" (newregs->regs[26])); + __asm__ __volatile__ ("stl $27, %0" : "=m" (newregs->regs[27])); + __asm__ __volatile__ ("stl $28, %0" : "=m" (newregs->regs[28])); + __asm__ __volatile__ ("stl $29, %0" : "=m" (newregs->regs[29])); + __asm__ __volatile__ ("stl $30, %0" : "=m" (newregs->regs[30])); newregs->pc = (unsigned long)current_text_addr(); } } diff --git a/arch/sw_64/include/asm/livepatch.h b/arch/sw_64/include/asm/livepatch.h index f93c2131113b..1feec0f6be76 100644 --- a/arch/sw_64/include/asm/livepatch.h +++ b/arch/sw_64/include/asm/livepatch.h @@ -15,8 +15,8 @@ static inline int klp_check_compiler_support(void) static inline void klp_arch_set_pc(struct pt_regs *regs, unsigned long ip) { - regs->r27 = ip; - regs->r28 = ip; + regs->regs[27] = ip; + regs->regs[28] = ip; } #endif /* _ASM_SW64_LIVEPATCH_H */ diff --git a/arch/sw_64/include/asm/processor.h b/arch/sw_64/include/asm/processor.h index c0b7b683492e..26394854071f 100644 --- a/arch/sw_64/include/asm/processor.h +++ b/arch/sw_64/include/asm/processor.h @@ -65,7 +65,7 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) -#define KSTK_ESP(tsk) (task_pt_regs(tsk)->sp) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[30]) #define cpu_relax() barrier() diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 3eedfcc4dad0..40f8096cfbc3 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -15,37 +15,7 @@ struct pt_regs { union { struct user_pt_regs user_regs; struct { - unsigned long r0; - unsigned long r1; - unsigned long r2; - unsigned long r3; - unsigned long r4; - unsigned long r5; - unsigned long r6; - unsigned long r7; - unsigned long r8; - unsigned long r9; - unsigned long r10; - unsigned long r11; - unsigned long r12; - unsigned long r13; - unsigned long r14; - unsigned long r15; - unsigned long r16; - unsigned long r17; - unsigned long r18; - unsigned long r19; - unsigned long r20; - unsigned long r21; - unsigned long r22; - unsigned long r23; - unsigned long r24; - unsigned long r25; - unsigned long r26; - unsigned long r27; - unsigned long r28; - unsigned long gp; - unsigned long sp; + unsigned long regs[31]; unsigned long pc; unsigned long ps; }; @@ -63,11 +33,11 @@ struct pt_regs { #define user_mode(regs) (((regs)->ps & 8) != 0) #define instruction_pointer(regs) ((regs)->pc) #define profile_pc(regs) instruction_pointer(regs) -#define user_stack_pointer(regs) (regs->sp) +#define user_stack_pointer(pt_regs) ((pt_regs)->regs[30]) #define kernel_stack_pointer(regs) ((unsigned long)((regs) + 1)) #define instruction_pointer_set(regs, val) ((regs)->pc = val) -#define force_successful_syscall_return() (current_pt_regs()->r0 = 0) +#define force_successful_syscall_return() (current_pt_regs()->regs[0] = 0) #define MAX_REG_OFFSET (offsetof(struct pt_regs, ps)) @@ -95,14 +65,14 @@ extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, static inline int is_syscall_success(struct pt_regs *regs) { - return !regs->r19; + return !regs->regs[19]; } static inline long regs_return_value(struct pt_regs *regs) { if (is_syscall_success(regs) || !user_mode(regs)) - return regs->r0; + return regs->regs[0]; else - return -regs->r0; + return -regs->regs[0]; } #endif /* _ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/include/asm/syscall.h b/arch/sw_64/include/asm/syscall.h index 0b1556a460b4..da662d14356d 100644 --- a/arch/sw_64/include/asm/syscall.h +++ b/arch/sw_64/include/asm/syscall.h @@ -8,19 +8,19 @@ extern void *sys_call_table[]; static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { - return regs->r0; + return regs->regs[0]; } static inline long syscall_get_error(struct task_struct *task, struct pt_regs *regs) { - return regs->r19 ? -regs->r0 : 0; + return regs->regs[19] ? -regs->regs[0] : 0; } static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { - return regs->r0; + return regs->regs[0]; } static inline void syscall_set_return_value(struct task_struct *task, @@ -28,11 +28,11 @@ static inline void syscall_set_return_value(struct task_struct *task, int error, long val) { if (error) { - regs->r0 = -error; - regs->r19 = -1; + regs->regs[0] = -error; + regs->regs[19] = -1; } else { - regs->r0 = val; - regs->r19 = 0; + regs->regs[0] = val; + regs->regs[19] = 0; } } @@ -47,24 +47,24 @@ static inline void syscall_get_arguments(struct task_struct *task, struct pt_regs *regs, unsigned long *args) { - *args++ = regs->r16; - *args++ = regs->r17; - *args++ = regs->r18; - *args++ = regs->r19; - *args++ = regs->r20; - *args = regs->r21; + *args++ = regs->regs[16]; + *args++ = regs->regs[17]; + *args++ = regs->regs[18]; + *args++ = regs->regs[19]; + *args++ = regs->regs[20]; + *args = regs->regs[21]; } static inline void syscall_set_arguments(struct task_struct *task, struct pt_regs *regs, const unsigned long *args) { - regs->r16 = *args++; - regs->r17 = *args++; - regs->r18 = *args++; - regs->r19 = *args++; - regs->r20 = *args++; - regs->r21 = *args; + regs->regs[16] = *args++; + regs->regs[17] = *args++; + regs->regs[18] = *args++; + regs->regs[19] = *args++; + regs->regs[20] = *args++; + regs->regs[21] = *args; } static inline int syscall_get_arch(struct task_struct *task) diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index fe46b42bc943..c7f9a922436b 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -68,37 +68,37 @@ void foo(void) BLANK(); DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs)); - DEFINE(PT_REGS_R0, offsetof(struct pt_regs, r0)); - DEFINE(PT_REGS_R1, offsetof(struct pt_regs, r1)); - DEFINE(PT_REGS_R2, offsetof(struct pt_regs, r2)); - DEFINE(PT_REGS_R3, offsetof(struct pt_regs, r3)); - DEFINE(PT_REGS_R4, offsetof(struct pt_regs, r4)); - DEFINE(PT_REGS_R5, offsetof(struct pt_regs, r5)); - DEFINE(PT_REGS_R6, offsetof(struct pt_regs, r6)); - DEFINE(PT_REGS_R7, offsetof(struct pt_regs, r7)); - DEFINE(PT_REGS_R8, offsetof(struct pt_regs, r8)); - DEFINE(PT_REGS_R9, offsetof(struct pt_regs, r9)); - DEFINE(PT_REGS_R10, offsetof(struct pt_regs, r10)); - DEFINE(PT_REGS_R11, offsetof(struct pt_regs, r11)); - DEFINE(PT_REGS_R12, offsetof(struct pt_regs, r12)); - DEFINE(PT_REGS_R13, offsetof(struct pt_regs, r13)); - DEFINE(PT_REGS_R14, offsetof(struct pt_regs, r14)); - DEFINE(PT_REGS_R15, offsetof(struct pt_regs, r15)); - DEFINE(PT_REGS_R16, offsetof(struct pt_regs, r16)); - DEFINE(PT_REGS_R17, offsetof(struct pt_regs, r17)); - DEFINE(PT_REGS_R18, offsetof(struct pt_regs, r18)); - DEFINE(PT_REGS_R19, offsetof(struct pt_regs, r19)); - DEFINE(PT_REGS_R20, offsetof(struct pt_regs, r20)); - DEFINE(PT_REGS_R21, offsetof(struct pt_regs, r21)); - DEFINE(PT_REGS_R22, offsetof(struct pt_regs, r22)); - DEFINE(PT_REGS_R23, offsetof(struct pt_regs, r23)); - DEFINE(PT_REGS_R24, offsetof(struct pt_regs, r24)); - DEFINE(PT_REGS_R25, offsetof(struct pt_regs, r25)); - DEFINE(PT_REGS_R26, offsetof(struct pt_regs, r26)); - DEFINE(PT_REGS_R27, offsetof(struct pt_regs, r27)); - DEFINE(PT_REGS_R28, offsetof(struct pt_regs, r28)); - DEFINE(PT_REGS_GP, offsetof(struct pt_regs, gp)); - DEFINE(PT_REGS_SP, offsetof(struct pt_regs, sp)); + DEFINE(PT_REGS_R0, offsetof(struct pt_regs, regs[0])); + DEFINE(PT_REGS_R1, offsetof(struct pt_regs, regs[1])); + DEFINE(PT_REGS_R2, offsetof(struct pt_regs, regs[2])); + DEFINE(PT_REGS_R3, offsetof(struct pt_regs, regs[3])); + DEFINE(PT_REGS_R4, offsetof(struct pt_regs, regs[4])); + DEFINE(PT_REGS_R5, offsetof(struct pt_regs, regs[5])); + DEFINE(PT_REGS_R6, offsetof(struct pt_regs, regs[6])); + DEFINE(PT_REGS_R7, offsetof(struct pt_regs, regs[7])); + DEFINE(PT_REGS_R8, offsetof(struct pt_regs, regs[8])); + DEFINE(PT_REGS_R9, offsetof(struct pt_regs, regs[9])); + DEFINE(PT_REGS_R10, offsetof(struct pt_regs, regs[10])); + DEFINE(PT_REGS_R11, offsetof(struct pt_regs, regs[11])); + DEFINE(PT_REGS_R12, offsetof(struct pt_regs, regs[12])); + DEFINE(PT_REGS_R13, offsetof(struct pt_regs, regs[13])); + DEFINE(PT_REGS_R14, offsetof(struct pt_regs, regs[14])); + DEFINE(PT_REGS_R15, offsetof(struct pt_regs, regs[15])); + DEFINE(PT_REGS_R16, offsetof(struct pt_regs, regs[16])); + DEFINE(PT_REGS_R17, offsetof(struct pt_regs, regs[17])); + DEFINE(PT_REGS_R18, offsetof(struct pt_regs, regs[18])); + DEFINE(PT_REGS_R19, offsetof(struct pt_regs, regs[19])); + DEFINE(PT_REGS_R20, offsetof(struct pt_regs, regs[20])); + DEFINE(PT_REGS_R21, offsetof(struct pt_regs, regs[21])); + DEFINE(PT_REGS_R22, offsetof(struct pt_regs, regs[22])); + DEFINE(PT_REGS_R23, offsetof(struct pt_regs, regs[23])); + DEFINE(PT_REGS_R24, offsetof(struct pt_regs, regs[24])); + DEFINE(PT_REGS_R25, offsetof(struct pt_regs, regs[25])); + DEFINE(PT_REGS_R26, offsetof(struct pt_regs, regs[26])); + DEFINE(PT_REGS_R27, offsetof(struct pt_regs, regs[27])); + DEFINE(PT_REGS_R28, offsetof(struct pt_regs, regs[28])); + DEFINE(PT_REGS_GP, offsetof(struct pt_regs, regs[29])); + DEFINE(PT_REGS_SP, offsetof(struct pt_regs, regs[30])); DEFINE(PT_REGS_PC, offsetof(struct pt_regs, pc)); DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); diff --git a/arch/sw_64/kernel/kgdb.c b/arch/sw_64/kernel/kgdb.c index 4142b2ada817..833f72a1577c 100644 --- a/arch/sw_64/kernel/kgdb.c +++ b/arch/sw_64/kernel/kgdb.c @@ -24,40 +24,40 @@ #include struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { - { "r0", 8, offsetof(struct pt_regs, r0)}, - { "r1", 8, offsetof(struct pt_regs, r1)}, - { "r2", 8, offsetof(struct pt_regs, r2)}, - { "r3", 8, offsetof(struct pt_regs, r3)}, - { "r4", 8, offsetof(struct pt_regs, r4)}, - { "r5", 8, offsetof(struct pt_regs, r5)}, - { "r6", 8, offsetof(struct pt_regs, r6)}, - { "r7", 8, offsetof(struct pt_regs, r7)}, - { "r8", 8, offsetof(struct pt_regs, r8)}, - - { "r9", 8, offsetof(struct pt_regs, r9)}, - { "r10", 8, offsetof(struct pt_regs, r10)}, - { "r11", 8, offsetof(struct pt_regs, r11)}, - { "r12", 8, offsetof(struct pt_regs, r12)}, - { "r13", 8, offsetof(struct pt_regs, r13)}, - { "r14", 8, offsetof(struct pt_regs, r14)}, - { "r15", 8, offsetof(struct pt_regs, r15)}, - - { "r16", 8, offsetof(struct pt_regs, r16)}, - { "r17", 8, offsetof(struct pt_regs, r17)}, - { "r18", 8, offsetof(struct pt_regs, r18)}, - - { "r19", 8, offsetof(struct pt_regs, r19)}, - { "r20", 8, offsetof(struct pt_regs, r20)}, - { "r21", 8, offsetof(struct pt_regs, r21)}, - { "r22", 8, offsetof(struct pt_regs, r22)}, - { "r23", 8, offsetof(struct pt_regs, r23)}, - { "r24", 8, offsetof(struct pt_regs, r24)}, - { "r25", 8, offsetof(struct pt_regs, r25)}, - { "r26", 8, offsetof(struct pt_regs, r26)}, - { "r27", 8, offsetof(struct pt_regs, r27)}, - { "at", 8, offsetof(struct pt_regs, r28)}, - { "gp", 8, offsetof(struct pt_regs, gp)}, - { "sp", 8, offsetof(struct pt_regs, sp)}, + { "r0", 8, offsetof(struct pt_regs, regs[0])}, + { "r1", 8, offsetof(struct pt_regs, regs[1])}, + { "r2", 8, offsetof(struct pt_regs, regs[2])}, + { "r3", 8, offsetof(struct pt_regs, regs[3])}, + { "r4", 8, offsetof(struct pt_regs, regs[4])}, + { "r5", 8, offsetof(struct pt_regs, regs[5])}, + { "r6", 8, offsetof(struct pt_regs, regs[6])}, + { "r7", 8, offsetof(struct pt_regs, regs[7])}, + { "r8", 8, offsetof(struct pt_regs, regs[8])}, + + { "r9", 8, offsetof(struct pt_regs, regs[9])}, + { "r10", 8, offsetof(struct pt_regs, regs[10])}, + { "r11", 8, offsetof(struct pt_regs, regs[11])}, + { "r12", 8, offsetof(struct pt_regs, regs[12])}, + { "r13", 8, offsetof(struct pt_regs, regs[13])}, + { "r14", 8, offsetof(struct pt_regs, regs[14])}, + { "r15", 8, offsetof(struct pt_regs, regs[15])}, + + { "r16", 8, offsetof(struct pt_regs, regs[16])}, + { "r17", 8, offsetof(struct pt_regs, regs[17])}, + { "r18", 8, offsetof(struct pt_regs, regs[18])}, + + { "r19", 8, offsetof(struct pt_regs, regs[19])}, + { "r20", 8, offsetof(struct pt_regs, regs[20])}, + { "r21", 8, offsetof(struct pt_regs, regs[21])}, + { "r22", 8, offsetof(struct pt_regs, regs[22])}, + { "r23", 8, offsetof(struct pt_regs, regs[23])}, + { "r24", 8, offsetof(struct pt_regs, regs[24])}, + { "r25", 8, offsetof(struct pt_regs, regs[25])}, + { "r26", 8, offsetof(struct pt_regs, regs[26])}, + { "r27", 8, offsetof(struct pt_regs, regs[27])}, + { "at", 8, offsetof(struct pt_regs, regs[28])}, + { "gp", 8, offsetof(struct pt_regs, regs[29])}, + { "sp", 8, offsetof(struct pt_regs, regs[30])}, { "zero", 8, -1 }, { "f0", 8, -1 }, diff --git a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c index 69fd38135cad..89d7dba9dc25 100644 --- a/arch/sw_64/kernel/kprobes/kprobes-ftrace.c +++ b/arch/sw_64/kernel/kprobes/kprobes-ftrace.c @@ -24,12 +24,12 @@ void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, if (kprobe_running()) { kprobes_inc_nmissed_count(p); } else { - regs->r28 -= MCOUNT_INSN_SIZE; + regs->regs[28] -= MCOUNT_INSN_SIZE; __this_cpu_write(current_kprobe, p); kcb->kprobe_status = KPROBE_HIT_ACTIVE; if (!p->pre_handler || !p->pre_handler(p, regs)) { - regs->r28 += MCOUNT_INSN_SIZE; + regs->regs[28] += MCOUNT_INSN_SIZE; if (unlikely(p->post_handler)) { kcb->kprobe_status = KPROBE_HIT_SSDONE; p->post_handler(p, regs, 0); diff --git a/arch/sw_64/kernel/kprobes/kprobes.c b/arch/sw_64/kernel/kprobes/kprobes.c index 7080c892a24d..3bacfb3b3a4c 100644 --- a/arch/sw_64/kernel/kprobes/kprobes.c +++ b/arch/sw_64/kernel/kprobes/kprobes.c @@ -267,11 +267,11 @@ void kretprobe_trampoline(void); void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { - ri->ret_addr = (kprobe_opcode_t *) regs->r26; + ri->ret_addr = (kprobe_opcode_t *) regs->regs[26]; ri->fp = NULL; /* Replace the return addr with trampoline addr */ - regs->r26 = (unsigned long)kretprobe_trampoline; + regs->regs[26] = (unsigned long)kretprobe_trampoline; } /* @@ -284,7 +284,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, orig_ret_address = __kretprobe_trampoline_handler(regs, kretprobe_trampoline, NULL); instruction_pointer(regs) = orig_ret_address; - regs->r26 = orig_ret_address; + regs->regs[26] = orig_ret_address; /* * By returning a non-zero value, we are telling diff --git a/arch/sw_64/kernel/perf_event.c b/arch/sw_64/kernel/perf_event.c index 336fab76d6f7..63922113de64 100644 --- a/arch/sw_64/kernel/perf_event.c +++ b/arch/sw_64/kernel/perf_event.c @@ -664,7 +664,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, perf_callchain_store(entry, regs->pc); - fp = (unsigned long __user *)regs->r15; + fp = (unsigned long __user *)regs->regs[15]; while (entry->nr < entry->max_stack && (unsigned long)fp < current->mm->start_stack) { if (!access_ok(fp, sizeof(frame))) diff --git a/arch/sw_64/kernel/process.c b/arch/sw_64/kernel/process.c index ad4b71d4037a..ad54129a48e5 100644 --- a/arch/sw_64/kernel/process.c +++ b/arch/sw_64/kernel/process.c @@ -22,7 +22,7 @@ start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) { regs->pc = pc; regs->ps = 8; - regs->sp = sp; + regs->regs[30] = sp; } EXPORT_SYMBOL(start_thread); @@ -91,14 +91,14 @@ copy_thread(unsigned long clone_flags, unsigned long usp, * application calling fork. */ if (clone_flags & CLONE_SETTLS) - childti->pcb.tp = regs->r20; + childti->pcb.tp = regs->regs[20]; else - regs->r20 = 0; + regs->regs[20] = 0; *childregs = *regs; if (usp) - childregs->sp = usp; - childregs->r0 = 0; - childregs->r19 = 0; + childregs->regs[30] = usp; + childregs->regs[0] = 0; + childregs->regs[19] = 0; p->thread.ra = (unsigned long) ret_from_fork; return 0; } @@ -115,7 +115,7 @@ void sw64_elf_core_copy_regs(elf_greg_t *dest, struct pt_regs *regs) ti = (void *)((__u64)regs & ~(THREAD_SIZE - 1)); for (i = 0; i < 31; i++) - dest[i] = *(__u64 *)((void *)regs + regoffsets[i]); + dest[i] = regs->regs[i]; dest[31] = regs->pc; dest[32] = ti->pcb.tp; } diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index 38116b893b5c..d53736985a37 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -47,14 +47,14 @@ * zero have no stack-slot and need to be treated specially (see * get_reg/put_reg below). */ -#define R(x) ((size_t) &((struct pt_regs *)0)->x) +#define R(x) ((size_t) &((struct pt_regs *)0)->regs[x]) short regoffsets[32] = { - R(r0), R(r1), R(r2), R(r3), R(r4), R(r5), R(r6), R(r7), R(r8), - R(r9), R(r10), R(r11), R(r12), R(r13), R(r14), R(r15), - R(r16), R(r17), R(r18), - R(r19), R(r20), R(r21), R(r22), R(r23), R(r24), R(r25), R(r26), - R(r27), R(r28), R(gp), R(sp), 0 + R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), + R(9), R(10), R(11), R(12), R(13), R(14), R(15), + R(16), R(17), R(18), + R(19), R(20), R(21), R(22), R(23), R(24), R(25), R(26), + R(27), R(28), R(29), R(30), 0 }; #undef R @@ -103,7 +103,7 @@ get_reg_addr(struct task_struct *task, unsigned long regno) addr = (void *)task_thread_info(task) + pcboff[regno]; break; case PT_REG_BASE ... PT_REG_END: - addr = (void *)task_pt_regs(task) + regoffsets[regno]; + addr = &task_pt_regs(task)->regs[regno]; break; case PT_FPREG_BASE ... PT_FPREG_END: fno = regno - PT_FPREG_BASE; @@ -396,9 +396,9 @@ asmlinkage unsigned long syscall_trace_enter(void) #endif if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) - trace_sys_enter(regs, regs->r0); - audit_syscall_entry(regs->r0, regs->r16, regs->r17, regs->r18, regs->r19); - return ret ?: current_pt_regs()->r0; + trace_sys_enter(regs, regs->regs[0]); + audit_syscall_entry(regs->regs[0], regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]); + return ret ?: current_pt_regs()->regs[0]; } asmlinkage void @@ -776,6 +776,11 @@ struct pt_regs_offset { int offset; }; +#define GPR_OFFSET_NAME(r) { \ + .name = "r" #r, \ + .offset = offsetof(struct pt_regs, regs[r]) \ +} + #define REG_OFFSET_NAME(r) { \ .name = #r, \ .offset = offsetof(struct pt_regs, r) \ @@ -787,37 +792,37 @@ struct pt_regs_offset { } static const struct pt_regs_offset regoffset_table[] = { - REG_OFFSET_NAME(r0), - REG_OFFSET_NAME(r1), - REG_OFFSET_NAME(r2), - REG_OFFSET_NAME(r3), - REG_OFFSET_NAME(r4), - REG_OFFSET_NAME(r5), - REG_OFFSET_NAME(r6), - REG_OFFSET_NAME(r7), - REG_OFFSET_NAME(r8), - REG_OFFSET_NAME(r9), - REG_OFFSET_NAME(r10), - REG_OFFSET_NAME(r11), - REG_OFFSET_NAME(r12), - REG_OFFSET_NAME(r13), - REG_OFFSET_NAME(r14), - REG_OFFSET_NAME(r15), - REG_OFFSET_NAME(r16), - REG_OFFSET_NAME(r17), - REG_OFFSET_NAME(r18), - REG_OFFSET_NAME(r19), - REG_OFFSET_NAME(r20), - REG_OFFSET_NAME(r21), - REG_OFFSET_NAME(r22), - REG_OFFSET_NAME(r23), - REG_OFFSET_NAME(r24), - REG_OFFSET_NAME(r25), - REG_OFFSET_NAME(r26), - REG_OFFSET_NAME(r27), - REG_OFFSET_NAME(r28), - REG_OFFSET_NAME(gp), - REG_OFFSET_NAME(sp), + GPR_OFFSET_NAME(0), + GPR_OFFSET_NAME(1), + GPR_OFFSET_NAME(2), + GPR_OFFSET_NAME(3), + GPR_OFFSET_NAME(4), + GPR_OFFSET_NAME(5), + GPR_OFFSET_NAME(6), + GPR_OFFSET_NAME(7), + GPR_OFFSET_NAME(8), + GPR_OFFSET_NAME(9), + GPR_OFFSET_NAME(10), + GPR_OFFSET_NAME(11), + GPR_OFFSET_NAME(12), + GPR_OFFSET_NAME(13), + GPR_OFFSET_NAME(14), + GPR_OFFSET_NAME(15), + GPR_OFFSET_NAME(16), + GPR_OFFSET_NAME(17), + GPR_OFFSET_NAME(18), + GPR_OFFSET_NAME(19), + GPR_OFFSET_NAME(20), + GPR_OFFSET_NAME(21), + GPR_OFFSET_NAME(22), + GPR_OFFSET_NAME(23), + GPR_OFFSET_NAME(24), + GPR_OFFSET_NAME(25), + GPR_OFFSET_NAME(26), + GPR_OFFSET_NAME(27), + GPR_OFFSET_NAME(28), + GPR_OFFSET_NAME(29), + GPR_OFFSET_NAME(30), REG_OFFSET_NAME(pc), REG_OFFSET_NAME(ps), REG_OFFSET_END, diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index cbf916a3da9d..8ef7e22183c4 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -98,7 +98,7 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) current->restart_block.fn = do_no_restart_syscall; - err |= __copy_from_user(regs, sc->sc_regs, sizeof(unsigned long) * 31); + err |= __copy_from_user(regs, sc->sc_regs, sizeof_field(struct pt_regs, regs)); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, offsetof(struct user_fpsimd_state, fpcr)); @@ -197,7 +197,7 @@ setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, err |= __put_user(regs->pc, &sc->sc_pc); err |= __put_user(8, &sc->sc_ps); - err |= __copy_to_user(sc->sc_regs, regs, sizeof(unsigned long) * 31); + err |= __copy_to_user(sc->sc_regs, regs, sizeof_field(struct pt_regs, regs)); err |= __put_user(0, sc->sc_regs+31); /* simd-fp */ __fpstate_save(current); @@ -214,7 +214,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) unsigned long err = 0; struct rt_sigframe __user *frame; - frame = get_sigframe(ksig, regs->sp, sizeof(*frame)); + frame = get_sigframe(ksig, regs->regs[30], sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) return -EFAULT; @@ -225,30 +225,30 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) err |= __put_user(0, &frame->uc.uc_flags); err |= __put_user(0, &frame->uc.uc_link); err |= __put_user(set->sig[0], &frame->uc.uc_old_sigmask); - err |= __save_altstack(&frame->uc.uc_stack, regs->sp); + err |= __save_altstack(&frame->uc.uc_stack, regs->regs[30]); err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, set->sig[0]); err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set)); if (err) return -EFAULT; /* "Return" to the handler */ - regs->r26 = VDSO_SYMBOL(current->mm->context.vdso, rt_sigreturn); - regs->r27 = regs->pc = (unsigned long) ksig->ka.sa.sa_handler; - regs->r16 = ksig->sig; /* a0: signal number */ + regs->regs[26] = VDSO_SYMBOL(current->mm->context.vdso, rt_sigreturn); + regs->regs[27] = regs->pc = (unsigned long) ksig->ka.sa.sa_handler; + regs->regs[16] = ksig->sig; /* a0: signal number */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) { /* a1: siginfo pointer, a2: ucontext pointer */ - regs->r17 = (unsigned long) &frame->info; - regs->r18 = (unsigned long) &frame->uc; + regs->regs[17] = (unsigned long) &frame->info; + regs->regs[18] = (unsigned long) &frame->uc; } else { /* a1: exception code, a2: sigcontext pointer */ - regs->r17 = 0; - regs->r18 = (unsigned long) &frame->uc.uc_mcontext; + regs->regs[17] = 0; + regs->regs[18] = (unsigned long) &frame->uc.uc_mcontext; } - regs->sp = (unsigned long) frame; + regs->regs[30] = (unsigned long) frame; #if DEBUG_SIG printk("SIG deliver (%s:%d): sp=%p pc=%p ra=%p\n", - current->comm, current->pid, frame, regs->pc, regs->r26); + current->comm, current->pid, frame, regs->pc, regs->regs[26]); #endif return 0; @@ -274,23 +274,23 @@ static inline void syscall_restart(unsigned long r0, unsigned long r19, struct pt_regs *regs, struct k_sigaction *ka) { - switch (regs->r0) { + switch (regs->regs[0]) { case ERESTARTSYS: if (!(ka->sa.sa_flags & SA_RESTART)) { - regs->r0 = EINTR; + regs->regs[0] = EINTR; break; } fallthrough; case ERESTARTNOINTR: - regs->r0 = r0; /* reset v0 and a3 and replay syscall */ - regs->r19 = r19; + regs->regs[0] = r0; /* reset v0 and a3 and replay syscall */ + regs->regs[19] = r19; regs->pc -= 4; break; case ERESTART_RESTARTBLOCK: - regs->r0 = EINTR; + regs->regs[0] = EINTR; break; case ERESTARTNOHAND: - regs->r0 = EINTR; + regs->regs[0] = EINTR; break; } } @@ -326,18 +326,18 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) } else { single_stepping |= ptrace_cancel_bpt(current); if (r0) { - switch (regs->r0) { + switch (regs->regs[0]) { case ERESTARTNOHAND: case ERESTARTSYS: case ERESTARTNOINTR: /* Reset v0 and a3 and replay syscall. */ - regs->r0 = r0; - regs->r19 = r19; + regs->regs[0] = r0; + regs->regs[19] = r19; regs->pc -= 4; break; case ERESTART_RESTARTBLOCK: /* Set v0 to the restart_syscall and replay */ - regs->r0 = __NR_restart_syscall; + regs->regs[0] = __NR_restart_syscall; regs->pc -= 4; break; } diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c index 57d4fcc35ec5..d7172e4ec78a 100644 --- a/arch/sw_64/kernel/stacktrace.c +++ b/arch/sw_64/kernel/stacktrace.c @@ -71,7 +71,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, if (regs) { unsigned long offset; pc = regs->pc; - fp = regs->r15; + fp = regs->regs[15]; if (kallsyms_lookup_size_offset(pc, NULL, &offset) && offset < 16) { /* call stack has not been setup @@ -79,7 +79,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, */ if (fn(pc, data)) return; - pc = regs->r26; + pc = regs->regs[26]; } } else if (tsk == current || tsk == NULL) { fp = (unsigned long)__builtin_frame_address(0); diff --git a/arch/sw_64/kernel/sys_sw64.c b/arch/sw_64/kernel/sys_sw64.c index 9793b6a89965..d0198aef554d 100644 --- a/arch/sw_64/kernel/sys_sw64.c +++ b/arch/sw_64/kernel/sys_sw64.c @@ -121,19 +121,19 @@ SYSCALL_DEFINE2(odd_getpriority, int, which, int, who) SYSCALL_DEFINE0(getxuid) { - current_pt_regs()->r20 = sys_geteuid(); + current_pt_regs()->regs[20] = sys_geteuid(); return sys_getuid(); } SYSCALL_DEFINE0(getxgid) { - current_pt_regs()->r20 = sys_getegid(); + current_pt_regs()->regs[20] = sys_getegid(); return sys_getgid(); } SYSCALL_DEFINE0(getxpid) { - current_pt_regs()->r20 = sys_getppid(); + current_pt_regs()->regs[20] = sys_getppid(); return sys_getpid(); } @@ -144,7 +144,7 @@ SYSCALL_DEFINE0(sw64_pipe) if (!res) { /* The return values are in $0 and $20. */ - current_pt_regs()->r20 = fd[1]; + current_pt_regs()->regs[20] = fd[1]; res = fd[0]; } return res; diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index c74b4b541192..f5100d7e5a16 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -47,32 +47,32 @@ void show_regs(struct pt_regs *regs) show_regs_print_info(KERN_DEFAULT); printk("pc = [<%016lx>] ra = [<%016lx>] ps = %04lx %s\n", - regs->pc, regs->r26, regs->ps, print_tainted()); + regs->pc, regs->regs[26], regs->ps, print_tainted()); printk("pc is at %pSR\n", (void *)regs->pc); - printk("ra is at %pSR\n", (void *)regs->r26); + printk("ra is at %pSR\n", (void *)regs->regs[26]); printk("v0 = %016lx t0 = %016lx t1 = %016lx\n", - regs->r0, regs->r1, regs->r2); + regs->regs[0], regs->regs[1], regs->regs[2]); printk("t2 = %016lx t3 = %016lx t4 = %016lx\n", - regs->r3, regs->r4, regs->r5); + regs->regs[3], regs->regs[4], regs->regs[5]); printk("t5 = %016lx t6 = %016lx t7 = %016lx\n", - regs->r6, regs->r7, regs->r8); + regs->regs[6], regs->regs[7], regs->regs[8]); printk("s0 = %016lx s1 = %016lx s2 = %016lx\n", - regs->r9, regs->r10, regs->r11); + regs->regs[9], regs->regs[10], regs->regs[11]); printk("s3 = %016lx s4 = %016lx s5 = %016lx\n", - regs->r12, regs->r13, regs->r14); + regs->regs[12], regs->regs[13], regs->regs[14]); printk("s6 = %016lx\n", - regs->r15); + regs->regs[15]); printk("a0 = %016lx a1 = %016lx a2 = %016lx\n", - regs->r16, regs->r17, regs->r18); + regs->regs[16], regs->regs[17], regs->regs[18]); printk("a3 = %016lx a4 = %016lx a5 = %016lx\n", - regs->r19, regs->r20, regs->r21); + regs->regs[19], regs->regs[20], regs->regs[21]); printk("t8 = %016lx t9 = %016lx t10 = %016lx\n", - regs->r22, regs->r23, regs->r24); + regs->regs[22], regs->regs[23], regs->regs[24]); printk("t11= %016lx pv = %016lx at = %016lx\n", - regs->r25, regs->r27, regs->r28); - printk("gp = %016lx sp = %016lx\n", regs->gp, regs->sp); + regs->regs[25], regs->regs[27], regs->regs[28]); + printk("gp = %016lx sp = %016lx\n", regs->regs[29], regs->regs[30]); } static void show_code(unsigned int *pc) @@ -225,7 +225,7 @@ do_entIF(unsigned long inst_type, unsigned long va, struct pt_regs *regs) case IF_GENTRAP: regs->pc -= 4; - switch ((long)regs->r16) { + switch ((long)regs->regs[16]) { case GEN_INTOVF: signo = SIGFPE; code = FPE_INTOVF; @@ -361,7 +361,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = tmp1 | tmp2; + regs->regs[reg] = tmp1 | tmp2; return; case 0x22: @@ -382,7 +382,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = (int)(tmp1 | tmp2); + regs->regs[reg] = (int)(tmp1 | tmp2); return; case 0x23: /* ldl */ @@ -403,7 +403,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, if (error) goto got_exception; - map_regs(reg) = tmp1 | tmp2; + regs->regs[reg] = tmp1 | tmp2; return; case 0x29: /* sth */ @@ -421,7 +421,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -453,7 +453,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -505,7 +505,7 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, ".previous" : "=r"(error), "=&r"(tmp1), "=&r"(tmp2), "=&r"(tmp3), "=&r"(tmp4), "=&r"(tmp5), "=&r"(tmp6), "=&r"(tmp7), "=&r"(tmp8) - : "r"(va), "r"(map_regs(reg)), "0"(0)); + : "r"(va), "r"(regs->regs[reg]), "0"(0)); if (error) goto got_exception; @@ -626,8 +626,7 @@ do_entUnaUser(void __user *va, unsigned long opcode, if ((1L << opcode) & OP_INT_MASK) { /* it's an integer load/store */ if (reg < 31) { - reg_addr = (unsigned long *) - ((char *)regs + regoffsets[reg]); + reg_addr = ®s->regs[reg]; } else { /* zero "register" */ fake_reg = 0; diff --git a/arch/sw_64/kernel/uprobes.c b/arch/sw_64/kernel/uprobes.c index 0e56134a8251..25da4cee7d5e 100644 --- a/arch/sw_64/kernel/uprobes.c +++ b/arch/sw_64/kernel/uprobes.c @@ -127,10 +127,10 @@ unsigned long arch_uretprobe_hijack_return_addr( { unsigned long ra; - ra = regs->r26; + ra = regs->regs[26]; /* Replace the return address with the trampoline address */ - regs->r26 = trampoline_vaddr; + regs->regs[26] = trampoline_vaddr; return ra; } @@ -178,5 +178,5 @@ void sw64_fix_uretprobe(struct pt_regs *regs, unsigned long exc_pc) * regs->pc has been changed to orig_ret_vaddr in handle_trampoline(). */ if (exc_pc == get_trampoline_vaddr()) - regs->r26 = regs->pc; + regs->regs[26] = regs->pc; } -- Gitee From 9e6c23fdc36375b36a94ab9137e17e8893bd794e Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:46 +0800 Subject: [PATCH 118/150] sw64: modify fixup_exception() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Now struct pt_regs have everything required, so implement fixup_exception() in a more standard way. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/extable.h | 16 ++-------------- arch/sw_64/kernel/ptrace.c | 12 ------------ arch/sw_64/kernel/traps.c | 11 ++--------- arch/sw_64/mm/Makefile | 2 +- arch/sw_64/mm/extable.c | 25 +++++++++++++++++++++++++ arch/sw_64/mm/fault.c | 9 +-------- 6 files changed, 31 insertions(+), 44 deletions(-) create mode 100644 arch/sw_64/mm/extable.c diff --git a/arch/sw_64/include/asm/extable.h b/arch/sw_64/include/asm/extable.h index 3680b4a918a6..42f872ce6c3b 100644 --- a/arch/sw_64/include/asm/extable.h +++ b/arch/sw_64/include/asm/extable.h @@ -34,26 +34,14 @@ struct exception_table_entry { } fixup; }; -/* Returns the new pc */ -#define fixup_exception(map_reg, _fixup, pc) \ -({ \ - if ((_fixup)->fixup.bits.valreg != 31) \ - map_reg((_fixup)->fixup.bits.valreg) = 0; \ - if ((_fixup)->fixup.bits.errreg != 31) \ - map_reg((_fixup)->fixup.bits.errreg) = -EFAULT; \ - (pc) + (_fixup)->fixup.bits.nextinsn; \ -}) - #define ARCH_HAS_RELATIVE_EXTABLE +extern int fixup_exception(struct pt_regs *regs, unsigned long pc); + #define swap_ex_entry_fixup(a, b, tmp, delta) \ do { \ (a)->fixup.unit = (b)->fixup.unit; \ (b)->fixup.unit = (tmp).fixup.unit; \ } while (0) -/* Macro for exception fixup code to access integer registers. */ -extern short regoffsets[]; -#define map_regs(r) (*(unsigned long *)((char *)regs + regoffsets[r])) - #endif /* _ASM_SW64_EXTABLE_H */ diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index d53736985a37..cb9ffde2ce39 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -47,18 +47,6 @@ * zero have no stack-slot and need to be treated specially (see * get_reg/put_reg below). */ -#define R(x) ((size_t) &((struct pt_regs *)0)->regs[x]) - -short regoffsets[32] = { - R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), - R(9), R(10), R(11), R(12), R(13), R(14), R(15), - R(16), R(17), R(18), - R(19), R(20), R(21), R(22), R(23), R(24), R(25), R(26), - R(27), R(28), R(29), R(30), 0 -}; - -#undef R - #define PCB_OFF(var) offsetof(struct pcb_struct, var) static int pcboff[] = { diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index f5100d7e5a16..7702233c4c0b 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -334,7 +334,6 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, long error; unsigned long tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; unsigned long pc = regs->pc - 4; - const struct exception_table_entry *fixup; /* * We don't want to use the generic get/put unaligned macros as @@ -520,15 +519,9 @@ do_entUna(void *va, unsigned long opcode, unsigned long reg, /* Ok, we caught the exception, but we don't want it. Is there * someone to pass it along to? */ - fixup = search_exception_tables(pc); - if (fixup != 0) { - unsigned long newpc; - - newpc = fixup_exception(map_regs, fixup, pc); + if (fixup_exception(regs, pc)) { printk("Forwarding unaligned exception at %lx (%lx)\n", - pc, newpc); - - regs->pc = newpc; + pc, regs->pc); return; } diff --git a/arch/sw_64/mm/Makefile b/arch/sw_64/mm/Makefile index 6f2ccec1d70c..8b9d6e4d2ebf 100644 --- a/arch/sw_64/mm/Makefile +++ b/arch/sw_64/mm/Makefile @@ -5,7 +5,7 @@ #ccflags-y := -Werror -obj-y := init.o fault.o physaddr.o mmap.o +obj-y := init.o fault.o physaddr.o mmap.o extable.o obj-$(CONFIG_NUMA) += numa.o ifeq ($(CONFIG_SUBARCH_C4),y) diff --git a/arch/sw_64/mm/extable.c b/arch/sw_64/mm/extable.c new file mode 100644 index 000000000000..d2678e12a1b1 --- /dev/null +++ b/arch/sw_64/mm/extable.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +int fixup_exception(struct pt_regs *regs, unsigned long pc) +{ + const struct exception_table_entry *fixup; + + fixup = search_exception_tables(pc); + if (fixup) { + unsigned int valreg = fixup->fixup.bits.valreg; + unsigned int errreg = fixup->fixup.bits.errreg; + + if (valreg != 31) + regs->regs[valreg] = 0; + if (errreg != 31) + regs->regs[errreg] = -EFAULT; + pc += fixup->fixup.bits.nextinsn; + regs->pc = pc; + + return 1; + } + return 0; +} diff --git a/arch/sw_64/mm/fault.c b/arch/sw_64/mm/fault.c index 26e16bd7f9ee..5c9a42c77fb2 100644 --- a/arch/sw_64/mm/fault.c +++ b/arch/sw_64/mm/fault.c @@ -138,7 +138,6 @@ do_page_fault(unsigned long address, unsigned long mmcsr, { struct vm_area_struct *vma; struct mm_struct *mm = current->mm; - const struct exception_table_entry *fixup; int si_code = SEGV_MAPERR; vm_fault_t fault; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; @@ -261,14 +260,8 @@ do_page_fault(unsigned long address, unsigned long mmcsr, no_context: /* Are we prepared to handle this fault as an exception? */ - fixup = search_exception_tables(regs->pc); - if (fixup != 0) { - unsigned long newpc; - - newpc = fixup_exception(map_regs, fixup, regs->pc); - regs->pc = newpc; + if (fixup_exception(regs, regs->pc)) return; - } /* * Oops. The kernel tried to access some bad page. We'll have to -- Gitee From 65318e6c7efb038e8490cfa1f1b56a3fe643c04d Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:47 +0800 Subject: [PATCH 119/150] sw64: modify do_[rt_]sigreturn Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCL0 -------------------------------- Since we have sp in struct pt_regs now, there's no need to pass it as a parameter. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/signal.c | 18 ++++++++++++------ arch/sw_64/kernel/vdso/vrt_sigreturn.S | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 8ef7e22183c4..e16acad8a52c 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -96,8 +96,6 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) { long err = __get_user(regs->pc, &sc->sc_pc); - current->restart_block.fn = do_no_restart_syscall; - err |= __copy_from_user(regs, sc->sc_regs, sizeof_field(struct pt_regs, regs)); /* simd-fp */ err |= __copy_from_user(¤t->thread.fpstate, &sc->sc_fpregs, @@ -116,12 +114,16 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) * registers and transfer control from userland. */ -asmlinkage void -do_sigreturn(struct sigcontext __user *sc) +asmlinkage void do_sigreturn(void) { struct pt_regs *regs = current_pt_regs(); + struct sigcontext __user *sc; sigset_t set; + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + sc = (struct sigcontext __user *)regs->regs[30]; /* Verify that it's a good sigcontext before using it */ if (!access_ok(sc, sizeof(*sc))) goto give_sigsegv; @@ -144,12 +146,16 @@ do_sigreturn(struct sigcontext __user *sc) force_sig(SIGSEGV); } -asmlinkage void -do_rt_sigreturn(struct rt_sigframe __user *frame) +asmlinkage void do_rt_sigreturn(void) { struct pt_regs *regs = current_pt_regs(); + struct rt_sigframe __user *frame; sigset_t set; + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + frame = (struct rt_sigframe __user *)regs->regs[30]; /* Verify that it's a good ucontext_t before using it */ if (!access_ok(&frame->uc, sizeof(frame->uc))) goto give_sigsegv; diff --git a/arch/sw_64/kernel/vdso/vrt_sigreturn.S b/arch/sw_64/kernel/vdso/vrt_sigreturn.S index d2d7295ffa7a..106b81f47664 100644 --- a/arch/sw_64/kernel/vdso/vrt_sigreturn.S +++ b/arch/sw_64/kernel/vdso/vrt_sigreturn.S @@ -63,7 +63,6 @@ nop ENTRY(__vdso_rt_sigreturn) - mov $sp, $16 ldi $0, __NR_rt_sigreturn sys_call HMC_callsys ENDPROC(__vdso_rt_sigreturn) -- Gitee From f19ac693dcbf27ad596899609c9d2d4357042233 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 27 Sep 2023 08:52:47 +0800 Subject: [PATCH 120/150] sw64: use call and br in kernel entry Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCM7 -------------------------------- Use call and br in kernel entry, so stacktrace can show the actual entry point. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/entry.S | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index 689cc58c30be..f7b51a5e204d 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -124,9 +124,9 @@ .ent entInt entInt: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $19 - call $31, do_entInt + call $26, do_entInt + br ret_from_sys_call .end entInt .align 4 @@ -134,9 +134,9 @@ entInt: .ent entArith entArith: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $18 - call $31, do_entArith + call $26, do_entArith + br ret_from_sys_call .end entArith .align 4 @@ -144,9 +144,9 @@ entArith: .ent entMM entMM: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $19 - call $31, do_page_fault + call $26, do_page_fault + br ret_from_sys_call .end entMM .align 4 @@ -154,9 +154,9 @@ entMM: .ent entIF entIF: SAVE_ALL - ldi $26, ret_from_sys_call mov $sp, $18 - call $31, do_entIF + call $26, do_entIF + br ret_from_sys_call .end entIF /* @@ -174,8 +174,8 @@ entUna: ldl $0, PT_REGS_PS($sp) and $0, 8, $0 /* user mode ? */ beq $0, 1f - ldi $26, ret_from_sys_call - call $31, do_entUnaUser /* return to ret_from_syscall */ + call $26, do_entUnaUser /* return to ret_from_syscall */ + br ret_from_sys_call 1: ldl $9, PT_REGS_GP($sp) call $26, do_entUna stl $9, PT_REGS_GP($sp) @@ -439,8 +439,8 @@ __switch_to: .align 4 .ent ret_from_fork ret_from_fork: - ldi $26, ret_from_sys_call - call $31, schedule_tail + call $26, schedule_tail + br ret_from_sys_call .end ret_from_fork /* -- Gitee From d2bdb31cb3a5409b3e83085f45a4059bb0833ea6 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Mon, 25 Sep 2023 15:31:38 +0800 Subject: [PATCH 121/150] sw64: fix default short clock rate Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCMP -------------------------------- This patch fixes error SHTCLK_RATE and SHTCLK_RATE_KHZ. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index 7b58c9450a7b..ecf4aef34028 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -13,8 +13,8 @@ #include #include -#define SHTCLK_RATE 2500000 -#define SHTCLK_RATE_KHZ 2500 +#define SHTCLK_RATE_KHZ 25000 +#define SHTCLK_RATE (SHTCLK_RATE_KHZ * 1000) #if defined(CONFIG_SUBARCH_C4) static u64 read_longtime(struct clocksource *cs) -- Gitee From c54a0a7d6451db1529474102ba213b1eeee2fcd6 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Mon, 25 Sep 2023 15:39:50 +0800 Subject: [PATCH 122/150] sw64: c3b: get MCLK rate dynamically according to specification Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCMP -------------------------------- This patch allows OS to obtain maintence clock frequency from specified memory location for a C3B based board. If it returns 0, fallback to default rate which is 25Mhz. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/clocksource/timer-sw64.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/timer-sw64.c b/drivers/clocksource/timer-sw64.c index ecf4aef34028..10fab75e449a 100644 --- a/drivers/clocksource/timer-sw64.c +++ b/drivers/clocksource/timer-sw64.c @@ -122,13 +122,20 @@ static struct clocksource clocksource_tc = { }; #endif /* SMP */ +#define DEFAULT_MCLK 25 /* Mhz */ + void __init sw64_setup_clocksource(void) { + unsigned int mclk = *((unsigned char *)__va(0x908001)); + + if (!mclk) + mclk = DEFAULT_MCLK; + #ifdef CONFIG_SMP if (is_in_host()) - clocksource_register_khz(&clocksource_longtime, 25000); + clocksource_register_khz(&clocksource_longtime, mclk * 1000); else - clocksource_register_khz(&clocksource_vtime, 25000); + clocksource_register_khz(&clocksource_vtime, DEFAULT_MCLK * 1000); #else clocksource_register_hz(&clocksource_tc, get_cpu_freq()); pr_info("Setup clocksource TC, mult = %d\n", clocksource_tc.mult); -- Gitee From 7f95a300a3d24774aefe886d32749d91ef9a2377 Mon Sep 17 00:00:00 2001 From: Hang Xiaoqian Date: Thu, 28 Sep 2023 12:28:49 +0800 Subject: [PATCH 123/150] sw64: add imemb at the end of fixup_hmcall() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCN4 -------------------------------- An imemb is added to ensure that subsequent hmcall is updated. Signed-off-by: Hang Xiaoqian Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hmcall.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/kernel/hmcall.c b/arch/sw_64/kernel/hmcall.c index e2be9f618e57..86fe4e3f7d51 100644 --- a/arch/sw_64/kernel/hmcall.c +++ b/arch/sw_64/kernel/hmcall.c @@ -101,6 +101,7 @@ void __init fixup_hmcall(void) fixup_wrasid(); fixup_rdktp(); fixup_wrktp(); + imemb(); #endif } -- Gitee From 1ecccb6a0c07066eb1d1bb3bdf74e66c69f2c56d Mon Sep 17 00:00:00 2001 From: Lu Feifei Date: Thu, 28 Sep 2023 11:06:48 +0800 Subject: [PATCH 124/150] sw64: kvm: delete kvm_sw64_ops structure Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Because the current code cannot be binary compatible with CORE3 and CORE4, in order to improve the readability of the code, struct kvm_sw64_ops is deleted, and the same function is implemented as a different code process under different subarch. Signed-off-by: Lu Feifei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/kvm_asm.h | 4 +- arch/sw_64/include/asm/kvm_host.h | 34 ++--- arch/sw_64/kvm/Kconfig | 2 +- arch/sw_64/kvm/kvm_core3.c | 220 ++++++++------------------- arch/sw_64/kvm/kvm_core4.c | 164 ++++---------------- arch/sw_64/kvm/mmu.c | 10 +- arch/sw_64/kvm/sw64.c | 243 ++++++++++++++---------------- arch/sw_64/kvm/vmem.c | 2 - 8 files changed, 222 insertions(+), 457 deletions(-) diff --git a/arch/sw_64/include/asm/kvm_asm.h b/arch/sw_64/include/asm/kvm_asm.h index 2e86c3745c9f..fd1b25018fc8 100644 --- a/arch/sw_64/include/asm/kvm_asm.h +++ b/arch/sw_64/include/asm/kvm_asm.h @@ -14,11 +14,9 @@ #define SW64_KVM_EXIT_RESTART 17 #define SW64_KVM_EXIT_APT_FAULT 18 #define SW64_KVM_EXIT_FATAL_ERROR 22 +#define SW64_KVM_EXIT_MEMHOTPLUG 23 #define SW64_KVM_EXIT_DEBUG 24 -#ifdef CONFIG_KVM_MEMHOTPLUG -#define SW64_KVM_EXIT_MEMHOTPLUG 23 -#endif #define kvm_sw64_exception_type \ {0, "HOST_INTR" }, \ diff --git a/arch/sw_64/include/asm/kvm_host.h b/arch/sw_64/include/asm/kvm_host.h index 9b6614773e54..eb90ca72ec38 100644 --- a/arch/sw_64/include/asm/kvm_host.h +++ b/arch/sw_64/include/asm/kvm_host.h @@ -32,10 +32,12 @@ #ifdef CONFIG_SUBARCH_C3B #define VPN_BITS 8 +#define GUEST_RESET_PC 0xffffffff80011100 #endif #ifdef CONFIG_SUBARCH_C4 #define VPN_BITS 10 +#define GUEST_RESET_PC 0xfff0000000011002 #endif #define VPN_FIRST_VERSION (1UL << VPN_BITS) @@ -79,27 +81,6 @@ struct kvm_arch { pgd_t *pgd; }; -struct kvm_sw64_ops { - unsigned long (*get_new_vpn_context)(struct kvm_vcpu *vcpu, long cpu); - void (*update_vpn)(struct kvm_vcpu *vcpu, unsigned long vpn); - int (*init_vm)(struct kvm *kvm); - void (*destroy_vm)(struct kvm *kvm); - int (*prepare_memory_region)(struct kvm *kvm, struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change); - void (*mmu_enable_log_dirty_pt_masked)(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask); - void (*commit_memory_region)(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, const struct kvm_memory_slot *new, - enum kvm_mr_change change); - void (*flush_shadow_memslot)(struct kvm *kvm, struct kvm_memory_slot *slot); - void (*flush_shadow_all)(struct kvm *kvm); - int (*vcpu_reset)(struct kvm_vcpu *vcpu); - int (*vcpu_run)(struct kvm_vcpu *vcpu, struct kvm_run *run); - void (*vcpu_free)(struct kvm_vcpu *vcpu); - long (*get_vcb)(struct file *filp, unsigned long arg); - long (*set_vcb)(struct file *filp, unsigned long arg); -}; - #define KVM_NR_MEM_OBJS 40 /* @@ -230,5 +211,14 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); int kvm_sw64_perf_init(void); int kvm_sw64_perf_teardown(void); - +void kvm_flush_tlb_all(void); +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn); +int kvm_sw64_init_vm(struct kvm *kvm); +void kvm_sw64_destroy_vm(struct kvm *kvm); +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu); +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg); +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg); + +void update_aptp(unsigned long); +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu); #endif /* _ASM_SW64_KVM_HOST_H */ diff --git a/arch/sw_64/kvm/Kconfig b/arch/sw_64/kvm/Kconfig index 94f1729fad1c..b7e43d0bae51 100644 --- a/arch/sw_64/kvm/Kconfig +++ b/arch/sw_64/kvm/Kconfig @@ -39,7 +39,7 @@ config KVM config KVM_MEMHOTPLUG bool "Memory hotplug support for guest" - depends on KVM && MEMORY_HOTPLUG + depends on KVM && MEMORY_HOTPLUG && SUBARCH_C3B help Provides memory hotplug support for SW64 guest. diff --git a/arch/sw_64/kvm/kvm_core3.c b/arch/sw_64/kvm/kvm_core3.c index f18d1f7a0336..0fab957bfd7b 100644 --- a/arch/sw_64/kvm/kvm_core3.c +++ b/arch/sw_64/kvm/kvm_core3.c @@ -56,43 +56,58 @@ static void bind_vcpu_exit(void) { } #endif -#define GUEST_RESET_PC 0xffffffff80011100 - static unsigned long longtime_offset; #ifdef CONFIG_KVM_MEMHOTPLUG -static u64 get_vpcr_memhp(u64 seg_base, u64 vpn) +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) { - return seg_base | ((vpn & VPN_MASK) << 44); + unsigned long base; + + base = virt_to_phys(vcpu->kvm->arch.seg_pgd); + return base | ((vpn & VPN_MASK) << 44); } #else -static u64 get_vpcr(u64 hpa_base, u64 mem_size, u64 vpn) +static unsigned long get_vpcr(struct kvm_vcpu *vcpu, u64 vpn) { - return (hpa_base >> 23) | ((mem_size >> 23) << 16) - | ((vpn & VPN_MASK) << 44); + unsigned long base, size; + + base = vcpu->kvm->arch.host_phys_addr; + size = vcpu->kvm->arch.size; + return (base >> 23) | ((size >> 23) << 16) | ((vpn & VPN_MASK) << 44); } #endif -static unsigned long core3_get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) { - unsigned long vpn = last_vpn(cpu); - unsigned long next = vpn + 1; + if (vcpu->arch.vcb.vpcr == 0) { + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); +#ifndef CONFIG_KVM_MEMHOTPLUG + if (unlikely(bind_vcpu_enabled)) { + int nid; + unsigned long end; - if ((vpn & VPN_MASK) >= VPN_MASK) { - tbia(); - next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; + nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); + if (pfn_to_nid(PHYS_PFN(end)) == nid) + set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); + } +#endif + vcpu->arch.vcb.upcr = 0x7; } - last_vpn(cpu) = next; - return next; } -static void core3_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +void kvm_flush_tlb_all(void) +{ + tbia(); +} + +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) { vcpu->arch.vcb.vpcr = ((vcpu->arch.vcb.vpcr) & (~(VPN_MASK << 44))) | (vpn << 44); vcpu->arch.vcb.dtb_vpcr = ((vcpu->arch.vcb.dtb_vpcr) & (~(VPN_MASK << VPN_SHIFT))) | (vpn << VPN_SHIFT); } -int kvm_core3_init_vm(struct kvm *kvm) +int kvm_sw64_init_vm(struct kvm *kvm) { #ifdef CONFIG_KVM_MEMHOTPLUG unsigned long *seg_pgd; @@ -111,7 +126,7 @@ int kvm_core3_init_vm(struct kvm *kvm) return 0; } -void kvm_core3_destroy_vm(struct kvm *kvm) +void kvm_sw64_destroy_vm(struct kvm *kvm) { int i; #ifdef CONFIG_KVM_MEMHOTPLUG @@ -150,7 +165,7 @@ static void setup_segment_table(struct kvm *kvm, } #endif -int kvm_core3_prepare_memory_region(struct kvm *kvm, +int kvm_arch_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *memslot, const struct kvm_userspace_memory_region *mem, enum kvm_mr_change change) @@ -259,9 +274,9 @@ void kvm_mark_migration(struct kvm *kvm, int mark) kvm_flush_remote_tlbs(kvm); } -void kvm_core3_commit_memory_region(struct kvm *kvm, +void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -284,7 +299,7 @@ void kvm_core3_commit_memory_region(struct kvm *kvm, kvm_mark_migration(kvm, 1); } -int kvm_core3_vcpu_reset(struct kvm_vcpu *vcpu) +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) { unsigned long addr = vcpu->kvm->arch.host_phys_addr; @@ -301,124 +316,7 @@ int kvm_core3_vcpu_reset(struct kvm_vcpu *vcpu) return 0; } -/* - * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on - * proper exit to userspace. - */ -int kvm_core3_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - int ret; - struct vcpucb *vcb = &(vcpu->arch.vcb); - struct hcall_args hargs; - int irq; - bool more; - sigset_t sigsaved; - - /* Set guest vcb */ - /* vpn will update later when vcpu is running */ - if (vcpu->arch.vcb.vpcr == 0) { -#ifndef CONFIG_KVM_MEMHOTPLUG - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - if (unlikely(bind_vcpu_enabled)) { - int nid; - unsigned long end; - - end = vcpu->kvm->arch.host_phys_addr + vcpu->kvm->arch.size; - nid = pfn_to_nid(PHYS_PFN(vcpu->kvm->arch.host_phys_addr)); - if (pfn_to_nid(PHYS_PFN(end)) == nid) - set_cpus_allowed_ptr(vcpu->arch.tsk, cpumask_of_node(nid)); - } -#else /* !CONFIG_KVM_MEMHOTPLUG */ - unsigned long seg_base = virt_to_phys(vcpu->kvm->arch.seg_pgd); - - vcpu->arch.vcb.vpcr = get_vpcr_memhp(seg_base, 0); -#endif /* CONFIG_KVM_MEMHOTPLUG */ - vcpu->arch.vcb.upcr = 0x7; - } - -#ifdef CONFIG_PERF_EVENTS - vcpu_load(vcpu); -#endif - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - if (run->exit_reason == KVM_EXIT_MMIO) - kvm_handle_mmio_return(vcpu, run); - - run->exit_reason = KVM_EXIT_UNKNOWN; - ret = 1; - while (ret > 0) { - /* Check conditions before entering the guest */ - cond_resched(); - - preempt_disable(); - local_irq_disable(); - - if (signal_pending(current)) { - ret = -EINTR; - run->exit_reason = KVM_EXIT_INTR; - vcpu->stat.signal_exits++; - } - - if (ret <= 0) { - local_irq_enable(); - preempt_enable(); - continue; - } - - memset(&hargs, 0, sizeof(hargs)); - - clear_vcpu_irq(vcpu); - - if (vcpu->arch.restart == 1) { - /* handle reset vCPU */ - vcpu->arch.regs.pc = GUEST_RESET_PC; - vcpu->arch.restart = 0; - } - - irq = interrupt_pending(vcpu, &more); - if (irq < SWVM_IRQS) - try_deliver_interrupt(vcpu, irq, more); - - vcpu->arch.halted = 0; - - sw64_kvm_switch_vpn(vcpu); - check_vcpu_requests(vcpu); - guest_enter_irqoff(); - - /* Enter the guest */ - trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); - vcpu->mode = IN_GUEST_MODE; - - ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); - - /* Back from guest */ - vcpu->mode = OUTSIDE_GUEST_MODE; - - vcpu->stat.exits++; - local_irq_enable(); - guest_exit_irqoff(); - - trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); - - preempt_enable(); - - /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ - ret = handle_exit(vcpu, run, ret, &hargs); - update_vcpu_stat_time(&vcpu->stat); - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - -#ifdef CONFIG_PERF_EVENTS - vcpu_put(vcpu); -#endif - return ret; -} - -static long kvm_core3_get_vcb(struct file *filp, unsigned long arg) +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; @@ -435,7 +333,7 @@ static long kvm_core3_get_vcb(struct file *filp, unsigned long arg) return 0; } -static long kvm_core3_set_vcb(struct file *filp, unsigned long arg) +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) { unsigned long result; struct kvm_vcpu *vcpu = filp->private_data; @@ -446,9 +344,7 @@ static long kvm_core3_set_vcb(struct file *filp, unsigned long arg) if (vcpu->arch.vcb.migration_mark) { /* updated vpcr needed by destination vm */ - vcpu->arch.vcb.vpcr - = get_vpcr(vcpu->kvm->arch.host_phys_addr, vcpu->kvm->arch.size, 0); - + vcpu->arch.vcb.vpcr = get_vpcr(vcpu, 0); /* synchronize the longtime of source and destination */ if (vcpu->arch.vcb.soft_cid == 0) { result = sw64_io_read(0, LONG_TIME); @@ -488,18 +384,28 @@ void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) } #endif -static struct kvm_sw64_ops core3_sw64_ops __ro_after_init = { - .get_new_vpn_context = core3_get_new_vpn_context, - .update_vpn = core3_update_vpn, - .init_vm = kvm_core3_init_vm, - .destroy_vm = kvm_core3_destroy_vm, - .prepare_memory_region = kvm_core3_prepare_memory_region, - .commit_memory_region = kvm_core3_commit_memory_region, - .vcpu_reset = kvm_core3_vcpu_reset, - .vcpu_run = kvm_core3_vcpu_ioctl_run, - .get_vcb = kvm_core3_get_vcb, - .set_vcb = kvm_core3_set_vcb, -}; +void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ +} + +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, gfn_t gfn_offset, + unsigned long mask) +{ +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) +{ +} + +void kvm_arch_flush_shadow_all(struct kvm *kvm) +{ +} + +void update_aptp(unsigned long pgd) +{ +} static int __init kvm_core3_init(void) { @@ -514,7 +420,7 @@ static int __init kvm_core3_init(void) for (i = 0; i < NR_CPUS; i++) last_vpn(i) = VPN_FIRST_VERSION; - ret = kvm_init(&core3_sw64_ops, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (likely(!ret)) return 0; diff --git a/arch/sw_64/kvm/kvm_core4.c b/arch/sw_64/kvm/kvm_core4.c index 5301abadcdc0..c1bc9c8f8595 100644 --- a/arch/sw_64/kvm/kvm_core4.c +++ b/arch/sw_64/kvm/kvm_core4.c @@ -21,34 +21,31 @@ #include "../kernel/pci_impl.h" -#define GUEST_RESET_PC 0xfff0000000011002 static unsigned long shtclock_offset; -static unsigned long core4_get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +void update_aptp(unsigned long pgd) { - unsigned long vpn = last_vpn(cpu); - unsigned long next = vpn + 1; - - if ((vpn & VPN_MASK) >= VPN_MASK) { - tbivpn(-1, 0, 0); - next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ - } - last_vpn(cpu) = next; - return next; + imemb(); + write_csr_imb(pgd, CSR_APTP); } -static void core4_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) +void kvm_sw64_update_vpn(struct kvm_vcpu *vcpu, unsigned long vpn) { vcpu->arch.vcb.vpcr = vpn << 44; vcpu->arch.vcb.dtb_vpcr = vpn; } -int kvm_core4_init_vm(struct kvm *kvm) +void kvm_flush_tlb_all(void) +{ + tbivpn(-1, 0, 0); +} + +int kvm_sw64_init_vm(struct kvm *kvm) { return kvm_alloc_addtional_stage_pgd(kvm); } -void kvm_core4_destroy_vm(struct kvm *kvm) +void kvm_sw64_destroy_vm(struct kvm *kvm) { int i; @@ -61,7 +58,7 @@ void kvm_core4_destroy_vm(struct kvm *kvm) atomic_set(&kvm->online_vcpus, 0); } -int kvm_core4_vcpu_reset(struct kvm_vcpu *vcpu) +int kvm_sw64_vcpu_reset(struct kvm_vcpu *vcpu) { if (vcpu->arch.has_run_once) apt_unmap_vm(vcpu->kvm); @@ -76,112 +73,7 @@ int kvm_core4_vcpu_reset(struct kvm_vcpu *vcpu) return 0; } -/* - * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on - * proper exit to userspace. - */ -int kvm_core4_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) -{ - int ret; - struct vcpucb *vcb = &(vcpu->arch.vcb); - struct hcall_args hargs; - int irq; - bool more; - sigset_t sigsaved; - -#ifdef CONFIG_PERF_EVENTS - vcpu_load(vcpu); -#endif - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - if (run->exit_reason == KVM_EXIT_MMIO) - kvm_handle_mmio_return(vcpu, run); - - run->exit_reason = KVM_EXIT_UNKNOWN; - ret = 1; - while (ret > 0) { - /* - * Check conditions before entering the guest - */ - cond_resched(); - - preempt_disable(); - local_irq_disable(); - - if (signal_pending(current)) { - ret = -EINTR; - run->exit_reason = KVM_EXIT_INTR; - vcpu->stat.signal_exits++; - } - - if (ret <= 0) { - local_irq_enable(); - preempt_enable(); - continue; - } - - memset(&hargs, 0, sizeof(hargs)); - - clear_vcpu_irq(vcpu); - - if (vcpu->arch.restart == 1) { - /* handle reset vCPU */ - vcpu->arch.regs.pc = GUEST_RESET_PC; - vcpu->arch.restart = 0; - } - - irq = interrupt_pending(vcpu, &more); - if (irq < SWVM_IRQS) - try_deliver_interrupt(vcpu, irq, more); - - vcpu->arch.halted = 0; - - sw64_kvm_switch_vpn(vcpu); - check_vcpu_requests(vcpu); - guest_enter_irqoff(); - - /* update aptp before the guest runs */ - imemb(); - write_csr_imb((unsigned long)vcpu->kvm->arch.pgd, CSR_APTP); - - /* Enter the guest */ - trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); - vcpu->mode = IN_GUEST_MODE; - - ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); - - /* Back from guest */ - vcpu->mode = OUTSIDE_GUEST_MODE; - - vcpu->stat.exits++; - local_irq_enable(); - guest_exit_irqoff(); - - trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); - - preempt_enable(); - - /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ - ret = handle_exit(vcpu, run, ret, &hargs); - update_vcpu_stat_time(&vcpu->stat); - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - -#ifdef CONFIG_PERF_EVENTS - vcpu_put(vcpu); -#endif - return ret; -} - -static void kvm_core4_vcpu_free(struct kvm_vcpu *vcpu) -{ - kvm_mmu_free_memory_caches(vcpu); -} - -static long kvm_core4_get_vcb(struct file *filp, unsigned long arg) +long kvm_sw64_get_vcb(struct file *filp, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; @@ -194,7 +86,7 @@ static long kvm_core4_get_vcb(struct file *filp, unsigned long arg) return 0; } -static long kvm_core4_set_vcb(struct file *filp, unsigned long arg) +long kvm_sw64_set_vcb(struct file *filp, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; struct vcpucb *kvm_vcb; @@ -213,20 +105,18 @@ static long kvm_core4_set_vcb(struct file *filp, unsigned long arg) return 0; } -static struct kvm_sw64_ops core4_sw64_ops __ro_after_init = { - .get_new_vpn_context = core4_get_new_vpn_context, - .update_vpn = core4_update_vpn, - .init_vm = kvm_core4_init_vm, - .destroy_vm = kvm_core4_destroy_vm, - .commit_memory_region = kvm_core4_commit_memory_region, - .flush_shadow_memslot = kvm_core4_flush_shadow_memslot, - .flush_shadow_all = kvm_core4_flush_shadow_all, - .vcpu_reset = kvm_core4_vcpu_reset, - .vcpu_run = kvm_core4_vcpu_ioctl_run, - .vcpu_free = kvm_core4_vcpu_free, - .get_vcb = kvm_core4_get_vcb, - .set_vcb = kvm_core4_set_vcb, -}; +int kvm_arch_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *memslot, + const struct kvm_userspace_memory_region *mem, + enum kvm_mr_change change) +{ + return 0; +} + +void vcpu_set_numa_affinity(struct kvm_vcpu *vcpu) +{ + return; +} static int __init kvm_core4_init(void) { @@ -235,7 +125,7 @@ static int __init kvm_core4_init(void) for (i = 0; i < NR_CPUS; i++) last_vpn(i) = VPN_FIRST_VERSION; - ret = kvm_init(&core4_sw64_ops, sizeof(struct kvm_vcpu), 0, THIS_MODULE); + ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); if (ret) return ret; diff --git a/arch/sw_64/kvm/mmu.c b/arch/sw_64/kvm/mmu.c index b0d15a1e549f..a0b1db924440 100644 --- a/arch/sw_64/kvm/mmu.c +++ b/arch/sw_64/kvm/mmu.c @@ -544,9 +544,9 @@ void kvm_mark_migration(struct kvm *kvm, int mark) vcpu->arch.migration_mark = mark; } -void kvm_core4_commit_memory_region(struct kvm *kvm, +void kvm_arch_commit_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region *mem, - const struct kvm_memory_slot *old, + struct kvm_memory_slot *old, const struct kvm_memory_slot *new, enum kvm_mr_change change) { @@ -570,7 +570,7 @@ void kvm_core4_commit_memory_region(struct kvm *kvm, } } -void kvm_core4_flush_shadow_memslot(struct kvm *kvm, +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { gpa_t gpa = slot->base_gfn << PAGE_SHIFT; @@ -635,7 +635,7 @@ void kvm_free_apt_pgd(struct kvm *kvm) free_pages_exact(pgd, PAGE_SIZE); } -void kvm_core4_flush_shadow_all(struct kvm *kvm) +void kvm_arch_flush_shadow_all(struct kvm *kvm) { kvm_free_apt_pgd(kvm); } @@ -1553,7 +1553,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to * enable dirty logging for them. */ -void kvm_core4_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, +void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { diff --git a/arch/sw_64/kvm/sw64.c b/arch/sw_64/kvm/sw64.c index 86788d548b19..9bb880f0d5fc 100644 --- a/arch/sw_64/kvm/sw64.c +++ b/arch/sw_64/kvm/sw64.c @@ -25,8 +25,18 @@ bool set_msi_flag; #define DFX_STAT(n, x, ...) \ { n, offsetof(struct kvm_vcpu_stat, x), DFX_STAT_U64, ## __VA_ARGS__ } -struct kvm_sw64_ops *kvm_sw64_ops __read_mostly; -EXPORT_SYMBOL_GPL(kvm_sw64_ops); +static unsigned long get_new_vpn_context(struct kvm_vcpu *vcpu, long cpu) +{ + unsigned long vpn = last_vpn(cpu); + unsigned long next = vpn + 1; + + if ((vpn & VPN_MASK) >= VPN_MASK) { + kvm_flush_tlb_all(); + next = (vpn & ~VPN_MASK) + VPN_FIRST_VERSION + 1; /* bypass 0 */ + } + last_vpn(cpu) = next; + return next; +} int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level) { @@ -69,7 +79,7 @@ void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) if ((vpnc ^ vpn) & ~VPN_MASK) { /* vpnc and cpu vpn not in the same version, get new vpnc and vpn */ - vpnc = kvm_sw64_ops->get_new_vpn_context(vcpu, cpu); + vpnc = get_new_vpn_context(vcpu, cpu); vcpu->arch.vpnc[cpu] = vpnc; } @@ -77,7 +87,7 @@ void sw64_kvm_switch_vpn(struct kvm_vcpu *vcpu) /* Always update vpn */ /* Just setup vcb, hardware CSR will be changed later in HMcode */ - kvm_sw64_ops->update_vpn(vcpu, vpn); + kvm_sw64_update_vpn(vcpu, vpn); /* * If vcpu migrate to a new physical cpu, the new physical cpu may keep @@ -218,22 +228,9 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } -void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, gfn_t gfn_offset, - unsigned long mask) -{ - if (kvm_sw64_ops->mmu_enable_log_dirty_pt_masked) - kvm_sw64_ops->mmu_enable_log_dirty_pt_masked(kvm, slot, gfn_offset, mask); -} - -int kvm_sw64_pending_timer(struct kvm_vcpu *vcpu) -{ - return test_bit(SW64_KVM_IRQ_TIMER, &vcpu->arch.irqs_pending); -} - int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - return kvm_sw64_pending_timer(vcpu); + return test_bit(SW64_KVM_IRQ_TIMER, &vcpu->arch.irqs_pending); } int kvm_arch_hardware_setup(void *opaque) @@ -246,16 +243,12 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (type) return -EINVAL; - if (kvm_sw64_ops->init_vm) - return kvm_sw64_ops->init_vm(kvm); - - return 0; + return kvm_sw64_init_vm(kvm); } void kvm_arch_destroy_vm(struct kvm *kvm) { - if (kvm_sw64_ops->destroy_vm) - return kvm_sw64_ops->destroy_vm(kvm); + return kvm_sw64_destroy_vm(kvm); } long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) @@ -269,44 +262,10 @@ int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, return 0; } -#ifdef CONFIG_KVM_MEMHOTPLUG -static void setup_segment_table(struct kvm *kvm, - struct kvm_memory_slot *memslot, unsigned long addr, size_t size) -{ - unsigned long *seg_pgd = kvm->arch.seg_pgd; - unsigned long num_of_entry; - unsigned long base_hpa = addr; - unsigned long i; - - num_of_entry = round_up(size, 1 << 30) >> 30; - - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + (i << 30); - seg_pgd++; - } - -} -#endif - -int kvm_arch_prepare_memory_region(struct kvm *kvm, - struct kvm_memory_slot *memslot, - const struct kvm_userspace_memory_region *mem, - enum kvm_mr_change change) -{ - if (kvm_sw64_ops->prepare_memory_region) - return kvm_sw64_ops->prepare_memory_region(kvm, memslot, - mem, change); - - return 0; -} - void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) { + kvm_mmu_free_memory_caches(vcpu); hrtimer_cancel(&vcpu->arch.hrt); - - if (kvm_sw64_ops->vcpu_free) - kvm_sw64_ops->vcpu_free(vcpu); - kfree(vcpu); } @@ -337,14 +296,6 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) return 0; } -int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) -{ - if (kvm_sw64_ops->vcpu_reset) - return kvm_sw64_ops->vcpu_reset(vcpu); - - return 0; -} - int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) @@ -456,7 +407,98 @@ void update_vcpu_stat_time(struct kvm_vcpu_stat *vcpu_stat) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { struct kvm_run *run = vcpu->run; - return kvm_sw64_ops->vcpu_run(vcpu, run); + struct vcpucb *vcb = &(vcpu->arch.vcb); + struct hcall_args hargs; + int irq, ret; + bool more; + sigset_t sigsaved; + + /* Set guest vcb */ + /* vpn will update later when vcpu is running */ + vcpu_set_numa_affinity(vcpu); +#ifdef CONFIG_PERF_EVENTS + vcpu_load(vcpu); +#endif + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + if (run->exit_reason == KVM_EXIT_MMIO) + kvm_handle_mmio_return(vcpu, run); + + run->exit_reason = KVM_EXIT_UNKNOWN; + ret = 1; + while (ret > 0) { + /* Check conditions before entering the guest */ + cond_resched(); + + preempt_disable(); + local_irq_disable(); + + if (signal_pending(current)) { + ret = -EINTR; + run->exit_reason = KVM_EXIT_INTR; + vcpu->stat.signal_exits++; + } + + if (ret <= 0) { + local_irq_enable(); + preempt_enable(); + continue; + } + + memset(&hargs, 0, sizeof(hargs)); + + clear_vcpu_irq(vcpu); + + if (vcpu->arch.restart == 1) { + /* handle reset vCPU */ + vcpu->arch.regs.pc = GUEST_RESET_PC; + vcpu->arch.restart = 0; + } + + irq = interrupt_pending(vcpu, &more); + if (irq < SWVM_IRQS) + try_deliver_interrupt(vcpu, irq, more); + + vcpu->arch.halted = 0; + + sw64_kvm_switch_vpn(vcpu); + check_vcpu_requests(vcpu); + guest_enter_irqoff(); + + /* update aptp before the guest runs */ + update_aptp((unsigned long)vcpu->kvm->arch.pgd); + + /* Enter the guest */ + trace_kvm_sw64_entry(vcpu->vcpu_id, vcpu->arch.regs.pc); + vcpu->mode = IN_GUEST_MODE; + + ret = __sw64_vcpu_run(__pa(vcb), &(vcpu->arch.regs), &hargs); + + /* Back from guest */ + vcpu->mode = OUTSIDE_GUEST_MODE; + + vcpu->stat.exits++; + local_irq_enable(); + guest_exit_irqoff(); + + trace_kvm_sw64_exit(ret, vcpu->arch.regs.pc); + + preempt_enable(); + + /* ret = 0 indicate interrupt in guest mode, ret > 0 indicate hcall */ + ret = handle_exit(vcpu, run, ret, &hargs); + update_vcpu_stat_time(&vcpu->stat); + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + +#ifdef CONFIG_PERF_EVENTS + vcpu_put(vcpu); +#endif + + return ret; } long kvm_arch_vcpu_ioctl(struct file *filp, @@ -467,13 +509,13 @@ long kvm_arch_vcpu_ioctl(struct file *filp, switch (ioctl) { case KVM_SW64_VCPU_INIT: - r = kvm_arch_vcpu_reset(vcpu); + r = kvm_sw64_vcpu_reset(vcpu); break; case KVM_SW64_GET_VCB: - r = kvm_sw64_ops->get_vcb(filp, arg); + r = kvm_sw64_get_vcb(filp, arg); break; case KVM_SW64_SET_VCB: - r = kvm_sw64_ops->set_vcb(filp, arg); + r = kvm_sw64_set_vcb(filp, arg); break; default: r = -EINVAL; @@ -504,26 +546,12 @@ long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) int kvm_arch_init(void *opaque) { - int r; - struct kvm_sw64_ops *ops = opaque; - - if (kvm_sw64_ops) { - printk(KERN_ERR "kvm: already loaded the other module\n"); - r = -EEXIST; - goto out; - } - - kvm_sw64_ops = ops; kvm_sw64_perf_init(); - return 0; -out: - return r; } void kvm_arch_exit(void) { - kvm_sw64_ops = NULL; kvm_sw64_perf_teardown(); } @@ -558,18 +586,6 @@ vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) -{ - if (kvm_sw64_ops->flush_shadow_memslot) - kvm_sw64_ops->flush_shadow_memslot(kvm, slot); -} - -void kvm_arch_flush_shadow_all(struct kvm *kvm) -{ - if (kvm_sw64_ops->flush_shadow_all) - kvm_sw64_ops->flush_shadow_all(kvm); -} - void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot) { @@ -577,15 +593,6 @@ void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, kvm_arch_flush_shadow_memslot(kvm, memslot); } -void kvm_arch_commit_memory_region(struct kvm *kvm, - const struct kvm_userspace_memory_region *mem, - struct kvm_memory_slot *old, - const struct kvm_memory_slot *new, - enum kvm_mr_change change) -{ - kvm_sw64_ops->commit_memory_region(kvm, mem, old, new, change); -} - int kvm_dev_ioctl_check_extension(long ext) { int r; @@ -605,30 +612,6 @@ int kvm_dev_ioctl_check_extension(long ext) return r; } -#ifdef CONFIG_KVM_MEMHOTPLUG -void vcpu_mem_hotplug(struct kvm_vcpu *vcpu, unsigned long start_addr) -{ - struct kvm *kvm = vcpu->kvm; - struct kvm_memory_slot *slot; - unsigned long start_pfn = start_addr >> PAGE_SHIFT; - - kvm_for_each_memslot(slot, kvm_memslots(kvm)) { - if (start_pfn == slot->base_gfn) { - unsigned long *seg_pgd; - unsigned long num_of_entry = slot->npages >> 17; - unsigned long base_hpa = slot->arch.host_phys_addr; - unsigned long i; - - seg_pgd = kvm->arch.seg_pgd + (start_pfn >> 17); - for (i = 0; i < num_of_entry; i++) { - *seg_pgd = base_hpa + (i << 30); - seg_pgd++; - } - } - } -} -#endif - void vcpu_send_ipi(struct kvm_vcpu *vcpu, int target_vcpuid, int type) { struct kvm_vcpu *target_vcpu = kvm_get_vcpu(vcpu->kvm, target_vcpuid); diff --git a/arch/sw_64/kvm/vmem.c b/arch/sw_64/kvm/vmem.c index f1afc3a0d602..fa893f5e26a7 100644 --- a/arch/sw_64/kvm/vmem.c +++ b/arch/sw_64/kvm/vmem.c @@ -70,7 +70,6 @@ static void vmem_vm_close(struct vm_area_struct *vma) size_t size; struct vmem_info *info; -#ifdef CONFIG_SUBARCH_C3B info = vma->vm_private_data; addr = info->start; size = round_up(info->size, 8 << 20); @@ -83,7 +82,6 @@ static void vmem_vm_close(struct vm_area_struct *vma) } kfree(info); } -#endif } const struct vm_operations_struct vmem_vm_ops = { -- Gitee From 985e00fe346ab429311ae54a6b4d12005b964aea Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 9 Oct 2023 13:47:07 +0800 Subject: [PATCH 125/150] sw64: expand struct pt_regs Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCP8 -------------------------------- Add three new members to pt_regs - orig_r0: stores the r0 before syscall - orig_r19: stores the r19 before syscall Since the syscall_nr is stored in r0, orig_r0 will be set to -1 in non-syscall context or when syscall is complete. Modify syscall process accordingly. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ptrace.h | 17 ++++- arch/sw_64/include/asm/syscall.h | 6 +- arch/sw_64/kernel/asm-offsets.c | 2 + arch/sw_64/kernel/entry.S | 95 +++++++-------------------- arch/sw_64/kernel/ptrace.c | 12 ++-- arch/sw_64/kernel/signal.c | 107 ++++++++++++++----------------- 6 files changed, 97 insertions(+), 142 deletions(-) diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index 40f8096cfbc3..c258ba17cda6 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -6,6 +6,11 @@ #include #include +#define NO_SYSCALL (-1) + +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ + /* * This struct defines the way the registers are stored on the * kernel stack during a system call or other kernel entry @@ -20,6 +25,8 @@ struct pt_regs { unsigned long ps; }; }; + unsigned long orig_r0; + unsigned long orig_r19; /* These are saved by HMcode: */ unsigned long hm_ps; unsigned long hm_pc; @@ -37,9 +44,9 @@ struct pt_regs { #define kernel_stack_pointer(regs) ((unsigned long)((regs) + 1)) #define instruction_pointer_set(regs, val) ((regs)->pc = val) -#define force_successful_syscall_return() (current_pt_regs()->regs[0] = 0) +#define force_successful_syscall_return() (current_pt_regs()->orig_r0 = NO_SYSCALL) -#define MAX_REG_OFFSET (offsetof(struct pt_regs, ps)) +#define MAX_REG_OFFSET (offsetof(struct pt_regs, orig_r0)) extern short regoffsets[]; @@ -70,9 +77,13 @@ static inline int is_syscall_success(struct pt_regs *regs) static inline long regs_return_value(struct pt_regs *regs) { - if (is_syscall_success(regs) || !user_mode(regs)) + if ((regs->orig_r0 == NO_SYSCALL) || is_syscall_success(regs)) return regs->regs[0]; else return -regs->regs[0]; } + +#endif /* !__ASSEMBLY__ */ +#endif /* __KERNEL__ */ + #endif /* _ASM_SW64_PTRACE_H */ diff --git a/arch/sw_64/include/asm/syscall.h b/arch/sw_64/include/asm/syscall.h index da662d14356d..403f607592c9 100644 --- a/arch/sw_64/include/asm/syscall.h +++ b/arch/sw_64/include/asm/syscall.h @@ -29,18 +29,18 @@ static inline void syscall_set_return_value(struct task_struct *task, { if (error) { regs->regs[0] = -error; - regs->regs[19] = -1; + regs->regs[19] = 1; } else { regs->regs[0] = val; regs->regs[19] = 0; } } - static inline void syscall_rollback(struct task_struct *task, struct pt_regs *regs) { - /* Do nothing */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; } static inline void syscall_get_arguments(struct task_struct *task, diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index c7f9a922436b..206152281a4a 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -101,6 +101,8 @@ void foo(void) DEFINE(PT_REGS_SP, offsetof(struct pt_regs, regs[30])); DEFINE(PT_REGS_PC, offsetof(struct pt_regs, pc)); DEFINE(PT_REGS_PS, offsetof(struct pt_regs, ps)); + DEFINE(PT_REGS_ORIG_R0, offsetof(struct pt_regs, orig_r0)); + DEFINE(PT_REGS_ORIG_R19, offsetof(struct pt_regs, orig_r19)); DEFINE(PT_REGS_HM_PS, offsetof(struct pt_regs, hm_ps)); DEFINE(PT_REGS_HM_PC, offsetof(struct pt_regs, hm_pc)); DEFINE(PT_REGS_HM_GP, offsetof(struct pt_regs, hm_gp)); diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index f7b51a5e204d..669a55d87cd0 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -8,6 +8,7 @@ #include #include #include +#include .text .set noat @@ -63,9 +64,11 @@ and $6, 0x8, $7 beq $7, 1f sys_call HMC_rdusp - br $31, 2f + br 2f 1: ldi $0, PT_REGS_SIZE($sp) 2: stl $0, PT_REGS_SP($sp) + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) sys_call HMC_rdktp .endm @@ -202,6 +205,8 @@ entSys: SAVE_ALL ldl $0, PT_REGS_R0($sp) + stl $0, PT_REGS_ORIG_R0($sp) + stl $19, PT_REGS_ORIG_R19($sp) ldi $4, NR_SYSCALLS($31) stl $16, PT_REGS_R16($sp) ldi $5, sys_call_table @@ -220,6 +225,10 @@ entSys: 1: call $26, ($27), ni_syscall ldgp $gp, 0($26) blt $0, $syscall_error /* the call failed */ + + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) +$syscall_success: stl $0, PT_REGS_R0($sp) stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ @@ -231,7 +240,6 @@ ret_from_sys_call: #endif br $27, 1f 1: ldgp $29, 0($27) - selne $26, 0, $18, $18 /* $18 = 0 => non-restartable */ ldl $0, PT_REGS_PS($sp) and $0, 8, $0 beq $0, ret_to_kernel @@ -246,7 +254,9 @@ ret_to_user: sys_call HMC_swpipl ldw $17, TI_FLAGS($8) and $17, _TIF_WORK_MASK, $2 - bne $2, work_pending + beq $2, restore_all + mov $sp, $16 + call $26, do_notify_resume restore_all: RESTORE_ALL sys_call HMC_rti @@ -256,7 +266,6 @@ ret_to_kernel: sys_call HMC_swpipl br restore_all - .align 3 $syscall_error: /* @@ -266,59 +275,16 @@ $syscall_error: * frame to indicate that a negative return value wasn't an * error number.. */ - ldl $18, PT_REGS_R0($sp) /* old syscall nr (zero if success) */ - beq $18, $ret_success + ldl $1, PT_REGS_ORIG_R0($sp) /* old syscall nr (-1 if success) */ + blt $1, $syscall_success - ldl $19, PT_REGS_R19($sp) /* .. and this a3 */ subl $31, $0, $0 /* with error in v0 */ addl $31, 1, $1 /* set a3 for errno return */ stl $0, PT_REGS_R0($sp) - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ stl $1, PT_REGS_R19($sp) /* a3 for return */ br ret_from_sys_call - - -$ret_success: - stl $0, PT_REGS_R0($sp) - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ - br ret_from_sys_call .end entSys -/* - * Do all cleanup when returning from all interrupts and system calls. - * - * Arguments: - * $8: current. - * $17: TI_FLAGS. - * $18: The old syscall number, or zero if this is not a return - * from a syscall that errored and is possibly restartable. - * $19: The old a3 value - */ - - .align 4 - .ent work_pending -work_pending: - and $17, _TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_UPROBE | _TIF_PATCH_PENDING, $2 - bne $2, $work_notifysig - -$work_resched: - /* - * We can get here only if we returned from syscall without SIGPENDING - * or got through work_notifysig already. Either case means no syscall - * restarts for us, so let $18 and $19 burn. - */ - call $26, schedule - mov 0, $18 - br ret_to_user - -$work_notifysig: - mov $sp, $16 - call $26, do_work_pending - br restore_all - .end work_pending - - - /* * PTRACE syscall handler */ @@ -327,12 +293,11 @@ $work_notifysig: .ent strace strace: /* set up signal stack, call syscall_trace */ - mov $0, $9 - mov $19, $10 call $26, syscall_trace_enter blt $0, $syscall_trace_failed /* get the system call number and the arguments back.. */ + stl $0, PT_REGS_ORIG_R0($sp) ldl $16, PT_REGS_R16($sp) ldl $17, PT_REGS_R17($sp) ldl $18, PT_REGS_R18($sp) @@ -355,38 +320,27 @@ ret_from_straced: /* check return.. */ blt $0, $strace_error /* the call failed */ - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ + + ldi $1, NO_SYSCALL + stl $1, PT_REGS_ORIG_R0($sp) $strace_success: stl $0, PT_REGS_R0($sp) /* save return value */ + stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ call $26, syscall_trace_leave - br $31, ret_from_sys_call + br ret_from_sys_call .align 3 $strace_error: - ldl $18, PT_REGS_R0($sp) /* old syscall nr (zero if success) */ - - beq $18, $strace_success - ldl $19, PT_REGS_R19($sp) /* .. and this a3 */ + ldl $1, PT_REGS_ORIG_R0($sp) /* old syscall nr (-1 if success) */ + blt $1, $strace_success subl $31, $0, $0 /* with error in v0 */ addl $31, 1, $1 /* set a3 for errno return */ stl $0, PT_REGS_R0($sp) stl $1, PT_REGS_R19($sp) /* a3 for return */ - mov $18, $9 /* save old syscall number */ - mov $19, $10 /* save old a3 */ - call $26, syscall_trace_leave - mov $9, $18 - mov $10, $19 - - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ - br ret_from_sys_call - $syscall_trace_failed: call $26, syscall_trace_leave - mov $9, $18 - mov $10, $19 - mov $31, $26 /* tell "ret_from_sys_call" we can restart */ br ret_from_sys_call .end strace @@ -454,8 +408,7 @@ ret_from_kernel_thread: mov $9, $27 mov $10, $16 call $26, ($9) - mov $31, $19 /* to disable syscall restarts */ - br $31, ret_to_user + br ret_to_user .end ret_from_kernel_thread .align 4 diff --git a/arch/sw_64/kernel/ptrace.c b/arch/sw_64/kernel/ptrace.c index cb9ffde2ce39..1911d5bd058b 100644 --- a/arch/sw_64/kernel/ptrace.c +++ b/arch/sw_64/kernel/ptrace.c @@ -374,19 +374,19 @@ asmlinkage unsigned long syscall_trace_enter(void) struct pt_regs *regs = current_pt_regs(); if (test_thread_flag(TIF_SYSCALL_TRACE) && - tracehook_report_syscall_entry(current_pt_regs())) - ret = -1UL; + tracehook_report_syscall_entry(regs)) + return NO_SYSCALL; #ifdef CONFIG_SECCOMP /* Do seccomp after ptrace, to catch any tracer changes. */ if (secure_computing() == -1) - return -1; + return NO_SYSCALL; #endif if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_enter(regs, regs->regs[0]); audit_syscall_entry(regs->regs[0], regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]); - return ret ?: current_pt_regs()->regs[0]; + return ret ?: regs->regs[0]; } asmlinkage void @@ -394,9 +394,9 @@ syscall_trace_leave(void) { struct pt_regs *regs = current_pt_regs(); - audit_syscall_exit(current_pt_regs()); + audit_syscall_exit(regs); if (test_thread_flag(TIF_SYSCALL_TRACE)) - tracehook_report_syscall_exit(current_pt_regs(), 0); + tracehook_report_syscall_exit(regs, 0); if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_exit(regs, regs_return_value(regs)); } diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index e16acad8a52c..9d3ad6a17b37 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -276,32 +276,6 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs) signal_setup_done(ret, ksig, 0); } -static inline void -syscall_restart(unsigned long r0, unsigned long r19, - struct pt_regs *regs, struct k_sigaction *ka) -{ - switch (regs->regs[0]) { - case ERESTARTSYS: - if (!(ka->sa.sa_flags & SA_RESTART)) { - regs->regs[0] = EINTR; - break; - } - fallthrough; - case ERESTARTNOINTR: - regs->regs[0] = r0; /* reset v0 and a3 and replay syscall */ - regs->regs[19] = r19; - regs->pc -= 4; - break; - case ERESTART_RESTARTBLOCK: - regs->regs[0] = EINTR; - break; - case ERESTARTNOHAND: - regs->regs[0] = EINTR; - break; - } -} - - /* * Note that 'init' is a special process: it doesn't get signals it doesn't * want to handle. Thus you cannot kill init even with a SIGKILL even by @@ -310,13 +284,9 @@ syscall_restart(unsigned long r0, unsigned long r19, * Note that we go through the signals twice: once to check the signals that * the kernel can handle, and then we build all the user-level signal handling * stack-frames in one go after that. - * - * "r0" and "r19" are the registers we need to restore for system call - * restart. "r0" is also used as an indicator whether we can restart at - * all (if we get here from anything but a syscall return, it will be 0) */ static void -do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) +do_signal(struct pt_regs *regs) { unsigned long single_stepping = ptrace_cancel_bpt(current); struct ksignal ksig; @@ -326,19 +296,38 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) /* ... so re-check the single stepping. */ single_stepping |= ptrace_cancel_bpt(current); /* Whee! Actually deliver the signal. */ - if (r0) - syscall_restart(r0, r19, regs, &ksig.ka); + if (regs->orig_r0 != NO_SYSCALL) { + switch (regs->regs[0]) { + case ERESTARTSYS: + if (!(ksig.ka.sa.sa_flags & SA_RESTART)) { + regs->regs[0] = EINTR; + break; + } + fallthrough; + case ERESTARTNOINTR: + /* reset v0 and a3 and replay syscall */ + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; + regs->pc -= 4; + break; + case ERESTARTNOHAND: + case ERESTART_RESTARTBLOCK: + regs->regs[0] = EINTR; + break; + } + regs->orig_r0 = NO_SYSCALL; + } handle_signal(&ksig, regs); } else { single_stepping |= ptrace_cancel_bpt(current); - if (r0) { + if (regs->orig_r0 != NO_SYSCALL) { switch (regs->regs[0]) { - case ERESTARTNOHAND: case ERESTARTSYS: case ERESTARTNOINTR: + case ERESTARTNOHAND: /* Reset v0 and a3 and replay syscall. */ - regs->regs[0] = r0; - regs->regs[19] = r19; + regs->regs[0] = regs->orig_r0; + regs->regs[19] = regs->orig_r19; regs->pc -= 4; break; case ERESTART_RESTARTBLOCK: @@ -347,6 +336,7 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) regs->pc -= 4; break; } + regs->orig_r0 = NO_SYSCALL; } restore_saved_sigmask(); } @@ -354,36 +344,35 @@ do_signal(struct pt_regs *regs, unsigned long r0, unsigned long r19) ptrace_set_bpt(current); /* re-set breakpoint */ } -void -do_work_pending(struct pt_regs *regs, unsigned long thread_flags, - unsigned long r0, unsigned long r19) +asmlinkage void +do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) { do { - if (thread_flags & _TIF_NEED_RESCHED) { + local_irq_enable(); + + if (thread_flags & _TIF_NEED_RESCHED) schedule(); - } else { - local_irq_enable(); - if (thread_flags & _TIF_UPROBE) { - unsigned long pc = regs->pc; + if (thread_flags & _TIF_UPROBE) { + unsigned long pc = regs->pc; - uprobe_notify_resume(regs); - sw64_fix_uretprobe(regs, pc - 4); - } + uprobe_notify_resume(regs); + sw64_fix_uretprobe(regs, pc - 4); + } - if (thread_flags & _TIF_PATCH_PENDING) - klp_update_patch_state(current); + if (thread_flags & _TIF_PATCH_PENDING) + klp_update_patch_state(current); - if (thread_flags & _TIF_SIGPENDING) { - do_signal(regs, r0, r19); - r0 = 0; - } else { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - rseq_handle_notify_resume(NULL, regs); - } + if (thread_flags & _TIF_SIGPENDING) + do_signal(regs); + + if (thread_flags & _TIF_NOTIFY_RESUME) { + clear_thread_flag(TIF_NOTIFY_RESUME); + tracehook_notify_resume(regs); + rseq_handle_notify_resume(NULL, regs); } + local_irq_disable(); - thread_flags = current_thread_info()->flags; + thread_flags = READ_ONCE(current_thread_info()->flags); } while (thread_flags & _TIF_WORK_MASK); } -- Gitee From 1a8e57127d1127ec35731b83959111f34ff32a29 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Mon, 9 Oct 2023 16:11:08 +0800 Subject: [PATCH 126/150] sw64: rewrite syscall invocation in C Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCP8 -------------------------------- Rewrite syscall invocation in C, leave only the minimum required assembly code in entry.S, just like interrupts and exceptions. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/ptrace.h | 3 + arch/sw_64/include/asm/syscall.h | 9 +- arch/sw_64/kernel/entry.S | 169 ++----------------------------- arch/sw_64/kernel/signal.c | 10 +- arch/sw_64/kernel/traps.c | 42 ++++++++ 5 files changed, 70 insertions(+), 163 deletions(-) diff --git a/arch/sw_64/include/asm/ptrace.h b/arch/sw_64/include/asm/ptrace.h index c258ba17cda6..62fc43696316 100644 --- a/arch/sw_64/include/asm/ptrace.h +++ b/arch/sw_64/include/asm/ptrace.h @@ -50,6 +50,9 @@ struct pt_regs { extern short regoffsets[]; +extern unsigned long syscall_trace_enter(void); +extern void syscall_trace_leave(void); + /** * regs_get_register() - get register value from its offset * @regs: pt_regs from which register value is gotten diff --git a/arch/sw_64/include/asm/syscall.h b/arch/sw_64/include/asm/syscall.h index 403f607592c9..a821bf68be16 100644 --- a/arch/sw_64/include/asm/syscall.h +++ b/arch/sw_64/include/asm/syscall.h @@ -4,7 +4,12 @@ #include -extern void *sys_call_table[]; +#ifndef __ASSEMBLY__ + +typedef long (*syscall_fn_t)(ulong, ulong, ulong, ulong, ulong, ulong); + +extern syscall_fn_t sys_call_table[]; + static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { @@ -72,4 +77,6 @@ static inline int syscall_get_arch(struct task_struct *task) return AUDIT_ARCH_SW64; } +#endif /* !__ASSEMBLY__ */ + #endif /* _ASM_SW64_SYSCALL_H */ diff --git a/arch/sw_64/kernel/entry.S b/arch/sw_64/kernel/entry.S index 669a55d87cd0..59c2ff4eb915 100644 --- a/arch/sw_64/kernel/entry.S +++ b/arch/sw_64/kernel/entry.S @@ -199,40 +199,20 @@ entUna: .align 4 .globl entSys - .globl ret_from_sys_call .ent entSys entSys: - SAVE_ALL - ldl $0, PT_REGS_R0($sp) - stl $0, PT_REGS_ORIG_R0($sp) - stl $19, PT_REGS_ORIG_R19($sp) - ldi $4, NR_SYSCALLS($31) stl $16, PT_REGS_R16($sp) - ldi $5, sys_call_table - ldi $27, sys_ni_syscall - cmpult $0, $4, $4 - ldw $3, TI_FLAGS($8) stl $17, PT_REGS_R17($sp) - s8addl $0, $5, $5 stl $18, PT_REGS_R18($sp) - ldi $6, _TIF_SYSCALL_WORK - and $3, $6, $3 - bne $3, strace - - beq $4, 1f - ldl $27, 0($5) -1: call $26, ($27), ni_syscall - ldgp $gp, 0($26) - blt $0, $syscall_error /* the call failed */ - - ldi $1, NO_SYSCALL - stl $1, PT_REGS_ORIG_R0($sp) -$syscall_success: - stl $0, PT_REGS_R0($sp) - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ + mov $sp, $16 + call $26, do_entSys + br ret_from_sys_call + .end entSys .align 4 + .globl ret_from_sys_call + .ent ret_from_sys_call ret_from_sys_call: #ifdef CONFIG_SUBARCH_C3B fillcs 0($sp) /* prefetch */ @@ -240,18 +220,14 @@ ret_from_sys_call: #endif br $27, 1f 1: ldgp $29, 0($27) - ldl $0, PT_REGS_PS($sp) - and $0, 8, $0 - beq $0, ret_to_kernel -ret_to_user: -#ifdef CONFIG_DEBUG_RSEQ - mov $sp, $16 - call $26, rseq_syscall -#endif /* Make sure need_resched and sigpending don't change between sampling and the rti. */ ldi $16, 7 sys_call HMC_swpipl + ldl $0, PT_REGS_PS($sp) + and $0, 8, $0 + beq $0, restore_all +ret_to_user: ldw $17, TI_FLAGS($8) and $17, _TIF_WORK_MASK, $2 beq $2, restore_all @@ -260,89 +236,7 @@ ret_to_user: restore_all: RESTORE_ALL sys_call HMC_rti - -ret_to_kernel: - ldi $16, 7 - sys_call HMC_swpipl - br restore_all - - .align 3 -$syscall_error: - /* - * Some system calls (e.g., ptrace) can return arbitrary - * values which might normally be mistaken as error numbers. - * Those functions must zero $0 (v0) directly in the stack - * frame to indicate that a negative return value wasn't an - * error number.. - */ - ldl $1, PT_REGS_ORIG_R0($sp) /* old syscall nr (-1 if success) */ - blt $1, $syscall_success - - subl $31, $0, $0 /* with error in v0 */ - addl $31, 1, $1 /* set a3 for errno return */ - stl $0, PT_REGS_R0($sp) - stl $1, PT_REGS_R19($sp) /* a3 for return */ - br ret_from_sys_call - .end entSys - -/* - * PTRACE syscall handler - */ - - .align 4 - .ent strace -strace: - /* set up signal stack, call syscall_trace */ - call $26, syscall_trace_enter - blt $0, $syscall_trace_failed - - /* get the system call number and the arguments back.. */ - stl $0, PT_REGS_ORIG_R0($sp) - ldl $16, PT_REGS_R16($sp) - ldl $17, PT_REGS_R17($sp) - ldl $18, PT_REGS_R18($sp) - ldl $19, PT_REGS_R19($sp) - ldl $20, PT_REGS_R20($sp) - ldl $21, PT_REGS_R21($sp) - - /* get the system call pointer.. */ - ldi $1, NR_SYSCALLS($31) - ldi $2, sys_call_table - ldi $27, ni_syscall - - cmpult $0, $1, $1 - s8addl $0, $2, $2 - beq $1, 1f - ldl $27, 0($2) -1: call $26, ($27), sys_gettimeofday -ret_from_straced: - ldgp $gp, 0($26) - - /* check return.. */ - blt $0, $strace_error /* the call failed */ - - ldi $1, NO_SYSCALL - stl $1, PT_REGS_ORIG_R0($sp) -$strace_success: - stl $0, PT_REGS_R0($sp) /* save return value */ - stl $31, PT_REGS_R19($sp) /* a3=0 => no error */ - call $26, syscall_trace_leave - br ret_from_sys_call - - .align 3 -$strace_error: - ldl $1, PT_REGS_ORIG_R0($sp) /* old syscall nr (-1 if success) */ - blt $1, $strace_success - - subl $31, $0, $0 /* with error in v0 */ - addl $31, 1, $1 /* set a3 for errno return */ - stl $0, PT_REGS_R0($sp) - stl $1, PT_REGS_R19($sp) /* a3 for return */ - -$syscall_trace_failed: - call $26, syscall_trace_leave - br ret_from_sys_call - .end strace + .end ret_from_sys_call /* * Integer register context switch @@ -410,44 +304,3 @@ ret_from_kernel_thread: call $26, ($9) br ret_to_user .end ret_from_kernel_thread - - .align 4 - .globl sys_sigreturn - .ent sys_sigreturn -sys_sigreturn: - .prologue 0 - ldi $9, ret_from_straced - cmpult $26, $9, $9 - call $26, do_sigreturn - bne $9, 1f - call $26, syscall_trace_leave -1: br ret_from_sys_call - .end sys_sigreturn - - .align 4 - .globl sys_rt_sigreturn - .ent sys_rt_sigreturn -sys_rt_sigreturn: - .prologue 0 - ldi $9, ret_from_straced - cmpult $26, $9, $9 - call $26, do_rt_sigreturn - bne $9, 1f - call $26, syscall_trace_leave -1: br ret_from_sys_call - .end sys_rt_sigreturn - - .align 4 - .globl ni_syscall - .ent ni_syscall -ni_syscall: - .prologue 0 - /* Special because it also implements overflow handling via - * syscall number 0. And if you recall, zero is a special - * trigger for "not an error". Store large non-zero there. - */ - ldi $0, -ENOSYS - unop - stl $0, PT_REGS_R0($sp) - ret - .end ni_syscall diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 9d3ad6a17b37..58f833e2b9c4 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -114,7 +114,7 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) * registers and transfer control from userland. */ -asmlinkage void do_sigreturn(void) +SYSCALL_DEFINE0(sigreturn) { struct pt_regs *regs = current_pt_regs(); struct sigcontext __user *sc; @@ -140,13 +140,14 @@ asmlinkage void do_sigreturn(void) force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); } - return; + return regs->regs[0]; give_sigsegv: force_sig(SIGSEGV); + return 0; } -asmlinkage void do_rt_sigreturn(void) +SYSCALL_DEFINE0(rt_sigreturn) { struct pt_regs *regs = current_pt_regs(); struct rt_sigframe __user *frame; @@ -175,10 +176,11 @@ asmlinkage void do_rt_sigreturn(void) force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); } - return; + return regs->regs[0]; give_sigsegv: force_sig(SIGSEGV); + return 0; } diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 7702233c4c0b..95ed6e6c95bf 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,8 @@ #include #include #include +#include +#include #include "proto.h" @@ -1481,6 +1484,45 @@ do_entUnaUser(void __user *va, unsigned long opcode, force_sig_fault(SIGBUS, BUS_ADRALN, va); } +asmlinkage void do_entSys(struct pt_regs *regs) +{ + long ret = -ENOSYS; + unsigned long nr; + unsigned long ti_flags = current_thread_info()->flags; + + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + nr = regs->regs[0]; + + if (ti_flags & _TIF_SYSCALL_WORK) { + nr = syscall_trace_enter(); + if (nr == NO_SYSCALL) + goto syscall_out; + regs->orig_r0 = regs->regs[0]; + regs->orig_r19 = regs->regs[19]; + } + + if (nr < __NR_syscalls) { + syscall_fn_t syscall_fn = sys_call_table[nr]; + + ret = syscall_fn(regs->regs[16], regs->regs[17], regs->regs[18], + regs->regs[19], regs->regs[20], regs->regs[21]); + } + + if ((nr != __NR_sigreturn) && (nr != __NR_rt_sigreturn)) { + if (likely((ret >= 0) || regs->orig_r0 == NO_SYSCALL)) + syscall_set_return_value(current, regs, 0, ret); + else + syscall_set_return_value(current, regs, ret, 0); + } + +syscall_out: + rseq_syscall(regs); + + if (ti_flags & _TIF_SYSCALL_WORK) + syscall_trace_leave(); +} + void trap_init(void) { -- Gitee From 38c524591afef8703cd6e208ea18e891b324e8ae Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 10 Oct 2023 17:30:22 +0800 Subject: [PATCH 127/150] sw64: optimize hmcall rdusp() and wrusp() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCPO -------------------------------- Remove the unnecessary memory barrier and instruction barrier. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/hmcall.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/sw_64/kernel/hmcall.c b/arch/sw_64/kernel/hmcall.c index 86fe4e3f7d51..d2054a930bd7 100644 --- a/arch/sw_64/kernel/hmcall.c +++ b/arch/sw_64/kernel/hmcall.c @@ -92,6 +92,22 @@ static inline void fixup_wrktp(void) entry[1] = 0x1ee00000; /* pri_ret $23 */ } +static inline void fixup_rdusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(rdusp)); + + entry[0] = 0x94161018; /* pri_ldl/p $0, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + +static inline void fixup_wrusp(void) +{ + unsigned int *entry = __va(HMCALL_ENTRY(wrusp)); + + entry[0] = 0xb6161018; /* pri_stl/p $16, VC__USP(vcpucb) */ + entry[1] = 0x1ee00000; /* pri_ret $23 */ +} + void __init fixup_hmcall(void) { #if defined(CONFIG_SUBARCH_C3B) @@ -101,6 +117,8 @@ void __init fixup_hmcall(void) fixup_wrasid(); fixup_rdktp(); fixup_wrktp(); + fixup_rdusp(); + fixup_wrusp(); imemb(); #endif } -- Gitee From 22b8299114d18a4e014be10d24161d38b02da9d5 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Tue, 17 Oct 2023 15:26:20 +0800 Subject: [PATCH 128/150] sw64: fix syscall sigreturn and rt_sigreturn Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCQ5 -------------------------------- Read sp from user input, because it may not be equal to sp in pt_regs due to the implementation of setcontext in glibc. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/signal.c | 8 ++------ arch/sw_64/kernel/vdso/vrt_sigreturn.S | 1 + 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 58f833e2b9c4..ac3ef83cfc5d 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -114,16 +114,14 @@ restore_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs) * registers and transfer control from userland. */ -SYSCALL_DEFINE0(sigreturn) +SYSCALL_DEFINE1(sigreturn, struct sigcontext __user *, sc) { struct pt_regs *regs = current_pt_regs(); - struct sigcontext __user *sc; sigset_t set; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - sc = (struct sigcontext __user *)regs->regs[30]; /* Verify that it's a good sigcontext before using it */ if (!access_ok(sc, sizeof(*sc))) goto give_sigsegv; @@ -147,16 +145,14 @@ SYSCALL_DEFINE0(sigreturn) return 0; } -SYSCALL_DEFINE0(rt_sigreturn) +SYSCALL_DEFINE1(rt_sigreturn, struct rt_sigframe __user *, frame) { struct pt_regs *regs = current_pt_regs(); - struct rt_sigframe __user *frame; sigset_t set; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; - frame = (struct rt_sigframe __user *)regs->regs[30]; /* Verify that it's a good ucontext_t before using it */ if (!access_ok(&frame->uc, sizeof(frame->uc))) goto give_sigsegv; diff --git a/arch/sw_64/kernel/vdso/vrt_sigreturn.S b/arch/sw_64/kernel/vdso/vrt_sigreturn.S index 106b81f47664..d2d7295ffa7a 100644 --- a/arch/sw_64/kernel/vdso/vrt_sigreturn.S +++ b/arch/sw_64/kernel/vdso/vrt_sigreturn.S @@ -63,6 +63,7 @@ nop ENTRY(__vdso_rt_sigreturn) + mov $sp, $16 ldi $0, __NR_rt_sigreturn sys_call HMC_callsys ENDPROC(__vdso_rt_sigreturn) -- Gitee From ffd9c055f5feab8200a72f249305cdaf50de7707 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Thu, 19 Oct 2023 03:33:40 +0000 Subject: [PATCH 129/150] sw64: iommu: upgrade sunway_iommu_v2 driver to make it work Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- Reconstruct the driver to fit the new driver framework. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/Kconfig | 16 +- drivers/iommu/sw64/sunway_iommu.c | 3 + drivers/iommu/sw64/sunway_iommu.h | 1 - drivers/iommu/sw64/sunway_iommu_v2.c | 401 ++++++++++++--------------- 4 files changed, 183 insertions(+), 238 deletions(-) diff --git a/drivers/iommu/sw64/Kconfig b/drivers/iommu/sw64/Kconfig index 39b0417ae98a..bffc464f2de8 100644 --- a/drivers/iommu/sw64/Kconfig +++ b/drivers/iommu/sw64/Kconfig @@ -1,20 +1,20 @@ # SPDX-License-Identifier: GPL-2.0-only # SW64 IOMMU SUPPORT config SUNWAY_IOMMU - bool "Sunway IOMMU Support" - select IOMMU_API - select IOMMU_IOVA - depends on SW64 && PCI - help - Support for IOMMU on SW64 platform. + bool "Sunway IOMMU Support" + select IOMMU_API + select IOMMU_IOVA + depends on SW64 && PCI && SUBARCH_C3B + help + Support for IOMMU on SW64 platform. It can enable or bypass specific device by + adding boot param "iommu_enable" and "iommu.passthrough". # SW64 IOMMU V2 SUPPORT config SUNWAY_IOMMU_V2 bool "Sunway IOMMU V2 Support" select IOMMU_API select IOMMU_IOVA - depends on SW64 && PCI - depends on !SUNWAY_IOMMU + depends on SW64 && PCI && SUBARCH_C4 help Support for IOMMU V2 on SW64 platform. It can enable or bypass specific device by adding boot param "iommu_enable" and "iommu.passthrough". diff --git a/drivers/iommu/sw64/sunway_iommu.c b/drivers/iommu/sw64/sunway_iommu.c index 89e56e4b9006..48ca66ec8476 100644 --- a/drivers/iommu/sw64/sunway_iommu.c +++ b/drivers/iommu/sw64/sunway_iommu.c @@ -1683,6 +1683,9 @@ const struct iommu_ops sunway_iommu_ops = { /***************************************************************************** * * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. * *****************************************************************************/ static int __init iommu_enable_setup(char *str) diff --git a/drivers/iommu/sw64/sunway_iommu.h b/drivers/iommu/sw64/sunway_iommu.h index b9cb68351cad..d2a7e96657e9 100644 --- a/drivers/iommu/sw64/sunway_iommu.h +++ b/drivers/iommu/sw64/sunway_iommu.h @@ -21,7 +21,6 @@ struct sunway_iommu { int node; /* NUMA node */ struct pci_controller *hose_pt; - struct pci_dev *pdev; /* PCI device to this IOMMU */ struct iommu_device iommu; /* IOMMU core code handle */ }; diff --git a/drivers/iommu/sw64/sunway_iommu_v2.c b/drivers/iommu/sw64/sunway_iommu_v2.c index 1d31f42a2b6d..3bc87fbdaa4d 100644 --- a/drivers/iommu/sw64/sunway_iommu_v2.c +++ b/drivers/iommu/sw64/sunway_iommu_v2.c @@ -11,12 +11,12 @@ #include #include #include -#include #include #include #include #include -#include +#include +#include #include #include #include @@ -95,24 +95,6 @@ struct dma_domain { const struct iommu_ops sunway_iommu_ops; static const struct dma_map_ops sunway_dma_ops; -struct pci_controller *get_hose_from_domain(struct sunway_iommu_domain *sdomain) -{ - struct pci_controller *hose = NULL; - struct sunway_iommu *iommu; - - if (!sdomain) - return NULL; - - iommu = sdomain->iommu; - - if (!iommu) - return NULL; - - hose = iommu->hose_pt; - - return hose; -} - /* flush helpers */ static void piu_flush_all(struct pci_controller *hose) @@ -122,90 +104,36 @@ static void piu_flush_all(struct pci_controller *hose) write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); } -struct sunway_iommu *get_iommu_from_device(struct device *dev) -{ - struct sunway_iommu *iommu; - struct pci_controller *hose; - - hose = to_pci_dev(dev)->sysdata; - iommu = hose->pci_iommu; - - return iommu; -} - -void domain_flush_all(struct sunway_iommu_domain *sdomain) -{ - struct pci_controller *hose; - - hose = get_hose_from_domain(sdomain); - if (!hose) - return; - - piu_flush_all(hose); -} - -void domain_flush_ptlb(struct sunway_iommu_domain *sdomain) -{ - struct pci_controller *hose; - - hose = get_hose_from_domain(sdomain); - if (!hose) - return; - - write_piu_ior0(hose->node, hose->index, PTLB_FLUSHALL, 0); - write_piu_ior0(hose->node, hose->index, PCACHE_FLUSHALL, 0); -} - -void dev_flush_dtlb(struct sunway_iommu_domain *sdomain, struct sunway_iommu_dev *sdev_data) +void flush_pcache_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; - u16 devid; + struct sunway_iommu_dev *sdev; - hose = get_hose_from_domain(sdomain); - if (!hose) - return; + list_for_each_entry(sdev, &sdomain->dev_list, list) { + hose = sdev->pdev->sysdata; - devid = sdev_data->devid; - write_piu_ior0(hose->node, hose->index, DTLB_FLUSHDEV, devid); + flush_addr = __pa(flush_addr); + /* Set memory bar here */ + mb(); + write_piu_ior0(hose->node, hose->index, + PCACHE_FLUSHPADDR, flush_addr); + } } -/* - * This function is designed to support IOMMU code only, - * as it only provides 2 specific types of flush ops - */ -void -flush_device_tlb(struct sunway_iommu_domain *sdomain, - unsigned long flush_addr, unsigned long hflush_addr) +void flush_ptlb_by_addr(struct sunway_iommu_domain *sdomain, unsigned long flush_addr) { struct pci_controller *hose; + struct sunway_iommu_dev *sdev; struct pci_dev *pdev; - struct sunway_iommu_dev *sdev_data; - hose = get_hose_from_domain(sdomain); - if (!hose) - return; + list_for_each_entry(sdev, &sdomain->dev_list, list) { + pdev = sdev->pdev; + hose = pdev->sysdata; - switch (hflush_addr) { - case PCACHE_FLUSHPADDR: - flush_addr = __pa(flush_addr) & PCACHE_FLUSHPADDR_MASK; - /* Set memory bar here */ - mb(); + flush_addr = (pdev->bus->number << 8) + | pdev->devfn | (flush_addr << 16); write_piu_ior0(hose->node, hose->index, - hflush_addr, flush_addr); - break; - - case PTLB_FLUSHVADDR: - list_for_each_entry(sdev_data, &sdomain->dev_list, list) { - pdev = sdev_data->pdev; - flush_addr = (pdev->bus->number << 8) - | pdev->devfn | (flush_addr << 16); - write_piu_ior0(hose->node, hose->index, - hflush_addr, flush_addr); - } - break; - - default: - break; + PTLB_FLUSHVADDR, flush_addr); } } @@ -396,8 +324,7 @@ set_dte_entry(struct sunway_iommu_dev *sdev, struct sunway_iommu_domain *sdomain return; sdev->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); - iommu = sdomain->iommu; - sdev->iommu = iommu; + iommu = sdev->iommu; dte_l1 = iommu->iommu_dtbr + (pdev->bus->number); dte_l1_val = *dte_l1; @@ -485,14 +412,14 @@ static void __detach_device(struct sunway_iommu_dev *sunway_dev_data) static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain) { - struct sunway_iommu_dev *sdev_data; + struct sunway_iommu_dev *sdev; unsigned long flags; int ret; - sdev_data = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); - ret = __attach_device(sdev_data, sdomain); + ret = __attach_device(sdev, sdomain); spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); return ret; @@ -501,17 +428,17 @@ static int attach_device(struct device *dev, struct sunway_iommu_domain *sdomain static void detach_device(struct device *dev) { struct sunway_iommu_domain *sunway_domain; - struct sunway_iommu_dev *sdev_data; + struct sunway_iommu_dev *sdev; unsigned long flags; - sdev_data = dev->archdata.iommu; - sunway_domain = sdev_data->domain; + sdev = dev_iommu_priv_get(dev); + sunway_domain = sdev->domain; - if (WARN_ON(!sdev_data->domain)) + if (WARN_ON(!sdev->domain)) return; spin_lock_irqsave(&sunway_iommu_device_table_lock, flags); - __detach_device(sdev_data); + __detach_device(sdev); spin_unlock_irqrestore(&sunway_iommu_device_table_lock, flags); if (!dev_is_pci(dev)) @@ -535,61 +462,6 @@ static struct sunway_iommu_dev *search_dev_data(u16 devid) return NULL; } -/* iommu device init/uninit ops */ -static int iommu_init_device(struct device *dev) -{ - struct sunway_iommu_dev *sdev_data; - struct sunway_iommu *iommu; - struct pci_dev *pdev; - struct pci_controller *hose; - - if (dev->archdata.iommu) - return 0; - - sdev_data = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); - if (!sdev_data) - return -ENOMEM; - - pdev = to_pci_dev(dev); - hose = pdev->sysdata; - llist_add(&sdev_data->dev_data_list, &dev_data_list); - sdev_data->pdev = pdev; - dev->archdata.iommu = sdev_data; - iommu = hose->pci_iommu; - iommu_device_link(&iommu->iommu, dev); - - return 0; -} - -static void init_iommu_group(struct device *dev) -{ - struct iommu_group *group; - - group = iommu_group_get_for_dev(dev); - if (IS_ERR(group)) - return; - - iommu_group_put(group); -} - -static void iommu_uninit_device(struct device *dev) -{ - struct sunway_iommu_dev *sdev; - struct sunway_iommu *iommu; - - sdev = dev->archdata.iommu; - if (!sdev) - return; - - if (sdev->domain) - detach_device(dev); - - iommu = sdev->iommu; - iommu_device_unlink(&iommu->iommu, dev); - iommu_group_remove_device(dev); - dev->dma_ops = NULL; -} - /* dma_ops helpers*/ static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) { @@ -602,7 +474,7 @@ static struct sunway_iommu_domain *get_sunway_domain(struct device *dev) if (!pdev) return ERR_PTR(-ENODEV); - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); sdomain = sdev->domain; if (sdomain == NULL) { domain = iommu_get_domain_for_dev(dev); @@ -855,7 +727,7 @@ static bool is_iommu_enable(struct pci_controller *hose) static struct iommu_domain *sunway_iommu_domain_alloc(unsigned int type); -int sunway_iommu_init(struct pci_controller *hose_head) +int sunway_iommu_init(void) { struct pci_controller *hose; struct sunway_iommu *iommu; @@ -900,7 +772,7 @@ int sunway_iommu_init(struct pci_controller *hose_head) return 1; } -EXPORT_SYMBOL(sunway_iommu_init); +device_initcall(sunway_iommu_init); /* iommu cpu syscore ops */ static int iommu_cpu_suspend(void) @@ -924,6 +796,17 @@ struct syscore_ops iommu_cpu_syscore_ops = { * ******************************************************************************/ +struct sunway_iommu *get_first_iommu_from_domain(struct sunway_iommu_domain *sdomain) +{ + struct sunway_iommu *iommu; + struct sunway_iommu_dev *entry; + + entry = list_first_entry(&sdomain->dev_list, struct sunway_iommu_dev, list); + iommu = entry->iommu; + + return iommu; +} + static unsigned long sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, unsigned long iova, unsigned long page_size) @@ -967,14 +850,14 @@ sunway_iommu_unmap_page(struct sunway_iommu_domain *sunway_domain, for (i = 0; i < 64; i++) { *(pte + i) = 0; - flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } } else { *pte = 0; - flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } - flush_device_tlb(sunway_domain, (iova >> PAGE_SHIFT), PTLB_FLUSHVADDR); + flush_ptlb_by_addr(sunway_domain, (iova >> PAGE_SHIFT)); break; } @@ -998,7 +881,7 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, int level = 0, current_level; int tmp = 1; - iommu = sunway_domain->iommu; + iommu = get_first_iommu_from_domain(sunway_domain); if (!iommu) return -1; iova_pfn = bus_addr >> PAGE_SHIFT; @@ -1047,13 +930,13 @@ int sunway_iommu_map_page(struct sunway_iommu_domain *sunway_domain, for (i = 0; i < 64; i++) { cmpxchg64((volatile u64 *)(pte + i), 0UL, pte_val); - flush_device_tlb(sunway_domain, (unsigned long)(pte + i), PCACHE_FLUSHPADDR); + flush_pcache_by_addr(sunway_domain, (unsigned long)(pte + i)); } } else { if (cmpxchg64((volatile u64 *)pte, 0UL, pte_val)) free_page((unsigned long)page_address(page)); else - flush_device_tlb(sunway_domain, (unsigned long)pte, PCACHE_FLUSHPADDR); + flush_pcache_by_addr(sunway_domain, (unsigned long)pte); } } @@ -1190,7 +1073,7 @@ static void *sunway_alloc_coherent(struct device *dev, if (!(hose->iommu_enable)) return cpu_addr; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) return cpu_addr; else if (sdev->passthrough) { @@ -1200,7 +1083,7 @@ static void *sunway_alloc_coherent(struct device *dev, } __free_pages(page, get_order(size)); - set_dma_ops(dev, &swiotlb_dma_ops); + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->alloc(dev, size, dma_addr, gfp, attrs); } @@ -1259,7 +1142,7 @@ sunway_free_coherent(struct device *dev, size_t size, if (!hose || !(hose->iommu_enable)) goto out_unmap; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) goto out_unmap; @@ -1300,7 +1183,7 @@ sunway_map_page(struct device *dev, struct page *page, if (!hose || !(hose->iommu_enable)) return paddr; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) return paddr; else if (sdev->passthrough) { @@ -1309,7 +1192,7 @@ sunway_map_page(struct device *dev, struct page *page, return paddr; } - set_dma_ops(dev, &swiotlb_dma_ops); + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->map_page(dev, page, offset, size, dir, attrs); } @@ -1341,7 +1224,7 @@ sunway_unmap_page(struct device *dev, dma_addr_t dma_addr, if (!hose->iommu_enable) return; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) return; @@ -1383,7 +1266,7 @@ sunway_map_sg(struct device *dev, struct scatterlist *sgl, if (!(hose->iommu_enable)) goto check; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough & DMA_MASK64) goto check; else if (sdev->passthrough) { @@ -1392,7 +1275,7 @@ sunway_map_sg(struct device *dev, struct scatterlist *sgl, goto check; } - set_dma_ops(dev, &swiotlb_dma_ops); + set_dma_ops(dev, get_arch_dma_ops(dev->bus)); return dev->dma_ops->map_sg(dev, sgl, nents, dir, attrs); } @@ -1440,7 +1323,7 @@ sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, if (!hose->iommu_enable) return; - sdev = dev->archdata.iommu; + sdev = dev_iommu_priv_get(dev); if (sdev->passthrough) return; @@ -1457,24 +1340,6 @@ sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, } } -static int sunway_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return dma_addr == 0; -} - -static int sunway_supported(struct device *dev, u64 mask) -{ - if (mask < DMA_BIT_MASK(32)) - return 0; - /* - * Upstream PCI/PCIe bridges or SoC interconnects may not carry - * as many DMA address bits as the device itself supports. - */ - if (dev->bus_dma_mask && mask > dev->bus_dma_mask) - return 0; - return 1; -} - static const struct dma_map_ops sunway_dma_ops = { .alloc = sunway_alloc_coherent, .free = sunway_free_coherent, @@ -1482,8 +1347,7 @@ static const struct dma_map_ops sunway_dma_ops = { .unmap_sg = sunway_unmap_sg, .map_page = sunway_map_page, .unmap_page = sunway_unmap_page, - .mapping_error = sunway_mapping_error, - .dma_supported = sunway_supported, + .dma_supported = dma_direct_supported, }; /********************************************************************** @@ -1586,7 +1450,7 @@ static void sunway_iommu_domain_free(struct iommu_domain *dom) static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *dev) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); - struct sunway_iommu_dev *sdev_data; + struct sunway_iommu_dev *sdev; struct pci_dev *pdev; struct pci_controller *hose; int ret; @@ -1602,14 +1466,11 @@ static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *d if (!hose->iommu_enable) return -EINVAL; - if (!sdomain->iommu) - sdomain->iommu = hose->pci_iommu; - - sdev_data = dev->archdata.iommu; - if (!sdev_data) + sdev = dev_iommu_priv_get(dev); + if (!sdev) return -EINVAL; - if (sdev_data->domain) + if (sdev->domain) detach_device(dev); ret = attach_device(dev, sdomain); @@ -1619,13 +1480,14 @@ static int sunway_iommu_attach_device(struct iommu_domain *dom, struct device *d static void sunway_iommu_detach_device(struct iommu_domain *dom, struct device *dev) { - struct sunway_iommu_dev *sunway_dev_data = dev->archdata.iommu; + struct sunway_iommu_dev *sdev; struct pci_dev *pdev = to_pci_dev(dev); if (!pdev) return; - if (sunway_dev_data->domain != NULL) + sdev = dev_iommu_priv_get(dev); + if (sdev->domain != NULL) detach_device(dev); } @@ -1692,7 +1554,7 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) static int sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, int iommu_prot) + phys_addr_t paddr, size_t page_size, int iommu_prot, gfp_t gfp) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); int ret; @@ -1713,7 +1575,9 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, } static size_t -sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t page_size) +sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, + size_t page_size, + struct iommu_iotlb_gather *gather) { struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; @@ -1728,49 +1592,114 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, size_t page_siz return unmap_size; } -static int sunway_iommu_add_device(struct device *dev) +static struct iommu_group *sunway_iommu_device_group(struct device *dev) +{ + return pci_device_group(dev); +} + +static void iommu_uninit_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (!sdev) + return; + + if (sdev->domain) + detach_device(dev); + + dev_iommu_priv_set(dev, NULL); +} + +static void sunway_iommu_release_device(struct device *dev) { struct pci_dev *pdev; struct pci_controller *hose; - int ret; pdev = to_pci_dev(dev); if (!pdev) - return 0; + return; - if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) - return 0; + hose = pdev->sysdata; + if (!hose->iommu_enable) + return; - if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) - return 0; + iommu_uninit_device(dev); +} - hose = pdev->sysdata; - if (!hose || !hose->iommu_enable) +static int iommu_init_device(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + struct sunway_iommu *iommu; + struct pci_dev *pdev; + struct pci_controller *hose; + + if (dev_iommu_priv_get(dev)) return 0; - ret = iommu_init_device(dev); - if (ret) { - pr_warn("Attention: %s %s failed, device may not work properly!\n", - dev_name(dev), __func__); - return ret; - } + sdev = kzalloc(sizeof(struct sunway_iommu_dev), GFP_KERNEL); + if (!sdev) + return -ENOMEM; - init_iommu_group(dev); + pdev = to_pci_dev(dev); + hose = pdev->sysdata; + iommu = hose->pci_iommu; + llist_add(&sdev->dev_data_list, &dev_data_list); + sdev->pdev = pdev; + sdev->iommu = iommu; - dev->dma_ops = &sunway_dma_ops; + dev_iommu_priv_set(dev, sdev); return 0; } -static void sunway_iommu_remove_device(struct device *dev) +static struct iommu_device *sunway_iommu_probe_device(struct device *dev) { struct pci_dev *pdev; + struct pci_controller *hose; + struct sunway_iommu *iommu; + int ret; pdev = to_pci_dev(dev); if (!pdev) - return; + return ERR_PTR(-ENODEV); - iommu_uninit_device(dev); + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + return ERR_PTR(-ENODEV); + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + return ERR_PTR(-ENODEV); + + hose = pdev->sysdata; + if (!hose) + return ERR_PTR(-ENODEV); + + if (!hose->iommu_enable) + return ERR_PTR(-ENODEV); + + if (dev_iommu_priv_get(dev)) { + iommu = hose->pci_iommu; + return &iommu->iommu; + } + + ret = iommu_init_device(dev); + if (ret) + return ERR_PTR(ret); + + iommu = hose->pci_iommu; + + return &iommu->iommu; +} + +static int sunway_iommu_def_domain_type(struct device *dev) +{ + struct sunway_iommu_dev *sdev; + + sdev = dev_iommu_priv_get(dev); + if (sdev->domain) + return 0; + + return sdev->domain->type; } static bool sunway_iommu_capable(enum iommu_cap cap) @@ -1783,24 +1712,38 @@ static bool sunway_iommu_capable(enum iommu_cap cap) } } +static void sunway_iommu_probe_finalize(struct device *dev) +{ + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (domain) + set_dma_ops(dev, &sunway_dma_ops); +} + const struct iommu_ops sunway_iommu_ops = { .capable = sunway_iommu_capable, .domain_alloc = sunway_iommu_domain_alloc, .domain_free = sunway_iommu_domain_free, .attach_dev = sunway_iommu_attach_device, .detach_dev = sunway_iommu_detach_device, + .probe_device = sunway_iommu_probe_device, + .probe_finalize = sunway_iommu_probe_finalize, + .release_device = sunway_iommu_release_device, .map = sunway_iommu_map, .unmap = sunway_iommu_unmap, .iova_to_phys = sunway_iommu_iova_to_phys, - .add_device = sunway_iommu_add_device, - .remove_device = sunway_iommu_remove_device, - .device_group = pci_device_group, + .device_group = sunway_iommu_device_group, .pgsize_bitmap = SW64_IOMMU_PGSIZES, + .def_domain_type = sunway_iommu_def_domain_type, }; /***************************************************************************** * * Boot param handle + * Each bit of iommu_enable bitmap represents an rc enable, and every 8 bits + * represents one cpu node. For example, iommu_enable=0x0100 means enabling + * rc0 for cpu node 1. * *****************************************************************************/ static int __init iommu_enable_setup(char *str) -- Gitee From 54bcfb29235c40d79b8294007c65363a123bd4c8 Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Mon, 23 Oct 2023 00:56:57 +0000 Subject: [PATCH 130/150] sw64: usb: redefine some macros Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- We define these macros ourselves instead of using common definitions. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/usb/host/pci-quirks.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/usb/host/pci-quirks.c b/drivers/usb/host/pci-quirks.c index 6bc756a6b63e..8d341adf5353 100644 --- a/drivers/usb/host/pci-quirks.c +++ b/drivers/usb/host/pci-quirks.c @@ -1288,12 +1288,11 @@ DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_ANY_ID, PCI_ANY_ID, #ifdef CONFIG_SW64 #include - -#define xhci_find_next_ext_cap xhci_find_next_ext_cap_2 // avoid redefinition -#include "xhci.h" -#undef xhci_find_next_ext_cap - -#define STS_RW1C_BITS (STS_FATAL | STS_EINT | STS_PORT | STS_SRE) +#define XHCI_STS_FATAL (1 << 2) +#define XHCI_STS_EINT (1 << 3) +#define XHCI_STS_PORT (1 << 4) +#define XHCI_STS_SRE (1 << 10) +#define STS_RW1C_BITS (XHCI_STS_FATAL | XHCI_STS_EINT | XHCI_STS_PORT | XHCI_STS_SRE) static void fixup_usb_xhci_reset(struct pci_dev *dev) -- Gitee From 77da0802d397b9f80d4d58ae10aee59d3053db48 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Thu, 19 Oct 2023 10:06:28 +0800 Subject: [PATCH 131/150] sw64: fix syscall restart Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCR7 -------------------------------- If syscall has completed successfully but r0 happens to be equal to one of ERESTART* codes, the syscall restart logic would be triggered incorrectly. To fix this, use syscall_get_error() to make sure we are using an errno, not return value. SW64 has a equivalent of arm bug fixed by commit 653d48b22166 ("arm: fix really nasty sigreturn bug"). If signal gets caught by an interrupt that hits when we have the right value in r0 (513) and r19 (1), *and* another signal gets delivered upon sigreturn() or rt_sigreturn() (e.g. included into the blocked mask for the first signal and posted while the handler had been running), the syscall restart logic will see regs->orig_r0 not equal to NO_SYSCALL (we are in a syscall, after all), and r0 and r19 already restored to its original value (513 and 1, which happens to indicate -ERESTARTNOINTR) and assume that we need to apply the usual syscall restart logics. To fix this, call force_successful_syscall_return() in sigreturn() and rt_sigreturn() so the syscall restart logic will not know we are in a syscall. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/signal.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index ac3ef83cfc5d..25eab83d1be1 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "proto.h" @@ -119,6 +120,8 @@ SYSCALL_DEFINE1(sigreturn, struct sigcontext __user *, sc) struct pt_regs *regs = current_pt_regs(); sigset_t set; + force_successful_syscall_return(); + /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; @@ -150,6 +153,8 @@ SYSCALL_DEFINE1(rt_sigreturn, struct rt_sigframe __user *, frame) struct pt_regs *regs = current_pt_regs(); sigset_t set; + force_successful_syscall_return(); + /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; @@ -295,21 +300,21 @@ do_signal(struct pt_regs *regs) single_stepping |= ptrace_cancel_bpt(current); /* Whee! Actually deliver the signal. */ if (regs->orig_r0 != NO_SYSCALL) { - switch (regs->regs[0]) { - case ERESTARTSYS: + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: if (!(ksig.ka.sa.sa_flags & SA_RESTART)) { regs->regs[0] = EINTR; break; } fallthrough; - case ERESTARTNOINTR: + case -ERESTARTNOINTR: /* reset v0 and a3 and replay syscall */ regs->regs[0] = regs->orig_r0; regs->regs[19] = regs->orig_r19; regs->pc -= 4; break; - case ERESTARTNOHAND: - case ERESTART_RESTARTBLOCK: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: regs->regs[0] = EINTR; break; } @@ -319,16 +324,16 @@ do_signal(struct pt_regs *regs) } else { single_stepping |= ptrace_cancel_bpt(current); if (regs->orig_r0 != NO_SYSCALL) { - switch (regs->regs[0]) { - case ERESTARTSYS: - case ERESTARTNOINTR: - case ERESTARTNOHAND: + switch (syscall_get_error(current, regs)) { + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: /* Reset v0 and a3 and replay syscall. */ regs->regs[0] = regs->orig_r0; regs->regs[19] = regs->orig_r19; regs->pc -= 4; break; - case ERESTART_RESTARTBLOCK: + case -ERESTART_RESTARTBLOCK: /* Set v0 to the restart_syscall and replay */ regs->regs[0] = __NR_restart_syscall; regs->pc -= 4; -- Gitee From 5ebb73b637b8aa53e9384d5d4262ad66c62ad866 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 25 Oct 2023 08:43:34 +0800 Subject: [PATCH 132/150] sw64: do not discard exit text at link time Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCRP -------------------------------- Discard .exit.text and .exit.data sections at runtime, not link time, to deal with referenced in section __jump_table. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/vmlinux.lds.S | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/sw_64/kernel/vmlinux.lds.S b/arch/sw_64/kernel/vmlinux.lds.S index 07bc3d8ee7e4..acdcf3c1ab1f 100644 --- a/arch/sw_64/kernel/vmlinux.lds.S +++ b/arch/sw_64/kernel/vmlinux.lds.S @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#define RUNTIME_DISCARD_EXIT #define EMITS_PT_NOTE #define RO_EXCEPTION_TABLE_ALIGN 16 @@ -39,6 +40,15 @@ SECTIONS __init_begin = ALIGN(PAGE_SIZE); INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) + /* we have to discard exit text and such at runtime, not link time */ + .exit.text : + { + EXIT_TEXT + } + .exit.data : + { + EXIT_DATA + } PERCPU_SECTION(L1_CACHE_BYTES) /* -- Gitee From 4bea8811d2eefa25a579171b09dd93f21d7d22c8 Mon Sep 17 00:00:00 2001 From: Zhou Xuemei Date: Wed, 25 Oct 2023 01:34:47 +0000 Subject: [PATCH 133/150] =?UTF-8?q?sw64:=20register=C2=A0suspend=C2=A0entr?= =?UTF-8?q?y=20for=20cpu=200=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56OLG -------------------------------- It used to register suspend entry for all of online cpus. Actually, only cpu 0 will be affected by different hardware design. The sleep codes are written in assembly, if all cpus jump to BIOS, it will become more complex, and more difficult to debug. So, only register suspend entry for cpu 0 to make the BIOS code more readable and easier to debug. Signed-off-by: Zhou Xuemei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/traps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/traps.c b/arch/sw_64/kernel/traps.c index 95ed6e6c95bf..cc14d914c7d4 100644 --- a/arch/sw_64/kernel/traps.c +++ b/arch/sw_64/kernel/traps.c @@ -1536,6 +1536,7 @@ trap_init(void) wrent(entUna, 4); wrent(entSys, 5); #ifdef CONFIG_EFI - wrent((void *)entSuspend, 6); + if (smp_processor_id() == 0) + wrent((void *)entSuspend, 6); #endif } -- Gitee From 21a00858475ee290d33a303217e45c2dcaae5e30 Mon Sep 17 00:00:00 2001 From: Wu Liliu Date: Wed, 25 Oct 2023 10:58:08 +0800 Subject: [PATCH 134/150] sw64: fix stack trace error Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8CCS9 -------------------------------- The stack tracer looks for a function that does not match the one stored on the stack, so cat /sys/kernel/debug/tracing/stack_trace is empty. Fix it. Signed-off-by: Wu Liliu Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/stacktrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/kernel/stacktrace.c b/arch/sw_64/kernel/stacktrace.c index d7172e4ec78a..1a67a8bb854b 100644 --- a/arch/sw_64/kernel/stacktrace.c +++ b/arch/sw_64/kernel/stacktrace.c @@ -130,7 +130,7 @@ void walk_stackframe(struct task_struct *tsk, struct pt_regs *regs, while (!kstack_end(ksp)) { if (__kernel_text_address(pc) && fn(pc, data)) break; - pc = (*ksp++) - 0x4; + pc = *ksp++; } } EXPORT_SYMBOL_GPL(walk_stackframe); -- Gitee From 88f9e2f495e74099b1f6fc976f159f3f3a2ab8f2 Mon Sep 17 00:00:00 2001 From: Gu Zitao Date: Tue, 24 Oct 2023 15:28:46 +0800 Subject: [PATCH 135/150] sw64: add support for TIF_NOTIFY_SIGNAL Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I56U83 -------------------------------- Wire up TIF_NOTIFY_SIGNAL handling for sw64. Signed-off-by: Gu Zitao Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/thread_info.h | 4 +++- arch/sw_64/kernel/signal.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/include/asm/thread_info.h b/arch/sw_64/include/asm/thread_info.h index 1fd97ce5fab2..38b06413121c 100644 --- a/arch/sw_64/include/asm/thread_info.h +++ b/arch/sw_64/include/asm/thread_info.h @@ -87,6 +87,7 @@ static __always_inline u64 rtid(void) #define TIF_SYSCALL_AUDIT 4 /* syscall audit active */ #define TIF_UPROBE 5 /* uprobe breakpoint or singlestep */ #define TIF_PATCH_PENDING 6 /* pending live patching update */ +#define TIF_NOTIFY_SIGNAL 7 /* signal notifications exist */ #define TIF_DIE_IF_KERNEL 9 /* dik recursion lock */ #define TIF_SYSCALL_TRACEPOINT 10 #define TIF_SECCOMP 11 /* secure computing */ @@ -101,13 +102,14 @@ static __always_inline u64 rtid(void) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_SECCOMP (1 << TIF_SECCOMP) +#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_UPROBE (1 << TIF_UPROBE) /* Work to do on interrupt/exception return. */ #define _TIF_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ - _TIF_PATCH_PENDING) + _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL) #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) diff --git a/arch/sw_64/kernel/signal.c b/arch/sw_64/kernel/signal.c index 25eab83d1be1..9ba151bb82d3 100644 --- a/arch/sw_64/kernel/signal.c +++ b/arch/sw_64/kernel/signal.c @@ -366,7 +366,7 @@ do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) if (thread_flags & _TIF_PATCH_PENDING) klp_update_patch_state(current); - if (thread_flags & _TIF_SIGPENDING) + if (thread_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); if (thread_flags & _TIF_NOTIFY_RESUME) { -- Gitee From c8259b67e24232e98edf6243cb73ba1ec31167cf Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Thu, 26 Oct 2023 08:03:17 +0000 Subject: [PATCH 136/150] sw64: iommu: fix dma_supported of dma_ops Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Since commit <6b3a638de236> ("sw64: correct the definition of MAX_DMA_ADDRESS"), the implementation of dma_supported is incorrect, and now we modify it to the public dma-direct implementation. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/sunway_iommu.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/iommu/sw64/sunway_iommu.c b/drivers/iommu/sw64/sunway_iommu.c index 48ca66ec8476..65d0ed8414fe 100644 --- a/drivers/iommu/sw64/sunway_iommu.c +++ b/drivers/iommu/sw64/sunway_iommu.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1298,14 +1299,6 @@ sunway_unmap_sg(struct device *dev, struct scatterlist *sgl, } } -static int sunway_supported(struct device *dev, u64 mask) -{ - if (MAX_DMA_ADDRESS - PAGE_OFFSET - 1 <= mask) - return 1; - - return 0; -} - static const struct dma_map_ops sunway_dma_ops = { .alloc = sunway_alloc_coherent, .free = sunway_free_coherent, @@ -1313,7 +1306,7 @@ static const struct dma_map_ops sunway_dma_ops = { .unmap_sg = sunway_unmap_sg, .map_page = sunway_map_page, .unmap_page = sunway_unmap_page, - .dma_supported = sunway_supported, + .dma_supported = dma_direct_supported, }; /********************************************************************** -- Gitee From dee75cd8266a1c76e742ce9c115937830bcb54f4 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 8 Nov 2023 09:20:31 +0800 Subject: [PATCH 137/150] sw64: expand vmemmap Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Expand vmemmap to make sure it can cover the whole physical memory range. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/pgtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/sw_64/include/asm/pgtable.h b/arch/sw_64/include/asm/pgtable.h index dea7e16c60b6..270baa15a829 100644 --- a/arch/sw_64/include/asm/pgtable.h +++ b/arch/sw_64/include/asm/pgtable.h @@ -83,7 +83,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) #define VMALLOC_END (-PGDIR_SIZE) #else #define VMEMMAP_END (-PGDIR_SIZE) -#define vmemmap ((struct page *)VMEMMAP_END - (1UL << (3 * (PAGE_SHIFT - 3)))) +#define vmemmap ((struct page *)VMEMMAP_END - (1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT))) #define VMALLOC_END ((unsigned long)vmemmap) #endif -- Gitee From aba4224a479da7ff0c6d363280c4096e28d32778 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Wed, 8 Nov 2023 09:22:39 +0800 Subject: [PATCH 138/150] sw64: do not allow assembler reorder in __copy_user() Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- To make sure the return value is always the left bytes to copy, we cannot allow assembler to reorder instructions in __copy_user(). Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/lib/deep-copy_user.S | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/sw_64/lib/deep-copy_user.S b/arch/sw_64/lib/deep-copy_user.S index 8f4ef09ab90e..b79f8f3f0f4a 100644 --- a/arch/sw_64/lib/deep-copy_user.S +++ b/arch/sw_64/lib/deep-copy_user.S @@ -25,6 +25,7 @@ .ent __copy_user __copy_user: .prologue 0 + .set noreorder bis $31, $31, $7 #if defined(CONFIG_SUBARCH_C3B) #include "deep-copy_template.S" -- Gitee From 59f5b447f13204973736b064b075e3731d96aaf7 Mon Sep 17 00:00:00 2001 From: Mao Minkai Date: Fri, 10 Nov 2023 08:41:12 +0800 Subject: [PATCH 139/150] sw64: use generated macro in vrt_sigreturn.S Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Generate RT_SIGFRAME_SIZE and RT_SIGFRAME_MCTX for vrt_sigreturn.S to avoid using hard coded numbers. Signed-off-by: Mao Minkai Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/asm-offsets.c | 5 ++++- arch/sw_64/kernel/vdso/vrt_sigreturn.S | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/sw_64/kernel/asm-offsets.c b/arch/sw_64/kernel/asm-offsets.c index 206152281a4a..41310a8a7af1 100644 --- a/arch/sw_64/kernel/asm-offsets.c +++ b/arch/sw_64/kernel/asm-offsets.c @@ -15,7 +15,7 @@ #include #include "traps.c" - +#include "signal.c" void foo(void) { @@ -234,4 +234,7 @@ void foo(void) OFFSET(TASK_THREAD_S6, task_struct, thread.s[6]); BLANK(); DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); + BLANK(); + DEFINE(RT_SIGFRAME_SIZE, sizeof(struct rt_sigframe)); + OFFSET(RT_SIGFRAME_MCTX, rt_sigframe, uc.uc_mcontext); } diff --git a/arch/sw_64/kernel/vdso/vrt_sigreturn.S b/arch/sw_64/kernel/vdso/vrt_sigreturn.S index d2d7295ffa7a..f1946c2bf476 100644 --- a/arch/sw_64/kernel/vdso/vrt_sigreturn.S +++ b/arch/sw_64/kernel/vdso/vrt_sigreturn.S @@ -20,9 +20,7 @@ #include #include #include - -#define RT_SIGFRAME_SIZE 1600 -#define RT_SIGFRAME_MCTX 176 +#include .text -- Gitee From 06a28a70462c3d901ad79f64e0c59688f27c9344 Mon Sep 17 00:00:00 2001 From: Tang Jinyang Date: Fri, 10 Nov 2023 17:02:12 +0800 Subject: [PATCH 140/150] sw64: fix cpu reduction bug in cpuautoplug Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- This patch fixs a critical bug in cpuautoplug subsystem related to cpufreq subsystem. With performance policy, the number of active cpus may be reduced to the minimum count because global variable curruent_policy is uninitialized. Signed-off-by: Tang Jinyang Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/cpufreq/sw64_cpufreq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/cpufreq/sw64_cpufreq.c b/drivers/cpufreq/sw64_cpufreq.c index 0ebab111e812..0cc10670e22d 100644 --- a/drivers/cpufreq/sw64_cpufreq.c +++ b/drivers/cpufreq/sw64_cpufreq.c @@ -90,6 +90,8 @@ static int sw64_cpufreq_cpu_init(struct cpufreq_policy *policy) policy->clk = cpuclk; + sw64_store_policy(policy); + cpufreq_generic_init(policy, freq_table, 0); return 0; -- Gitee From 5c2d6d571c8841f3b1e215b79eac8065ecf83463 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Fri, 10 Nov 2023 08:31:19 +0000 Subject: [PATCH 141/150] sw64: use generic dma_direct_ops Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Our original dma_ops implementation is missing map_resource and several other functions. However, new kernel will check if there are any arch-dependent dma_ops first and ignore missing functions instead of using generic implementation, which has caused bad value returned. After some evaluation, we decide to switch back to generic implementation, which has been taken by most architectures now. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/Makefile | 1 - arch/sw_64/kernel/pci.c | 38 +++++++ arch/sw_64/kernel/pci_common.c | 178 --------------------------------- 3 files changed, 38 insertions(+), 179 deletions(-) delete mode 100644 arch/sw_64/kernel/pci_common.c diff --git a/arch/sw_64/kernel/Makefile b/arch/sw_64/kernel/Makefile index 25a299daab1f..b71be5783812 100644 --- a/arch/sw_64/kernel/Makefile +++ b/arch/sw_64/kernel/Makefile @@ -30,7 +30,6 @@ obj-$(CONFIG_SUSPEND) += suspend_asm.o suspend.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HIBERNATION) += hibernate_asm.o hibernate.o obj-$(CONFIG_AUDIT) += audit.o -obj-$(CONFIG_PCI) += pci_common.o obj-$(CONFIG_RELOCATABLE) += relocate.o obj-$(CONFIG_DEBUG_FS) += segvdbg.o unaligned.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 0c4dca96ccb4..8bd7afc3464a 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -758,3 +758,41 @@ static void fix_bus_dma_limit(struct pci_dev *dev) pr_info("Set zx200 bus_dma_limit to 32-bit\n"); } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_ZHAOXIN, 0x071f, fix_bus_dma_limit); + +const struct dma_map_ops *dma_ops; +EXPORT_SYMBOL(dma_ops); + +#ifdef CONFIG_DCA +static void enable_sw_dca(struct pci_dev *dev) +{ + struct pci_controller *hose = (struct pci_controller *)dev->sysdata; + unsigned long node, rc_index, dca_ctl, dca_conf; + int i; + + if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) + return; + + node = hose->node; + rc_index = hose->index; + + for (i = 0; i < 256; i++) { + dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); + if (dca_conf >> 63) + continue; + else { + dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; + pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); + write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); + break; + } + } + + dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); + if (dca_ctl & 0x1) { + dca_ctl = 0x2; + write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); + pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); + } +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); +#endif diff --git a/arch/sw_64/kernel/pci_common.c b/arch/sw_64/kernel/pci_common.c deleted file mode 100644 index 56c9be523461..000000000000 --- a/arch/sw_64/kernel/pci_common.c +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/sw_64/kernel/pci_iommu.c - */ - -#include -#include -#include -#include -#include - -static dma_addr_t sw64_direct_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t dma_addr = page_to_phys(page) + offset; - - if (unlikely(swiotlb_force == SWIOTLB_FORCE)) - return swiotlb_map(dev, dma_addr, size, dir, attrs); - - if (unlikely(!dma_capable(dev, dma_addr, size, true))) { - if (swiotlb_force != SWIOTLB_NO_FORCE) - return swiotlb_map(dev, dma_addr, size, dir, attrs); - - dev_WARN_ONCE(dev, 1, - "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", - &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); - return DMA_MAPPING_ERROR; - } - - return dma_addr; -} - -static inline void sw64_direct_unmap_page(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if (unlikely(is_swiotlb_buffer(addr))) - swiotlb_tbl_unmap_single(dev, addr, size, size, dir, attrs); -} - -static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) -{ - return phys + size - 1 <= - min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); -} - -static void *sw64_direct_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp, - unsigned long attrs) -{ - struct page *page; - void *ret; - u64 dma_limit; - - size = PAGE_ALIGN(size); - if (attrs & DMA_ATTR_NO_WARN) - gfp |= __GFP_NOWARN; - - dma_limit = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); - if (dma_limit <= DMA_BIT_MASK(32)) - gfp |= GFP_DMA32; - - /* we always manually zero the memory once we are done */ - gfp &= ~__GFP_ZERO; -again: - page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); - if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { - dma_free_contiguous(dev, page, size); - page = NULL; - - if (IS_ENABLED(CONFIG_ZONE_DMA32) && - dma_limit < DMA_BIT_MASK(64) && - !(gfp & (GFP_DMA32 | GFP_DMA))) { - gfp |= GFP_DMA32; - goto again; - } - } - - if (!page) - return NULL; - - ret = page_address(page); - memset(ret, 0, size); - *dma_addrp = page_to_phys(page); - - return ret; -} - -static void sw64_direct_free_coherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) { - /* cpu_addr is a struct page cookie, not a kernel address */ - dma_free_contiguous(dev, cpu_addr, size); - return; - } - - free_pages((unsigned long)cpu_addr, get_order(size)); -} - -static void sw64_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - struct scatterlist *sg; - int i; - - for_each_sg(sgl, sg, nents, i) - sw64_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir, - attrs); -} - -static int sw64_direct_map_sg(struct device *dev, struct scatterlist *sgl, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sgl, sg, nents, i) { - sg_dma_address(sg) = sw64_direct_map_page(dev, sg_page(sg), - sg->offset, sg->length, dir, attrs); - if (sg->dma_address == DMA_MAPPING_ERROR) - goto out_unmap; - sg_dma_len(sg) = sg->length; - } - return nents; - -out_unmap: - sw64_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); - return 0; -} - -const struct dma_map_ops sw64_dma_direct_ops = { - .alloc = sw64_direct_alloc_coherent, - .free = sw64_direct_free_coherent, - .map_page = sw64_direct_map_page, - .unmap_page = sw64_direct_unmap_page, - .map_sg = sw64_direct_map_sg, - .unmap_sg = sw64_direct_unmap_sg, - .dma_supported = dma_direct_supported, -}; - -const struct dma_map_ops *dma_ops = &sw64_dma_direct_ops; -EXPORT_SYMBOL(dma_ops); - -#ifdef CONFIG_DCA -static void enable_sw_dca(struct pci_dev *dev) -{ - struct pci_controller *hose = (struct pci_controller *)dev->sysdata; - unsigned long node, rc_index, dca_ctl, dca_conf; - int i; - - if (dev->class >> 8 != PCI_CLASS_NETWORK_ETHERNET) - return; - node = hose->node; - rc_index = hose->index; - for (i = 0; i < 256; i++) { - dca_conf = read_piu_ior1(node, rc_index, DEVICEID0 + (i << 7)); - if (dca_conf >> 63) - continue; - else { - dca_conf = (1UL << 63) | (dev->bus->number << 8) | dev->devfn; - pr_info("dca device index %d, dca_conf = %#lx\n", i, dca_conf); - write_piu_ior1(node, rc_index, DEVICEID0 + (i << 7), dca_conf); - break; - } - } - dca_ctl = read_piu_ior1(node, rc_index, DCACONTROL); - if (dca_ctl & 0x1) { - dca_ctl = 0x2; - write_piu_ior1(node, rc_index, DCACONTROL, dca_ctl); - pr_info("Node %ld RC %ld enable DCA 1.0\n", node, rc_index); - } -} -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, enable_sw_dca); -#endif -- Gitee From f0f2811d24a24bdb17ba8c0338e0aa9e9e15cf9e Mon Sep 17 00:00:00 2001 From: Zheng Chongzhen Date: Mon, 13 Nov 2023 06:07:57 +0000 Subject: [PATCH 142/150] sw64: msi: split the msi file Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Since C3B and C4 have different msi interrupt mechanisms, we decided to implement them separately. Signed-off-by: Zheng Chongzhen Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/Makefile | 6 + drivers/irqchip/irq-sunway-msi-v2.c | 517 ++++++++++++++++++++++++++++ drivers/irqchip/irq-sunway-msi.c | 216 ++---------- 3 files changed, 560 insertions(+), 179 deletions(-) create mode 100644 drivers/irqchip/irq-sunway-msi-v2.c diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index b40c75f5b4cf..eaa41878ed29 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -30,7 +30,13 @@ obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_SW64_INTC_V2) += irq-sw64-intc-v2.o obj-$(CONFIG_SW64_LPC_INTC) += irq-sw64-lpc-intc.o obj-$(CONFIG_SW64_IRQ_CPU) += irq-sunway-cpu.o + +ifeq ($(CONFIG_UNCORE_XUELANG),y) obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi.o +else +obj-$(CONFIG_SW64_IRQ_MSI) += irq-sunway-msi-v2.o +endif + obj-$(CONFIG_SW64_IRQ_MSI_VT) += irq-sunway-msi-vt.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o diff --git a/drivers/irqchip/irq-sunway-msi-v2.c b/drivers/irqchip/irq-sunway-msi-v2.c new file mode 100644 index 000000000000..213a22f2b54b --- /dev/null +++ b/drivers/irqchip/irq-sunway-msi-v2.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include + +static struct irq_domain *msi_default_domain; +static DEFINE_RAW_SPINLOCK(vector_lock); +DEFINE_PER_CPU(vector_irq_t, vector_irq) = { + [0 ... PERCPU_MSI_IRQS - 1] = 0, +}; + +static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_data) +{ + struct sw64_msi_chip_data *data; + int node; + + node = irq_data_get_node(irq_data); + data = kzalloc_node(sizeof(*data), GFP_KERNEL, node); + if (!data) + return NULL; + spin_lock_init(&data->cdata_lock); + return data; +} + +static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct sw64_msi_chip_data *chip_data; + int rcid; + + chip_data = irq_data_get_irq_chip_data(data->parent_data); + rcid = cpu_to_rcid(chip_data->dst_cpu); + msg->address_hi = MSI_ADDR_BASE_HI; + msg->address_lo = + (unsigned int)chip_data->msiaddr | + (rcid_to_msicid(rcid) << MSI_ADDR_DEST_ID_SHIFT); + msg->data = chip_data->vector; +} + +bool find_free_cpu_vector(const struct cpumask *search_mask, + int *found_cpu, int *found_vector) +{ + int vector, max_vector, cpu; + bool find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + if (is_guest_or_emul()) { + vector = IRQ_PENDING_MSI_VECTORS_SHIFT; + max_vector = SWVM_IRQS; + } else { + vector = 0; + max_vector = 256; + } + for (; vector < max_vector; vector++) { + while (per_cpu(vector_irq, cpu)[vector]) { + cpu = cpumask_next(cpu, search_mask); + if (cpu >= nr_cpu_ids) { + if (vector == 255) { + if (find_once_global) { + printk("No global free vector\n"); + return false; + } + printk("No local free vector\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } + cpu = cpumask_first(search_mask); + break; + } + } + if (!per_cpu(vector_irq, cpu)[vector]) + break; + } + + *found_cpu = cpu; + *found_vector = vector; + return true; +} + +static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_cpu, int *found_vector, unsigned int nr_irqs) +{ + int i, vector, cpu; + bool found = false, find_once_global = false; + + cpu = cpumask_first(search_mask); +try_again: + for (vector = 0; vector < 256; vector++) { + for (i = 0; i < nr_irqs; i++) + if (per_cpu(vector_irq, cpu)[vector + i]) + break; + + if (i == nr_irqs) { + found = true; + *found_cpu = cpu; + *found_vector = vector; + return found; + } + + vector += i; + } + + cpu = cpumask_next(cpu, search_mask); + if (cpu < nr_cpu_ids) + goto try_again; + else { + if (find_once_global) { + printk("No global free vectors\n"); + return found; + } + printk("No local free vectors\n"); + search_mask = cpu_online_mask; + cpu = cpumask_first(search_mask); + find_once_global = true; + goto try_again; + } +} + +static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) +{ + struct sw64_msi_chip_data *cdata; + struct irq_data *irqd; + struct msi_desc *entry; + struct cpumask searchmask; + unsigned long flags; + int vector, cpu; + int i; + struct msi_msg msg; + + /* Is this valid ? */ + if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) + return -EINVAL; + + irqd = irq_domain_get_irq_data(msi_default_domain->parent, d->irq); + /* Don't do anything if the interrupt isn't started */ + if (!irqd_is_started(irqd)) + return IRQ_SET_MASK_OK; + + cdata = irqd->chip_data; + if (!cdata) + return -ENOMEM; + + /* + * If existing target cpu is already in the new mask and is online + * then do nothing. + */ + if (cpu_online(cdata->dst_cpu) && cpumask_test_cpu(cdata->dst_cpu, cpumask)) + return IRQ_SET_MASK_OK; + + raw_spin_lock_irqsave(&vector_lock, flags); + + cpumask_and(&searchmask, cpumask, cpu_online_mask); + if (cdata->multi_msi > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, cdata->multi_msi)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } else { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; + } + } + + /* update new setting */ + entry = irq_get_msi_desc(irqd->irq); + spin_lock(&cdata->cdata_lock); + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; + BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); + __pci_write_msi_msg(entry, &msg); + cdata->prev_vector = cdata->vector; + cdata->prev_cpu = cdata->dst_cpu; + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->move_in_progress = true; + spin_unlock(&cdata->cdata_lock); + cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); + + raw_spin_unlock_irqrestore(&vector_lock, flags); + + return 0; +} + +static void chip_irq_ack(struct irq_data *data) +{ +} + +static struct irq_chip pci_msi_controller = { + .name = "PCI-MSI", + .irq_unmask = pci_msi_unmask_irq, + .irq_mask = pci_msi_mask_irq, + .irq_ack = chip_irq_ack, + .irq_compose_msi_msg = irq_msi_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_affinity = sw64_set_affinity, +}; + +static int __assign_irq_vector(int virq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + struct irq_data *irq_data; + const struct cpumask *mask; + struct cpumask searchmask; + struct sw64_msi_chip_data *cdata; + int node; + int i, vector, cpu; + unsigned long msiaddr; + + if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) + nr_irqs = __roundup_pow_of_two(nr_irqs); + + irq_data = irq_domain_get_irq_data(domain, virq); + BUG_ON(!irq_data); + irq_data->chip = &pci_msi_controller; + + if (irqd_affinity_is_managed(irq_data)) { + mask = irq_data_get_affinity_mask(irq_data); + cpumask_and(&searchmask, mask, cpu_online_mask); + } else { + node = irq_data_get_node(irq_data); + cpumask_copy(&searchmask, cpumask_of_node(node)); + } + + if (cpumask_first(&searchmask) >= nr_cpu_ids) + cpumask_copy(&searchmask, cpu_online_mask); + + if (type == IRQ_ALLOC_TYPE_MSI && nr_irqs > 1) { + if (!find_free_cpu_vectors(&searchmask, &cpu, + &vector, nr_irqs)) + return -ENOSPC; + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + printk("error alloc irq chip data\n"); + return -ENOMEM; + } + + for (i = 0; i < nr_irqs; i++) { + per_cpu(vector_irq, cpu)[vector + i] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + irq_data->chip_data = cdata; + } + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = nr_irqs; + cdata->move_in_progress = false; + } else { + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) + return -ENOSPC; + + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + + cdata = alloc_sw_msi_chip_data(irq_data); + if (!cdata) { + printk("error alloc irq chip data\n"); + return -ENOMEM; + } + + irq_data->chip_data = cdata; + + cdata->dst_cpu = cpu; + cdata->vector = vector; + cdata->msiaddr = MSIX_MSG_ADDR; + cdata->prev_cpu = cpu; + cdata->prev_vector = vector; + cdata->multi_msi = 1; + cdata->move_in_progress = false; + } + } + return 0; +} + +static int assign_irq_vector(int irq, unsigned int nr_irqs, + struct irq_domain *domain, enum irq_alloc_type type) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector(irq, nr_irqs, domain, type); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +static void sw64_vector_free_irqs(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i, j; + struct irq_data *irq_data; + unsigned long flags; + unsigned int multi_msi; + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + if (irq_data && irq_data->chip_data) { + struct sw64_msi_chip_data *cdata; + + raw_spin_lock_irqsave(&vector_lock, flags); + cdata = irq_data->chip_data; + irq_domain_reset_irq_data(irq_data); + multi_msi = cdata->multi_msi; + for (j = 0; j < multi_msi; j++) + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; + kfree(cdata); + raw_spin_unlock_irqrestore(&vector_lock, flags); + if (multi_msi > 1) + break; + } + } +} + +static void sw64_irq_free_descs(unsigned int virq, unsigned int nr_irqs) +{ + if (is_guest_or_emul()) { + vt_sw64_vector_free_irqs(virq, nr_irqs); + return irq_free_descs(virq, nr_irqs); + } + + return irq_domain_free_irqs(virq, nr_irqs); +} + +void arch_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + int i; + + for_each_pci_msi_entry(desc, dev) { + if (desc->irq) { + for (i = 0; i < desc->nvec_used; i++) + sw64_irq_free_descs(desc->irq + i, 1); + desc->irq = 0; + } + } +} + +static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int err; + struct irq_alloc_info *info = arg; + enum irq_alloc_type msi_type; + + if (arg == NULL) + return -ENODEV; + msi_type = info->type; + err = assign_irq_vector(virq, nr_irqs, domain, msi_type); + if (err) + goto error; + return 0; +error: + sw64_vector_free_irqs(domain, virq, nr_irqs); + return err; +} + +static int pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *arg) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct msi_desc *desc = first_pci_msi_entry(pdev); + + memset(arg, 0, sizeof(*arg)); + arg->msi_dev = pdev; + if (desc->msi_attrib.is_msix) + arg->type = IRQ_ALLOC_TYPE_MSIX; + else + arg->type = IRQ_ALLOC_TYPE_MSI; + return 0; +} + +static struct msi_domain_ops pci_msi_domain_ops = { + .msi_prepare = pci_msi_prepare, +}; + +static struct msi_domain_info pci_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .ops = &pci_msi_domain_ops, + .chip = &pci_msi_controller, + .handler = handle_edge_irq, + .handler_name = "edge", +}; + +static int sw64_irq_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hw) +{ + irq_set_chip_and_handler(virq, &sw64_irq_chip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); + return 0; +} + +const struct irq_domain_ops sw64_msi_domain_ops = { + .map = sw64_irq_map, + .alloc = sw64_vector_alloc_irqs, + .free = sw64_vector_free_irqs, +}; + +int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + struct irq_domain *domain; + int err; + + if (is_guest_or_emul()) + return sw64_setup_vt_msi_irqs(pdev, nvec, type); + + domain = msi_default_domain; + if (domain == NULL) + return -ENOSYS; + err = msi_domain_alloc_irqs(domain, &pdev->dev, nvec); + return err; +} + +void arch_init_msi_domain(struct irq_domain *parent) +{ + struct irq_domain *sw64_irq_domain; + + if (is_guest_or_emul()) + return; + + sw64_irq_domain = irq_domain_add_tree(NULL, &sw64_msi_domain_ops, NULL); + BUG_ON(sw64_irq_domain == NULL); + irq_set_default_host(sw64_irq_domain); + msi_default_domain = pci_msi_create_irq_domain(NULL, + &pci_msi_domain_info, sw64_irq_domain); + if (!msi_default_domain) + pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); +} + +static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vector) +{ + if (likely(!cdata->move_in_progress)) + return; + if (cdata->dst_cpu == cpu) { + if (vector >= cdata->vector && + vector < cdata->vector + cdata->multi_msi) { + int i; + + raw_spin_lock(&vector_lock); + cdata->move_in_progress = false; + for (i = 0; i < cdata->multi_msi; i++) + per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector + i] = 0; + raw_spin_unlock(&vector_lock); + } + } +} + +void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) +{ + int i, irq, msi_index = 0; + int cpu, vector_index = 0; + unsigned long int_pci_msi[3]; + unsigned long *ptr; + struct irq_data *irq_data; + struct sw64_msi_chip_data *cdata; + + if (is_guest_or_emul()) { + cpu = smp_processor_id(); + irq = per_cpu(vector_irq, cpu)[vector]; + handle_irq(irq); + return; + } + + ptr = (unsigned long *)pci_msi1_addr; + int_pci_msi[0] = *ptr; + int_pci_msi[1] = *(ptr + 1); + int_pci_msi[2] = *(ptr + 2); + + cpu = smp_processor_id(); + + for (i = 0; i < 4; i++) { + vector_index = i * 64; + while (vector != 0) { + int irq = 0; + + msi_index = find_next_bit(&vector, 64, msi_index); + if (msi_index == 64) { + msi_index = 0; + continue; + } + + irq = per_cpu(vector_irq, cpu)[vector_index + msi_index]; + irq_data = irq_domain_get_irq_data(msi_default_domain->parent, irq); + cdata = irq_data_get_irq_chip_data(irq_data); + spin_lock(&cdata->cdata_lock); + irq_move_complete(cdata, cpu, vector_index + msi_index); + spin_unlock(&cdata->cdata_lock); + handle_irq(irq); + + vector = vector & (~(1UL << msi_index)); + } + + vector = int_pci_msi[i % 3]; + } +} + +MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-sunway-msi.c b/drivers/irqchip/irq-sunway-msi.c index 505c935132c6..3f8fd9fc775b 100644 --- a/drivers/irqchip/irq-sunway-msi.c +++ b/drivers/irqchip/irq-sunway-msi.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -30,22 +29,12 @@ static struct sw64_msi_chip_data *alloc_sw_msi_chip_data(struct irq_data *irq_da static void irq_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { struct sw64_msi_chip_data *chip_data; - int rcid __maybe_unused; chip_data = irq_data_get_irq_chip_data(data->parent_data); -#if defined(CONFIG_UNCORE_XUELANG) + msg->address_hi = MSI_ADDR_BASE_HI; msg->address_lo = MSI_ADDR_BASE_LO; msg->data = chip_data->msi_config_index; -#elif defined(CONFIG_UNCORE_JUNZHANG) - rcid = cpu_to_rcid(chip_data->dst_cpu); - msg->address_hi = MSI_ADDR_BASE_HI; - msg->address_lo = - (unsigned int)chip_data->msiaddr | - (rcid_to_msicid(rcid) << MSI_ADDR_DEST_ID_SHIFT); - msg->data = chip_data->vector; - printk("%s msg->address_lo:%#x msg->data:%#x\n", __func__, msg->address_lo, msg->data); -#endif } bool find_free_cpu_vector(const struct cpumask *search_mask, @@ -91,45 +80,6 @@ bool find_free_cpu_vector(const struct cpumask *search_mask, return true; } -static bool find_free_cpu_vectors(const struct cpumask *search_mask, int *found_cpu, int *found_vector, unsigned int nr_irqs) -{ - int i, vector, cpu; - bool found = false, find_once_global = false; - - cpu = cpumask_first(search_mask); -try_again: - for (vector = 0; vector < 256; vector++) { - for (i = 0; i < nr_irqs; i++) - if (per_cpu(vector_irq, cpu)[vector + i]) - break; - - if (i == nr_irqs) { - found = true; - *found_cpu = cpu; - *found_vector = vector; - return found; - } - - vector += i; - } - - cpu = cpumask_next(cpu, search_mask); - if (cpu < nr_cpu_ids) - goto try_again; - else { - if (find_once_global) { - printk("No global free vectors\n"); - return found; - } - printk("No local free vectors\n"); - search_mask = cpu_online_mask; - cpu = cpumask_first(search_mask); - find_once_global = true; - goto try_again; - } -} - -#ifdef CONFIG_UNCORE_XUELANG static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, int msiconf_index, int vector) { @@ -147,20 +97,16 @@ static unsigned long set_piu_msi_config(struct pci_controller *hose, int cpu, return msi_config; } -#endif static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) { struct sw64_msi_chip_data *cdata; + struct pci_controller *hose; struct irq_data *irqd; struct msi_desc *entry; struct cpumask searchmask; - unsigned long flags; + unsigned long flags, msi_config; int vector, cpu; - struct pci_controller *hose __maybe_unused; - unsigned long msi_config __maybe_unused; - int i __maybe_unused; - struct msi_msg msg __maybe_unused; /* Is this valid ? */ if (cpumask_any_and(cpumask, cpu_online_mask) >= nr_cpu_ids) @@ -185,37 +131,22 @@ static int sw64_set_affinity(struct irq_data *d, const struct cpumask *cpumask, raw_spin_lock_irqsave(&vector_lock, flags); cpumask_and(&searchmask, cpumask, cpu_online_mask); - if (cdata->multi_msi > 1) { - if (!find_free_cpu_vectors(&searchmask, &cpu, - &vector, cdata->multi_msi)) { - raw_spin_unlock_irqrestore(&vector_lock, flags); - return -ENOSPC; - } - } else { - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { - raw_spin_unlock_irqrestore(&vector_lock, flags); - return -ENOSPC; - } + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) { + raw_spin_unlock_irqrestore(&vector_lock, flags); + return -ENOSPC; } /* update new setting */ entry = irq_get_msi_desc(irqd->irq); - spin_lock(&cdata->cdata_lock); -#if defined(CONFIG_UNCORE_XUELANG) hose = (struct pci_controller *)msi_desc_to_pci_sysdata(entry); + spin_lock(&cdata->cdata_lock); per_cpu(vector_irq, cpu)[vector] = irqd->irq; msi_config = set_piu_msi_config(hose, cpu, cdata->msi_config_index, vector); - cdata->msi_config = msi_config; -#elif defined(CONFIG_UNCORE_JUNZHANG) - for (i = 0; i < cdata->multi_msi; i++) - per_cpu(vector_irq, cpu)[vector + i] = entry->irq + i; - BUG_ON(irq_chip_compose_msi_msg(irqd, &msg)); - __pci_write_msi_msg(entry, &msg); -#endif cdata->prev_vector = cdata->vector; cdata->prev_cpu = cdata->dst_cpu; cdata->dst_cpu = cpu; cdata->vector = vector; + cdata->msi_config = msi_config; cdata->move_in_progress = true; spin_unlock(&cdata->cdata_lock); cpumask_copy(irq_data_get_affinity_mask(irqd), &searchmask); @@ -240,24 +171,20 @@ static struct irq_chip pci_msi_controller = { }; static int __assign_irq_vector(int virq, unsigned int nr_irqs, - struct irq_domain *domain, struct pci_controller *hose, - enum irq_alloc_type type) + struct irq_domain *domain, struct pci_controller *hose) { struct irq_data *irq_data; const struct cpumask *mask; struct cpumask searchmask; struct sw64_msi_chip_data *cdata; - int node; + int msiconf_index, node; int i, vector, cpu; - unsigned long msi_config __maybe_unused; - unsigned long msiaddr __maybe_unused; - int msiconf_index __maybe_unused; - int start_index __maybe_unused; + unsigned long msi_config; + int start_index; if (unlikely((nr_irqs > 1) && (!is_power_of_2(nr_irqs)))) nr_irqs = __roundup_pow_of_two(nr_irqs); -#if defined(CONFIG_UNCORE_XUELANG) msiconf_index = bitmap_find_next_zero_area(hose->piu_msiconfig, 256, 0, nr_irqs, nr_irqs - 1); @@ -267,9 +194,6 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, } start_index = msiconf_index; -#elif defined(CONFIG_UNCORE_JUNZHANG) - msiaddr = read_piu_ior0(hose->node, hose->index, MSIADDR); -#endif irq_data = irq_domain_get_irq_data(domain, virq); BUG_ON(!irq_data); irq_data->chip = &pci_msi_controller; @@ -285,87 +209,48 @@ static int __assign_irq_vector(int virq, unsigned int nr_irqs, if (cpumask_first(&searchmask) >= nr_cpu_ids) cpumask_copy(&searchmask, cpu_online_mask); - if (type == IRQ_ALLOC_TYPE_MSI && nr_irqs > 1) { - if (!find_free_cpu_vectors(&searchmask, &cpu, - &vector, nr_irqs)) + for (i = 0; i < nr_irqs; i++) { + if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) return -ENOSPC; + per_cpu(vector_irq, cpu)[vector] = virq + i; + + if (i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + irq_data->chip = &pci_msi_controller; + } + cdata = alloc_sw_msi_chip_data(irq_data); if (!cdata) { printk("error alloc irq chip data\n"); return -ENOMEM; } - for (i = 0; i < nr_irqs; i++) { - per_cpu(vector_irq, cpu)[vector + i] = virq + i; - - if (i) { - irq_data = irq_domain_get_irq_data(domain, virq + i); - irq_data->chip = &pci_msi_controller; - } - - irq_data->chip_data = cdata; - } + irq_data->chip_data = cdata; + msiconf_index = start_index + i; + msi_config = set_piu_msi_config(hose, cpu, msiconf_index, vector); cdata->dst_cpu = cpu; cdata->vector = vector; - cdata->msiaddr = msiaddr; + cdata->rc_index = hose->index; + cdata->rc_node = hose->node; + cdata->msi_config = msi_config; + cdata->msi_config_index = msiconf_index; cdata->prev_cpu = cpu; cdata->prev_vector = vector; - cdata->multi_msi = nr_irqs; cdata->move_in_progress = false; - printk("TYPE_MSI cpu:%#x vector:%#x msiaddr:%#lx virq:%d\n", - cpu, vector, msiaddr, virq); - } else { - for (i = 0; i < nr_irqs; i++) { - if (!find_free_cpu_vector(&searchmask, &cpu, &vector)) - return -ENOSPC; - - per_cpu(vector_irq, cpu)[vector] = virq + i; - - if (i) { - irq_data = irq_domain_get_irq_data(domain, virq + i); - irq_data->chip = &pci_msi_controller; - } - - cdata = alloc_sw_msi_chip_data(irq_data); - if (!cdata) { - printk("error alloc irq chip data\n"); - return -ENOMEM; - } - - irq_data->chip_data = cdata; - - cdata->dst_cpu = cpu; - cdata->vector = vector; - cdata->rc_index = hose->index; - cdata->rc_node = hose->node; -#if defined(CONFIG_UNCORE_XUELANG) - msiconf_index = start_index + i; - msi_config = set_piu_msi_config(hose, cpu, msiconf_index, vector); - cdata->msi_config = msi_config; - cdata->msi_config_index = msiconf_index; -#elif defined(CONFIG_UNCORE_JUNZHANG) - cdata->msiaddr = msiaddr; -#endif - cdata->prev_cpu = cpu; - cdata->prev_vector = vector; - cdata->multi_msi = 1; - cdata->move_in_progress = false; - } } return 0; } static int assign_irq_vector(int irq, unsigned int nr_irqs, - struct irq_domain *domain, struct pci_controller *hose, - enum irq_alloc_type type) + struct irq_domain *domain, struct pci_controller *hose) { int err; unsigned long flags; raw_spin_lock_irqsave(&vector_lock, flags); - err = __assign_irq_vector(irq, nr_irqs, domain, hose, type); + err = __assign_irq_vector(irq, nr_irqs, domain, hose); raw_spin_unlock_irqrestore(&vector_lock, flags); return err; } @@ -373,33 +258,28 @@ static int assign_irq_vector(int irq, unsigned int nr_irqs, static void sw64_vector_free_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { - int i, j; + int i; struct irq_data *irq_data; unsigned long flags; - struct msi_desc *entry __maybe_unused; - struct pci_controller *hose __maybe_unused; for (i = 0; i < nr_irqs; i++) { irq_data = irq_domain_get_irq_data(domain, virq + i); if (irq_data && irq_data->chip_data) { struct sw64_msi_chip_data *cdata; + struct msi_desc *entry; + struct pci_controller *hose; raw_spin_lock_irqsave(&vector_lock, flags); cdata = irq_data->chip_data; -#ifdef CONFIG_UNCORE_XUELANG entry = irq_get_msi_desc(virq + i); if (entry) { hose = (struct pci_controller *)msi_desc_to_pci_sysdata(entry); clear_bit(cdata->msi_config_index, hose->piu_msiconfig); } -#endif irq_domain_reset_irq_data(irq_data); - for (j = 0; j < cdata->multi_msi; j++) - per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector + j] = 0; + per_cpu(vector_irq, cdata->dst_cpu)[cdata->vector] = 0; kfree(cdata); raw_spin_unlock_irqrestore(&vector_lock, flags); - if (cdata->multi_msi > 1) - break; } } } @@ -434,13 +314,11 @@ static int sw64_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, int err; struct irq_alloc_info *info = arg; struct pci_controller *hose; - enum irq_alloc_type msi_type; if (arg == NULL) return -ENODEV; hose = info->msi_dev->sysdata; - msi_type = info->type; - err = assign_irq_vector(virq, nr_irqs, domain, hose, msi_type); + err = assign_irq_vector(virq, nr_irqs, domain, hose); if (err) goto error; return 0; @@ -525,39 +403,23 @@ static void irq_move_complete(struct sw64_msi_chip_data *cdata, int cpu, int vec { if (likely(!cdata->move_in_progress)) return; -#if defined(CONFIG_UNCORE_XUELANG) if (vector == cdata->vector && cdata->dst_cpu == cpu) { raw_spin_lock(&vector_lock); cdata->move_in_progress = 0; per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector] = 0; raw_spin_unlock(&vector_lock); } -#elif defined(UNCORE_JUNZHANG) - if (cdata->dst_cpu == cpu) { - if (vector >= cdata->vector && - vector < cdata->vector + cdata->multi_msi) { - int i; - - raw_spin_lock(&vector_lock); - cdata->move_in_progress = false; - for (i = 0; i < cdata->multi_msi; i++) - per_cpu(vector_irq, cdata->prev_cpu)[cdata->prev_vector + i] = 0; - raw_spin_unlock(&vector_lock); - } - } -#endif } void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned long pci_msi1_addr) { - int i, irq, msi_index = 0; + int i, irq, piu_index, msi_index = 0; int cpu, vector_index = 0; + unsigned long value = 0; unsigned long int_pci_msi[3]; unsigned long *ptr; struct irq_data *irq_data; struct sw64_msi_chip_data *cdata; - int piu_index __maybe_unused; - unsigned long value __maybe_unused = 0; if (is_guest_or_emul()) { cpu = smp_processor_id(); @@ -576,8 +438,6 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned for (i = 0; i < 4; i++) { vector_index = i * 64; while (vector != 0) { - int irq = 0; - msi_index = find_next_bit(&vector, 64, msi_index); if (msi_index == 64) { msi_index = 0; @@ -589,11 +449,9 @@ void handle_pci_msi_interrupt(unsigned long type, unsigned long vector, unsigned cdata = irq_data_get_irq_chip_data(irq_data); spin_lock(&cdata->cdata_lock); irq_move_complete(cdata, cpu, vector_index + msi_index); -#ifdef CONFIG_UNCORE_XUELANG piu_index = cdata->msi_config_index; value = cdata->msi_config | (1UL << 63); write_piu_ior0(cdata->rc_node, cdata->rc_index, MSICONFIG0 + (piu_index << 7), value); -#endif spin_unlock(&cdata->cdata_lock); handle_irq(irq); -- Gitee From 1bc99f566c4dc5e31ea1d7c3006fd637ced44397 Mon Sep 17 00:00:00 2001 From: Xu Yiwei Date: Tue, 14 Nov 2023 02:14:01 +0000 Subject: [PATCH 143/150] sw64: iommu: adjust iova map space Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Device Passthrough on QEMU is causing iova overlaps on an unexpected address, which happens to be the lower boundary of our BAR address. This is caused by QEMU visiting that address intentionally during its hot-plug initalization. The quickest way to fix this is by including that address in our current IOVA check boundary, so applications will stop mapping IOVA on that address. Signed-off-by: Xu Yiwei Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/iommu/sw64/sunway_iommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/sw64/sunway_iommu.c b/drivers/iommu/sw64/sunway_iommu.c index 65d0ed8414fe..77780c608a00 100644 --- a/drivers/iommu/sw64/sunway_iommu.c +++ b/drivers/iommu/sw64/sunway_iommu.c @@ -1464,7 +1464,7 @@ sunway_iommu_iova_to_phys(struct iommu_domain *dom, dma_addr_t iova) struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); unsigned long paddr, grn; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return iova; paddr = fetch_pte(sdomain, iova, PTE_LEVEL2_VAL); @@ -1501,7 +1501,7 @@ sunway_iommu_map(struct iommu_domain *dom, unsigned long iova, * and pci device BAR, check should be introduced manually * to avoid VFIO trying to map pci config space. */ - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return 0; mutex_lock(&sdomain->api_lock); @@ -1519,7 +1519,7 @@ sunway_iommu_unmap(struct iommu_domain *dom, unsigned long iova, struct sunway_iommu_domain *sdomain = to_sunway_domain(dom); size_t unmap_size; - if (iova > SW64_BAR_ADDRESS) + if (iova >= SW64_BAR_ADDRESS) return page_size; mutex_lock(&sdomain->api_lock); -- Gitee From 25664756199c095224dfc3b387137c2bb49b3357 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 9 Oct 2023 10:16:32 +0800 Subject: [PATCH 144/150] sw64: fix kernel crash caused by unaligned access exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sunway inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- If ACPI enabled, unaligned access occurs when parsing ACPI tables in function acpi_boot_table_init(). since SW64 does not support handling unaligned access exception via hardware, the related trap entry should be initialized first. A feasible solution for this issue is that BIOS register the trap entry and make it as a runtime service. But this solution causes the kernel to have an ugly dependence on BIOS. To address this issue within the kernel, this commit invoke trap_init() at the beginning of setup_arch(). BTW, there is a redundant call to the function trap_init() for the bootstrap core, which may be deprecated some day. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/setup.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/sw_64/kernel/setup.c b/arch/sw_64/kernel/setup.c index ec8980e3c0df..9118c440bd16 100644 --- a/arch/sw_64/kernel/setup.c +++ b/arch/sw_64/kernel/setup.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -764,6 +765,12 @@ void __init sw64_kvm_reserve(void) void __init setup_arch(char **cmdline_p) { + /** + * Work around the unaligned access exception to parse ACPI + * tables in the following function acpi_boot_table_init(). + */ + trap_init(); + jump_label_init(); setup_cpu_info(); setup_run_mode(); -- Gitee From dcfe32c6ab94876d1b6edbab0568644d4e0d0fac Mon Sep 17 00:00:00 2001 From: Sergio Paracuellos Date: Tue, 7 Dec 2021 11:49:20 +0100 Subject: [PATCH 145/150] pci: let pcibios_root_bridge_prepare() access bridge->windows Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- This is backport of commit 661c4c4f2693("PCI: Let pci_bios_root_bridge_prepare() access bridge->windows") When pci_register_host_bridge() is called, bridge->windows are already available. However these windows are being moved temporarily from there. To let pcibios_root_bridge_prepare() have access to these windows, move the windows movement after calling this function. This is useful for the MIPS ralink mt7621 platform so it can set up I/O coherence units and avoid custom MIPS code in the mt7621 PCIe controller driver. Link: https://lore.kernel.org/r/20211207104924.21327-2-sergio.paracuellos@gmail.com Signed-off-by: Sergio Paracuellos Signed-off-by: Bjorn Helgaas Acked-by: Arnd Bergmann Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/pci/probe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 5a926c883a89..63c02b346e93 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -891,8 +891,6 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) bridge->bus = bus; - /* Temporarily move resources off the list */ - list_splice_init(&bridge->windows, &resources); bus->sysdata = bridge->sysdata; bus->msi = bridge->msi; bus->ops = bridge->ops; @@ -917,6 +915,8 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge) if (err) goto free; + /* Temporarily move resources off the list */ + list_splice_init(&bridge->windows, &resources); err = device_add(&bridge->dev); if (err) { put_device(&bridge->dev); -- Gitee From e52aabcaad53ec67df03b9d6ac89085e220baf38 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 18 Sep 2023 15:15:35 +0800 Subject: [PATCH 146/150] sw64: acpi: disable all ACPI features by default Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Set the default values of acpi_noirq and acpi_pci_disabled to be the same as acpi_disabled, making all ACPI features disabled by default. This change also makes the control logic of ACPI features clearer. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/acpi.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 61f2948f1781..586465425341 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -8,8 +8,9 @@ int acpi_disabled = 1; EXPORT_SYMBOL(acpi_disabled); -int acpi_noirq; /* skip ACPI IRQ initialization */ -int acpi_pci_disabled; /* skip ACPI PCI scan and IRQ initialization */ + +int acpi_noirq = 1; /* skip ACPI IRQ initialization */ +int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); int acpi_strict; u64 arch_acpi_wakeup_start; -- Gitee From ba29c4ce89454d27f29df45d6fbae6d45a7c36c0 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Mon, 18 Sep 2023 15:23:00 +0800 Subject: [PATCH 147/150] sw64: acpi: use early param to control whether ACPI is enabled Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Using early param "acpi=on" to enable ACPI. If "acpi=on" is not explicitly passed or "acpi=off" is passed, ACPI will be disabled. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/include/asm/acpi.h | 14 +++++++++++++- arch/sw_64/kernel/acpi.c | 34 ++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/arch/sw_64/include/asm/acpi.h b/arch/sw_64/include/asm/acpi.h index 38615b969555..a74db7667e09 100644 --- a/arch/sw_64/include/asm/acpi.h +++ b/arch/sw_64/include/asm/acpi.h @@ -43,6 +43,7 @@ extern int acpi_pci_disabled; ACPI_PDC_C_C2C3_FFH) #define ACPI_TABLE_UPGRADE_MAX_PHYS (max_low_pfn_mapped << PAGE_SHIFT) + static inline void disable_acpi(void) { acpi_disabled = 1; @@ -50,7 +51,18 @@ static inline void disable_acpi(void) acpi_noirq = 1; } -static inline void acpi_noirq_set(void) { acpi_noirq = 1; } +static inline void enable_acpi(void) +{ + acpi_disabled = 0; + acpi_pci_disabled = 0; + acpi_noirq = 0; +} + +static inline void acpi_noirq_set(void) +{ + acpi_noirq = 1; +} + static inline void acpi_disable_pci(void) { acpi_pci_disabled = 1; diff --git a/arch/sw_64/kernel/acpi.c b/arch/sw_64/kernel/acpi.c index 586465425341..71c28b0fc735 100644 --- a/arch/sw_64/kernel/acpi.c +++ b/arch/sw_64/kernel/acpi.c @@ -12,6 +12,10 @@ EXPORT_SYMBOL(acpi_disabled); int acpi_noirq = 1; /* skip ACPI IRQ initialization */ int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); + +static bool param_acpi_on __initdata; +static bool param_acpi_off __initdata; + int acpi_strict; u64 arch_acpi_wakeup_start; u64 acpi_saved_sp_s3; @@ -116,13 +120,14 @@ static int __init parse_acpi(char *arg) if (!arg) return -EINVAL; - /* "acpi=off" disables both ACPI table parsing and interpreter */ - if (strcmp(arg, "off") == 0) { - disable_acpi(); - } else { - /* Core will printk when we return error. */ - return -EINVAL; - } + /* disable both ACPI table parsing and interpreter */ + if (strcmp(arg, "off") == 0) + param_acpi_off = true; + else if (strcmp(arg, "on") == 0) /* prefer ACPI over device tree */ + param_acpi_on = true; + else + return -EINVAL; /* Core will printk when we return error. */ + return 0; } early_param("acpi", parse_acpi); @@ -365,16 +370,25 @@ EXPORT_SYMBOL(acpi_unmap_cpu); #endif /* CONFIG_ACPI_HOTPLUG_CPU */ void __init acpi_boot_table_init(void) - { + /** + * ACPI is disabled by default. + * ACPI is only enabled when firmware passes ACPI table + * and sets boot parameter "acpi=on". + */ + if (param_acpi_on) + enable_acpi(); + /* * If acpi_disabled, bail out */ if (!acpi_disabled) { + pr_warn("Currently, ACPI is an experimental feature!\n"); if (acpi_table_init()) { pr_err("Failed to init ACPI tables\n"); disable_acpi(); - } - pr_info("Enable ACPI support\n"); + } else + pr_info("Successfully parsed ACPI table\n"); } } + -- Gitee From 2b995cf3d977bdcadb9ca635be3a79a88ac3a96b Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 13 Sep 2023 19:56:00 +0800 Subject: [PATCH 148/150] sw64: pci: bypass legacy PCI initialization when ACPI enabled Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- To avoid conflicts between legacy PCI initialization and ACPI-PCI initialization, just bypass legacy PCI initialization when ACPI enabled. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/kernel/pci.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/sw_64/kernel/pci.c b/arch/sw_64/kernel/pci.c index 8bd7afc3464a..6e635c05bbac 100644 --- a/arch/sw_64/kernel/pci.c +++ b/arch/sw_64/kernel/pci.c @@ -169,7 +169,8 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, static int __init pcibios_init(void) { - sw64_init_pci(); + if (acpi_disabled) + sw64_init_pci(); return 0; } subsys_initcall(pcibios_init); @@ -644,7 +645,6 @@ void __init sw64_init_arch(void) char id[8], msg[64]; int i; - pr_info("SW arch PCI initialize!\n"); cpu_num = sw64_chip->get_cpu_num(); for (node = 0; node < cpu_num; node++) { @@ -652,6 +652,13 @@ void __init sw64_init_arch(void) set_devint_wken(node); set_adr_int(node); } + } + + if (!acpi_disabled) + return; + + pr_info("SW arch PCI initialize!\n"); + for (node = 0; node < cpu_num; node++) { rc_enable = sw64_chip_init->pci_init.get_rc_enable(node); if (rc_enable == 0) { printk("PCIe is disabled on node %ld\n", node); -- Gitee From 4a5f74fa5049fba9d68fa603957f6655806e4165 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Wed, 13 Sep 2023 20:41:21 +0800 Subject: [PATCH 149/150] sw64: acpi: disable INTC initialization via ACPI MADT table Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- Currently, ACPI MADT table for SW64 is not supported. Remove related codes to avoid erroneous interrupt controller initialization. Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- drivers/irqchip/irq-sw64-intc-v2.c | 60 ------------------------------ 1 file changed, 60 deletions(-) diff --git a/drivers/irqchip/irq-sw64-intc-v2.c b/drivers/irqchip/irq-sw64-intc-v2.c index 070c12ede959..bc2c8ef3ed2f 100644 --- a/drivers/irqchip/irq-sw64-intc-v2.c +++ b/drivers/irqchip/irq-sw64-intc-v2.c @@ -34,66 +34,6 @@ static const struct irq_domain_ops sw64_intc_domain_ops = { .map = sw64_intc_domain_map, }; -#ifdef CONFIG_ACPI -static int __init -intc_parse_madt(union acpi_subtable_headers *header, - const unsigned long end) -{ - struct acpi_madt_io_sapic *its_entry; - static struct irq_domain *root_domain; - int intc_irqs = 8, irq_base = NR_IRQS_LEGACY; - irq_hw_number_t hwirq_base = 0; - int irq_start = -1; - - its_entry = (struct acpi_madt_io_sapic *)header; - - intc_irqs -= hwirq_base; /* calculate # of irqs to allocate */ - - irq_base = irq_alloc_descs(irq_start, 16, intc_irqs, - numa_node_id()); - if (irq_base < 0) { - WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n", - irq_start); - irq_base = irq_start; - } - - root_domain = irq_domain_add_legacy(NULL, intc_irqs, irq_base, - hwirq_base, &sw64_intc_domain_ops, NULL); - - if (!root_domain) - pr_err("Failed to create irqdomain"); - - irq_set_default_host(root_domain); - - sw64_io_write(0, MCU_DVC_INT_EN, 0xff); - - return 0; -} - -static int __init acpi_intc_init(void) -{ - int count = 0; - - count = acpi_table_parse_madt(ACPI_MADT_TYPE_IO_SAPIC, - intc_parse_madt, 0); - - if (count <= 0) { - pr_err("No valid intc entries exist\n"); - return -EINVAL; - } - return 0; -} - -static int __init intc_init(void) -{ - acpi_intc_init(); - - return 0; -} - -subsys_initcall(intc_init); -#endif - #ifdef CONFIG_OF static struct irq_domain *root_domain; -- Gitee From b606868aeea0c61c119cdb828d25f366747d1707 Mon Sep 17 00:00:00 2001 From: Jing Li Date: Fri, 1 Sep 2023 17:48:29 +0800 Subject: [PATCH 150/150] sw64: pci: add new files to help support ACPI-based PCI initialization Sunway inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IBDJNZ -------------------------------- To avoid break legacy codes, add the following new files to help support ACPI-based PCI initialization: - acpi.c : pci-acpi related codes - pci.c : sw64 pci common codes Signed-off-by: Jing Li Reviewed-by: He Sheng Signed-off-by: Gu Zitao --- arch/sw_64/Makefile | 1 + arch/sw_64/pci/Makefile | 8 ++++++++ arch/sw_64/pci/acpi.c | 7 +++++++ arch/sw_64/pci/pci.c | 8 ++++++++ 4 files changed, 24 insertions(+) create mode 100644 arch/sw_64/pci/Makefile create mode 100644 arch/sw_64/pci/acpi.c create mode 100644 arch/sw_64/pci/pci.c diff --git a/arch/sw_64/Makefile b/arch/sw_64/Makefile index 57b899929b83..7b2ef781b1b1 100644 --- a/arch/sw_64/Makefile +++ b/arch/sw_64/Makefile @@ -40,6 +40,7 @@ head-y := arch/sw_64/kernel/head.o core-y += arch/sw_64/ drivers-$(CONFIG_OPROFILE) += arch/sw_64/oprofile/ +drivers-$(CONFIG_PCI) += arch/sw_64/pci/ libs-y += arch/sw_64/lib/ # export what is needed by arch/sw_64/boot/Makefile diff --git a/arch/sw_64/pci/Makefile b/arch/sw_64/pci/Makefile new file mode 100644 index 000000000000..abc6553fd2c7 --- /dev/null +++ b/arch/sw_64/pci/Makefile @@ -0,0 +1,8 @@ +PDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux kernel. +# + +obj-y += pci.o +obj-$(CONFIG_ACPI) += acpi.o + diff --git a/arch/sw_64/pci/acpi.c b/arch/sw_64/pci/acpi.c new file mode 100644 index 000000000000..c8666b9d82ae --- /dev/null +++ b/arch/sw_64/pci/acpi.c @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + diff --git a/arch/sw_64/pci/pci.c b/arch/sw_64/pci/pci.c new file mode 100644 index 000000000000..510f3e72de75 --- /dev/null +++ b/arch/sw_64/pci/pci.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#include +#include + -- Gitee