From b3c1b47b493f6b786707163fa9d2ac866828725a Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 09:20:16 +0800 Subject: [PATCH 01/16] LoongArch: Add kexec support LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Add three new files, kexec.h, machine_kexec.c and relocate_kernel.S to the LoongArch architecture, so as to add support for the kexec re-boot mechanism (CONFIG_KEXEC) on LoongArch platforms. Kexec supports loading vmlinux.elf in ELF format and vmlinux.efi in PE format. I tested kexec on LoongArch machines (Loongson-3A5000) and it works as expected: $ sudo kexec -l /boot/vmlinux.efi --reuse-cmdline $ sudo kexec -e Signed-off-by: Youling Tang Change-Id: I09ee6548f45f899736a9579845626e93f49760c0 --- arch/loongarch/Kconfig | 11 ++ arch/loongarch/include/asm/kexec.h | 60 +++++++ arch/loongarch/kernel/Makefile | 2 + arch/loongarch/kernel/head.S | 6 +- arch/loongarch/kernel/machine_kexec.c | 216 ++++++++++++++++++++++++ arch/loongarch/kernel/relocate_kernel.S | 106 ++++++++++++ 6 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 arch/loongarch/include/asm/kexec.h create mode 100644 arch/loongarch/kernel/machine_kexec.c create mode 100644 arch/loongarch/kernel/relocate_kernel.S diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 34eff7c52582..5a7b9049a6b3 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -463,6 +463,17 @@ config ARCH_IOREMAP protection support. However, you can enable LoongArch DMW-based ioremap() for better performance. +config KEXEC + bool "Kexec system call" + select KEXEC_CORE + help + kexec is a system call that implements the ability to shutdown your + current kernel, and to start another kernel. It is like a reboot + but it is independent of the system firmware. And like a reboot + you can start any kernel with it, not just Linux. + + The name comes from the similarity to the exec system call. + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/include/asm/kexec.h b/arch/loongarch/include/asm/kexec.h new file mode 100644 index 000000000000..cf95cd3eb2de --- /dev/null +++ b/arch/loongarch/include/asm/kexec.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * kexec.h for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_KEXEC_H +#define _ASM_KEXEC_H + +#include +#include + +/* Maximum physical address we can use pages from */ +#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) +/* Maximum address we can reach in physical address mode */ +#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL) + /* Maximum address we can use for the control code buffer */ +#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL) + +/* Reserve a page for the control code buffer */ +#define KEXEC_CONTROL_PAGE_SIZE PAGE_SIZE + +/* The native architecture */ +#define KEXEC_ARCH KEXEC_ARCH_LOONGARCH + +static inline void crash_setup_regs(struct pt_regs *newregs, + struct pt_regs *oldregs) +{ + if (oldregs) + memcpy(newregs, oldregs, sizeof(*newregs)); + else + prepare_frametrace(newregs); +} + +#define ARCH_HAS_KIMAGE_ARCH + +struct kimage_arch { + unsigned long efi_boot; + unsigned long cmdline_ptr; + unsigned long systable_ptr; +}; + +typedef void (*do_kexec_t)(unsigned long efi_boot, + unsigned long cmdline_ptr, + unsigned long systable_ptr, + unsigned long start_addr, + unsigned long first_ind_entry); + +struct kimage; +extern const unsigned char relocate_new_kernel[]; +extern const size_t relocate_new_kernel_size; +extern void kexec_reboot(void); + +#ifdef CONFIG_SMP +extern atomic_t kexec_ready_to_reboot; +extern const unsigned char kexec_smp_wait[]; +#endif + +#endif /* !_ASM_KEXEC_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 6c940e41d676..6aad35760644 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -26,6 +26,8 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o + obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 58254fd1999d..5536ffc7825c 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -20,7 +20,11 @@ _head: .word MZ_MAGIC /* "MZ", MS-DOS header */ - .org 0x3c /* 0x04 ~ 0x3b reserved */ + .org 0x8 + .dword kernel_entry /* Kernel entry point */ + .dword _end - _text /* Kernel image effective size */ + .quad 0 /* Kernel image load offset from start of RAM */ + .org 0x3c /* 0x20 ~ 0x3b reserved */ .long pe_header - _head /* Offset to the PE header */ pe_header: diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c new file mode 100644 index 000000000000..d5037573ed66 --- /dev/null +++ b/arch/loongarch/kernel/machine_kexec.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * machine_kexec.c for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* 0x100000 ~ 0x200000 is safe */ +#define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL) +#define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL) + +static unsigned long reboot_code_buffer; + +#ifdef CONFIG_SMP +static void (*relocated_kexec_smp_wait)(void *); +atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); +#endif + +static unsigned long efi_boot; +static unsigned long cmdline_ptr; +static unsigned long systable_ptr; +static unsigned long start_addr; +static unsigned long first_ind_entry; + +static void kexec_image_info(const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("kexec kimage info:\n"); + pr_debug("\ttype: %d\n", kimage->type); + pr_debug("\tstart: %lx\n", kimage->start); + pr_debug("\thead: %lx\n", kimage->head); + pr_debug("\tnr_segments: %lu\n", kimage->nr_segments); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug("\t segment[%lu]: %016lx - %016lx", i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz); + pr_debug("\t\t0x%lx bytes, %lu pages\n", + (unsigned long)kimage->segment[i].memsz, + (unsigned long)kimage->segment[i].memsz / PAGE_SIZE); + } +} + +int machine_kexec_prepare(struct kimage *kimage) +{ + int i; + char *bootloader = "kexec"; + void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; + + kexec_image_info(kimage); + + kimage->arch.efi_boot = fw_arg0; + kimage->arch.systable_ptr = fw_arg2; + + /* Find the command line */ + for (i = 0; i < kimage->nr_segments; i++) { + if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { + if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) + kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; + break; + } + } + + if (!kimage->arch.cmdline_ptr) { + pr_err("Command line not included in the provided image\n"); + return -EINVAL; + } + + /* kexec need a safe page to save reboot_code_buffer */ + kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE); + + reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page); + memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); + +#ifdef CONFIG_SMP + /* All secondary cpus now may jump to kexec_smp_wait cycle */ + relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel); +#endif + + return 0; +} + +void machine_kexec_cleanup(struct kimage *kimage) +{ +} + +void kexec_reboot(void) +{ + do_kexec_t do_kexec = NULL; + + /* + * We know we were online, and there will be no incoming IPIs at + * this point. + */ + set_cpu_online(smp_processor_id(), true); + + /* Ensure remote CPUs observe that we're online before rebooting. */ + smp_mb__after_atomic(); + + /* + * Make sure we get correct instructions written by the + * machine_kexec_prepare() CPU. + */ + __asm__ __volatile__ ("\tibar 0\n"::); + +#ifdef CONFIG_SMP + /* All secondary cpus go to kexec_smp_wait */ + if (smp_processor_id() > 0) { + relocated_kexec_smp_wait(NULL); + unreachable(); + } +#endif + + do_kexec = (void *)reboot_code_buffer; + do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry); + + unreachable(); +} + + +#ifdef CONFIG_SMP +static void kexec_shutdown_secondary(void *regs) +{ + int cpu = smp_processor_id(); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} +#endif + +void machine_shutdown(void) +{ + int cpu; + + /* All CPUs go to reboot_code_buffer */ + for_each_possible_cpu(cpu) + if (!cpu_online(cpu)) + cpu_device_up(get_cpu_device(cpu)); + +#ifdef CONFIG_SMP + smp_call_function(kexec_shutdown_secondary, NULL, 0); +#endif +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ +} + +void machine_kexec(struct kimage *image) +{ + unsigned long entry, *ptr; + struct kimage_arch *internal = &image->arch; + + efi_boot = internal->efi_boot; + cmdline_ptr = internal->cmdline_ptr; + systable_ptr = internal->systable_ptr; + + start_addr = (unsigned long)phys_to_virt(image->start); + + first_ind_entry = (unsigned long)phys_to_virt(image->head & PAGE_MASK); + + /* + * The generic kexec code builds a page list with physical + * addresses. they are directly accessible through XKPRANGE + * hence the phys_to_virt() call. + */ + for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); + ptr = (entry & IND_INDIRECTION) ? + phys_to_virt(entry & PAGE_MASK) : ptr + 1) { + if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION || + *ptr & IND_DESTINATION) + *ptr = (unsigned long) phys_to_virt(*ptr); + } + + /* Mark offline before disabling local irq. */ + set_cpu_online(smp_processor_id(), false); + + /* We do not want to be bothered. */ + local_irq_disable(); + + pr_notice("EFI boot flag 0x%lx\n", efi_boot); + pr_notice("Command line at 0x%lx\n", cmdline_ptr); + pr_notice("System table at 0x%lx\n", systable_ptr); + pr_notice("We will call new kernel at 0x%lx\n", start_addr); + pr_notice("Bye ...\n"); + + /* Make reboot code buffer available to the boot CPU. */ + flush_cache_all(); + +#ifdef CONFIG_SMP + atomic_set(&kexec_ready_to_reboot, 1); +#endif + + kexec_reboot(); +} diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S new file mode 100644 index 000000000000..48362f38c6bf --- /dev/null +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * relocate_kernel.S for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#include + +#include +#include +#include +#include +#include +#include + +SYM_CODE_START(relocate_new_kernel) + /* + * a0: EFI boot flag for the new kernel + * a1: Command line pointer for the new kernel + * a2: System table pointer for the new kernel + * a3: Start address to jump to after relocation + * a4: Pointer to the current indirection page entry + */ + move s0, a4 + +process_entry: + PTR_L s1, s0, 0 + PTR_ADDI s0, s0, SZREG + + /* destination page */ + andi s2, s1, IND_DESTINATION + beqz s2, 1f + li.w t0, ~0x1 + and s3, s1, t0 /* store destination addr in s3 */ + b process_entry + +1: + /* indirection page, update s0 */ + andi s2, s1, IND_INDIRECTION + beqz s2, 1f + li.w t0, ~0x2 + and s0, s1, t0 + b process_entry + +1: + /* done page */ + andi s2, s1, IND_DONE + beqz s2, 1f + b done + +1: + /* source page */ + andi s2, s1, IND_SOURCE + beqz s2, process_entry + li.w t0, ~0x8 + and s1, s1, t0 + li.w s5, (1 << _PAGE_SHIFT) / SZREG + +copy_word: + /* copy page word by word */ + REG_L s4, s1, 0 + REG_S s4, s3, 0 + PTR_ADDI s3, s3, SZREG + PTR_ADDI s1, s1, SZREG + LONG_ADDI s5, s5, -1 + beqz s5, process_entry + b copy_word + b process_entry + +done: + ibar 0 + dbar 0 + + /* + * Jump to the new kernel, + * make sure the values of a0, a1, a2 and a3 are not changed. + */ + jr a3 +SYM_CODE_END(relocate_new_kernel) + +#ifdef CONFIG_SMP +/* + * Other CPUs should wait until code is relocated and + * then start at the entry point from LOONGARCH_IOCSR_MBUF0. + */ +SYM_CODE_START(kexec_smp_wait) +1: li.w t0, 0x100 /* wait for init loop */ +2: addi.w t0, t0, -1 /* limit mailbox access */ + bnez t0, 2b + li.w t1, LOONGARCH_IOCSR_MBUF0 + iocsrrd.w s0, t1 /* check PC as an indicator */ + beqz s0, 1b + iocsrrd.d s0, t1 /* get PC via mailbox */ + + li.d t0, CACHE_BASE + or s0, s0, t0 /* s0 = TO_CACHE(s0) */ + jr s0 /* jump to initial PC */ +SYM_CODE_END(kexec_smp_wait) +#endif + +relocate_new_kernel_end: + +SYM_DATA_START(relocate_new_kernel_size) + PTR relocate_new_kernel_end - relocate_new_kernel +SYM_DATA_END(relocate_new_kernel_size) -- Gitee From 530b3d21d029677757d3da04c491cc8504f83ab5 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 09:20:56 +0800 Subject: [PATCH 02/16] LoongArch: Add kdump support LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- This patch adds support for kdump. In kdump case the normal kernel will reserve a region for the crash kernel and jump there on panic. Arch-specific functions are added to allow for implementing a crash dump file interface, /proc/vmcore, which can be viewed as a ELF file. A user-space tool, such as kexec-tools, is responsible for allocating a separate region for the core's ELF header within the crash kdump kernel memory and filling it in when executing kexec_load(). Then, its location will be advertised to the crash dump kernel via a command line argument "elfcorehdr=", and the crash dump kernel will preserve this region for later use with arch_reserve_vmcore() at boot time. At the same time, the crash kdump kernel is also limited within the "crashkernel" area via a command line argument "mem=", so as not to destroy the original kernel dump data. In the crash dump kernel environment, /proc/vmcore is used to access the primary kernel's memory with copy_oldmem_page(). I tested kdump on LoongArch machines (Loongson-3A5000) and it works as expected (suggested crashkernel parameter is "crashkernel=512M@2560M"), you may test it by triggering a crash through /proc/sysrq-trigger: $ sudo kexec -p /boot/vmlinux-kdump --reuse-cmdline --append="nr_cpus=1" # echo c > /proc/sysrq-trigger Signed-off-by: Youling Tang Change-Id: I2afa7501ad3108a18a3dacdc742bf9ca239b6745 --- arch/loongarch/Kconfig | 22 ++++++ arch/loongarch/Makefile | 4 + arch/loongarch/kernel/Makefile | 1 + arch/loongarch/kernel/crash_dump.c | 30 ++++++++ arch/loongarch/kernel/machine_kexec.c | 98 +++++++++++++++++++++++-- arch/loongarch/kernel/mem.c | 3 - arch/loongarch/kernel/relocate_kernel.S | 6 ++ arch/loongarch/kernel/setup.c | 74 +++++++++++++++++++ arch/loongarch/kernel/traps.c | 4 + 9 files changed, 234 insertions(+), 8 deletions(-) create mode 100644 arch/loongarch/kernel/crash_dump.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 5a7b9049a6b3..b68aea70b563 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -474,6 +474,28 @@ config KEXEC The name comes from the similarity to the exec system call. +config CRASH_DUMP + bool "Build kdump crash kernel" + help + Generate crash dump after being started by kexec. This should + be normally only set in special crash dump kernels which are + loaded in the main kernel with kexec-tools into a specially + reserved region and then later executed after a crash by + kdump/kexec. + + For more details see Documentation/admin-guide/kdump/kdump.rst + +config PHYSICAL_START + hex "Physical address where the kernel is loaded" + default "0x90000000a0000000" + depends on CRASH_DUMP + help + This gives the XKPRANGE address where the kernel is loaded. + If you plan to use kernel for capturing the crash dump change + this value to start of the reserved region (the "X" value as + specified in the "crashkernel=YM@XM" command line boot parameter + passed to the panic-ed kernel). + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index cb14e7f96401..1c5fe78ab973 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -66,7 +66,11 @@ endif cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) +ifndef CONFIG_PHYSICAL_START load-y = 0x9000000000200000 +else +load-y = $(CONFIG_PHYSICAL_START) +endif bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) drivers-$(CONFIG_PCI) += arch/loongarch/pci/ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 6aad35760644..4ed6981bf8ea 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -27,6 +27,7 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o diff --git a/arch/loongarch/kernel/crash_dump.c b/arch/loongarch/kernel/crash_dump.c new file mode 100644 index 000000000000..db280e4b1ec1 --- /dev/null +++ b/arch/loongarch/kernel/crash_dump.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +ssize_t copy_oldmem_page(unsigned long pfn, char *buf, + size_t csize, unsigned long offset, int userbuf) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB); + if (!vaddr) + return -ENOMEM; + + if (!userbuf) { + memcpy(buf, vaddr + offset, csize); + } else { + if (copy_to_user(buf, vaddr + offset, csize)) { + memunmap(vaddr); + csize = -EFAULT; + } + } + + memunmap(vaddr); + + return csize; +} diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c index d5037573ed66..2dcb9e003657 100644 --- a/arch/loongarch/kernel/machine_kexec.c +++ b/arch/loongarch/kernel/machine_kexec.c @@ -7,10 +7,15 @@ #include #include #include -#include +#include #include +#include #include +#include #include +#include +#include +#include #include #include @@ -21,6 +26,7 @@ #define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL) static unsigned long reboot_code_buffer; +static cpumask_t cpus_in_crash = CPU_MASK_NONE; #ifdef CONFIG_SMP static void (*relocated_kexec_smp_wait)(void *); @@ -78,7 +84,7 @@ int machine_kexec_prepare(struct kimage *kimage) return -EINVAL; } - /* kexec need a safe page to save reboot_code_buffer */ + /* kexec/kdump need a safe page to save reboot_code_buffer */ kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE); reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page); @@ -102,7 +108,8 @@ void kexec_reboot(void) /* * We know we were online, and there will be no incoming IPIs at - * this point. + * this point. Mark online again before rebooting so that the crash + * analysis tool will see us correctly. */ set_cpu_online(smp_processor_id(), true); @@ -147,7 +154,74 @@ static void kexec_shutdown_secondary(void *regs) kexec_reboot(); } -#endif + +static void crash_shutdown_secondary(void *passed_regs) +{ + int cpu = smp_processor_id(); + struct pt_regs *regs = passed_regs; + + /* + * If we are passed registers, use those. Otherwise get the + * regs from the last interrupt, which should be correct, as + * we are in an interrupt. But if the regs are not there, + * pull them from the top of the stack. They are probably + * wrong, but we need something to keep from crashing again. + */ + if (!regs) + regs = get_irq_regs(); + if (!regs) + regs = task_pt_regs(current); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + if (!cpumask_test_cpu(cpu, &cpus_in_crash)) + crash_save_cpu(regs, cpu); + cpumask_set_cpu(cpu, &cpus_in_crash); + + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} + +void crash_smp_send_stop(void) +{ + unsigned int ncpus; + unsigned long timeout; + static int cpus_stopped; + + /* + * This function can be called twice in panic path, but obviously + * we should execute this only once. + */ + if (cpus_stopped) + return; + + cpus_stopped = 1; + + /* Excluding the panic cpu */ + ncpus = num_online_cpus() - 1; + + smp_call_function(crash_shutdown_secondary, NULL, 0); + smp_wmb(); + + /* + * The crash CPU sends an IPI and wait for other CPUs to + * respond. Delay of at least 10 seconds. + */ + timeout = MSEC_PER_SEC * 10; + pr_emerg("Sending IPI to other cpus...\n"); + while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) { + mdelay(1); + cpu_relax(); + } +} +#endif /* defined(CONFIG_SMP) */ void machine_shutdown(void) { @@ -165,6 +239,19 @@ void machine_shutdown(void) void machine_crash_shutdown(struct pt_regs *regs) { + int crashing_cpu; + + local_irq_disable(); + + crashing_cpu = smp_processor_id(); + crash_save_cpu(regs, crashing_cpu); + +#ifdef CONFIG_SMP + crash_smp_send_stop(); +#endif + cpumask_set_cpu(crashing_cpu, &cpus_in_crash); + + pr_info("Starting crashdump kernel...\n"); } void machine_kexec(struct kimage *image) @@ -178,7 +265,8 @@ void machine_kexec(struct kimage *image) start_addr = (unsigned long)phys_to_virt(image->start); - first_ind_entry = (unsigned long)phys_to_virt(image->head & PAGE_MASK); + first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ? + (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0; /* * The generic kexec code builds a page list with physical diff --git a/arch/loongarch/kernel/mem.c b/arch/loongarch/kernel/mem.c index b273bf22a7b6..f672943b0bb4 100644 --- a/arch/loongarch/kernel/mem.c +++ b/arch/loongarch/kernel/mem.c @@ -80,7 +80,4 @@ void __init memblock_init(void) /* Reserve the kernel text/data/bss */ memblock_reserve(__pa_symbol(&_text), __pa_symbol(&_end) - __pa_symbol(&_text)); - - /* Reserve the initrd */ - reserve_initrd_mem(); } diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S index 48362f38c6bf..d13252553a7c 100644 --- a/arch/loongarch/kernel/relocate_kernel.S +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -24,6 +24,12 @@ SYM_CODE_START(relocate_new_kernel) */ move s0, a4 + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beqz s0, done + process_entry: PTR_L s1, s0, 0 PTR_ADDI s0, s0, SZREG diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 075fd4f778ea..d19df19d67d5 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -221,8 +223,70 @@ static void __init set_pcie_wakeup(void) } +static void __init arch_reserve_vmcore(void) +{ +#ifdef CONFIG_PROC_VMCORE + u64 i; + phys_addr_t start, end; + + if (!is_kdump_kernel()) + return; + + if (!elfcorehdr_size) { + for_each_mem_range(i, &start, &end) { + if (elfcorehdr_addr >= start && elfcorehdr_addr < end) { + /* + * Reserve from the elf core header to the end of + * the memory segment, that should all be kdump + * reserved memory. + */ + elfcorehdr_size = end - elfcorehdr_addr; + break; + } + } + } + + if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) { + pr_warn("elfcorehdr is overlapped\n"); + return; + } + + memblock_reserve(elfcorehdr_addr, elfcorehdr_size); + + pr_info("Reserving %llu KiB of memory at 0x%llx for elfcorehdr\n", + elfcorehdr_size >> 10, elfcorehdr_addr); +#endif +} + +static void __init arch_parse_crashkernel(void) +{ +#ifdef CONFIG_KEXEC + int ret; + unsigned long long start; + unsigned long long total_mem; + unsigned long long crash_base, crash_size; + + total_mem = memblock_phys_mem_size(); + ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base); + if (ret < 0 || crash_size <= 0) + return; + + start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size); + if (start != crash_base) { + pr_warn("Invalid memory region reserved for crash kernel\n"); + return; + } + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; +#endif +} + void __init platform_init(void) { + arch_reserve_vmcore(); + arch_parse_crashkernel(); + #ifdef CONFIG_ACPI_TABLE_UPGRADE acpi_table_upgrade(); #endif @@ -326,6 +390,15 @@ static void __init resource_init(void) request_resource(res, &data_resource); request_resource(res, &bss_resource); } + +#ifdef CONFIG_KEXEC + if (crashk_res.start < crashk_res.end) { + insert_resource(&iomem_resource, &crashk_res); + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n", + (unsigned long)((crashk_res.end - crashk_res.start + 1) >> 20), + (unsigned long)(crashk_res.start >> 20)); + } +#endif } static int __init reserve_memblock_reserved_regions(void) @@ -388,6 +461,7 @@ void __init setup_arch(char **cmdline_p) memblock_init(); pagetable_init(); parse_early_param(); + reserve_initrd_mem(); platform_init(); arch_mem_init(cmdline_p); diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index 4840358c5341..044bfbf8e7d1 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -246,6 +247,9 @@ void __noreturn die(const char *str, struct pt_regs *regs) oops_exit(); + if (regs && kexec_should_crash(current)) + crash_kexec(regs); + if (in_interrupt()) panic("Fatal exception in interrupt"); -- Gitee From b817dbdbb03b016cacb36f2ef670949ce3c40532 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 6 Feb 2023 09:47:35 +0800 Subject: [PATCH 03/16] LoongArch: Flush TLB earlier at initialization LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Move local_flush_tlb_all() earlier (just after setup_ptwalker() and before page allocation). This can avoid stale TLB entries misguiding the later page allocation. Without this patch the second kernel of kexec/kdump fails to boot SMP. Signed-off-by: Huacai Chen Signed-off-by: Youling Tang Change-Id: I118e0c8aafabfef28bc47a0d06286db850b6e79e --- arch/loongarch/mm/tlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 9818ce11546b..e18e2aadab65 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -258,6 +258,7 @@ extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; void setup_tlb_handler(int cpu) { setup_ptwalker(); + local_flush_tlb_all(); output_pgtable_bits_defines(); /* The tlb handlers are generated only once */ @@ -302,5 +303,4 @@ void tlb_init(int cpu) write_csr_stlbpgsize(PS_DEFAULT_SIZE); write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE); setup_tlb_handler(cpu); - local_flush_tlb_all(); } -- Gitee From 28fc51d6a8178cbd21e0b59d9119edcdacdb8542 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 09:22:17 +0800 Subject: [PATCH 04/16] LoongArch: kexec: Add compatibility with old interfaces LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Old interface: a0 = argc, a1 = argv, a2 = envp New interface: a0 = efi flag, a2 = cmdline, a3 = systemtab The following interfaces are not supported: a0 = efi flag, a2 = small fdt Signed-off-by: Youling Tang Change-Id: If490ebe181ea75c955b93a69d28994f326938d93 --- arch/loongarch/kernel/machine_kexec.c | 45 +++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c index 2dcb9e003657..561706cb1e6d 100644 --- a/arch/loongarch/kernel/machine_kexec.c +++ b/arch/loongarch/kernel/machine_kexec.c @@ -59,6 +59,9 @@ static void kexec_image_info(const struct kimage *kimage) } } +#define MAX_ARGS 64 +#define KEXEC_CMDLINE_SIZE (COMMAND_LINE_SIZE * 2) + int machine_kexec_prepare(struct kimage *kimage) { int i; @@ -70,11 +73,49 @@ int machine_kexec_prepare(struct kimage *kimage) kimage->arch.efi_boot = fw_arg0; kimage->arch.systable_ptr = fw_arg2; + if (!fw_arg2) + pr_err("Small fdt mode is not supported!\n"); + /* Find the command line */ for (i = 0; i < kimage->nr_segments; i++) { if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { - if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) - kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; + if (fw_arg0 < 2) { + /* New firmware */ + if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) + kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; + } else { + /* Old firmware */ + int argc = 0; + long offt; + char *ptr, *str; + unsigned long *argv; + + /* + * convert command line string to array + * of parameters (as bootloader does). + */ + argv = (unsigned long *)kmalloc(KEXEC_CMDLINE_SIZE, GFP_KERNEL); + argv[argc++] = (unsigned long)(KEXEC_CMDLINE_ADDR + KEXEC_CMDLINE_SIZE/2); + str = (char *)argv + KEXEC_CMDLINE_SIZE/2; + + if (copy_from_user(str, kimage->segment[i].buf, KEXEC_CMDLINE_SIZE/2)) + return -EINVAL; + + ptr = strchr(str, ' '); + + while (ptr && (argc < MAX_ARGS)) { + *ptr = '\0'; + if (ptr[1] != ' ') { + offt = (long)(ptr - str + 1); + argv[argc++] = (unsigned long)argv + KEXEC_CMDLINE_SIZE/2 + offt; + } + ptr = strchr(ptr + 1, ' '); + } + + kimage->arch.efi_boot = argc; + kimage->arch.cmdline_ptr = (unsigned long)argv; + break; + } break; } } -- Gitee From 37b09006e621dce876410bb30540273c57a99656 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 6 Feb 2023 09:49:14 +0800 Subject: [PATCH 05/16] LoongArch: Integrate initrd operation LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Integrate initrd operations to avoid reserving initrd space multiple times during kexec/kdump. Signed-off-by: Youling Tang Change-Id: Idb8a10c2143e7ff4682328f620de8d5b3e35b105 --- arch/loongarch/kernel/legacy_boot.c | 91 +++-------------------------- arch/loongarch/kernel/legacy_boot.h | 2 + arch/loongarch/kernel/setup.c | 2 +- 3 files changed, 12 insertions(+), 83 deletions(-) diff --git a/arch/loongarch/kernel/legacy_boot.c b/arch/loongarch/kernel/legacy_boot.c index 93ce1a6a6a8c..15940f045dbe 100644 --- a/arch/loongarch/kernel/legacy_boot.c +++ b/arch/loongarch/kernel/legacy_boot.c @@ -303,79 +303,9 @@ int setup_legacy_IRQ(void) * Manage initrd */ #ifdef CONFIG_BLK_DEV_INITRD -static unsigned long init_initrd(unsigned long ps, unsigned long z) -{ - static int initalized; - - if (!ps || !z) - return 0; - - initrd_start = (unsigned long)__va(ps); - initrd_end = initrd_start + z; - /* - * Board specific code or command line parser should have - * already set up initrd_start and initrd_end. In these cases - * perfom sanity checks and use them if all looks good. - */ - if (initrd_start < PAGE_OFFSET || initrd_end <= initrd_start) { - pr_err("initrd start load address error!"); - goto disable; - } - - if (initrd_start & ~PAGE_MASK) { - pr_err("initrd start must be page aligned\n"); - goto disable; - } - - memblock_reserve(__pa(initrd_start), z); - initrd_below_start_ok = 1; - - if (!initalized) - pr_info("Initial ramdisk at: 0x%lx (%lu bytes)\n", - initrd_start, z); - initalized = 1; - - return 0; -disable: - printk(KERN_CONT " - disabling initrd\n"); - initrd_start = 0; - initrd_end = 0; - return 0; -} - -static int early_initrd(char *p) -{ - unsigned long start, size; - char *endp; - - if (!efi_bp) - return 0; - start = memparse(p, &endp); - if (*endp == ',') - size = memparse(endp + 1, NULL); - - if (start + size > PFN_PHYS(max_low_pfn)) { - pr_err(KERN_INFO "Initrd physical address is out of memory!"); - return 0; - } - - init_initrd(start, size); - - return 0; -} -early_param("initrd", early_initrd); - static int rd_start_early(char *p) { - unsigned long start; - - if (!efi_bp) - return 0; - - start = memparse(p, &p); - initrd_start = start; - initrd_end += start; - init_initrd(__pa(start), initrd_end - start); + phys_initrd_start = __pa(memparse(p, &p)); return 0; } @@ -383,24 +313,21 @@ early_param("rd_start", rd_start_early); static int rd_size_early(char *p) { - unsigned long size; - - if (!efi_bp) - return 0; - size = memparse(p, &p); - initrd_end += size; + phys_initrd_size = memparse(p, &p); - init_initrd(__pa(initrd_start), size); return 0; } early_param("rd_size", rd_size_early); +#endif -#else /* !CONFIG_BLK_DEV_INITRD */ -static unsigned long init_initrd(void) +void __init loongarch_reserve_initrd_mem(void) { - return 0; + /* The small fdt method should be skipped directly to avoid two reserved operations. */ + if (!fw_arg2) + return; + + reserve_initrd_mem(); } -#endif void fw_init_cmdline(unsigned long argc, unsigned long cmdp) { diff --git a/arch/loongarch/kernel/legacy_boot.h b/arch/loongarch/kernel/legacy_boot.h index 1d6b42a26460..b52d8aa262dc 100644 --- a/arch/loongarch/kernel/legacy_boot.h +++ b/arch/loongarch/kernel/legacy_boot.h @@ -87,4 +87,6 @@ extern int __init pch_msi_parse_madt(union acpi_subtable_headers *header, const unsigned long end); extern struct irq_domain *get_pchpic_irq_domain(void); + +extern void __init loongarch_reserve_initrd_mem(void); #endif diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index d19df19d67d5..9c2196e166d2 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -461,7 +461,7 @@ void __init setup_arch(char **cmdline_p) memblock_init(); pagetable_init(); parse_early_param(); - reserve_initrd_mem(); + loongarch_reserve_initrd_mem(); platform_init(); arch_mem_init(cmdline_p); -- Gitee From 418173eeee1566a45e4e361f5fdf5eb3902a4448 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 6 Feb 2023 09:51:10 +0800 Subject: [PATCH 06/16] LoongArch: kdump: Add memory reservation for old kernel LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- After moving the old kernel memory reservation to elfcorehdr operation, avoid the elfcorehdr space from being destroyed. Signed-off-by: Youling Tang Change-Id: I94d5c38e24dd6d3ab3fddc3945580780d81e6922 --- arch/loongarch/kernel/setup.c | 45 +++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 9c2196e166d2..fac223d533e0 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -65,6 +65,8 @@ EXPORT_SYMBOL(cpu_data); struct loongson_board_info b_info; static const char dmi_empty_string[] = " "; +static phys_addr_t crashmem_start, crashmem_size; + /* * Setup information * @@ -180,16 +182,6 @@ static int __init early_parse_mem(char *p) return -EINVAL; } - /* - * If a user specifies memory size, we - * blow away any automatically generated - * size. - */ - if (usermem == 0) { - usermem = 1; - memblock_remove(memblock_start_of_DRAM(), - memblock_end_of_DRAM() - memblock_start_of_DRAM()); - } start = 0; size = memparse(p, &p); if (*p == '@') @@ -199,6 +191,23 @@ static int __init early_parse_mem(char *p) return -EINVAL; } + /* + * If a user specifies memory size, we + * blow away any automatically generated + * size. + */ + if (usermem == 0) { + usermem = 1; + if (!strstr(boot_command_line, "elfcorehdr")) { + memblock_remove(memblock_start_of_DRAM(), + memblock_end_of_DRAM() - memblock_start_of_DRAM()); + } else { + crashmem_start = start; + crashmem_size = size; + return 0; + } + } + if (!IS_ENABLED(CONFIG_NUMA)) memblock_add(start, size); else @@ -282,10 +291,26 @@ static void __init arch_parse_crashkernel(void) #endif } +/* + * After the kdump operation is performed to enter the capture kernel, the + * memory area used by the previous production kernel should be reserved to + * avoid destroy to the captured data. + */ +static void reserve_oldmem_region(void) +{ +#ifdef CONFIG_CRASH_DUMP + if (!is_kdump_kernel()) + return; + + memblock_cap_memory_range(crashmem_start, crashmem_size); +#endif +} + void __init platform_init(void) { arch_reserve_vmcore(); arch_parse_crashkernel(); + reserve_oldmem_region(); #ifdef CONFIG_ACPI_TABLE_UPGRADE acpi_table_upgrade(); -- Gitee From 05a63c8b6d85d2de5d3f73ca4c08124767c78764 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Tue, 17 Jan 2023 19:32:55 +0800 Subject: [PATCH 07/16] LoongArch: efistub: Modify the address where efistub jumps to the main kernel LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Signed-off-by: Youling Tang Change-Id: I0c7f5360cea6837017de0bd5e3ec3220dfc68add --- arch/loongarch/kernel/head.S | 1 + arch/loongarch/kernel/image-vars.h | 2 +- drivers/firmware/efi/libstub/loongarch-stub.c | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 5536ffc7825c..9bfdc718d9f7 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -30,6 +30,7 @@ _head: pe_header: __EFI_PE_HEADER +SYM_DATA(kernel_entry_rel, .quad kernel_entry); SYM_DATA(kernel_asize, .long _end - _text); SYM_DATA(kernel_fsize, .long _edata - _text); SYM_DATA(kernel_offset, .long kernel_offset - _text); diff --git a/arch/loongarch/kernel/image-vars.h b/arch/loongarch/kernel/image-vars.h index 88f5d81702df..e7e760a02687 100644 --- a/arch/loongarch/kernel/image-vars.h +++ b/arch/loongarch/kernel/image-vars.h @@ -16,7 +16,7 @@ __efistub_strncat = strncat; __efistub_strnstr = strnstr; __efistub_strnlen = strnlen; __efistub_strrchr = strrchr; -__efistub_kernel_entry = kernel_entry; +__efistub_kernel_entry = kernel_entry_rel; __efistub_kernel_asize = kernel_asize; __efistub_kernel_fsize = kernel_fsize; __efistub_kernel_offset = kernel_offset; diff --git a/drivers/firmware/efi/libstub/loongarch-stub.c b/drivers/firmware/efi/libstub/loongarch-stub.c index 8727daa90d64..481807fe2b29 100644 --- a/drivers/firmware/efi/libstub/loongarch-stub.c +++ b/drivers/firmware/efi/libstub/loongarch-stub.c @@ -49,8 +49,7 @@ void __noreturn efi_enter_kernel(unsigned long entrypoint, unsigned long fdt, un csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); - real_kernel_entry = (kernel_entry_t) - ((unsigned long)&kernel_entry - entrypoint + VMLINUX_LOAD_ADDRESS); + real_kernel_entry = (kernel_entry_t) kernel_entry; real_kernel_entry(true, fdt, 0); } -- Gitee From fd5abfada56685901c38858cf63e598adaccda36 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 09:34:17 +0800 Subject: [PATCH 08/16] LoongArch: Use la.pcrel instead of la.abs when it's trivially possible LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Let's start to kill la.abs in preparation for the subsequent support of the PIE kernel. BTW, Re-tab the indention in arch/loongarch/kernel/entry.S for alignment. Signed-off-by: Xi Ruoyao Signed-off-by: Huacai Chen Signed-off-by: Youling Tang Change-Id: Ia36815100690c2272528843870fa626221a5640f --- arch/loongarch/include/asm/stackframe.h | 2 +- arch/loongarch/include/asm/uaccess.h | 1 - arch/loongarch/kernel/head.S | 2 +- arch/loongarch/mm/tlbex.S | 3 +-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 733dc9e96241..d9bd03cdf64c 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -90,7 +90,7 @@ .endm .macro set_saved_sp stackp temp temp2 - la.abs \temp, kernelsp + la.pcrel \temp, kernelsp #ifdef CONFIG_SMP LONG_ADD \temp, \temp, u0 #endif diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 306bef8f4972..775e7ef014c9 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -21,7 +21,6 @@ extern u64 __ua_limit; #define __UA_ADDR ".dword" -#define __UA_LA "la.abs" #define __UA_LIMIT __ua_limit /* diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 9bfdc718d9f7..df6e396ec4ae 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -115,7 +115,7 @@ SYM_CODE_START(smpboot_entry) li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 csrwr t0, LOONGARCH_CSR_EUEN - la.abs t0, cpuboot_data + la.pcrel t0, cpuboot_data ld.d sp, t0, CPU_BOOT_STACK ld.d tp, t0, CPU_BOOT_TINFO diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 39743337999e..2ece41bdce85 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -17,8 +17,7 @@ move a0, sp REG_S a2, sp, PT_BVADDR li.w a1, \write - la.abs t0, do_page_fault - jirl ra, t0, 0 + bl do_page_fault RESTORE_ALL_AND_RET SYM_FUNC_END(tlb_do_page_fault_\write) .endm -- Gitee From c066ef2f99d3fd941970751dca8bfdb35312dd96 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 09:42:38 +0800 Subject: [PATCH 09/16] LoongArch: Add JUMP_VIRT_ADDR macro implementation to avoid using la.abs LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Add JUMP_VIRT_ADDR macro implementation to avoid using la.abs directly. This is a preparation for subsequent patches. Signed-off-by: Youling Tang Change-Id: I4b64da6d5ab46b6bc561b74f39224765f6892758 --- arch/loongarch/include/asm/stackframe.h | 9 +++++++++ arch/loongarch/kernel/head.S | 12 ++++-------- arch/loongarch/power/hibernate_asm.S | 2 +- arch/loongarch/power/suspend_asm.S | 5 ++--- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index d9bd03cdf64c..880b43a61d08 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -36,6 +37,14 @@ cfi_restore \reg \offset \docfi .endm +/* Jump to the runtime virtual address. */ + .macro JUMP_VIRT_ADDR temp1 temp2 + li.d \temp1, CACHE_BASE + pcaddi \temp2, 0 + or \temp1, \temp1, \temp2 + jirl zero, \temp1, 0xc + .endm + .macro BACKUP_T0T1 csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index df6e396ec4ae..8de2f307bf89 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -48,11 +48,8 @@ SYM_CODE_START(kernel_entry) # kernel entry point li.d t0, CSR_DMW1_INIT # CA, PLV0, 0x9000 xxxx xxxx xxxx csrwr t0, LOONGARCH_CSR_DMWIN1 - /* We might not get launched at the address the kernel is linked to, - so we jump there. */ - la.abs t0, 0f - jr t0 -0: + JUMP_VIRT_ADDR t0, t1 + /* Enable PG */ li.w t0, 0xb0 # PLV=0, IE=0, PG=1 csrwr t0, LOONGARCH_CSR_CRMD @@ -104,9 +101,8 @@ SYM_CODE_START(smpboot_entry) li.d t0, CSR_DMW1_INIT # CA, PLV0 csrwr t0, LOONGARCH_CSR_DMWIN1 - la.abs t0, 0f - jr t0 -0: + JUMP_VIRT_ADDR t0, t1 + /* Enable PG */ li.w t0, 0xb0 # PLV=0, IE=0, PG=1 csrwr t0, LOONGARCH_CSR_CRMD diff --git a/arch/loongarch/power/hibernate_asm.S b/arch/loongarch/power/hibernate_asm.S index 1874e473b293..a8a22afb2e4d 100644 --- a/arch/loongarch/power/hibernate_asm.S +++ b/arch/loongarch/power/hibernate_asm.S @@ -16,7 +16,7 @@ .text SYM_FUNC_START(swsusp_arch_save) - la.abs t0, saved_regs + la.pcrel t0, saved_regs PTR_S ra, t0, PT_R1 PTR_S sp, t0, PT_R3 PTR_S fp, t0, PT_R22 diff --git a/arch/loongarch/power/suspend_asm.S b/arch/loongarch/power/suspend_asm.S index ceac577c3794..cb5e77c72713 100644 --- a/arch/loongarch/power/suspend_asm.S +++ b/arch/loongarch/power/suspend_asm.S @@ -113,9 +113,8 @@ SYM_CODE_START(loongarch_wakeup_start) li.d t0, CSR_DMW1_INIT # CA, PLV0 csrwr t0, LOONGARCH_CSR_DMWIN1 - la.abs t0, 0f - jirl zero, t0, 0 -0: + JUMP_VIRT_ADDR t0, t1 + la.pcrel t0, acpi_saved_sp ld.d sp, t0, 0 SETUP_WAKEUP -- Gitee From d40a9b0a3e29d3f3c7a6e1a8ecd6f4ec4b637c57 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 10:04:26 +0800 Subject: [PATCH 10/16] LoongArch: Add la_abs macro implementation LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Use the "la_abs macro" instead of the "la.abs pseudo instruction" to prepare for the subsequent PIE kernel. When PIE is not enabled, la_abs is equivalent to la.abs. Signed-off-by: Youling Tang Change-Id: I640026372b7fe03af662aa7fa980427ab860fe5c --- arch/loongarch/include/asm/asmmacro.h | 4 ++++ arch/loongarch/include/asm/stackframe.h | 2 +- arch/loongarch/kernel/genex.S | 8 ++++---- arch/loongarch/mm/tlbex.S | 14 +++++++------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index 3a629b545174..294bf26753c3 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -898,4 +898,8 @@ nor \dst, \src, zero .endm +.macro la_abs reg, sym + la.abs \reg, \sym +.endm + #endif /* _ASM_ASMMACRO_H */ diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h index 880b43a61d08..cfa3951a3e60 100644 --- a/arch/loongarch/include/asm/stackframe.h +++ b/arch/loongarch/include/asm/stackframe.h @@ -86,7 +86,7 @@ * new value in sp. */ .macro get_saved_sp docfi=0 - la.abs t1, kernelsp + la_abs t1, kernelsp #ifdef CONFIG_SMP csrrd t0, PERCPU_BASE_KS LONG_ADD t1, t1, t0 diff --git a/arch/loongarch/kernel/genex.S b/arch/loongarch/kernel/genex.S index 75e5be807a0d..4661cdde96cc 100644 --- a/arch/loongarch/kernel/genex.S +++ b/arch/loongarch/kernel/genex.S @@ -34,7 +34,7 @@ SYM_FUNC_END(__arch_cpu_idle) SYM_FUNC_START(handle_vint) BACKUP_T0T1 SAVE_ALL - la.abs t1, __arch_cpu_idle + la_abs t1, __arch_cpu_idle LONG_L t0, sp, PT_ERA /* 32 byte rollback region */ ori t0, t0, 0x1f @@ -43,7 +43,7 @@ SYM_FUNC_START(handle_vint) LONG_S t0, sp, PT_ERA 1: move a0, sp move a1, sp - la.abs t0, do_vint + la_abs t0, do_vint jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_vint) @@ -71,7 +71,7 @@ SYM_FUNC_END(except_vec_cex) SAVE_ALL build_prep_\prep move a0, sp - la.abs t0, do_\handler + la_abs t0, do_\handler jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_\exception) @@ -90,6 +90,6 @@ SYM_FUNC_END(except_vec_cex) BUILD_HANDLER reserved reserved none /* others */ SYM_FUNC_START(handle_sys) - la.abs t0, handle_syscall + la_abs t0, handle_syscall jr t0 SYM_FUNC_END(handle_sys) diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 2ece41bdce85..7d108e0d1cff 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -32,7 +32,7 @@ SYM_FUNC_START(handle_tlb_protect) move a1, zero csrrd a2, LOONGARCH_CSR_BADV REG_S a2, sp, PT_BVADDR - la.abs t0, do_page_fault + la_abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_tlb_protect) @@ -122,7 +122,7 @@ leave_load: ertn #ifdef CONFIG_64BIT vmalloc_load: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_load #endif @@ -196,7 +196,7 @@ tlb_huge_update_load: nopage_tlb_load: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_0 + la_abs t0, tlb_do_page_fault_0 jr t0 SYM_FUNC_END(handle_tlb_load) @@ -288,7 +288,7 @@ leave_store: ertn #ifdef CONFIG_64BIT vmalloc_store: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_store #endif @@ -364,7 +364,7 @@ tlb_huge_update_store: nopage_tlb_store: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 + la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_store) @@ -453,7 +453,7 @@ leave_modify: ertn #ifdef CONFIG_64BIT vmalloc_modify: - la.abs t1, swapper_pg_dir + la_abs t1, swapper_pg_dir b vmalloc_done_modify #endif @@ -523,7 +523,7 @@ tlb_huge_update_modify: nopage_tlb_modify: dbar 0 csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 + la_abs t0, tlb_do_page_fault_1 jr t0 SYM_FUNC_END(handle_tlb_modify) -- Gitee From 002f6a5ed1aa79dabd822149b05da340c7046f41 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 10:10:00 +0800 Subject: [PATCH 11/16] LoongArch: Add support for kernel relocation LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- This config allows to compile kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR. Runtime relocation is possible since relocation metadata are embedded into the kernel. Signed-off-by: Youling Tang Signed-off-by: Xi Ruoyao # Use arch_initcall Signed-off-by: Jinyang He # Provide la_abs relocation code Signed-off-by: Huacai Chen Change-Id: I0b6310cdc4e0b93b6a02691bae36e347f8109df3 --- arch/loongarch/Kconfig | 8 ++ arch/loongarch/Makefile | 5 ++ arch/loongarch/include/asm/asmmacro.h | 13 ++++ arch/loongarch/include/asm/setup.h | 16 ++++ arch/loongarch/kernel/Makefile | 2 + arch/loongarch/kernel/head.S | 4 + arch/loongarch/kernel/relocate.c | 106 ++++++++++++++++++++++++++ arch/loongarch/kernel/vmlinux.lds.S | 18 +++++ 8 files changed, 172 insertions(+) create mode 100644 arch/loongarch/kernel/relocate.c diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index b68aea70b563..8b8156a9c21e 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -496,6 +496,14 @@ config PHYSICAL_START specified in the "crashkernel=YM@XM" command line boot parameter passed to the panic-ed kernel). +config RELOCATABLE + bool "Relocatable kernel" + help + This builds the kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required, so as to relocate + the kernel binary at runtime to a different virtual address from + its link address. + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index 1c5fe78ab973..b774357c8c99 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -63,6 +63,11 @@ KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs endif +ifeq ($(CONFIG_RELOCATABLE),y) +KBUILD_CFLAGS_KERNEL += -fPIE +LDFLAGS_vmlinux += -static -pie --no-dynamic-linker -z notext +endif + cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h index 294bf26753c3..938b3189bb70 100644 --- a/arch/loongarch/include/asm/asmmacro.h +++ b/arch/loongarch/include/asm/asmmacro.h @@ -899,7 +899,20 @@ .endm .macro la_abs reg, sym +#ifndef CONFIG_RELOCATABLE la.abs \reg, \sym +#else + 766: + lu12i.w \reg, 0 + ori \reg, \reg, 0 + lu32i.d \reg, 0 + lu52i.d \reg, \reg, 0 + .pushsection ".la_abs", "aw", %progbits + 768: + .dword 768b-766b + .dword \sym + .popsection +#endif .endm #endif /* _ASM_ASMMACRO_H */ diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 6d7d2a3e23dd..36a2beb6b4df 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -18,4 +18,20 @@ extern void per_cpu_trap_init(int cpu); extern void set_handler(unsigned long offset, void *addr, unsigned long len); extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len); +#ifdef CONFIG_RELOCATABLE + +struct rela_la_abs { + long offset; + long symvalue; +}; + +extern long __la_abs_begin; +extern long __la_abs_end; +extern long __rela_dyn_begin; +extern long __rela_dyn_end; + +extern void __init relocate_kernel(void); + +#endif + #endif /* __SETUP_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index 4ed6981bf8ea..c39f35231035 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -26,6 +26,8 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o +obj-$(CONFIG_RELOCATABLE) += relocate.o + obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 8de2f307bf89..d8eae27d6467 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -85,6 +85,10 @@ SYM_CODE_START(kernel_entry) # kernel entry point set_saved_sp sp, t0, t1 PTR_ADDI sp, sp, -4 * SZREG # init stack pointer +#ifdef CONFIG_RELOCATABLE + bl relocate_kernel +#endif + bl start_kernel SYM_CODE_END(kernel_entry) diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c new file mode 100644 index 000000000000..015f7c9d2a62 --- /dev/null +++ b/arch/loongarch/kernel/relocate.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Kernel relocation at boot time + * + * Copyright (C) 2023 Loongson Technology Corporation Limited + */ + +#include +#include +#include +#include +#include +#include + +#define RELOCATED(x) ((void *)((long)x + reloc_offset)) + +static unsigned long reloc_offset; + +static inline __init void relocate_relative(void) +{ + Elf64_Rela *rela, *rela_end; + rela = (Elf64_Rela *)&__rela_dyn_begin; + rela_end = (Elf64_Rela *)&__rela_dyn_end; + + for ( ; rela < rela_end; rela++) { + Elf64_Addr addr = rela->r_offset; + Elf64_Addr relocated_addr = rela->r_addend; + + if (rela->r_info != R_LARCH_RELATIVE) + continue; + + if (relocated_addr >= VMLINUX_LOAD_ADDRESS) + relocated_addr = (Elf64_Addr)RELOCATED(relocated_addr); + + *(Elf64_Addr *)RELOCATED(addr) = relocated_addr; + } +} + +static inline void __init relocate_la_abs(void) +{ + void *begin, *end; + struct rela_la_abs *p; + + begin = &__la_abs_begin; + end = &__la_abs_end; + + for (p = begin; (void *)p < end; p++) { + long v = p->symvalue; + uint32_t lu12iw, ori, lu32id, lu52id; + union loongarch_instruction *insn = (void *)p - p->offset; + + lu12iw = (v >> 12) & 0xfffff; + ori = v & 0xfff; + lu32id = (v >> 32) & 0xfffff; + lu52id = v >> 52; + + insn[0].reg1i20_format.immediate = lu12iw; + insn[1].reg2i12_format.immediate = ori; + insn[2].reg1i20_format.immediate = lu32id; + insn[3].reg2i12_format.immediate = lu52id; + } +} + +void __init relocate_kernel(void) +{ + reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; + + if (reloc_offset) + relocate_relative(); + + relocate_la_abs(); +} + +/* + * Show relocation information on panic. + */ +static void show_kernel_relocation(const char *level) +{ + if (reloc_offset > 0) { + printk(level); + pr_cont("Kernel relocated by 0x%lx\n", reloc_offset); + pr_cont(" .text @ 0x%px\n", _text); + pr_cont(" .data @ 0x%px\n", _sdata); + pr_cont(" .bss @ 0x%px\n", __bss_start); + } +} + +static int kernel_location_notifier_fn(struct notifier_block *self, + unsigned long v, void *p) +{ + show_kernel_relocation(KERN_EMERG); + return NOTIFY_DONE; +} + +static struct notifier_block kernel_location_notifier = { + .notifier_call = kernel_location_notifier_fn +}; + +static int __init register_kernel_offset_dumper(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &kernel_location_notifier); + return 0; +} + +arch_initcall(register_kernel_offset_dumper); diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index c673347a80bb..754a67dab7f6 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -66,6 +66,17 @@ SECTIONS __alt_instructions_end = .; } +#ifdef CONFIG_RELOCATABLE + . = ALIGN(8); + .la_abs : AT(ADDR(.la_abs) - LOAD_OFFSET) { + __la_abs_begin = .; + *(.la_abs) + __la_abs_end = .; + } +#endif + + .data.rel : { *(.data.rel*) } + . = ALIGN(PECOFF_SEGMENT_ALIGN); __init_begin = .; __inittext_begin = .; @@ -101,6 +112,12 @@ SECTIONS RO_DATA(4096) RW_DATA(1 << CONFIG_L1_CACHE_SHIFT, PAGE_SIZE, THREAD_SIZE) + .rela.dyn : ALIGN(8) { + __rela_dyn_begin = .; + *(.rela.dyn) *(.rela*) + __rela_dyn_end = .; + } + .sdata : { *(.sdata) } @@ -127,6 +144,7 @@ SECTIONS DISCARDS /DISCARD/ : { + *(.dynamic .dynsym .dynstr .hash .gnu.hash) *(.gnu.attributes) *(.options) *(.eh_frame) -- Gitee From a836141a61ac970d8584a920eed843295d7e1562 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 10:35:26 +0800 Subject: [PATCH 12/16] LoongArch: Add support for kernel address space layout randomization (KASLR) LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- This patch adds support for relocating the kernel to a random address. Entropy is derived from the banner, which will change every build and random_get_entropy() which should provide additional runtime entropy. The kernel is relocated by up to RANDOMIZE_BASE_MAX_OFFSET bytes from its link address. Because relocation happens so early in the kernel boot, the amount of physical memory has not yet been determined. This means the only way to limit relocation within the available memory is via Kconfig. Limit the maximum value of RANDOMIZE_BASE_MAX_OFFSET to 256M(0x10000000) because our memory layout has many holes. KERNELOFFSET (kaslr_offset) is added to vmcoreinfo in the future, for crash --kaslr support. Signed-off-by: Youling Tang Change-Id: I1e2e5cda0445e6f2c431638694fe822794b63442 --- arch/loongarch/Kconfig | 25 +++++ arch/loongarch/include/asm/setup.h | 2 +- arch/loongarch/kernel/head.S | 12 ++ arch/loongarch/kernel/legacy_boot.c | 1 + arch/loongarch/kernel/relocate.c | 167 +++++++++++++++++++++++++++- 5 files changed, 201 insertions(+), 6 deletions(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 8b8156a9c21e..82322a138503 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -504,6 +504,31 @@ config RELOCATABLE the kernel binary at runtime to a different virtual address from its link address. +config RANDOMIZE_BASE + bool "Randomize the address of the kernel (KASLR)" + depends on RELOCATABLE + help + Randomizes the physical and virtual address at which the + kernel image is loaded, as a security feature that + deters exploit attempts relying on knowledge of the location + of kernel internals. + + The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET. + + If unsure, say N. + +config RANDOMIZE_BASE_MAX_OFFSET + hex "Maximum KASLR offset" if EXPERT + depends on RANDOMIZE_BASE + range 0x0 0x10000000 + default "0x01000000" + help + When KASLR is active, this provides the maximum offset that will + be applied to the kernel image. It should be set according to the + amount of physical RAM available in the target system. + + This is limited by the size of the lower address memory, 256MB. + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 36a2beb6b4df..7a1986dba4d9 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -30,7 +30,7 @@ extern long __la_abs_end; extern long __rela_dyn_begin; extern long __rela_dyn_end; -extern void __init relocate_kernel(void); +extern void * __init relocate_kernel(void); #endif diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index d8eae27d6467..5f7d50073806 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -86,7 +86,19 @@ SYM_CODE_START(kernel_entry) # kernel entry point PTR_ADDI sp, sp, -4 * SZREG # init stack pointer #ifdef CONFIG_RELOCATABLE + bl relocate_kernel + +#ifdef CONFIG_RANDOMIZE_BASE + /* Repoint the sp into the new kernel */ + PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE) + PTR_ADD sp, sp, tp + set_saved_sp sp, t0, t1 +#endif + + /* relocate_kernel() returns the new kernel entry point */ + jr a0 + #endif bl start_kernel diff --git a/arch/loongarch/kernel/legacy_boot.c b/arch/loongarch/kernel/legacy_boot.c index 15940f045dbe..c46303a7b585 100644 --- a/arch/loongarch/kernel/legacy_boot.c +++ b/arch/loongarch/kernel/legacy_boot.c @@ -344,6 +344,7 @@ void fw_init_cmdline(unsigned long argc, unsigned long cmdp) } strlcat(boot_command_line, arcs_cmdline, COMMAND_LINE_SIZE); } +EXPORT_SYMBOL_GPL(fw_init_cmdline); static u8 ext_listhdr_checksum(u8 *buffer, u32 length) { diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c index 015f7c9d2a62..36823b0be952 100644 --- a/arch/loongarch/kernel/relocate.c +++ b/arch/loongarch/kernel/relocate.c @@ -8,11 +8,18 @@ #include #include #include +#include +#include +#include #include #include #include +#include #define RELOCATED(x) ((void *)((long)x + reloc_offset)) +#define RELOCATED_KASLR(x) ((void *)((long)x + random_offset)) + +extern void fw_init_cmdline(unsigned long argc, unsigned long cmdp); static unsigned long reloc_offset; @@ -36,13 +43,13 @@ static inline __init void relocate_relative(void) } } -static inline void __init relocate_la_abs(void) +static inline void __init relocate_la_abs(long random_offset) { void *begin, *end; struct rela_la_abs *p; - begin = &__la_abs_begin; - end = &__la_abs_end; + begin = RELOCATED_KASLR(&__la_abs_begin); + end = RELOCATED_KASLR(&__la_abs_end); for (p = begin; (void *)p < end; p++) { long v = p->symvalue; @@ -61,14 +68,164 @@ static inline void __init relocate_la_abs(void) } } -void __init relocate_kernel(void) +#ifdef CONFIG_RANDOMIZE_BASE +static inline __init unsigned long rotate_xor(unsigned long hash, + const void *area, size_t size) +{ + size_t i, diff; + const typeof(hash) *ptr = PTR_ALIGN(area, sizeof(hash)); + + diff = (void *)ptr - area; + if (size < diff + sizeof(hash)) + return hash; + + size = ALIGN_DOWN(size - diff, sizeof(hash)); + + for (i = 0; i < size / sizeof(hash); i++) { + /* Rotate by odd number of bits and XOR. */ + hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); + hash ^= ptr[i]; + } + + return hash; +} + +static inline __init unsigned long get_random_boot(void) +{ + unsigned long hash = 0; + unsigned long entropy = random_get_entropy(); + + /* Attempt to create a simple but unpredictable starting entropy. */ + hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); + + /* Add in any runtime entropy we can get */ + hash = rotate_xor(hash, &entropy, sizeof(entropy)); + + return hash; +} + +static inline __init bool kaslr_disabled(void) { + char *str; + const char *builtin_cmdline = CONFIG_CMDLINE; + + str = strstr(builtin_cmdline, "nokaslr"); + if (str == builtin_cmdline || (str > builtin_cmdline && *(str - 1) == ' ')) + return true; + + str = strstr(boot_command_line, "nokaslr"); + if (str == boot_command_line || (str > boot_command_line && *(str - 1) == ' ')) + return true; + + return false; +} + +/* Choose a new address for the kernel */ +static inline void __init *determine_relocation_address(void) +{ + unsigned long kernel_length; + unsigned long random_offset; + void *destination = _text; + + if (kaslr_disabled()) + return destination; + + kernel_length = (long)_end - (long)_text; + + random_offset = get_random_boot() << 16; + random_offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); + if (random_offset < kernel_length) + random_offset += ALIGN(kernel_length, 0xffff); + + return RELOCATED_KASLR(destination); +} + +static inline int __init relocation_addr_valid(void *location_new) +{ + if ((unsigned long)location_new & 0x00000ffff) + return 0; /* Inappropriately aligned new location */ + + if ((unsigned long)location_new < (unsigned long)_end) + return 0; /* New location overlaps original kernel */ + + return 1; +} +#endif + +static inline void __init update_reloc_offset(unsigned long *addr, long random_offset) +{ + unsigned long *new_addr = (unsigned long *)RELOCATED_KASLR(addr); + + *new_addr = (unsigned long)reloc_offset; +} + +void * __init relocate_kernel(void) +{ + unsigned long kernel_length; + unsigned long random_offset = 0; + void *location_new = _text; /* Default to original kernel start */ + void *kernel_entry = start_kernel; /* Default to original kernel entry point */ + char *cmdline; + + /* Get the command line */ + if (!fw_arg2) { + /* a0 = efi flag, a1 = small fdt */ + early_init_dt_scan(early_ioremap(fw_arg1, SZ_64K)); + } else if (fw_arg0 == 1 || fw_arg0 == 0) { + /* a0 = efi flag, a1 = cmdline, a2 = systemtab */ + cmdline = early_ioremap(fw_arg1, COMMAND_LINE_SIZE); + strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); + early_iounmap(cmdline, COMMAND_LINE_SIZE); + } else { + /* a0 = argc, a1 = argv, a3 = envp */ + fw_init_cmdline(fw_arg0, fw_arg1); + } + +#ifdef CONFIG_RANDOMIZE_BASE + location_new = determine_relocation_address(); + + /* Sanity check relocation address */ + if (relocation_addr_valid(location_new)) + random_offset = (unsigned long)location_new - (unsigned long)(_text); +#endif reloc_offset = (unsigned long)_text - VMLINUX_LOAD_ADDRESS; + /* Reset the command line now so we don't end up with a duplicate */ + boot_command_line[0] = '\0'; + + if (random_offset) { + kernel_length = (long)(_end) - (long)(_text); + + /* Copy the kernel to it's new location */ + memcpy(location_new, _text, kernel_length); + + /* Sync the caches ready for execution of new kernel */ + __asm__ __volatile__ ( + "ibar 0 \t\n" + "dbar 0 \t\n" + ::: "memory"); + + reloc_offset += random_offset; + + /* Return the new kernel's entry point */ + kernel_entry = RELOCATED_KASLR(start_kernel); + + /* The current thread is now within the relocated kernel */ + __asm__ __volatile__ ( + "move $t0, %0\t\n" + "add.d $tp, $tp, $t0\t\n" + ::"r" (random_offset) + :); + + update_reloc_offset(&reloc_offset, random_offset); + } + if (reloc_offset) relocate_relative(); - relocate_la_abs(); + relocate_la_abs(random_offset); + + return kernel_entry; } /* -- Gitee From 45cad9a617c3df59d9c3ba610fa69194516207d4 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 10:14:41 +0800 Subject: [PATCH 13/16] LoongArch: kdump: Add single kernel image implementation LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- This feature depends on the kernel being relocatable. Enable using single kernel image for kdump, and then no longer need to build two kernels (production kernel and capture kernel share a single kernel image). Signed-off-by: Youling Tang Change-Id: I28a3f91cef4ea9e54eacb573754f0c16ec744bf6 --- arch/loongarch/Kconfig | 12 +----------- arch/loongarch/Makefile | 4 ---- arch/loongarch/include/asm/addrspace.h | 2 ++ arch/loongarch/kernel/head.S | 2 +- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 82322a138503..aeb69a46d915 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -476,6 +476,7 @@ config KEXEC config CRASH_DUMP bool "Build kdump crash kernel" + select RELOCATABLE help Generate crash dump after being started by kexec. This should be normally only set in special crash dump kernels which are @@ -485,17 +486,6 @@ config CRASH_DUMP For more details see Documentation/admin-guide/kdump/kdump.rst -config PHYSICAL_START - hex "Physical address where the kernel is loaded" - default "0x90000000a0000000" - depends on CRASH_DUMP - help - This gives the XKPRANGE address where the kernel is loaded. - If you plan to use kernel for capturing the crash dump change - this value to start of the reserved region (the "X" value as - specified in the "crashkernel=YM@XM" command line boot parameter - passed to the panic-ed kernel). - config RELOCATABLE bool "Relocatable kernel" help diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index b774357c8c99..345dc10576d4 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -71,11 +71,7 @@ endif cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) -ifndef CONFIG_PHYSICAL_START load-y = 0x9000000000200000 -else -load-y = $(CONFIG_PHYSICAL_START) -endif bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) drivers-$(CONFIG_PCI) += arch/loongarch/pci/ diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h index ba2d71b74a04..035d80db58e5 100644 --- a/arch/loongarch/include/asm/addrspace.h +++ b/arch/loongarch/include/asm/addrspace.h @@ -126,4 +126,6 @@ extern unsigned long vm_map_base; #define IO_SPACE_LIMIT (PCI_IOSIZE - 1) #define ISA_PHY_IOBASE LOONGSON_LIO_BASE +#define PHYS_LINK_KADDR PHYSADDR(VMLINUX_LOAD_ADDRESS) + #endif /* _ASM_ADDRSPACE_H */ diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 5f7d50073806..877da878d07f 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -23,7 +23,7 @@ _head: .org 0x8 .dword kernel_entry /* Kernel entry point */ .dword _end - _text /* Kernel image effective size */ - .quad 0 /* Kernel image load offset from start of RAM */ + .quad PHYS_LINK_KADDR /* Kernel image load offset from start of RAM */ .org 0x3c /* 0x20 ~ 0x3b reserved */ .long pe_header - _head /* Offset to the PE header */ -- Gitee From ab4bfe57a0c5fea8f0ef13dcb32195db53051344 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 10:15:31 +0800 Subject: [PATCH 14/16] LoongArch: kdump: Add crashkernel=YM handling LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- When the kernel crashkernel parameter is specified with just a size, we are supposed to allocate a region from RAM to store the crashkernel, "crashkernel=512M" would be recommended for kdump. Fix this by lifting similar code from x86, importing it to LoongArch with LoongArch specific parameters added. We allocate the crashkernel region from the first 4GB of physical memory (because SWIOTLB should be allocated below 4GB). However, LoongArch currently does not implement crashkernel_low and crashkernel_high the same as x86. When X is not specified, crash_base defaults to 0 (crashkernel=YM@XM). Signed-off-by: Youling Tang Change-Id: Id744bf55c2ccbe32e7d44eac8901b8849586baa3 --- arch/loongarch/kernel/setup.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index fac223d533e0..5239d1e60101 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -267,11 +267,14 @@ static void __init arch_reserve_vmcore(void) #endif } +/* 2MB alignment for crash kernel regions */ +#define CRASH_ALIGN SZ_2M +#define CRASH_ADDR_MAX SZ_4G + static void __init arch_parse_crashkernel(void) { #ifdef CONFIG_KEXEC int ret; - unsigned long long start; unsigned long long total_mem; unsigned long long crash_base, crash_size; @@ -280,8 +283,13 @@ static void __init arch_parse_crashkernel(void) if (ret < 0 || crash_size <= 0) return; - start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size); - if (start != crash_base) { + if (crash_base <= 0) { + crash_base = memblock_phys_alloc_range(crash_size, CRASH_ALIGN, CRASH_ALIGN, CRASH_ADDR_MAX); + if (!crash_base) { + pr_warn("crashkernel reservation failed - No suitable area found.\n"); + return; + } + } else if (!memblock_phys_alloc_range(crash_size, CRASH_ALIGN, crash_base, crash_base + crash_size)) { pr_warn("Invalid memory region reserved for crash kernel\n"); return; } -- Gitee From 12d99260171114aedfc7837e8bdc28d9ba2cea15 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Mon, 13 Mar 2023 19:07:13 +0800 Subject: [PATCH 15/16] LoongArch: configs: Enable kexec/kdump LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- Signed-off-by: Youling Tang Change-Id: Ia09d55c506f16f622ed52c2e7e17d5c490635f7d --- arch/loongarch/configs/loongson3_defconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index ed606e85e72c..a4b13cc652c7 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -38,6 +38,8 @@ CONFIG_PERF_EVENTS=y CONFIG_CPU_HAS_LSX=y CONFIG_CPU_HAS_LASX=y CONFIG_NUMA=y +CONFIG_KEXEC=y +CONFIG_CRASH_DUMP=y CONFIG_HIBERNATION=y CONFIG_ACPI_SPCR_TABLE=y CONFIG_ACPI_DOCK=y -- Gitee From f45a5a9978a3d6064fa69f1432a469649afa1e92 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Tue, 21 Mar 2023 09:55:25 +0800 Subject: [PATCH 16/16] LoongArch: Fix kdump failure on v40 interface specification LoongArch inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I736HO -------------------------------- The old memory should be reserved after efi_runtime_init() to avoid destroying the EFI space and causing failure when executing svam(). Fix the following problems when executing kdump: [ 0.000000] The BIOS Version: Loongson-UDK2018-V2.0.04082-beta7 [ 0.000000] CPU 0 Unable to handle kernel paging request at virtual address 00000000fdeb0e7c, era == 00000000fdeb0e7c, ra == 90000000dae6585c [ 0.000000] Oops[#1]: [ 0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 5.10.137+ #86 [ 0.000000] Hardware name: Loongson Loongson-3A5000-7A1000-1w-A2101/Loongson-LS3A5000-7A1000-1w-A2101, BIOS vUDK2018-LoongArch-V2.0.pre-beta8 06/15/2022 [ 0.000000] $ 0 : 0000000000000000 90000000dae6585c 90000000db200000 90000000db203840 [ 0.000000] $ 4 : 0000000000000078 0000000000000028 0000000000000001 00000000db203860 [ 0.000000] $ 8 : 0000000000000000 0000000000000040 90000000db203680 0000000000000000 [ 0.000000] $12 : 00000000fdeb0e7c ffffffffffffffc0 00000000fbffffff 0000000020000000 [ 0.000000] $16 : 000000000003e780 0000000020000000 90000000dad8c348 0000000000003fff [ 0.000000] $20 : 0000000000000018 90000000dad8bdd0 90000000db203850 0000000000000040 [ 0.000000] $24 : 000000000000000f 90000000db21a570 90000000daeb07a0 90000000db217000 [ 0.000000] $28 : 90000000db203858 0000000001ffffff 90000000db2171b0 0000000000000040 [ 0.000000] era : 00000000fdeb0e7c 0xfdeb0e7c [ 0.000000] ra : 90000000dae6585c set_virtual_map.isra.0+0x23c/0x394 [ 0.000000] CSR crmd: 90000000db21a570 [ 0.000000] CSR prmd: 00000000 [ 0.000000] CSR euen: 00000000 [ 0.000000] CSR ecfg: 90000000db203850 [ 0.000000] CSR estat: 90000000dae65800 [ 0.000000] ExcCode : 26 (SubCode 16b) [ 0.000000] PrId : 0014c012 (Loongson-64bit) [ 0.000000] Modules linked in: [ 0.000000] Process swapper (pid: 0, threadinfo=(____ptrval____), task=(____ptrval____)) [ 0.000000] Stack : 0000000000000001 00000000fdeb0e7c 0000000000036780 000000000003e780 [ 0.000000] 0000000000000006 0000000010000000 8000000010000000 0000000000010000 [ 0.000000] 8000000000000001 0000000000000005 00000000fde40000 90000000fde40000 [ 0.000000] 0000000000000100 800000000000000f 0000000000000006 00000000fdf40000 [ 0.000000] 90000000fdf40000 0000000000000300 800000000000000f 00000000000000b0 [ 0.000000] 0000000000000001 90000000da094cf0 0000000000000000 ffffffffffffffea [ 0.000000] 90000000db2039b8 ffff0a1000000609 0000000000000035 0000000000000030 [ 0.000000] 90000000dad7b258 0000000000000400 00000000000000b0 ffff0a1000000609 [ 0.000000] 90000000db2039a8 90000000db095730 000000007fffffff ffff0a1000000609 [ 0.000000] 90000000db203a90 90000000db203a30 90000000db2039d8 90000000db09570b [ 0.000000] ... [ 0.000000] Call Trace: [ 0.000000] [ 0.000000] Code: (Bad address in era) [ 0.000000] [ 0.000000] Signed-off-by: Youling Tang Change-Id: I1cdf06f520fd16fa2c2dcc6b852d76d038771a86 --- arch/loongarch/kernel/setup.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 5239d1e60101..45cb89308fcf 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -316,10 +316,6 @@ static void reserve_oldmem_region(void) void __init platform_init(void) { - arch_reserve_vmcore(); - arch_parse_crashkernel(); - reserve_oldmem_region(); - #ifdef CONFIG_ACPI_TABLE_UPGRADE acpi_table_upgrade(); #endif @@ -356,6 +352,10 @@ static void __init check_kernel_sections_mem(void) */ static void __init arch_mem_init(char **cmdline_p) { + arch_reserve_vmcore(); + arch_parse_crashkernel(); + reserve_oldmem_region(); + if (usermem) pr_info("User-defined physical RAM map overwrite\n"); -- Gitee