diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 26c1d2ced4ce18f85ae3f0e33a7a931072f4e4ff..d7635f74610054dd4742737644f2e1c00e8cb446 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -31,7 +31,7 @@ void handle_IRQ(unsigned int, struct pt_regs *); #ifdef CONFIG_SMP #include -extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace #endif diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 3431c0553f45cdf5f0b1b8dbb59bf38003da9995..9ffc6c2536fa30903203337f1266314540f403d3 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -846,7 +846,8 @@ static void raise_nmi(cpumask_t *mask) __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_nmi); + return true; } diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index fac08e18bcd512eec9bbafe7fa79a38ccea1d48c..9e926b4f7d47614ee4fd53cba48c04fbc09d421b 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -6,6 +6,12 @@ #include +#ifdef CONFIG_SMP +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, + bool exclude_self); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace +#endif + struct pt_regs; int set_handle_irq(void (*handle_irq)(struct pt_regs *)); diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h new file mode 100644 index 0000000000000000000000000000000000000000..4cd14b6af88b9658c7552a73618fb5a99f87d93e --- /dev/null +++ b/arch/arm64/include/asm/nmi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NMI_H +#define __ASM_NMI_H + +#ifndef __ASSEMBLER__ + +#include + +extern bool arm64_supports_nmi(void); +extern void arm64_send_nmi(cpumask_t *mask); + +void set_smp_dynamic_ipi(int ipi); +void dynamic_ipi_setup(int cpu); +void dynamic_ipi_teardown(int cpu); + +#endif /* !__ASSEMBLER__ */ +#endif diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 59fa5d6e6f3e632aa4cf38ee78503b6cce35dbd9..187685f57980532e5c2d9ae95c463a275083426b 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,7 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idreg-override.o idle.o \ - patching.o + patching.o ipi_nmi.o obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/ipi_nmi.c b/arch/arm64/kernel/ipi_nmi.c new file mode 100644 index 0000000000000000000000000000000000000000..898d69c3742982891bf94bb48f3f7d7225227a91 --- /dev/null +++ b/arch/arm64/kernel/ipi_nmi.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NMI support for IPIs + * + * Copyright (C) 2020 Linaro Limited + * Author: Sumit Garg + */ + +#include +#include +#include +#include +#include + +#include + +static struct irq_desc *ipi_nmi_desc __read_mostly; +static int ipi_nmi_id __read_mostly; + +bool arm64_supports_nmi(void) +{ + if (ipi_nmi_desc) + return true; + + return false; +} + +void arm64_send_nmi(cpumask_t *mask) +{ + if (WARN_ON_ONCE(!ipi_nmi_desc)) + return; + + __ipi_send_mask(ipi_nmi_desc, mask); +} + +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) +{ + if (!ipi_nmi_desc) + return false; + + nmi_trigger_cpumask_backtrace(mask, exclude_self, arm64_send_nmi); + + return true; +} + +static irqreturn_t ipi_nmi_handler(int irq, void *data) +{ + irqreturn_t ret = IRQ_NONE; + unsigned int cpu = smp_processor_id(); + + if (nmi_cpu_backtrace(get_irq_regs())) + ret = IRQ_HANDLED; + + if (!kgdb_nmicallback(cpu, get_irq_regs())) + ret = IRQ_HANDLED; + + return ret; +} + +void dynamic_ipi_setup(int cpu) +{ + if (!ipi_nmi_desc) + return; + + if (!prepare_percpu_nmi(ipi_nmi_id)) + enable_percpu_nmi(ipi_nmi_id, IRQ_TYPE_NONE); +} + +void dynamic_ipi_teardown(int cpu) +{ + if (!ipi_nmi_desc) + return; + + disable_percpu_nmi(ipi_nmi_id); + teardown_percpu_nmi(ipi_nmi_id); +} + +void __init set_smp_dynamic_ipi(int ipi) +{ + if (!request_percpu_nmi(ipi, ipi_nmi_handler, "IPI", &cpu_number)) { + ipi_nmi_desc = irq_to_desc(ipi); + ipi_nmi_id = ipi; + } +} diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index 4e1f983df3d1c22df953e9249312683aa3e0d04e..98d8c1027f06c10b6b6c42d87b7d67812c8900d8 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -356,3 +357,20 @@ int kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt) return aarch64_insn_write((void *)bpt->bpt_addr, *(u32 *)bpt->saved_instr); } + +void kgdb_roundup_cpus(void) +{ + struct cpumask mask; + + if (!arm64_supports_nmi()) { + kgdb_smp_call_nmi_hook(); + return; + } + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(raw_smp_processor_id(), &mask); + if (cpumask_empty(&mask)) + return; + + arm64_send_nmi(&mask); +} diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 960b98b43506dd2246aac628e2a92ab455f44043..8288549ca5f77de0059db1836369bdc0a8aeab24 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -935,6 +936,8 @@ static void ipi_setup(int cpu) for (i = 0; i < nr_ipi; i++) enable_percpu_irq(ipi_irq_base + i, 0); + + dynamic_ipi_setup(cpu); } #ifdef CONFIG_HOTPLUG_CPU @@ -947,6 +950,8 @@ static void ipi_teardown(int cpu) for (i = 0; i < nr_ipi; i++) disable_percpu_irq(ipi_irq_base + i); + + dynamic_ipi_teardown(cpu); } #endif @@ -968,6 +973,9 @@ void __init set_smp_ipi_range(int ipi_base, int n) irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); } + if (n > nr_ipi) + set_smp_dynamic_ipi(ipi_base + nr_ipi); + ipi_irq_base = ipi_base; /* Setup the boot CPU immediately */ diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index 3a848e7e69f71102c925df2dd2be1003419a9bfd..7923ba290e2d12b9a777a79fe68caeca7bac3b46 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -76,7 +76,7 @@ extern int cp0_fdc_irq; extern int get_c0_fdc_int(void); -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, int exclude_cpu); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 5387ed0a51862b66609d990cb039b27261a5c234..e2e57dde25702445f888d6197ddfb9db771c2395 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -750,9 +750,10 @@ static void raise_backtrace(cpumask_t *mask) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_backtrace); + return true; } int mips_get_process_fp_mode(struct task_struct *task) diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index 49a75340c3e07f37d4e1c344f63a191476f164aa..daf2d9fcc7a0ed844edf6eb14110c4513e29df6f 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -9,6 +9,12 @@ void watchdog_hardlockup_set_timeout_pct(u64 pct); static inline void watchdog_hardlockup_set_timeout_pct(u64 pct) {} #endif +#ifdef CONFIG_NMI_IPI +extern bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, + int exclude_cpu); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace +#endif + extern void hv_nmi_check_nonrecoverable(struct pt_regs *regs); #endif /* _ASM_NMI_H */ diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c index e6a958a5da2763deca47037b1246e2d5a83576d5..3d1c373c1cee05e71175cb0a6410931d9d39c765 100644 --- a/arch/powerpc/kernel/stacktrace.c +++ b/arch/powerpc/kernel/stacktrace.c @@ -204,8 +204,9 @@ static void raise_backtrace_ipi(cpumask_t *mask) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { nmi_trigger_cpumask_backtrace(mask, exclude_cpu, raise_backtrace_ipi); + return true; } #endif /* defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_NMI_IPI) */ diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 8c4c0c87f99808356b20a0c7172c5096fd7201e5..4daf266971007d804b06757cfb4f9eae856bc789 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -86,7 +86,7 @@ static inline unsigned long get_softint(void) return retval; } -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, int exclude_cpu); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 529adfecd58ca1ee98199c64e30ea00e232b439f..5bcb86ec0c0708b4c33ce9100e4c9b7f505885bc 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -236,7 +236,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { struct thread_info *tp = current_thread_info(); struct pt_regs *regs = get_irq_regs(); @@ -291,6 +291,8 @@ void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); + + return true; } #ifdef CONFIG_MAGIC_SYSRQ diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 836c170d308755fe205b802e34b51c7ffd592017..e1cadd9a439db7976df4893e2fa83c02b0f85a78 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -41,7 +41,7 @@ extern void __handle_irq(struct irq_desc *desc, struct pt_regs *regs); extern void init_ISA_irqs(void); #ifdef CONFIG_X86_LOCAL_APIC -void arch_trigger_cpumask_backtrace(const struct cpumask *mask, +bool arch_trigger_cpumask_backtrace(const struct cpumask *mask, int exclude_cpu); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index 45af535c44a07de0c4198acd4aa66fbd0c045e93..3c7ebee0d491c48d2d7df9ed59eb37c68ea78cb8 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -36,10 +36,11 @@ static void nmi_raise_cpu_backtrace(cpumask_t *mask) __apic_send_IPI_mask(mask, NMI_VECTOR); } -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +bool arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { nmi_trigger_cpumask_backtrace(mask, exclude_cpu, nmi_raise_cpu_backtrace); + return true; } static int nmi_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index f59ac9586b7b1d2240ea66aec29a4e0dde676919..874fbc4a2da69c62bedf124d75bc9bec183a57d8 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -527,6 +527,7 @@ static u32 gic_get_ppi_index(struct irq_data *d) static int gic_irq_nmi_setup(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); + u32 idx; if (!gic_supports_nmi()) return -EINVAL; @@ -544,16 +545,22 @@ static int gic_irq_nmi_setup(struct irq_data *d) return -EINVAL; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); + switch (get_intid_range(d)) { + case SGI_RANGE: + break; + case PPI_RANGE: + case EPPI_RANGE: + idx = gic_get_ppi_index(d); /* Setting up PPI as NMI, only switch handler for first NMI */ if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { refcount_set(&ppi_nmi_refs[idx], 1); desc->handle_irq = handle_percpu_devid_fasteoi_nmi; } - } else { + break; + default: desc->handle_irq = handle_fasteoi_nmi; + break; } gic_irq_set_prio(d, GICD_INT_NMI_PRI); @@ -564,6 +571,7 @@ static int gic_irq_nmi_setup(struct irq_data *d) static void gic_irq_nmi_teardown(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); + u32 idx; if (WARN_ON(!gic_supports_nmi())) return; @@ -581,14 +589,20 @@ static void gic_irq_nmi_teardown(struct irq_data *d) return; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); + switch (get_intid_range(d)) { + case SGI_RANGE: + break; + case PPI_RANGE: + case EPPI_RANGE: + idx = gic_get_ppi_index(d); /* Tearing down NMI, only switch handler for last NMI */ if (refcount_dec_and_test(&ppi_nmi_refs[idx])) desc->handle_irq = handle_percpu_devid_irq; - } else { + break; + default: desc->handle_irq = handle_fasteoi_irq; + break; } gic_irq_set_prio(d, GICD_INT_DEF_PRI); @@ -2074,6 +2088,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_dist_init(); gic_cpu_init(); + gic_enable_nmi_support(); gic_smp_init(); gic_cpu_pm_init(); @@ -2086,8 +2101,6 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gicv2m_init(handle, gic_data.domain); } - gic_enable_nmi_support(); - return 0; out_free: diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index 76e891ee9e37126e47bb238f7fb6127b86b88594..f28bea121891a3d70b4e0ac19cdcba1f251bb512 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -199,6 +199,18 @@ kgdb_arch_handle_qxfer_pkt(char *remcom_in_buffer, extern void kgdb_call_nmi_hook(void *ignored); +/** + * kgdb_smp_call_nmi_hook - Provide default fallback mechanism to + * round-up CPUs + * + * If you're using the default implementation of kgdb_roundup_cpus() + * this function will be called. And if an arch detects at runtime to + * not support NMI based roundup then it can fallback to default + * mechanism using this API. + */ + +extern void kgdb_smp_call_nmi_hook(void); + /** * kgdb_roundup_cpus - Get other CPUs into a holding pattern * diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 7bd446acad24ab70c7cd66bae1fb8cd069f13105..692f2759be55bff58addc2d6efadff9878772b7a 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -157,26 +157,22 @@ static inline void touch_nmi_watchdog(void) #ifdef arch_trigger_cpumask_backtrace static inline bool trigger_all_cpu_backtrace(void) { - arch_trigger_cpumask_backtrace(cpu_online_mask, -1); - return true; + return arch_trigger_cpumask_backtrace(cpu_online_mask, -1); } static inline bool trigger_allbutcpu_cpu_backtrace(int exclude_cpu) { - arch_trigger_cpumask_backtrace(cpu_online_mask, exclude_cpu); - return true; + return arch_trigger_cpumask_backtrace(cpu_online_mask, exclude_cpu); } static inline bool trigger_cpumask_backtrace(struct cpumask *mask) { - arch_trigger_cpumask_backtrace(mask, -1); - return true; + return arch_trigger_cpumask_backtrace(mask, -1); } static inline bool trigger_single_cpu_backtrace(int cpu) { - arch_trigger_cpumask_backtrace(cpumask_of(cpu), -1); - return true; + return arch_trigger_cpumask_backtrace(cpumask_of(cpu), -1); } /* generic implementation */ diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index ce1bb2301c061d9695d85f6508ed23380c4cb4ce..46aa536205bc88de26123b1bf5899883b422c9af 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -238,7 +238,7 @@ NOKPROBE_SYMBOL(kgdb_call_nmi_hook); static DEFINE_PER_CPU(call_single_data_t, kgdb_roundup_csd) = CSD_INIT(kgdb_call_nmi_hook, NULL); -void __weak kgdb_roundup_cpus(void) +void kgdb_smp_call_nmi_hook(void) { call_single_data_t *csd; int this_cpu = raw_smp_processor_id(); @@ -269,6 +269,12 @@ void __weak kgdb_roundup_cpus(void) kgdb_info[cpu].rounding_up = false; } } +NOKPROBE_SYMBOL(kgdb_smp_call_nmi_hook); + +void __weak kgdb_roundup_cpus(void) +{ + kgdb_smp_call_nmi_hook(); +} NOKPROBE_SYMBOL(kgdb_roundup_cpus); #endif