diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 74b925c8d534e76de6a3dcd41bba3ab739ad642c..b1c6cbe02db7a2c950a143279c95e80eec33ed2d 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -1813,7 +1813,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, int syscall_trace_enter(struct pt_regs *regs) { - unsigned long flags = READ_ONCE(current_thread_info()->flags); + unsigned long flags = read_thread_flags(); if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER); @@ -1836,7 +1836,7 @@ int syscall_trace_enter(struct pt_regs *regs) void syscall_trace_exit(struct pt_regs *regs) { - unsigned long flags = READ_ONCE(current_thread_info()->flags); + unsigned long flags = read_thread_flags(); audit_syscall_exit(regs); diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 17cb54d1e420bc90a23ab90812bf34301de65786..3a99b096abec6f6b8ffd4e19e1210e78b0f42952 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -724,7 +724,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, } local_daif_mask(); - thread_flags = READ_ONCE(current_thread_info()->flags); + thread_flags = read_thread_flags(); } while (thread_flags & _TIF_WORK_MASK); } diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 2a106e67a4cbf1c85ffc6146b6c3811da484b1c0..330cec1f7768b35b2303ab1388cfdb58f61a75fb 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -109,7 +109,7 @@ static void cortex_a76_erratum_1463225_svc_handler(void) { } static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, const syscall_fn_t syscall_table[]) { - unsigned long flags = current_thread_info()->flags; + unsigned long flags = read_thread_flags(); regs->orig_x0 = regs->regs[0]; regs->syscallno = scno; @@ -176,11 +176,9 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, * exit regardless, as the old entry assembly did. */ if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { - local_daif_mask(); - flags = current_thread_info()->flags; + flags = read_thread_flags(); if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP)) return; - local_daif_restore(DAIF_PROCCTX); } trace_exit: diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 76b37d1bcac16861ed16636a959c5350336fceda..7f2f299bff8d2365d1e23306e9b6d4db161e1941 100755 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -370,7 +370,7 @@ void arch_setup_new_exec(void) clear_thread_flag(TIF_SSBD); task_clear_spec_ssb_disable(current); task_clear_spec_ssb_noexec(current); - speculation_ctrl_update(task_thread_info(current)->flags); + speculation_ctrl_update(read_thread_flags()); } } @@ -622,7 +622,7 @@ static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk) clear_tsk_thread_flag(tsk, TIF_SPEC_IB); } /* Return the updated threadinfo flags*/ - return task_thread_info(tsk)->flags; + return read_task_thread_flags(tsk); } void speculation_ctrl_update(unsigned long tif) @@ -658,8 +658,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p) { unsigned long tifp, tifn; - tifn = READ_ONCE(task_thread_info(next_p)->flags); - tifp = READ_ONCE(task_thread_info(prev_p)->flags); + tifn = read_task_thread_flags(next_p); + tifp = read_task_thread_flags(prev_p); switch_to_bitmap(tifp); diff --git a/arch/x86/kernel/process.h b/arch/x86/kernel/process.h index 1d0797b2338a291a9ba6ec89f0ec7aead1b4fbd0..76b547b832320b0570ff8e0aa4a6ad1171db5348 100644 --- a/arch/x86/kernel/process.h +++ b/arch/x86/kernel/process.h @@ -13,8 +13,8 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p); static inline void switch_to_extra(struct task_struct *prev, struct task_struct *next) { - unsigned long next_tif = task_thread_info(next)->flags; - unsigned long prev_tif = task_thread_info(prev)->flags; + unsigned long next_tif = read_task_thread_flags(next); + unsigned long prev_tif = read_task_thread_flags(prev); if (IS_ENABLED(CONFIG_SMP)) { /* diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index f4b162f273f52ca0ca6f404f6c30920886dab7fd..86a66efa26dc3394aa5269b4886f00669b2ab1ac 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -318,7 +318,7 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, static unsigned long mm_mangle_tif_spec_ib(struct task_struct *next) { - unsigned long next_tif = task_thread_info(next)->flags; + unsigned long next_tif = read_task_thread_flags(next); unsigned long ibpb = (next_tif >> TIF_SPEC_IB) & LAST_USER_MM_IBPB; return (unsigned long)next->mm | ibpb; diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 6cee83bf2e5e2dc4d42d3e7b05fcb3d80eff2a07..ef7ddfefcd00df93b1555cff1cad1a3430bb9fab 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1299,7 +1299,7 @@ static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) * Ensure that stores to Normal memory are visible to the * other CPUs before issuing the IPI. */ - wmb(); + dsb(ishst); for_each_cpu(cpu, mask) { u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); diff --git a/include/linux/entry-kvm.h b/include/linux/entry-kvm.h index 859fdfd7d46c97fd7b50dc3ee5fb09fe4c198a31..4fe38c2d27066ac136bd53efa7ee062e850ce536 100644 --- a/include/linux/entry-kvm.h +++ b/include/linux/entry-kvm.h @@ -60,7 +60,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu); */ static inline bool __xfer_to_guest_mode_work_pending(void) { - unsigned long ti_work = READ_ONCE(current_thread_info()->flags); + unsigned long ti_work = read_thread_flags(); return !!(ti_work & XFER_TO_GUEST_MODE_WORK); } diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index 19f76d87f20fcd6aca9f00441ef5954c7c9ba1a5..af3ca6429d77e0e7957cb496b1eb1444a1c5fd85 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -36,6 +36,21 @@ static inline long set_restart_fn(struct restart_block *restart, #define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO) +/* + * This may be used in noinstr code, and needs to be __always_inline to prevent + * inadvertent instrumentation. + */ +static __always_inline unsigned long read_ti_thread_flags(struct thread_info *ti) +{ + return READ_ONCE(ti->flags); +} + +#define read_thread_flags() \ + read_ti_thread_flags(current_thread_info()) + +#define read_task_thread_flags(t) \ + read_ti_thread_flags(task_thread_info(t)) + #define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) #ifndef CONFIG_HAVE_ARCH_WITHIN_STACK_FRAMES diff --git a/kernel/entry/common.c b/kernel/entry/common.c index a028b28daed5b587334d5880b1680adfb2ee2625..7e4fc453da7da5fea335b2b894c605309800a960 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -186,7 +186,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, /* Check if any of the above work has queued a deferred wakeup */ rcu_nocb_flush_deferred_wakeup(); - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); } /* Return the latest work state for arch_exit_to_user_mode() */ @@ -195,7 +195,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, static void exit_to_user_mode_prepare(struct pt_regs *regs) { - unsigned long ti_work = READ_ONCE(current_thread_info()->flags); + unsigned long ti_work = read_thread_flags(); lockdep_assert_irqs_disabled(); diff --git a/kernel/entry/kvm.c b/kernel/entry/kvm.c index 049fd06b4c3de348d3e47a9691880c3272a81c8f..6995b7ee63b9784c2105cf0174c166a503ab5a61 100644 --- a/kernel/entry/kvm.c +++ b/kernel/entry/kvm.c @@ -28,7 +28,7 @@ static int xfer_to_guest_mode_work(struct kvm_vcpu *vcpu, unsigned long ti_work) if (ret) return ret; - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); } while (ti_work & XFER_TO_GUEST_MODE_WORK || need_resched()); return 0; } @@ -45,7 +45,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu) * disabled in the inner loop before going into guest mode. No need * to disable interrupts here. */ - ti_work = READ_ONCE(current_thread_info()->flags); + ti_work = read_thread_flags(); if (!(ti_work & XFER_TO_GUEST_MODE_WORK)) return 0; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 20efc7cccf199e417e338d59a1b4b55aff1e3652..81b58ce710ba8de04792ee48f9a37630619f8656 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7524,7 +7524,7 @@ void sched_show_task(struct task_struct *p) rcu_read_unlock(); pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n", free, task_pid_nr(p), ppid, - (unsigned long)task_thread_info(p)->flags); + read_task_thread_flags(p)); print_worker_info(KERN_INFO, p); show_stack(p, NULL, KERN_INFO); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d3c4b945c0196204c80ff36f51189171d5592768..fbabad1e193aa7a7deb8fb31814c708f24d138b3 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6190,9 +6190,11 @@ static inline bool test_idle_cores(int cpu, bool def) { struct sched_domain_shared *sds; - sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); - if (sds) - return READ_ONCE(sds->has_idle_cores); + if (static_branch_likely(&sched_smt_present)) { + sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); + if (sds) + return READ_ONCE(sds->has_idle_cores); + } return def; }