From 50c0a0b47f935929ddcc9231148855439b8f5f12 Mon Sep 17 00:00:00 2001 From: Zhiteng Qiu Date: Fri, 9 Jan 2026 10:04:56 +0800 Subject: [PATCH] anolis: x86/fpu: Resolve FPU state corruption between atomic and non-atomic contexts ANBZ: #29391 Fix data inconsistency caused by race condition between kernel_fpu_begin_nonatomic (used for NT copy) and kernel_fpu_begin. Introduce separate FPU context flags to properly isolate atomic and non-atomic FPU usage paths, ensuring state integrity during preemption and context switches. Signed-off-by: Zhiteng Qiu --- arch/x86/include/asm/fpu/api.h | 25 ------ arch/x86/include/asm/fpu/internal.h | 12 +-- arch/x86/include/asm/uaccess_64.h | 22 +++-- arch/x86/kernel/cpu/bugs.c | 16 ++++ arch/x86/kernel/cpu/hygon.c | 27 ++---- arch/x86/kernel/fpu/core.c | 135 +++++++++++++++++++--------- arch/x86/kernel/fpu/init.c | 11 +++ arch/x86/kernel/fpu/xstate.c | 9 ++ arch/x86/kernel/process_64.c | 6 +- 9 files changed, 164 insertions(+), 99 deletions(-) diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h index 4cffcba00e54..cd2c7f15995d 100644 --- a/arch/x86/include/asm/fpu/api.h +++ b/arch/x86/include/asm/fpu/api.h @@ -45,34 +45,9 @@ extern void kernel_fpu_end_nonatomic(void); /* Code that is unaware of kernel_fpu_begin_nonatomic_mask() can use this */ static inline int kernel_fpu_begin_nonatomic(void) { -#ifdef CONFIG_X86_64 - /* - * Any 64-bit code that uses 387 instructions must explicitly request - * KFPU_387. - */ - return kernel_fpu_begin_nonatomic_mask(KFPU_MXCSR); -#else - /* - * 32-bit kernel code may use 387 operations as well as SSE2, etc, - * as long as it checks that the CPU has the required capability. - */ return kernel_fpu_begin_nonatomic_mask(KFPU_387 | KFPU_MXCSR); -#endif -} - -/* - * It means we call kernel_fpu_end after kernel_fpu_begin_nonatomic - * func, but before kernel_fpu_end_nonatomic - */ -static inline void check_using_kernel_fpu(void) -{ - if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) - WARN_ON_ONCE(test_thread_flag(TIF_USING_FPU_NONATOMIC)); } -#else -static inline void check_using_kernel_fpu(void) { } - #endif /* diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 727d667d1c19..be9e21599eb8 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -16,12 +16,12 @@ #if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ defined(CONFIG_X86_HYGON_LMC_AVX2_ON) extern void save_fpregs_to_fpkernelstate(struct fpu *kfpu); +extern unsigned long get_fpu_registers_pos(struct fpu *fpu, unsigned int off); static inline void switch_kernel_fpu_prepare(struct task_struct *prev, int cpu) { struct fpu *old_fpu = &prev->thread.fpu; - if ((boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) || - !test_thread_flag(TIF_USING_FPU_NONATOMIC)) + if (!test_thread_flag(TIF_USING_FPU_NONATOMIC)) return; if (static_cpu_has(X86_FEATURE_FPU) && !(prev->flags & PF_KTHREAD)) @@ -31,8 +31,8 @@ static inline void switch_kernel_fpu_prepare(struct task_struct *prev, int cpu) /* Internal helper for switch_kernel_fpu_finish() and signal frame setup */ static inline void fpregs_restore_kernelregs(struct fpu *kfpu) { - kernel_fpu_states_restore(NULL, &kfpu->fpstate->kernel_state, - sizeof(kfpu->fpstate->kernel_state)); + kernel_fpu_states_restore(NULL, (void *)get_fpu_registers_pos(kfpu, MAX_FPU_CTX_SIZE), + MAX_FPU_CTX_SIZE); } /* Loading of the complete FPU state immediately. */ @@ -40,8 +40,7 @@ static inline void switch_kernel_fpu_finish(struct task_struct *next) { struct fpu *new_fpu = &next->thread.fpu; - if ((next->flags & PF_KTHREAD) || - (boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)) + if (next->flags & PF_KTHREAD) return; if (cpu_feature_enabled(X86_FEATURE_FPU) && @@ -49,6 +48,7 @@ static inline void switch_kernel_fpu_finish(struct task_struct *next) TIF_USING_FPU_NONATOMIC)) fpregs_restore_kernelregs(new_fpu); } + #else static inline void switch_kernel_fpu_prepare(struct task_struct *prev, int cpu) { diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index ea7cc9dab376..63b81cd2c32b 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -11,11 +11,14 @@ #include #include #include +#include #if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ defined(CONFIG_X86_HYGON_LMC_AVX2_ON) #include #endif +extern struct static_key_false hygon_lmc_key; + /* * Copy To/From Userspace */ @@ -38,6 +41,9 @@ void fpu_restore_xmm0_3(void *to, const void *from, unsigned len); __must_check unsigned long copy_user_sse2_opt_string(void *to, const void *from, unsigned len); +#define MAX_FPU_CTX_SIZE 64 +#define KERNEL_FPU_NONATOMIC_SIZE (2 * (MAX_FPU_CTX_SIZE)) + #define copy_user_large_memory_generic_string copy_user_sse2_opt_string #endif @@ -52,6 +58,9 @@ void fpu_restore_ymm0_7(void *to, const void *from, unsigned len); __must_check unsigned long copy_user_avx2_pf64_nt_string(void *to, const void *from, unsigned len); +#define MAX_FPU_CTX_SIZE 256 +#define KERNEL_FPU_NONATOMIC_SIZE (2 * (MAX_FPU_CTX_SIZE)) + #define copy_user_large_memory_generic_string copy_user_avx2_pf64_nt_string #endif @@ -62,8 +71,7 @@ static inline bool Hygon_LMC_check(unsigned len) { unsigned int nt_blk_cpy_mini_len = get_nt_block_copy_mini_len(); - if ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && - ((nt_blk_cpy_mini_len) && (nt_blk_cpy_mini_len <= len) && + if (((nt_blk_cpy_mini_len) && (nt_blk_cpy_mini_len <= len) && (system_state == SYSTEM_RUNNING) && (!kernel_fpu_begin_nonatomic()))) return true; @@ -97,9 +105,13 @@ copy_user_generic(void *to, const void *from, unsigned len) unsigned ret; /* Check if Hygon large memory copy support enabled. */ - if (Hygon_LMC_check(len)) { - ret = copy_large_memory_generic_string(to, from, len); - return ret; + if (static_branch_unlikely(&hygon_lmc_key)) { + if (Hygon_LMC_check(len)) { + unsigned ret; + + ret = copy_large_memory_generic_string(to, from, len); + return ret; + } } /* If CPU has ERMS feature, use copy_user_enhanced_fast_string. diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 51dfedc68d6a..703664bb7a41 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -122,6 +122,20 @@ EXPORT_SYMBOL_GPL(mds_idle_clear); DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear); EXPORT_SYMBOL_GPL(mmio_stale_data_clear); +DEFINE_STATIC_KEY_FALSE(hygon_lmc_key); +EXPORT_SYMBOL_GPL(hygon_lmc_key); + +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) +static inline void update_lmc_branch_cond(void) +{ + if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + static_branch_enable(&hygon_lmc_key); +} +#else +static inline void update_lmc_branch_cond(void) { } +#endif + void __init check_bugs(void) { identify_boot_cpu(); @@ -181,6 +195,8 @@ void __init check_bugs(void) arch_smt_update(); + update_lmc_branch_cond(); + #ifdef CONFIG_X86_32 /* * Check whether we are able to run this kernel safely on SMP. diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index f57257465d0c..1fb867463220 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -491,9 +491,7 @@ struct hygon_c86_info { unsigned int nt_cpy_mini_len; }; -static struct hygon_c86_info hygon_c86_data = { - .nt_cpy_mini_len = 0 -}; +static struct hygon_c86_info hygon_c86_data = { .nt_cpy_mini_len = 0 }; void set_c86_features_para_invalid(void) { @@ -502,7 +500,9 @@ void set_c86_features_para_invalid(void) unsigned int get_nt_block_copy_mini_len(void) { - return hygon_c86_data.nt_cpy_mini_len; + unsigned int mini_len = hygon_c86_data.nt_cpy_mini_len; + + return mini_len; } EXPORT_SYMBOL_GPL(get_nt_block_copy_mini_len); @@ -528,13 +528,11 @@ static ssize_t store_nt_cpy_mini_len(struct kobject *kobj, return count; } -static struct kobj_attribute nt_cpy_mini_len_attribute = - __ATTR(nt_cpy_mini_len, 0600, show_nt_cpy_mini_len, - store_nt_cpy_mini_len); +static struct kobj_attribute nt_cpy_mini_len_attribute = __ATTR( + nt_cpy_mini_len, 0600, show_nt_cpy_mini_len, store_nt_cpy_mini_len); static struct attribute *c86_default_attrs[] = { - &nt_cpy_mini_len_attribute.attr, - NULL + &nt_cpy_mini_len_attribute.attr, NULL }; const struct attribute_group hygon_c86_attr_group = { @@ -569,15 +567,6 @@ static int __init kobject_hygon_c86_init(void) return -1; } -module_init(kobject_hygon_c86_init); - -static void __exit kobject_hygon_c86_exit(void) -{ - if (c86_features_kobj) { - sysfs_remove_group(c86_features_kobj, &hygon_c86_attr_group); - kobject_del(c86_features_kobj); - } -} -module_exit(kobject_hygon_c86_exit); +subsys_initcall(kobject_hygon_c86_init); #endif diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 8eb9e7c7295f..b9548d79450d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -13,7 +13,7 @@ #include #include #include - +#include #include #include @@ -44,12 +44,15 @@ struct fpstate init_fpstate __ro_after_init; /* Track in-kernel FPU usage */ static DEFINE_PER_CPU(bool, in_kernel_fpu); +static DEFINE_PER_CPU(bool, in_kernel_fpu_nonatomic); /* * Track which context is using the FPU on the CPU: */ DEFINE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); +extern struct static_key_false hygon_lmc_key; + /* * Can we use the FPU in kernel mode with the * whole "kernel_fpu_begin/end()" sequence? @@ -406,56 +409,37 @@ int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate); #endif /* CONFIG_KVM */ -void kernel_fpu_begin_mask(unsigned int kfpu_mask) -{ - preempt_disable(); - - check_using_kernel_fpu(); - - WARN_ON_FPU(!irq_fpu_usable()); - WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); - - this_cpu_write(in_kernel_fpu, true); - - if (!(current->flags & PF_KTHREAD) && - !test_thread_flag(TIF_NEED_FPU_LOAD)) { - set_thread_flag(TIF_NEED_FPU_LOAD); - save_fpregs_to_fpstate(¤t->thread.fpu); - } - __cpu_invalidate_fpregs_state(); - - /* Put sane initial values into the control registers. */ - if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM)) - ldmxcsr(MXCSR_DEFAULT); +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) - if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU)) - asm volatile ("fninit"); +extern unsigned int fpu_kernel_nonatomic_xstate_size; +unsigned int get_fpustate_free_space(struct fpu *fpu) +{ + if ((fpu_kernel_cfg.default_size + fpu_kernel_nonatomic_xstate_size) > + sizeof(fpu->fpstate->regs)) + return 0; + return fpu_kernel_nonatomic_xstate_size; } -EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask); -void kernel_fpu_end(void) +unsigned long get_fpu_registers_pos(struct fpu *fpu, unsigned int off) { - check_using_kernel_fpu(); - - WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); + unsigned long addr = 0; - this_cpu_write(in_kernel_fpu, false); - preempt_enable(); + if (fpu && (fpu_kernel_nonatomic_xstate_size > off)) { + addr = (unsigned long)&fpu->__fpstate.regs.__padding[0]; + addr += fpu_kernel_cfg.default_size + off; + } + return addr; } -EXPORT_SYMBOL_GPL(kernel_fpu_end); -#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ - defined(CONFIG_X86_HYGON_LMC_AVX2_ON) /* * We can call kernel_fpu_begin_nonatomic in non-atomic task context. */ int kernel_fpu_begin_nonatomic_mask(unsigned int kfpu_mask) { - preempt_disable(); - /* we not support Nested call */ - if (test_thread_flag(TIF_USING_FPU_NONATOMIC)) - goto err; + unsigned long flags; + preempt_disable(); /* * This means we call kernel_fpu_begin_nonatomic after kernel_fpu_begin, * but before kernel_fpu_end. @@ -463,12 +447,17 @@ int kernel_fpu_begin_nonatomic_mask(unsigned int kfpu_mask) if (this_cpu_read(in_kernel_fpu)) goto err; + if (this_cpu_read(in_kernel_fpu_nonatomic)) + goto err; + if (in_interrupt()) goto err; if (current->flags & PF_KTHREAD) goto err; + local_irq_save(flags); + if (!test_thread_flag(TIF_NEED_FPU_LOAD)) { set_thread_flag(TIF_NEED_FPU_LOAD); save_fpregs_to_fpstate(¤t->thread.fpu); @@ -476,8 +465,12 @@ int kernel_fpu_begin_nonatomic_mask(unsigned int kfpu_mask) /* Set thread flag: TIC_USING_FPU_NONATOMIC */ set_thread_flag(TIF_USING_FPU_NONATOMIC); + this_cpu_write(in_kernel_fpu_nonatomic, true); + __cpu_invalidate_fpregs_state(); + local_irq_restore(flags); + /* Put sane initial values into the control registers. */ if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM)) ldmxcsr(MXCSR_DEFAULT); @@ -503,22 +496,80 @@ void kernel_fpu_end_nonatomic(void) * This means we call kernel_fpu_end_nonatomic after kernel_fpu_begin, * but before kernel_fpu_end. */ - WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); + WARN_ON_FPU(!this_cpu_read(in_kernel_fpu_nonatomic)); - WARN_ON_FPU(!test_thread_flag(TIF_USING_FPU_NONATOMIC)); + this_cpu_write(in_kernel_fpu_nonatomic, false); clear_thread_flag(TIF_USING_FPU_NONATOMIC); + preempt_enable(); } EXPORT_SYMBOL_GPL(kernel_fpu_end_nonatomic); void save_fpregs_to_fpkernelstate(struct fpu *kfpu) { - kernel_fpu_states_save(&kfpu->fpstate->kernel_state, NULL, - sizeof(kfpu->fpstate->kernel_state)); + kernel_fpu_states_save((void *)get_fpu_registers_pos(kfpu, + MAX_FPU_CTX_SIZE), + NULL, MAX_FPU_CTX_SIZE); +} + +/* + * It means we call kernel_fpu_end after kernel_fpu_begin_nonatomic + * func, but before kernel_fpu_end_nonatomic + */ +static inline void check_using_kernel_fpu(bool flag) +{ + if (this_cpu_read(in_kernel_fpu_nonatomic)) { + struct fpu *current_fpu = ¤t->thread.fpu; + if (flag) + save_fpregs_to_fpkernelstate(current_fpu); + else + fpregs_restore_kernelregs(current_fpu); + } } +#else +static inline void check_using_kernel_fpu(bool flag) { } #endif +void kernel_fpu_begin_mask(unsigned int kfpu_mask) +{ + preempt_disable(); + + if (static_branch_unlikely(&hygon_lmc_key)) + check_using_kernel_fpu(true); + + WARN_ON_FPU(!irq_fpu_usable()); + WARN_ON_FPU(this_cpu_read(in_kernel_fpu)); + + this_cpu_write(in_kernel_fpu, true); + + if (!(current->flags & PF_KTHREAD) && + !test_thread_flag(TIF_NEED_FPU_LOAD)) { + set_thread_flag(TIF_NEED_FPU_LOAD); + save_fpregs_to_fpstate(¤t->thread.fpu); + } + __cpu_invalidate_fpregs_state(); + + /* Put sane initial values into the control registers. */ + if (likely(kfpu_mask & KFPU_MXCSR) && boot_cpu_has(X86_FEATURE_XMM)) + ldmxcsr(MXCSR_DEFAULT); + + if (unlikely(kfpu_mask & KFPU_387) && boot_cpu_has(X86_FEATURE_FPU)) + asm volatile ("fninit"); +} +EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask); + +void kernel_fpu_end(void) +{ + if (static_branch_unlikely(&hygon_lmc_key)) + check_using_kernel_fpu(false); + + WARN_ON_FPU(!this_cpu_read(in_kernel_fpu)); + + this_cpu_write(in_kernel_fpu, false); + preempt_enable(); +} +EXPORT_SYMBOL_GPL(kernel_fpu_end); /* * Sync the FPU register state to current's memory register state when the * current task owns the FPU. The hardware register state is preserved. diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 851eb13edc01..ce1a7b2b825b 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -133,6 +133,12 @@ static void __init fpu__init_system_generic(void) fpu__init_system_mxcsr(); } +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) +unsigned int fpu_kernel_nonatomic_xstate_size; +EXPORT_SYMBOL_GPL(fpu_kernel_nonatomic_xstate_size); +#endif + /* * Enforce that 'MEMBER' is the last field of 'TYPE'. * @@ -161,6 +167,11 @@ static void __init fpu__init_task_struct_size(void) * size. */ task_size += fpu_kernel_cfg.default_size; +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) + if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) + task_size += fpu_kernel_nonatomic_xstate_size; +#endif /* * We dynamically size 'struct fpu', so we require that diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 37607b6132a3..b9df72be9a32 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -679,6 +679,11 @@ static unsigned int __init get_xsave_size_user(void) static int __init init_xstate_size(void) { +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) + extern unsigned int fpu_kernel_nonatomic_xstate_size; +#endif + /* Recompute the context size for enabled features: */ unsigned int user_size, kernel_size, kernel_default_size; bool compacted = cpu_feature_enabled(X86_FEATURE_XSAVES); @@ -711,6 +716,10 @@ static int __init init_xstate_size(void) fpu_user_cfg.default_size = xstate_calculate_size(fpu_user_cfg.default_features, false); +#if defined(CONFIG_X86_HYGON_LMC_SSE2_ON) || \ + defined(CONFIG_X86_HYGON_LMC_AVX2_ON) + fpu_kernel_nonatomic_xstate_size = KERNEL_FPU_NONATOMIC_SIZE; +#endif return 0; } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 6b89477f4e2f..eae0a448bbe0 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -569,7 +569,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) if (!test_thread_flag(TIF_NEED_FPU_LOAD)) switch_fpu_prepare(prev_fpu, cpu); - switch_kernel_fpu_prepare(prev_p, cpu); + if (static_branch_unlikely(&hygon_lmc_key)) + switch_kernel_fpu_prepare(prev_p, cpu); /* We must save %fs and %gs before load_TLS() because * %fs and %gs may be cleared by load_TLS(). @@ -625,7 +626,8 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) switch_fpu_finish(); - switch_kernel_fpu_finish(next_p); + if (static_branch_unlikely(&hygon_lmc_key)) + switch_kernel_fpu_finish(next_p); /* Reload sp0. */ update_task_stack(next_p); -- Gitee