diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c25c2f0eceb7364d5967e0a0a341c184b1bf4ae8..47f2a20fc6b783d6b2b86c289b52994d88f29393 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1545,6 +1545,21 @@ config NODES_SHIFT Specify the maximum number of NUMA Nodes available on the target system. Increases memory reserved to accommodate various tables. +config NUMA_AWARE_SPINLOCKS + bool "Numa-aware spinlocks" + depends on NUMA + depends on QUEUED_SPINLOCKS + default n + help + Introduce NUMA (Non Uniform Memory Access) awareness into + the slow path of spinlocks. + + In this variant of qspinlock, the kernel will try to keep the lock + on the same node, thus reducing the number of remote cache misses, + while trading some of the short term fairness for better performance. + + Say N if you want absolute first come first serve fairness. + source "kernel/Kconfig.hz" config ARCH_SPARSEMEM_ENABLE diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 00e8677d750b2329b7e77b67461230de89b2898d..9a0ac002bbeec7f7d1a865aed5e4ead5d0314b44 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -461,6 +461,7 @@ CONFIG_HOTPLUG_CPU=y # CONFIG_ARM64_BOOTPARAM_HOTPLUG_CPU0 is not set CONFIG_NUMA=y CONFIG_NODES_SHIFT=8 +CONFIG_NUMA_AWARE_SPINLOCKS=y # CONFIG_HZ_100 is not set CONFIG_HZ_250=y # CONFIG_HZ_300 is not set diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index 5c8ee5a541d2047c10c8440e3691d6b1b9f97c25..d16ee8095366326ae55d0939d31424dca21f6528 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -2,7 +2,6 @@ generic-y += early_ioremap.h generic-y += mcs_spinlock.h generic-y += qrwlock.h -generic-y += qspinlock.h generic-y += parport.h generic-y += user.h diff --git a/arch/arm64/include/asm/qspinlock.h b/arch/arm64/include/asm/qspinlock.h new file mode 100644 index 0000000000000000000000000000000000000000..8cc7d00b8c67b4ac865cf60f116876de9b8adfb7 --- /dev/null +++ b/arch/arm64/include/asm/qspinlock.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_ARM64_QSPINLOCK_H +#define _ASM_ARM64_QSPINLOCK_H + +#ifdef CONFIG_NUMA_AWARE_SPINLOCKS +#include + +extern void cna_configure_spin_lock_slowpath(void); + +extern void (*cna_queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val); +extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val); + +#define queued_spin_unlock queued_spin_unlock +/** + * queued_spin_unlock - release a queued spinlock + * @lock : Pointer to queued spinlock structure + * + * A smp_store_release() on the least-significant byte. + */ +static inline void native_queued_spin_unlock(struct qspinlock *lock) +{ + smp_store_release(&lock->locked, 0); +} + +static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +{ + cna_queued_spin_lock_slowpath(lock, val); +} + +static inline void queued_spin_unlock(struct qspinlock *lock) +{ + native_queued_spin_unlock(lock); +} +#endif + +#include + +#endif /* _ASM_ARM64_QSPINLOCK_H */ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 3683252da8fb3f37d5774623b301c81d5e79feb1..aae7456ece070002c5f73bd68866fc2ef083a35e 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1612,10 +1612,6 @@ void __init alternative_instructions(void) */ paravirt_set_cap(); -#if defined(CONFIG_NUMA_AWARE_SPINLOCKS) - cna_configure_spin_lock_slowpath(); -#endif - /* * First patch paravirt functions, such that we overwrite the indirect * call with the direct call. diff --git a/init/main.c b/init/main.c index ba7da8fe83ea1dc526b312e4b2e6d81e183719eb..4a04a1c1d69e9f25378c635c7a9901e4cf4d8ff5 100644 --- a/init/main.c +++ b/init/main.c @@ -1013,7 +1013,9 @@ void start_kernel(void) panic_param); lockdep_init(); - +#if defined(CONFIG_NUMA_AWARE_SPINLOCKS) + cna_configure_spin_lock_slowpath(); +#endif /* * Need to run this when irqs are enabled, because it wants * to self-test [hard/soft]-irqs on/off lock inversion bugs diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index d3f99060b60f1cab0961c94e154153e7604b2594..c1818fec34f2a5f9ed86faf238e404a1247d6a6e 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -289,7 +289,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, #define pv_kick_node __pv_kick_node #define pv_wait_head_or_lock __pv_wait_head_or_lock -#ifdef CONFIG_PARAVIRT_SPINLOCKS +#if defined(CONFIG_PARAVIRT_SPINLOCKS) || defined(CONFIG_NUMA_AWARE_SPINLOCKS) #define queued_spin_lock_slowpath native_queued_spin_lock_slowpath #endif diff --git a/kernel/locking/qspinlock_cna.h b/kernel/locking/qspinlock_cna.h index 3983505c1118fa956458d160080d65721232d594..b4951b7a59304ae26fc63a902b22ca6c87ab9270 100644 --- a/kernel/locking/qspinlock_cna.h +++ b/kernel/locking/qspinlock_cna.h @@ -376,6 +376,14 @@ static inline void cna_lock_handoff(struct mcs_spinlock *node, arch_mcs_lock_handoff(&next->locked, val); } +#ifdef CONFIG_PARAVIRT_SPINLOCKS +#define cna_queued_spin_lock_slowpath pv_ops.lock.queued_spin_lock_slowpath +#else +void (*cna_queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val) = + native_queued_spin_lock_slowpath; +EXPORT_SYMBOL(cna_queued_spin_lock_slowpath); +#endif + /* * Constant (boot-param configurable) flag selecting the NUMA-aware variant * of spinlock. Possible values: -1 (off, default) / 0 (auto) / 1 (on). @@ -413,13 +421,13 @@ void __init cna_configure_spin_lock_slowpath(void) return; if (numa_spinlock_flag == 0 && (nr_node_ids < 2 || - pv_ops.lock.queued_spin_lock_slowpath != + cna_queued_spin_lock_slowpath != native_queued_spin_lock_slowpath)) return; cna_init_nodes(); - pv_ops.lock.queued_spin_lock_slowpath = __cna_queued_spin_lock_slowpath; + cna_queued_spin_lock_slowpath = __cna_queued_spin_lock_slowpath; pr_info("Enabling CNA spinlock\n"); }