From 0974fbc181b2d671b88e92a85d2cdc38e8ab9f16 Mon Sep 17 00:00:00 2001 From: Ze Zuo Date: Mon, 23 Jun 2025 09:46:43 +0800 Subject: [PATCH 1/3] mm/mem_sampling: add trace event for spe based damon record hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICGGS3 CVE: NA -------------------------------- This patch adds a new DAMON access tracking mechanism using ARM Statistical Profiling Extension (SPE). By parsing memory access samples from SPE, DAMON can infer access patterns with low overhead and higher precision on supported ARM platforms. Signed-off-by: Ze Zuo --- include/trace/events/kmem.h | 21 +++++++++++++++++++++ mm/mem_sampling.c | 1 + 2 files changed, 22 insertions(+) diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index 28b9679c474d..3e78e6bd6e18 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -523,6 +523,27 @@ TRACE_EVENT(mm_mem_sampling_access_record, __entry->cpuid, __entry->pid) ); #endif /* CONFIG_NUMABALANCING_MEM_SAMPLING */ + +#ifdef CONFIG_DAMON_MEM_SAMPLING +TRACE_EVENT(mm_mem_sampling_damon_record, + + TP_PROTO(u64 vaddr, int pid), + + TP_ARGS(vaddr, pid), + + TP_STRUCT__entry( + __field(u64, vaddr) + __field(int, pid) + ), + + TP_fast_assign( + __entry->vaddr = vaddr; + __entry->pid = pid; + ), + + TP_printk("vaddr=%llx pid=%d", __entry->vaddr, __entry->pid) +); +#endif /* CONFIG_DAMON_MEM_SAMPLING */ #endif /* _TRACE_KMEM_H */ /* This part must be outside protection */ diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 9ee68e15d1f6..8d79e83e64f0 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -316,6 +316,7 @@ static void damon_mem_sampling_record_cb(struct mem_sampling_record *record) mmput(mm); domon_record.vaddr = record->virt_addr; + trace_mm_mem_sampling_damon_record(record->virt_addr, (pid_t)record->context_id); /* only the proc under monitor now has damon_fifo */ if (damon_fifo) { -- Gitee From 9492910364371e723aefb3029aa7e372c6d425ad Mon Sep 17 00:00:00 2001 From: Ze Zuo Date: Mon, 23 Jun 2025 09:46:44 +0800 Subject: [PATCH 2/3] mm/mem_sampling: Prevent mem_sampling from being enabled if SPE init failed hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/ICGGS3 CVE: NA -------------------------------- When the ARM Statistical Profiling Extension (SPE) PMU driver is not properly initialized or has already been unloaded, it is not safe to enable the `mem_sampling` feature. Add a check to ensure that `mem_sampling` can only be enabled if `spe_probe_status == SPE_INIT_SUCC`. This prevents enabling memory sampling in an invalid state, which may otherwise lead to undefined behavior or system instability. Fixs: 5e73787bd539 ("mm/numa: Use mem_sampling framework for NUMA balancing") Signed-off-by: Ze Zuo --- drivers/arm/mm_monitor/mm_spe.c | 2 +- mm/mem_sampling.c | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/arm/mm_monitor/mm_spe.c b/drivers/arm/mm_monitor/mm_spe.c index 192f1f9c6adb..3c0438ab54e4 100644 --- a/drivers/arm/mm_monitor/mm_spe.c +++ b/drivers/arm/mm_monitor/mm_spe.c @@ -114,8 +114,8 @@ void mm_spe_buffer_free(void) continue; mm_spe_percpu_buffer_free(cpu); } - spe_probe_status -= 1; set_mem_sampling_state(false); + spe_probe_status -= 1; } EXPORT_SYMBOL_GPL(mm_spe_buffer_free); diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 8d79e83e64f0..74f95e4611fe 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -278,6 +278,9 @@ static void numa_balancing_mem_sampling_cb_unregister(void) } static void set_numabalancing_mem_sampling_state(bool enabled) { + if (!mem_sampling_ops.sampling_start || !mm_spe_enabled()) + return; + if (enabled) { numa_balancing_mem_sampling_cb_register(); static_branch_enable(&sched_numabalancing_mem_sampling); @@ -342,6 +345,9 @@ static void damon_mem_sampling_record_cb_unregister(void) static void set_damon_mem_sampling_state(bool enabled) { + if (!mem_sampling_ops.sampling_start || !mm_spe_enabled()) + return; + if (enabled) { damon_mem_sampling_record_cb_register(); static_branch_enable(&mm_damon_mem_sampling); @@ -412,14 +418,15 @@ static void __set_mem_sampling_state(bool enabled) void set_mem_sampling_state(bool enabled) { + if (!mem_sampling_ops.sampling_start || !mm_spe_enabled()) + return; + if (mem_sampling_saved_state != MEM_SAMPLING_STATE_EMPTY) { mem_sampling_saved_state = enabled ? MEM_SAMPLING_STATE_ENABLE : MEM_SAMPLING_STATE_DISABLE; return; } - if (!mem_sampling_ops.sampling_start || !mm_spe_enabled()) - return; if (enabled) sysctl_mem_sampling_mode = MEM_SAMPLING_NORMAL; else -- Gitee From 99f2fe2318e068672069d9d266e1a719c35a56a4 Mon Sep 17 00:00:00 2001 From: Ze Zuo Date: Mon, 23 Jun 2025 09:46:45 +0800 Subject: [PATCH 3/3] mm/mem_sampling: Fix inaccurate sampling for NUMA balancing and DAMON hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/ICGGS3 CVE: NA -------------------------------- SPE memory access records can be used by NUMA balancing and DAMON for accurate kernel-side memory profiling. However, the current implementation suffers from inaccuracies in kernel-side consumers due to two key issues: 1. The `mem_sampling_record` exposed to kernel consumers is not correctly synchronized with `struct arm_spe_record`. This leads to fields such as physical address, virtual address, or task-related metadata being outdated or inconsistent. 2. In dual-buffer configurations, the function `mm_spe_getbuf_addr()` fails to return the correct `record_base` when the backup buffer is in use. As a result, consumers may fetch stale or invalid sampling data. This patch addresses both problems by: - Explicitly copying all relevant fields from `struct arm_spe_record` into `mem_sampling_record` before it is passed to users. - Updating `mm_spe_getbuf_addr()` to check both active and backup buffer state and return the correct `record_base` accordingly. With this fix, NUMA auto-balancing and DAMON-based sampling can operate on accurate and up-to-date SPE sample data in all buffer modes. Fixs: 5e73787bd539 ("mm/numa: Use mem_sampling framework for NUMA balancing") Signed-off-by: Ze Zuo --- drivers/arm/mm_monitor/mm_spe.c | 4 ++-- include/linux/mem_sampling.h | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/arm/mm_monitor/mm_spe.c b/drivers/arm/mm_monitor/mm_spe.c index 3c0438ab54e4..73fa4acdc998 100644 --- a/drivers/arm/mm_monitor/mm_spe.c +++ b/drivers/arm/mm_monitor/mm_spe.c @@ -366,11 +366,11 @@ void mm_spe_decoding(void) arm_spe_decode_buf(spe_buf->cur, spe_buf->size); } -struct mm_spe_buf *mm_spe_getbuf_addr(void) +void *mm_spe_getbuf_addr(void) { struct mm_spe_buf *spe_buf = this_cpu_ptr(&per_cpu_spe_buf); - return spe_buf; + return spe_buf->record_base; } int mm_spe_getnum_record(void) diff --git a/include/linux/mem_sampling.h b/include/linux/mem_sampling.h index 71d098fd3c05..b325c9955d31 100644 --- a/include/linux/mem_sampling.h +++ b/include/linux/mem_sampling.h @@ -49,6 +49,7 @@ struct mem_sampling_record { u64 context_id; u64 boost_spe_addr[8]; u64 rem_addr; + u16 boost_spe_idx; u16 source; }; @@ -57,7 +58,7 @@ struct mem_sampling_ops_struct { void (*sampling_stop)(void); void (*sampling_continue)(void); void (*sampling_decoding)(void); - struct mm_spe_buf* (*mm_spe_getbuf_addr)(void); + void* (*mm_spe_getbuf_addr)(void); int (*mm_spe_getnum_record)(void); }; @@ -83,7 +84,7 @@ void mm_spe_stop(void); void mm_spe_continue(void); void mm_spe_decoding(void); int mm_spe_getnum_record(void); -struct mm_spe_buf *mm_spe_getbuf_addr(void); +void *mm_spe_getbuf_addr(void); int mm_spe_enabled(void); void arm_spe_set_probe_status(int status); #else @@ -93,7 +94,7 @@ static inline void mm_spe_decoding(void) { } static inline void arm_spe_set_probe_status(int status) { } static inline int mm_spe_start(void) { return 0; } static inline int mm_spe_getnum_record(void) { return 0; } -static inline struct mm_spe_buf *mm_spe_getbuf_addr(void) { return NULL; } +static inline void *mm_spe_getbuf_addr(void) { return NULL; } static inline int mm_spe_enabled(void) { return 0; } #endif /* CONFIG_ARM_SPE_MEM_SAMPLING */ -- Gitee