From cc70cbc1616ef88d339ebaca813deabfbc0e59a1 Mon Sep 17 00:00:00 2001 From: Jie Liu Date: Wed, 25 Oct 2023 16:03:36 +0800 Subject: [PATCH 1/9] Revert "sched/fair: Scan cluster before scanning LLC in wake-up path" kunpeng inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA ---------------------------------------------------------------------- This reverts commit 0c3a4f986962ed94da6e26ba3ec0bdf700945894. Signed-off-by: Jie Liu --- kernel/sched/fair.c | 30 +++--------------------------- kernel/sched/sched.h | 1 - kernel/sched/topology.c | 10 ---------- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6f6ced57cf77..bbe54206b8ce 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6663,30 +6663,6 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t } } - if (static_branch_unlikely(&sched_cluster_active)) { - struct sched_domain *sdc = rcu_dereference(per_cpu(sd_cluster, target)); - - if (sdc) { - for_each_cpu_wrap(cpu, sched_domain_span(sdc), target) { - if (!cpumask_test_cpu(cpu, cpus)) - continue; - - if (smt) { - i = select_idle_core(p, cpu, cpus, &idle_cpu); - if ((unsigned int)i < nr_cpumask_bits) - return i; - } else { - if (--nr <= 0) - return -1; - idle_cpu = __select_idle_cpu(cpu, p); - if ((unsigned int)idle_cpu < nr_cpumask_bits) - return idle_cpu; - } - } - cpumask_andnot(cpus, cpus, sched_domain_span(sdc)); - } - } - for_each_cpu_wrap(cpu, cpus, target) { if (smt) { i = select_idle_core(p, cpu, cpus, &idle_cpu); @@ -6694,7 +6670,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t return i; } else { - if (--nr <= 0) + if (!--nr) return -1; idle_cpu = __select_idle_cpu(cpu, p); if ((unsigned int)idle_cpu < nr_cpumask_bits) @@ -6806,7 +6782,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) /* * If the previous CPU is cache affine and idle, don't be stupid: */ - if (prev != target && cpus_share_lowest_cache(prev, target) && + if (prev != target && cpus_share_cache(prev, target) && (available_idle_cpu(prev) || sched_idle_cpu(prev)) && #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY cpumask_test_cpu(prev, p->select_cpus) && @@ -6837,7 +6813,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) recent_used_cpu = p->recent_used_cpu; if (recent_used_cpu != prev && recent_used_cpu != target && - cpus_share_lowest_cache(recent_used_cpu, target) && + cpus_share_cache(recent_used_cpu, target) && (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && #ifdef CONFIG_QOS_SCHED_DYNAMIC_AFFINITY cpumask_test_cpu(p->recent_used_cpu, p->select_cpus) && diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 91ae933c20be..00730c971591 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1851,7 +1851,6 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); extern struct static_key_false sched_asym_cpucapacity; -extern struct static_key_false sched_cluster_active; struct sched_group_capacity { atomic_t ref; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0058681e7f43..d93adc223242 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -653,9 +653,7 @@ DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); - DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); -DEFINE_STATIC_KEY_FALSE(sched_cluster_active); static void update_top_cache_domain(int cpu) { @@ -2344,7 +2342,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att struct rq *rq = NULL; int i, ret = -ENOMEM; bool has_asym = false; - bool has_cluster = false; if (WARN_ON(cpumask_empty(cpu_map))) goto error; @@ -2366,7 +2363,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att sd = build_sched_domain(tl, cpu_map, attr, sd, i); has_asym |= sd->flags & SD_ASYM_CPUCAPACITY; - has_cluster |= sd->flags & SD_CLUSTER; if (tl == sched_domain_topology) *per_cpu_ptr(d.sd, i) = sd; @@ -2480,9 +2476,6 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att if (has_asym) static_branch_inc_cpuslocked(&sched_asym_cpucapacity); - if (has_cluster) - static_branch_inc_cpuslocked(&sched_cluster_active); - if (rq && sched_debug_enabled) { pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n", cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); @@ -2583,9 +2576,6 @@ static void detach_destroy_domains(const struct cpumask *cpu_map) if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu))) static_branch_dec_cpuslocked(&sched_asym_cpucapacity); - if (rcu_access_pointer(per_cpu(sd_cluster, cpu))) - static_branch_dec_cpuslocked(&sched_cluster_active); - rcu_read_lock(); for_each_cpu(i, cpu_map) cpu_attach_domain(NULL, &def_root_domain, i); -- Gitee From 358ab24d13c3d3a4ff26ba707845dcd00a272a30 Mon Sep 17 00:00:00 2001 From: Jie Liu Date: Wed, 25 Oct 2023 16:48:23 +0800 Subject: [PATCH 2/9] Revert "sched: Add per_cpu cluster domain info and cpus_share_lowest_cache API" kunpeng inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA ---------------------------------------------------------------------- This reverts commit 53ad6bf76d9c646e3c8494ed82d90f304c50de1f. Signed-off-by: Jie Liu --- include/linux/sched/sd_flags.h | 7 ------- include/linux/sched/topology.h | 8 +------- kernel/sched/core.c | 12 ------------ kernel/sched/sched.h | 2 -- kernel/sched/topology.c | 15 --------------- 5 files changed, 1 insertion(+), 43 deletions(-) diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 42ed454e8b18..57bde66d95f7 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -109,13 +109,6 @@ SD_FLAG(SD_ASYM_CPUCAPACITY_FULL, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) */ SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) -/* - * Domain members share CPU cluster (LLC tags or L2 cache) - * - * NEEDS_GROUPS: Clusters are shared between groups. - */ -SD_FLAG(SD_CLUSTER, SDF_NEEDS_GROUPS) - /* * Domain members share CPU package resources (i.e. caches) * diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 62a1e22b057f..02bd32c31616 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -46,7 +46,7 @@ static inline int cpu_smt_flags(void) #ifdef CONFIG_SCHED_CLUSTER static inline int cpu_cluster_flags(void) { - return SD_CLUSTER | SD_SHARE_PKG_RESOURCES; + return SD_SHARE_PKG_RESOURCES; } #endif @@ -189,7 +189,6 @@ cpumask_var_t *alloc_sched_domains(unsigned int ndoms); void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); -bool cpus_share_lowest_cache(int this_cpu, int that_cpu); typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); typedef int (*sched_domain_flags_f)(void); @@ -244,11 +243,6 @@ static inline bool cpus_share_cache(int this_cpu, int that_cpu) return true; } -static inline bool cpus_share_lowest_cache(int this_cpu, int that_cpu) -{ - return true; -} - #endif /* !CONFIG_SMP */ #ifndef arch_scale_cpu_capacity diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8c8c946ee1de..b834f419aa60 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3021,18 +3021,6 @@ bool cpus_share_cache(int this_cpu, int that_cpu) return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } -/* - * Whether CPUs are share lowest cache, which means LLC on non-cluster - * machines and LLC tag or L2 on machines with clusters. - */ -bool cpus_share_lowest_cache(int this_cpu, int that_cpu) -{ - if (this_cpu == that_cpu) - return true; - - return per_cpu(sd_lowest_cache_id, this_cpu) == per_cpu(sd_lowest_cache_id, that_cpu); -} - static inline bool ttwu_queue_cond(int cpu) { /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 00730c971591..be25382b0184 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1844,9 +1844,7 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); -DECLARE_PER_CPU(int, sd_lowest_cache_id); DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); -DECLARE_PER_CPU(struct sched_domain __rcu *, sd_cluster); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index d93adc223242..bf5ccdb0d7b0 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -647,8 +647,6 @@ static void destroy_sched_domains(struct sched_domain *sd) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); -DEFINE_PER_CPU(int, sd_lowest_cache_id); -DEFINE_PER_CPU(struct sched_domain __rcu *, sd_cluster); DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); @@ -684,18 +682,6 @@ static void update_top_cache_domain(int cpu) per_cpu(sd_llc_id, cpu) = id; rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); - sd = lowest_flag_domain(cpu, SD_CLUSTER); - if (sd) - id = cpumask_first(sched_domain_span(sd)); - rcu_assign_pointer(per_cpu(sd_cluster, cpu), sd); - - /* - * This assignment should be placed after the sd_llc_id as - * we want this id equals to cluster id on cluster machines - * but equals to LLC id on non-Cluster machines. - */ - per_cpu(sd_lowest_cache_id, cpu) = id; - sd = lowest_flag_domain(cpu, SD_NUMA); rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); @@ -1530,7 +1516,6 @@ int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; */ #define TOPOLOGY_SD_FLAGS \ (SD_SHARE_CPUCAPACITY | \ - SD_CLUSTER | \ SD_SHARE_PKG_RESOURCES | \ SD_NUMA | \ SD_ASYM_PACKING) -- Gitee From eb54b9e56ceb3afc5626f4203fc59c56570ae819 Mon Sep 17 00:00:00 2001 From: Ricardo Neri Date: Fri, 10 Sep 2021 18:18:15 -0700 Subject: [PATCH 3/9] sched/topology: Introduce sched_group::flags mainline inclusion from mainline-v5.15-rc5 commit 16d364ba6ef2aa59b409df70682770f3ed23f7c0 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=16d364ba6ef2aa59b409df70682770f3ed23f7c0 ---------------------------------------------------------------------- There exist situations in which the load balance needs to know the properties of the CPUs in a scheduling group. When using asymmetric packing, for instance, the load balancer needs to know not only the state of dst_cpu but also of its SMT siblings, if any. Use the flags of the child scheduling domains to initialize scheduling group flags. This will reflect the properties of the CPUs in the group. A subsequent changeset will make use of these new flags. No functional changes are introduced. Originally-by: Peter Zijlstra (Intel) Signed-off-by: Ricardo Neri Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Joel Fernandes (Google) Reviewed-by: Len Brown Reviewed-by: Vincent Guittot Link: https://lkml.kernel.org/r/20210911011819.12184-3-ricardo.neri-calderon@linux.intel.com Signed-off-by: Jie Liu --- kernel/sched/sched.h | 1 + kernel/sched/topology.c | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index be25382b0184..00395f42f6ce 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1876,6 +1876,7 @@ struct sched_group { unsigned int group_weight; struct sched_group_capacity *sgc; int asym_prefer_cpu; /* CPU of highest priority in group */ + int flags; KABI_RESERVE(1) KABI_RESERVE(2) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index bf5ccdb0d7b0..800991cb8694 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -729,8 +729,20 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) tmp = sd; sd = sd->parent; destroy_sched_domain(tmp); - if (sd) + if (sd) { + struct sched_group *sg = sd->groups; + + /* + * sched groups hold the flags of the child sched + * domain for convenience. Clear such flags since + * the child is being destroyed. + */ + do { + sg->flags = 0; + } while (sg != sd->groups); + sd->child = NULL; + } } for (tmp = sd; tmp; tmp = tmp->parent) @@ -929,10 +941,12 @@ build_group_from_child_sched_domain(struct sched_domain *sd, int cpu) return NULL; sg_span = sched_group_span(sg); - if (sd->child) + if (sd->child) { cpumask_copy(sg_span, sched_domain_span(sd->child)); - else + sg->flags = sd->child->flags; + } else { cpumask_copy(sg_span, sched_domain_span(sd)); + } atomic_inc(&sg->ref); return sg; @@ -1182,6 +1196,7 @@ static struct sched_group *get_group(int cpu, struct sd_data *sdd) if (child) { cpumask_copy(sched_group_span(sg), sched_domain_span(child)); cpumask_copy(group_balance_mask(sg), sched_group_span(sg)); + sg->flags = child->flags; } else { cpumask_set_cpu(cpu, sched_group_span(sg)); cpumask_set_cpu(cpu, group_balance_mask(sg)); -- Gitee From 84b410cbd469836559d4fca3562c94c52fac2dc9 Mon Sep 17 00:00:00 2001 From: Tim C Chen Date: Thu, 4 May 2023 09:09:51 -0700 Subject: [PATCH 4/9] sched/topology: Propagate SMT flags when removing degenerate domain mainline inclusion from mainline-v6.4-rc2 commit bf2dc42d6beb890c995b8b09f881ef1b37259107 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=bf2dc42d6beb890c995b8b09f881ef1b37259107 ---------------------------------------------------------------------- When a degenerate cluster domain for core with SMT CPUs is removed, the SD_SHARE_CPUCAPACITY flag in the local child sched group was not propagated to the new parent. We need this flag to properly determine whether the local sched group is SMT. Set the flag in the local child sched group of the new parent sched domain. Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ricardo Neri Link: https://lkml.kernel.org/r/73cf0959eafa53c02e7ef6bf805d751d9190e55d.1683156492.git.tim.c.chen@linux.intel.com Signed-off-by: Jie Liu --- kernel/sched/topology.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 800991cb8694..eff132d1e00d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -711,8 +711,13 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) if (sd_parent_degenerate(tmp, parent)) { tmp->parent = parent->parent; - if (parent->parent) + + if (parent->parent) { parent->parent->child = tmp; + if (tmp->flags & SD_SHARE_CPUCAPACITY) + parent->parent->groups->flags |= SD_SHARE_CPUCAPACITY; + } + /* * Transfer SD_PREFER_SIBLING down in case of a * degenerate parent; the spans match for this -- Gitee From 727ea34d582b7ab5660fee6a106c393990f56eac Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Thu, 13 Jul 2023 09:31:33 +0800 Subject: [PATCH 5/9] sched/topology: Align group flags when removing degenerate domain mainline inclusion from mainline-v6.5-rc4 commit 4efcc8bc7e08c09c58a2f5cbc2096fbda5b7cf5e category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=4efcc8bc7e08c09c58a2f5cbc2096fbda5b7cf5e ---------------------------------------------------------------------- The flags of the child of a given scheduling domain are used to initialize the flags of its scheduling groups. When the child of a scheduling domain is degenerated, the flags of its local scheduling group need to be updated to align with the flags of its new child domain. The flag SD_SHARE_CPUCAPACITY was aligned in Commit bf2dc42d6beb ("sched/topology: Propagate SMT flags when removing degenerate domain"). Further generalize this alignment so other flags can be used later, such as in cluster-based task wakeup. [1] Reported-by: Yicong Yang Suggested-by: Ricardo Neri Signed-off-by: Chen Yu Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Tim Chen Reviewed-by: Yicong Yang Link: https://lore.kernel.org/r/20230713013133.2314153-1-yu.c.chen@intel.com Signed-off-by: Jie Liu --- kernel/sched/topology.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index eff132d1e00d..27a04a164b92 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -714,8 +714,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) if (parent->parent) { parent->parent->child = tmp; - if (tmp->flags & SD_SHARE_CPUCAPACITY) - parent->parent->groups->flags |= SD_SHARE_CPUCAPACITY; + parent->parent->groups->flags = tmp->flags; } /* -- Gitee From ceff622c41255fbc2728405e5d6f66c3f17bb176 Mon Sep 17 00:00:00 2001 From: Barry Song Date: Thu, 19 Oct 2023 11:33:21 +0800 Subject: [PATCH 6/9] sched: Add cpus_share_resources API mainline inclusion from mainline-v6.7-rc1 commit b95303e0aeaf446b65169dd4142cacdaeb7d4c8b category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=b95303e0aeaf446b65169dd4142cacdaeb7d4c8b ---------------------------------------------------------------------- Add cpus_share_resources() API. This is the preparation for the optimization of select_idle_cpu() on platforms with cluster scheduler level. On a machine with clusters cpus_share_resources() will test whether two cpus are within the same cluster. On a non-cluster machine it will behaves the same as cpus_share_cache(). So we use "resources" here for cache resources. Signed-off-by: Barry Song Signed-off-by: Yicong Yang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Gautham R. Shenoy Reviewed-by: Tim Chen Reviewed-by: Vincent Guittot Tested-and-reviewed-by: Chen Yu Tested-by: K Prateek Nayak Link: https://lkml.kernel.org/r/20231019033323.54147-2-yangyicong@huawei.com Signed-off-by: Jie Liu --- include/linux/sched/sd_flags.h | 7 +++++++ include/linux/sched/topology.h | 8 +++++++- kernel/sched/core.c | 12 ++++++++++++ kernel/sched/sched.h | 1 + kernel/sched/topology.c | 13 +++++++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 57bde66d95f7..42ed454e8b18 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -109,6 +109,13 @@ SD_FLAG(SD_ASYM_CPUCAPACITY_FULL, SDF_SHARED_PARENT | SDF_NEEDS_GROUPS) */ SD_FLAG(SD_SHARE_CPUCAPACITY, SDF_SHARED_CHILD | SDF_NEEDS_GROUPS) +/* + * Domain members share CPU cluster (LLC tags or L2 cache) + * + * NEEDS_GROUPS: Clusters are shared between groups. + */ +SD_FLAG(SD_CLUSTER, SDF_NEEDS_GROUPS) + /* * Domain members share CPU package resources (i.e. caches) * diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 02bd32c31616..c60eea1c805e 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -46,7 +46,7 @@ static inline int cpu_smt_flags(void) #ifdef CONFIG_SCHED_CLUSTER static inline int cpu_cluster_flags(void) { - return SD_SHARE_PKG_RESOURCES; + return SD_CLUSTER | SD_SHARE_PKG_RESOURCES; } #endif @@ -189,6 +189,7 @@ cpumask_var_t *alloc_sched_domains(unsigned int ndoms); void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); bool cpus_share_cache(int this_cpu, int that_cpu); +bool cpus_share_resources(int this_cpu, int that_cpu); typedef const struct cpumask *(*sched_domain_mask_f)(int cpu); typedef int (*sched_domain_flags_f)(void); @@ -243,6 +244,11 @@ static inline bool cpus_share_cache(int this_cpu, int that_cpu) return true; } +static inline bool cpus_share_resources(int this_cpu, int that_cpu) +{ + return true; +} + #endif /* !CONFIG_SMP */ #ifndef arch_scale_cpu_capacity diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b834f419aa60..ebd8c3a6a964 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3021,6 +3021,18 @@ bool cpus_share_cache(int this_cpu, int that_cpu) return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } +/* + * Whether CPUs are share cache resources, which means LLC on non-cluster + * machines and LLC tag or L2 on machines with clusters. + */ +bool cpus_share_resources(int this_cpu, int that_cpu) +{ + if (this_cpu == that_cpu) + return true; + + return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu); +} + static inline bool ttwu_queue_cond(int cpu) { /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 00395f42f6ce..5258ea882a6a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1844,6 +1844,7 @@ static inline struct sched_domain *lowest_flag_domain(int cpu, int flag) DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc); DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); +DECLARE_PER_CPU(int, sd_share_id); DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 27a04a164b92..8ef91ef86798 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -647,6 +647,7 @@ static void destroy_sched_domains(struct sched_domain *sd) DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc); DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id); +DEFINE_PER_CPU(int, sd_share_id); DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); @@ -682,6 +683,17 @@ static void update_top_cache_domain(int cpu) per_cpu(sd_llc_id, cpu) = id; rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds); + sd = lowest_flag_domain(cpu, SD_CLUSTER); + if (sd) + id = cpumask_first(sched_domain_span(sd)); + + /* + * This assignment should be placed after the sd_llc_id as + * we want this id equals to cluster id on cluster machines + * but equals to LLC id on non-Cluster machines. + */ + per_cpu(sd_share_id, cpu) = id; + sd = lowest_flag_domain(cpu, SD_NUMA); rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); @@ -1535,6 +1547,7 @@ int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE; */ #define TOPOLOGY_SD_FLAGS \ (SD_SHARE_CPUCAPACITY | \ + SD_CLUSTER | \ SD_SHARE_PKG_RESOURCES | \ SD_NUMA | \ SD_ASYM_PACKING) -- Gitee From d99534fa9b0adb98f51a0aed8607cc327e8fbb5f Mon Sep 17 00:00:00 2001 From: Barry Song Date: Thu, 19 Oct 2023 11:33:22 +0800 Subject: [PATCH 7/9] sched/fair: Scan cluster before scanning LLC in wake-up path mainline inclusion from mainline-v6.7-rc1 commit 8881e1639f1f899b64e9bccf6cc14d51c1d3c822 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=8881e1639f1f899b64e9bccf6cc14d51c1d3c822 ---------------------------------------------------------------------- For platforms having clusters like Kunpeng920, CPUs within the same cluster have lower latency when synchronizing and accessing shared resources like cache. Thus, this patch tries to find an idle cpu within the cluster of the target CPU before scanning the whole LLC to gain lower latency. This will be implemented in 2 steps in select_idle_sibling(): 1. When the prev_cpu/recent_used_cpu are good wakeup candidates, use them if they're sharing cluster with the target CPU. Otherwise trying to scan for an idle CPU in the target's cluster. 2. Scanning the cluster prior to the LLC of the target CPU for an idle CPU to wakeup. Testing has been done on Kunpeng920 by pinning tasks to one numa and two numa. On Kunpeng920, Each numa has 8 clusters and each cluster has 4 CPUs. With this patch, We noticed enhancement on tbench and netperf within one numa or cross two numa on top of tip-sched-core commit 9b46f1abc6d4 ("sched/debug: Print 'tgid' in sched_show_task()") tbench results (node 0): baseline patched 1: 327.2833 372.4623 ( 13.80%) 4: 1320.5933 1479.8833 ( 12.06%) 8: 2638.4867 2921.5267 ( 10.73%) 16: 5282.7133 5891.5633 ( 11.53%) 32: 9810.6733 9877.3400 ( 0.68%) 64: 7408.9367 7447.9900 ( 0.53%) 128: 6203.2600 6191.6500 ( -0.19%) tbench results (node 0-1): baseline patched 1: 332.0433 372.7223 ( 12.25%) 4: 1325.4667 1477.6733 ( 11.48%) 8: 2622.9433 2897.9967 ( 10.49%) 16: 5218.6100 5878.2967 ( 12.64%) 32: 10211.7000 11494.4000 ( 12.56%) 64: 13313.7333 16740.0333 ( 25.74%) 128: 13959.1000 14533.9000 ( 4.12%) netperf results TCP_RR (node 0): baseline patched 1: 76546.5033 90649.9867 ( 18.42%) 4: 77292.4450 90932.7175 ( 17.65%) 8: 77367.7254 90882.3467 ( 17.47%) 16: 78519.9048 90938.8344 ( 15.82%) 32: 72169.5035 72851.6730 ( 0.95%) 64: 25911.2457 25882.2315 ( -0.11%) 128: 10752.6572 10768.6038 ( 0.15%) netperf results TCP_RR (node 0-1): baseline patched 1: 76857.6667 90892.2767 ( 18.26%) 4: 78236.6475 90767.3017 ( 16.02%) 8: 77929.6096 90684.1633 ( 16.37%) 16: 77438.5873 90502.5787 ( 16.87%) 32: 74205.6635 88301.5612 ( 19.00%) 64: 69827.8535 71787.6706 ( 2.81%) 128: 25281.4366 25771.3023 ( 1.94%) netperf results UDP_RR (node 0): baseline patched 1: 96869.8400 110800.8467 ( 14.38%) 4: 97744.9750 109680.5425 ( 12.21%) 8: 98783.9863 110409.9637 ( 11.77%) 16: 99575.0235 110636.2435 ( 11.11%) 32: 95044.7250 97622.8887 ( 2.71%) 64: 32925.2146 32644.4991 ( -0.85%) 128: 12859.2343 12824.0051 ( -0.27%) netperf results UDP_RR (node 0-1): baseline patched 1: 97202.4733 110190.1200 ( 13.36%) 4: 95954.0558 106245.7258 ( 10.73%) 8: 96277.1958 105206.5304 ( 9.27%) 16: 97692.7810 107927.2125 ( 10.48%) 32: 79999.6702 103550.2999 ( 29.44%) 64: 80592.7413 87284.0856 ( 8.30%) 128: 27701.5770 29914.5820 ( 7.99%) Note neither Kunpeng920 nor x86 Jacobsville supports SMT, so the SMT branch in the code has not been tested but it supposed to work. Chen Yu also noticed this will improve the performance of tbench and netperf on a 24 CPUs Jacobsville machine, there are 4 CPUs in one cluster sharing L2 Cache. [https://lore.kernel.org/lkml/Ytfjs+m1kUs0ScSn@worktop.programming.kicks-ass.net] Suggested-by: Peter Zijlstra Signed-off-by: Barry Song Signed-off-by: Yicong Yang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Tim Chen Reviewed-by: Chen Yu Reviewed-by: Gautham R. Shenoy Reviewed-by: Vincent Guittot Tested-and-reviewed-by: Chen Yu Tested-by: Yicong Yang Link: https://lkml.kernel.org/r/20231019033323.54147-3-yangyicong@huawei.com Signed-off-by: Jie Liu --- kernel/sched/fair.c | 37 ++++++++++++++++++++++++++++++++++--- kernel/sched/sched.h | 1 + kernel/sched/topology.c | 12 ++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bbe54206b8ce..f894102750e1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6663,6 +6663,30 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t } } + if (static_branch_unlikely(&sched_cluster_active)) { + struct sched_group *sg = sd->groups; + + if (sg->flags & SD_CLUSTER) { + for_each_cpu_wrap(cpu, sched_group_span(sg), target) { + if (!cpumask_test_cpu(cpu, cpus)) + continue; + + if (smt) { + i = select_idle_core(p, cpu, cpus, &idle_cpu); + if ((unsigned int)i < nr_cpumask_bits) + return i; + } else { + if (--nr <= 0) + return -1; + idle_cpu = __select_idle_cpu(cpu, p); + if ((unsigned int)idle_cpu < nr_cpumask_bits) + return idle_cpu; + } + } + cpumask_andnot(cpus, cpus, sched_group_span(sg)); + } + } + for_each_cpu_wrap(cpu, cpus, target) { if (smt) { i = select_idle_core(p, cpu, cpus, &idle_cpu); @@ -6670,7 +6694,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t return i; } else { - if (!--nr) + if (--nr <= 0) return -1; idle_cpu = __select_idle_cpu(cpu, p); if ((unsigned int)idle_cpu < nr_cpumask_bits) @@ -6789,7 +6813,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) #endif asym_fits_capacity(task_util, prev)) { SET_STAT(found_idle_cpu_easy); - return prev; + + if (!static_branch_unlikely(&sched_cluster_active) || + cpus_share_resources(prev, target)) + return prev; } /* @@ -6827,7 +6854,11 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ SET_STAT(found_idle_cpu_easy); p->recent_used_cpu = prev; - return recent_used_cpu; + + if (!static_branch_unlikely(&sched_cluster_active) || + cpus_share_resources(recent_used_cpu, target)) + return recent_used_cpu; + } /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 5258ea882a6a..ccf98c56ecba 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1850,6 +1850,7 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); extern struct static_key_false sched_asym_cpucapacity; +extern struct static_key_false sched_cluster_active; struct sched_group_capacity { atomic_t ref; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 8ef91ef86798..e7413d6dd75b 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -652,7 +652,9 @@ DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); + DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); +DEFINE_STATIC_KEY_FALSE(sched_cluster_active); static void update_top_cache_domain(int cpu) { @@ -2359,6 +2361,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att struct rq *rq = NULL; int i, ret = -ENOMEM; bool has_asym = false; + bool has_cluster = false; if (WARN_ON(cpumask_empty(cpu_map))) goto error; @@ -2487,12 +2490,18 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig); cpu_attach_domain(sd, d.rd, i); + + if (lowest_flag_domain(i, SD_CLUSTER)) + has_cluster = true; } rcu_read_unlock(); if (has_asym) static_branch_inc_cpuslocked(&sched_asym_cpucapacity); + if (has_cluster) + static_branch_inc_cpuslocked(&sched_cluster_active); + if (rq && sched_debug_enabled) { pr_info("root domain span: %*pbl (max cpu_capacity = %lu)\n", cpumask_pr_args(cpu_map), rq->rd->max_cpu_capacity); @@ -2593,6 +2602,9 @@ static void detach_destroy_domains(const struct cpumask *cpu_map) if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu))) static_branch_dec_cpuslocked(&sched_asym_cpucapacity); + if (static_branch_unlikely(&sched_cluster_active)) + static_branch_dec_cpuslocked(&sched_cluster_active); + rcu_read_lock(); for_each_cpu(i, cpu_map) cpu_attach_domain(NULL, &def_root_domain, i); -- Gitee From 258145eefc5433e49b93d3474590170e575a4bae Mon Sep 17 00:00:00 2001 From: Yicong Yang Date: Thu, 19 Oct 2023 11:33:23 +0800 Subject: [PATCH 8/9] sched/fair: Use candidate prev/recent_used CPU if scanning failed for cluster wakeup mainline inclusion from mainline-v6.7-rc1 commit 22165f61d0c4092adf40f967c899e5d8b8a0d703 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=22165f61d0c4092adf40f967c899e5d8b8a0d703 ---------------------------------------------------------------------- Chen Yu reports a hackbench regression of cluster wakeup when hackbench threads equal to the CPU number [1]. Analysis shows it's because we wake up more on the target CPU even if the prev_cpu is a good wakeup candidate and leads to the decrease of the CPU utilization. Generally if the task's prev_cpu is idle we'll wake up the task on it without scanning. On cluster machines we'll try to wake up the task in the same cluster of the target for better cache affinity, so if the prev_cpu is idle but not sharing the same cluster with the target we'll still try to find an idle CPU within the cluster. This will improve the performance at low loads on cluster machines. But in the issue above, if the prev_cpu is idle but not in the cluster with the target CPU, we'll try to scan an idle one in the cluster. But since the system is busy, we're likely to fail the scanning and use target instead, even if the prev_cpu is idle. Then leads to the regression. This patch solves this in 2 steps: o record the prev_cpu/recent_used_cpu if they're good wakeup candidates but not sharing the cluster with the target. o on scanning failure use the prev_cpu/recent_used_cpu if they're recorded as idle [1] https://lore.kernel.org/all/ZGzDLuVaHR1PAYDt@chenyu5-mobl1/ Closes: https://lore.kernel.org/all/ZGsLy83wPIpamy6x@chenyu5-mobl1/ Reported-by: Chen Yu Signed-off-by: Yicong Yang Tested-and-reviewed-by: Chen Yu Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Vincent Guittot Link: https://lkml.kernel.org/r/20231019033323.54147-4-yangyicong@huawei.com Signed-off-by: Jie Liu --- kernel/sched/fair.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f894102750e1..8698094ef8ce 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6778,7 +6778,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) { struct sched_domain *sd; unsigned long task_util; - int i, recent_used_cpu; + int i, recent_used_cpu, prev_aff = -1; /* * On asymmetric system, update task utilization because we will check @@ -6817,6 +6817,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (!static_branch_unlikely(&sched_cluster_active) || cpus_share_resources(prev, target)) return prev; + + prev_aff = prev; } /* @@ -6859,6 +6861,8 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) cpus_share_resources(recent_used_cpu, target)) return recent_used_cpu; + } else { + recent_used_cpu = -1; } /* @@ -6895,6 +6899,18 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) } SET_STAT(nofound_idle_cpu); + + /* + * For cluster machines which have lower sharing cache like L2 or + * LLC Tag, we tend to find an idle CPU in the target's cluster + * first. But prev_cpu or recent_used_cpu may also be a good candidate, + * use them if possible when no idle CPU found in select_idle_cpu(). + */ + if ((unsigned int)prev_aff < nr_cpumask_bits) + return prev_aff; + if ((unsigned int)recent_used_cpu < nr_cpumask_bits) + return recent_used_cpu; + return target; } -- Gitee From b126dff5843662845ca150eabc501f6557646802 Mon Sep 17 00:00:00 2001 From: Jie Liu Date: Wed, 8 Nov 2023 10:17:23 +0800 Subject: [PATCH 9/9] sched/topology: fix kabi change in sched_group kunpeng inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I8E8NN CVE: NA ---------------------------------------------------------------------- Fix kabi change for sched_group->flags. Signed-off-by: Jie Liu --- kernel/sched/sched.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index ccf98c56ecba..2c82deee946a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1878,9 +1878,8 @@ struct sched_group { unsigned int group_weight; struct sched_group_capacity *sgc; int asym_prefer_cpu; /* CPU of highest priority in group */ - int flags; - KABI_RESERVE(1) + KABI_USE(1, int flags) KABI_RESERVE(2) /* * The CPUs this group covers. -- Gitee