diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index ef427ee787a99bce2a66e7cb93d1d13046374fcf..774b46cdf1c04b169d4d1ffe0c364414cf5622e6 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -188,9 +188,38 @@ static const struct attribute_group crash_note_cpu_attr_group = { }; #endif +#ifdef CONFIG_CPU_ISOLATION_OPT +static ssize_t isolate_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cpu *cpu = container_of(dev, struct cpu, dev); + ssize_t rc; + int cpuid = cpu->dev.id; + unsigned int isolated = cpu_isolated(cpuid); + + rc = sysfs_emit(buf, "%d\n", isolated); + + return rc; +} + +static DEVICE_ATTR_RO(isolate); + +static struct attribute *cpu_isolated_attrs[] = { + &dev_attr_isolate.attr, + NULL +}; + +static struct attribute_group cpu_isolated_attr_group = { + .attrs = cpu_isolated_attrs, +}; +#endif + static const struct attribute_group *common_cpu_attr_groups[] = { #ifdef CONFIG_KEXEC_CORE &crash_note_cpu_attr_group, +#endif +#ifdef CONFIG_CPU_ISOLATION_OPT + &cpu_isolated_attr_group, #endif NULL }; @@ -198,6 +227,9 @@ static const struct attribute_group *common_cpu_attr_groups[] = { static const struct attribute_group *hotplugable_cpu_attr_groups[] = { #ifdef CONFIG_KEXEC_CORE &crash_note_cpu_attr_group, +#endif +#ifdef CONFIG_CPU_ISOLATION_OPT + &cpu_isolated_attr_group, #endif NULL }; @@ -228,6 +260,9 @@ static struct cpu_attr cpu_attrs[] = { _CPU_ATTR(online, &__cpu_online_mask), _CPU_ATTR(possible, &__cpu_possible_mask), _CPU_ATTR(present, &__cpu_present_mask), +#ifdef CONFIG_CPU_ISOLATION_OPT + _CPU_ATTR(core_ctl_isolated, &__cpu_isolated_mask), +#endif }; /* @@ -492,6 +527,9 @@ static struct attribute *cpu_root_attrs[] = { &cpu_attrs[0].attr.attr, &cpu_attrs[1].attr.attr, &cpu_attrs[2].attr.attr, +#ifdef CONFIG_CPU_ISOLATION_OPT + &cpu_attrs[3].attr.attr, +#endif &dev_attr_kernel_max.attr, &dev_attr_offline.attr, &dev_attr_isolated.attr, diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 624d4a38c358a08f2ca417523058bc1c6a319a8d..b205c4820a59728c721658fd622c0a6f1877e7a5 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -111,6 +111,9 @@ enum cpuhp_state { CPUHP_SLAB_PREPARE, CPUHP_MD_RAID5_PREPARE, CPUHP_RCUTREE_PREP, +#ifdef CONFIG_SCHED_CORE_CTRL + CPUHP_CORE_CTL_ISOLATION_DEAD, +#endif CPUHP_CPUIDLE_COUPLED_PREPARE, CPUHP_POWERPC_PMAC_PREPARE, CPUHP_POWERPC_MMU_CTX_PREPARE, diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index f10fb87d49dbe38bf220857202c9a2ece8e9f31e..f0d8f305eaea470a19c19df42f3da9d3a528439b 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -94,6 +94,7 @@ static inline void set_nr_cpu_ids(unsigned int nr) * cpu_present_mask - has bit 'cpu' set iff cpu is populated * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler * cpu_active_mask - has bit 'cpu' set iff cpu available to migration + * cpu_isolated_mask- has bit 'cpu' set iff cpu isolated * * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online. * @@ -132,9 +133,28 @@ extern struct cpumask __cpu_dying_mask; #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) +#ifdef CONFIG_CPU_ISOLATION_OPT +extern struct cpumask __cpu_isolated_mask; +#define cpu_isolated_mask ((const struct cpumask *)&__cpu_isolated_mask) +#endif extern atomic_t __num_online_cpus; +#if defined(CONFIG_CPU_ISOLATION_OPT) && NR_CPUS > 1 +#define num_isolated_cpus() cpumask_weight(cpu_isolated_mask) +#define num_online_uniso_cpus() \ +({ \ + cpumask_t mask; \ + \ + cpumask_andnot(&mask, cpu_online_mask, cpu_isolated_mask); \ + cpumask_weight(&mask); \ +}) +#define cpu_isolated(cpu) cpumask_test_cpu((cpu), cpu_isolated_mask) +#else /* !CONFIG_CPU_ISOLATION_OPT || NR_CPUS == 1 */ +#define num_isolated_cpus() 0U +#define num_online_uniso_cpus() num_online_cpus() +#define cpu_isolated(cpu) 0U +#endif extern cpumask_t cpus_booted_once_mask; static __always_inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits) @@ -974,6 +994,9 @@ extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS); #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask) #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask) #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask) +#ifdef CONFIG_CPU_ISOLATION_OPT +#define for_each_isolated_cpu(cpu) for_each_cpu((cpu), cpu_isolated_mask) +#endif #endif /* Wrappers for arch boot code to manipulate normally-constant masks */ @@ -1024,6 +1047,17 @@ set_cpu_dying(unsigned int cpu, bool dying) cpumask_clear_cpu(cpu, &__cpu_dying_mask); } +#ifdef CONFIG_CPU_ISOLATION_OPT +static inline void +set_cpu_isolated(unsigned int cpu, bool isolated) +{ + if (isolated) + cpumask_set_cpu(cpu, &__cpu_isolated_mask); + else + cpumask_clear_cpu(cpu, &__cpu_isolated_mask); +} +#endif + /** * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask * * @bitmap: the bitmap diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 254d4a898179c0ad9c3a4b41a07b8e2ed9b1741b..3f9f51ea6eef3914f86b4e1f747511cb458c5f8d 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -74,6 +74,7 @@ enum hrtimer_restart { * * 0x00 inactive * 0x01 enqueued into rbtree + * 0x02 timer is pinned to a cpu * * The callback state is not part of the timer->state because clearing it would * mean touching the timer after the callback, this makes it impossible to free @@ -93,6 +94,8 @@ enum hrtimer_restart { */ #define HRTIMER_STATE_INACTIVE 0x00 #define HRTIMER_STATE_ENQUEUED 0x01 +#define HRTIMER_PINNED_SHIFT 1 +#define HRTIMER_STATE_PINNED (1 << HRTIMER_PINNED_SHIFT) /** * struct hrtimer - the basic hrtimer structure @@ -369,6 +372,12 @@ static inline void hrtimer_cancel_wait_running(struct hrtimer *timer) #endif /* Exported timer functions: */ +#ifdef CONFIG_CPU_ISOLATION_OPT +/* To be used from cpusets, only */ +extern void hrtimer_quiesce_cpu(void *cpup); +#else +static inline void hrtimer_quiesce_cpu(void *cpup) { } +#endif /* Initialize timers: */ extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, diff --git a/include/linux/nmi.h b/include/linux/nmi.h index e92e378df000fb1eca1e082ebc889fe7849a19d2..66af92ddef4098da76256d4437f3dc3e78677a62 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -15,6 +15,11 @@ #ifdef CONFIG_LOCKUP_DETECTOR void lockup_detector_init(void); +#ifdef CONFIG_CPU_ISOLATION_OPT +extern void watchdog_enable(unsigned int cpu); +extern void watchdog_disable(unsigned int cpu); +extern bool watchdog_configured(unsigned int cpu); +#endif void lockup_detector_retry_init(void); void lockup_detector_soft_poweroff(void); void lockup_detector_cleanup(void); @@ -38,6 +43,22 @@ static inline void lockup_detector_init(void) { } static inline void lockup_detector_retry_init(void) { } static inline void lockup_detector_soft_poweroff(void) { } static inline void lockup_detector_cleanup(void) { } +#ifdef CONFIG_CPU_ISOLATION_OPT +static inline void watchdog_enable(unsigned int cpu) +{ +} +static inline void watchdog_disable(unsigned int cpu) +{ +} +static inline bool watchdog_configured(unsigned int cpu) +{ + /* + * Pretend the watchdog is always configured. + * We will be waiting for the watchdog to be enabled in core isolation + */ + return true; +} +#endif #endif /* !CONFIG_LOCKUP_DETECTOR */ #ifdef CONFIG_SOFTLOCKUP_DETECTOR diff --git a/include/linux/sched.h b/include/linux/sched.h index 89869b515c64c0fb797a3ca8c55ee8e28afa1cd0..7943095a925d771c50ea922b416fdfc35425cd98 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -300,6 +300,41 @@ enum task_event { IRQ_UPDATE = 5, }; +#ifdef CONFIG_CPU_ISOLATION_OPT +extern int sched_isolate_count(const cpumask_t *mask, bool include_offline); +extern int sched_isolate_cpu(int cpu); +extern int sched_unisolate_cpu(int cpu); +extern int sched_unisolate_cpu_unlocked(int cpu); +#else +static inline int sched_isolate_count(const cpumask_t *mask, + bool include_offline) +{ + cpumask_t count_mask; + + if (include_offline) + cpumask_andnot(&count_mask, mask, cpu_online_mask); + else + return 0; + + return cpumask_weight(&count_mask); +} + +static inline int sched_isolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu(int cpu) +{ + return 0; +} + +static inline int sched_unisolate_cpu_unlocked(int cpu) +{ + return 0; +} +#endif + extern void scheduler_tick(void); #define MAX_SCHEDULE_TIMEOUT LONG_MAX diff --git a/include/linux/sched/core_ctl.h b/include/linux/sched/core_ctl.h new file mode 100755 index 0000000000000000000000000000000000000000..ca321b7b0b08449b2952fca5554233f35ed087ca --- /dev/null +++ b/include/linux/sched/core_ctl.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2016, 2019-2020, The Linux Foundation. All rights reserved. + */ + +#ifndef __CORE_CTL_H +#define __CORE_CTL_H + +#ifdef CONFIG_SCHED_CORE_CTRL +extern void core_ctl_check(u64 wallclock); +#else +static inline void core_ctl_check(u64 wallclock) { } +#endif +#endif diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index fe1a46f30d2409ff0fe0906bd12af567ae225f0e..3ec3a0ec6e8ddf04245b4bf00fa5059d957ea209 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -29,10 +29,25 @@ extern void __init housekeeping_init(void); #else +#ifdef CONFIG_CPU_ISOLATION_OPT +static inline int housekeeping_any_cpu(enum hk_type type) +{ + cpumask_t available; + int cpu; + + cpumask_andnot(&available, cpu_online_mask, cpu_isolated_mask); + cpu = cpumask_any(&available); + if (cpu >= nr_cpu_ids) + cpu = smp_processor_id(); + + return cpu; +} +#else static inline int housekeeping_any_cpu(enum hk_type type) { return smp_processor_id(); } +#endif static inline const struct cpumask *housekeeping_cpumask(enum hk_type type) { @@ -61,7 +76,11 @@ static inline bool housekeeping_cpu(int cpu, enum hk_type type) if (static_branch_unlikely(&housekeeping_overridden)) return housekeeping_test_cpu(cpu, type); #endif +#ifdef CONFIG_CPU_ISOLATION_OPT + return !cpu_isolated(cpu); +#else return true; +#endif } static inline bool cpu_is_isolated(int cpu) diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index ea7a74ea7389318627bae6f5410576e00135ca76..243e661e3231973c3c28f13aba85c1a5cd5c3140 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -33,6 +33,9 @@ int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg); bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf); +#ifdef CONFIG_CPU_ISOLATION_OPT +int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg); +#endif void stop_machine_park(int cpu); void stop_machine_unpark(int cpu); void stop_machine_yield(const struct cpumask *cpumask); @@ -83,6 +86,14 @@ static inline bool stop_one_cpu_nowait(unsigned int cpu, return false; } +static inline int stop_cpus(const struct cpumask *cpumask, + cpu_stop_fn_t fn, void *arg) +{ + if (cpumask_test_cpu(raw_smp_processor_id(), cpumask)) + return stop_one_cpu(raw_smp_processor_id(), fn, arg); + return -ENOENT; +} + static inline void print_stop_info(const char *log_lvl, struct task_struct *task) { } #endif /* CONFIG_SMP */ diff --git a/include/linux/timer.h b/include/linux/timer.h index 9162f275819a780def8efeb98cc4b553e1c44d28..9993326e1015af2ac753cf372f1ba007432bbdee 100644 --- a/include/linux/timer.h +++ b/include/linux/timer.h @@ -179,6 +179,13 @@ extern int timer_reduce(struct timer_list *timer, unsigned long expires); */ #define NEXT_TIMER_MAX_DELTA ((1UL << 30) - 1) +/* To be used from cpusets, only */ +#ifdef CONFIG_CPU_ISOLATION_OPT +extern void timer_quiesce_cpu(void *cpup); +#else +static inline void timer_quiesce_cpu(void *cpup) { } +#endif + extern void add_timer(struct timer_list *timer); extern int try_to_del_timer_sync(struct timer_list *timer); diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index fbb99a61f714cbebb91ba9280ce44f812ece32de..df0ace51c578dca877b208446cbc5a96a1d54ba7 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -685,6 +686,165 @@ TRACE_EVENT(sched_wake_idle_without_ipi, TP_printk("cpu=%d", __entry->cpu) ); +#ifdef CONFIG_SCHED_CORE_CTRL +TRACE_EVENT(core_ctl_eval_need, + + TP_PROTO(unsigned int cpu, unsigned int old_need, + unsigned int new_need, unsigned int updated), + TP_ARGS(cpu, old_need, new_need, updated), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, old_need) + __field(u32, new_need) + __field(u32, updated) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->old_need = old_need; + __entry->new_need = new_need; + __entry->updated = updated; + ), + TP_printk("cpu=%u, old_need=%u, new_need=%u, updated=%u", __entry->cpu, + __entry->old_need, __entry->new_need, __entry->updated) +); + +TRACE_EVENT(core_ctl_set_busy, + + TP_PROTO(unsigned int cpu, unsigned int busy, + unsigned int old_is_busy, unsigned int is_busy, int high_irqload), + TP_ARGS(cpu, busy, old_is_busy, is_busy, high_irqload), + TP_STRUCT__entry( + __field(u32, cpu) + __field(u32, busy) + __field(u32, old_is_busy) + __field(u32, is_busy) + __field(bool, high_irqload) + ), + TP_fast_assign( + __entry->cpu = cpu; + __entry->busy = busy; + __entry->old_is_busy = old_is_busy; + __entry->is_busy = is_busy; + __entry->high_irqload = high_irqload; + ), + TP_printk("cpu=%u, busy=%u, old_is_busy=%u, new_is_busy=%u high_irqload=%d", + __entry->cpu, __entry->busy, __entry->old_is_busy, + __entry->is_busy, __entry->high_irqload) +); + +TRACE_EVENT(core_ctl_set_boost, + + TP_PROTO(u32 refcount, s32 ret), + TP_ARGS(refcount, ret), + TP_STRUCT__entry( + __field(u32, refcount) + __field(s32, ret) + ), + TP_fast_assign( + __entry->refcount = refcount; + __entry->ret = ret; + ), + TP_printk("refcount=%u, ret=%d", __entry->refcount, __entry->ret) +); + +TRACE_EVENT(core_ctl_update_nr_need, + + TP_PROTO(int cpu, int nr_need, int prev_misfit_need, + int nrrun, int max_nr, int nr_prev_assist), + + TP_ARGS(cpu, nr_need, prev_misfit_need, nrrun, max_nr, nr_prev_assist), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr_need) + __field(int, prev_misfit_need) + __field(int, nrrun) + __field(int, max_nr) + __field(int, nr_prev_assist) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr_need = nr_need; + __entry->prev_misfit_need = prev_misfit_need; + __entry->nrrun = nrrun; + __entry->max_nr = max_nr; + __entry->nr_prev_assist = nr_prev_assist; + ), + + TP_printk("cpu=%d nr_need=%d prev_misfit_need=%d nrrun=%d max_nr=%d nr_prev_assist=%d", + __entry->cpu, __entry->nr_need, __entry->prev_misfit_need, + __entry->nrrun, __entry->max_nr, __entry->nr_prev_assist) +); +#endif + +#ifdef CONFIG_SCHED_RUNNING_AVG +/* + * Tracepoint for sched_get_nr_running_avg + */ +TRACE_EVENT(sched_get_nr_running_avg, + + TP_PROTO(int cpu, int nr, int nr_misfit, int nr_max), + + TP_ARGS(cpu, nr, nr_misfit, nr_max), + + TP_STRUCT__entry( + __field(int, cpu) + __field(int, nr) + __field(int, nr_misfit) + __field(int, nr_max) + ), + + TP_fast_assign( + __entry->cpu = cpu; + __entry->nr = nr; + __entry->nr_misfit = nr_misfit; + __entry->nr_max = nr_max; + ), + + TP_printk("cpu=%d nr=%d nr_misfit=%d nr_max=%d", + __entry->cpu, __entry->nr, __entry->nr_misfit, __entry->nr_max) +); +#endif + +#ifdef CONFIG_CPU_ISOLATION_OPT +/* + * sched_isolate - called when cores are isolated/unisolated + * + * @acutal_mask: mask of cores actually isolated/unisolated + * @req_mask: mask of cores requested isolated/unisolated + * @online_mask: cpu online mask + * @time: amount of time in us it took to isolate/unisolate + * @isolate: 1 if isolating, 0 if unisolating + * + */ +TRACE_EVENT(sched_isolate, + + TP_PROTO(unsigned int requested_cpu, unsigned int isolated_cpus, + u64 start_time, unsigned char isolate), + + TP_ARGS(requested_cpu, isolated_cpus, start_time, isolate), + + TP_STRUCT__entry( + __field(u32, requested_cpu) + __field(u32, isolated_cpus) + __field(u32, time) + __field(unsigned char, isolate) + ), + + TP_fast_assign( + __entry->requested_cpu = requested_cpu; + __entry->isolated_cpus = isolated_cpus; + __entry->time = div64_u64(sched_clock() - start_time, 1000); + __entry->isolate = isolate; + ), + + TP_printk("iso cpu=%u cpus=0x%x time=%u us isolated=%d", + __entry->requested_cpu, __entry->isolated_cpus, + __entry->time, __entry->isolate) +); +#endif + /* * Following tracepoints are not exported in tracefs and provide hooking * mechanisms only for testing and debugging purposes. diff --git a/init/Kconfig b/init/Kconfig index b12f17a062e8b5a6041b060ad1ce5f22f12d8092..da8d4664f1a4776ff1184cc8c6706debceaeb483 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -698,6 +698,32 @@ config CPU_ISOLATION Say Y if unsure. +config SCHED_RUNNING_AVG + bool "per-rq and per-cluster running average statistics" + default n + +config CPU_ISOLATION_OPT + bool "CPU isolation optimization" + depends on SMP + default n + help + This option enables cpu isolation optimization, which allows + to isolate cpu dynamically. The isolated cpu will be unavailable + to scheduler and load balancer, and all its non-pinned timers, + IRQs and tasks will be migrated to other cpus, only pinned + kthread and IRQS are still allowed to run, this achieves + similar effect as hotplug but at lower latency cost. + +config SCHED_CORE_CTRL + bool "Core control" + depends on CPU_ISOLATION_OPT + select SCHED_RUNNING_AVG + default n + help + This option enables the core control functionality in + the scheduler. Core control automatically isolate and + unisolate cores based on cpu load and utilization. + source "kernel/rcu/Kconfig" config IKCONFIG diff --git a/kernel/cpu.c b/kernel/cpu.c index 72e0f5380bf68121d826b1191bc05d2b41e3d774..0fc0c2af8864ca00e90c0b6b8116c49e69e53581 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1441,6 +1441,11 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, if (!cpu_present(cpu)) return -EINVAL; +#ifdef CONFIG_CPU_ISOLATION_OPT + if (!tasks_frozen && !cpu_isolated(cpu) && num_online_uniso_cpus() == 1) + return -EBUSY; +#endif + cpus_write_lock(); cpuhp_tasks_frozen = tasks_frozen; @@ -3127,6 +3132,11 @@ EXPORT_SYMBOL(__cpu_active_mask); struct cpumask __cpu_dying_mask __read_mostly; EXPORT_SYMBOL(__cpu_dying_mask); +#ifdef CONFIG_CPU_ISOLATION_OPT +struct cpumask __cpu_isolated_mask __read_mostly; +EXPORT_SYMBOL(__cpu_isolated_mask); +#endif + atomic_t __num_online_cpus __read_mostly; EXPORT_SYMBOL(__num_online_cpus); @@ -3145,6 +3155,13 @@ void init_cpu_online(const struct cpumask *src) cpumask_copy(&__cpu_online_mask, src); } +#ifdef CONFIG_CPU_ISOLATION_OPT +void init_cpu_isolated(const struct cpumask *src) +{ + cpumask_copy(&__cpu_isolated_mask, src); +} +#endif + void set_cpu_online(unsigned int cpu, bool online) { /* diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c index 1ed2b1739363b855bee1404e35f5c036056bdd1f..72db2795ad7d1f9efdf94c828c6fe4b127a4bab9 100644 --- a/kernel/irq/cpuhotplug.c +++ b/kernel/irq/cpuhotplug.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "internals.h" @@ -58,6 +59,9 @@ static bool migrate_one_irq(struct irq_desc *desc) const struct cpumask *affinity; bool brokeaff = false; int err; +#ifdef CONFIG_CPU_ISOLATION_OPT + struct cpumask available_cpus; +#endif /* * IRQ chip might be already torn down, but the irq descriptor is @@ -110,7 +114,17 @@ static bool migrate_one_irq(struct irq_desc *desc) if (maskchip && chip->irq_mask) chip->irq_mask(d); +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_copy(&available_cpus, affinity); + cpumask_andnot(&available_cpus, &available_cpus, cpu_isolated_mask); + affinity = &available_cpus; +#endif + if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) { +#ifdef CONFIG_CPU_ISOLATION_OPT + const struct cpumask *default_affinity; +#endif + /* * If the interrupt is managed, then shut it down and leave * the affinity untouched. @@ -120,7 +134,38 @@ static bool migrate_one_irq(struct irq_desc *desc) irq_shutdown_and_deactivate(desc); return false; } + +#ifdef CONFIG_CPU_ISOLATION_OPT + default_affinity = desc->affinity_hint ? : irq_default_affinity; + /* + * The order of preference for selecting a fallback CPU is + * + * (1) online and un-isolated CPU from default affinity + * (2) online and un-isolated CPU + * (3) online CPU + */ + cpumask_andnot(&available_cpus, cpu_online_mask, + cpu_isolated_mask); + if (cpumask_intersects(&available_cpus, default_affinity)) + cpumask_and(&available_cpus, &available_cpus, + default_affinity); + else if (cpumask_empty(&available_cpus)) + affinity = cpu_online_mask; + + /* + * We are overriding the affinity with all online and + * un-isolated cpus. irq_set_affinity_locked() call + * below notify this mask to PM QOS affinity listener. + * That results in applying the CPU_DMA_LATENCY QOS + * to all the CPUs specified in the mask. But the low + * level irqchip driver sets the affinity of an irq + * to only one CPU. So pick only one CPU from the + * prepared mask while overriding the user affinity. + */ + affinity = cpumask_of(cpumask_any(affinity)); +#else affinity = cpu_online_mask; +#endif brokeaff = true; } /* @@ -129,7 +174,11 @@ static bool migrate_one_irq(struct irq_desc *desc) * mask and therefore might keep/reassign the irq to the outgoing * CPU. */ +#ifdef CONFIG_CPU_ISOLATION_OPT + err = irq_set_affinity_locked(d, affinity, false); +#else err = irq_do_set_affinity(d, affinity, false); +#endif if (err) { pr_warn_ratelimited("IRQ%u: set affinity failed(%d).\n", d->irq, err); diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 623b8136e9af3b86040102392e63c1ff4a5a04aa..9ae83868989218c27a7cdce8e6c171938c42a3bc 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -154,6 +154,12 @@ static ssize_t write_irq_affinity(int type, struct file *file, if (err) goto free_cpumask; +#ifdef CONFIG_CPU_ISOLATION_OPT + if (cpumask_subset(new_value, cpu_isolated_mask)) { + err = -EINVAL; + goto free_cpumask; + } +#endif /* * Do not allow disabling IRQs completely - it's a too easy * way to make the system unusable accidentally :-) At least diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index c46379af99d98f11f5fbba1e0686d3a6613fb5f1..ea4f0b3c4773e6c17b84a2385a37d336fcbfbb14 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -34,3 +34,4 @@ obj-y += build_policy.o obj-y += build_utility.o obj-$(CONFIG_SCHED_WALT) += walt.o obj-$(CONFIG_SCHED_RUNNING_AVG) += sched_avg.o +obj-$(CONFIG_SCHED_CORE_CTRL) += core_ctl.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ecb69c975d4ba3a5c0c7402121e97a423aa58d3f..3803325df56f73097849000948712642401c21d4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -64,6 +64,8 @@ #include #include #include +#include +#include #ifdef CONFIG_PREEMPT_DYNAMIC # ifdef CONFIG_GENERIC_ENTRY @@ -92,6 +94,7 @@ #include "smp.h" #include "stats.h" #include "walt.h" +#include "rtg/rtg.h" #include "../workqueue_internal.h" #include "../../io_uring/io-wq.h" @@ -3172,6 +3175,9 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p, bool kthread = p->flags & PF_KTHREAD; unsigned int dest_cpu; int ret = 0; +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_t allowed_mask; +#endif update_rq_clock(rq); @@ -3217,7 +3223,20 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p, goto out; } } +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&allowed_mask, ctx->new_mask, cpu_isolated_mask); + cpumask_and(&allowed_mask, &allowed_mask, cpu_valid_mask); + dest_cpu = cpumask_any(&allowed_mask); + if (dest_cpu >= nr_cpu_ids) { + cpumask_and(&allowed_mask, cpu_valid_mask, ctx->new_mask); + dest_cpu = cpumask_any(&allowed_mask); + if (!cpumask_intersects(ctx->new_mask, cpu_valid_mask)) { + ret = -EINVAL; + goto out; + } + } +#else /* * Picking a ~random cpu helps in cases where we are changing affinity * for groups of tasks (ie. cpuset), so that load balancing is not @@ -3228,10 +3247,16 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p, ret = -EINVAL; goto out; } +#endif __do_set_cpus_allowed(p, ctx); +#ifdef CONFIG_CPU_ISOLATION_OPT + if (cpumask_test_cpu(task_cpu(p), &allowed_mask)) + goto out; +#else return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); +#endif out: task_rq_unlock(rq, p, rf); @@ -3606,12 +3631,19 @@ EXPORT_SYMBOL_GPL(kick_process); * select_task_rq() below may allow selection of !active CPUs in order * to satisfy the above rules. */ +#ifdef CONFIG_CPU_ISOLATION_OPT +static int select_fallback_rq(int cpu, struct task_struct *p, bool allow_iso) +#else static int select_fallback_rq(int cpu, struct task_struct *p) +#endif { int nid = cpu_to_node(cpu); const struct cpumask *nodemask = NULL; - enum { cpuset, possible, fail } state = cpuset; + enum { cpuset, possible, fail, bug } state = cpuset; int dest_cpu; +#ifdef CONFIG_CPU_ISOLATION_OPT + int isolated_candidate = -1; +#endif /* * If the node that the CPU is on has been offlined, cpu_to_node() @@ -3623,6 +3655,8 @@ static int select_fallback_rq(int cpu, struct task_struct *p) /* Look for allowed, online CPU in same node. */ for_each_cpu(dest_cpu, nodemask) { + if (cpu_isolated(dest_cpu)) + continue; if (is_cpu_allowed(p, dest_cpu)) return dest_cpu; } @@ -3633,7 +3667,18 @@ static int select_fallback_rq(int cpu, struct task_struct *p) for_each_cpu(dest_cpu, p->cpus_ptr) { if (!is_cpu_allowed(p, dest_cpu)) continue; +#ifdef CONFIG_CPU_ISOLATION_OPT + if (cpu_isolated(dest_cpu)) { + if (allow_iso) + isolated_candidate = dest_cpu; + continue; + } + goto out; + } + if (isolated_candidate != -1) { + dest_cpu = isolated_candidate; +#endif goto out; } @@ -3656,6 +3701,15 @@ static int select_fallback_rq(int cpu, struct task_struct *p) state = fail; break; case fail: +#ifdef CONFIG_CPU_ISOLATION_OPT + allow_iso = true; + state = bug; + break; +#else + /* fall through; */ +#endif + + case bug: BUG(); break; } @@ -3683,6 +3737,10 @@ static int select_fallback_rq(int cpu, struct task_struct *p) static inline int select_task_rq(struct task_struct *p, int cpu, int wake_flags) { +#ifdef CONFIG_CPU_ISOLATION_OPT + bool allow_isolated = (p->flags & PF_KTHREAD); +#endif + lockdep_assert_held(&p->pi_lock); if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) @@ -3700,8 +3758,14 @@ int select_task_rq(struct task_struct *p, int cpu, int wake_flags) * [ this allows ->select_task() to simply return task_cpu(p) and * not worry about this generic constraint ] */ +#ifdef CONFIG_CPU_ISOLATION_OPT + if (unlikely(!is_cpu_allowed(p, cpu)) || + (cpu_isolated(cpu) && !allow_isolated)) + cpu = select_fallback_rq(task_cpu(p), p, allow_isolated); +#else if (unlikely(!is_cpu_allowed(p, cpu))) cpu = select_fallback_rq(task_cpu(p), p); +#endif return cpu; } @@ -5595,7 +5659,7 @@ void sched_exec(void) if (dest_cpu == smp_processor_id()) return; - if (unlikely(!cpu_active(dest_cpu))) + if (unlikely(!cpu_active(dest_cpu) && likely(!cpu_isolated(dest_cpu)))) return; arg = (struct migration_arg){ p, dest_cpu }; @@ -8421,7 +8485,10 @@ __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx) { int retval; cpumask_var_t cpus_allowed, new_mask; - +#ifdef CONFIG_CPU_ISOLATION_OPT + int dest_cpu; + cpumask_t allowed_mask; +#endif if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) return -ENOMEM; @@ -8439,11 +8506,19 @@ __sched_setaffinity(struct task_struct *p, struct affinity_context *ctx) retval = dl_task_check_affinity(p, new_mask); if (retval) goto out_free_new_mask; - +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&allowed_mask, new_mask, cpu_isolated_mask); + dest_cpu = cpumask_any_and(cpu_active_mask, &allowed_mask); + if (dest_cpu < nr_cpu_ids) { +#endif retval = __set_cpus_allowed_ptr(p, ctx); if (retval) goto out_free_new_mask; - +#ifdef CONFIG_CPU_ISOLATION_OPT + } else { + retval = -EINVAL; + } +#endif cpuset_cpus_allowed(p, cpus_allowed); if (!cpumask_subset(new_mask, cpus_allowed)) { /* @@ -8484,6 +8559,10 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) struct cpumask *user_mask; struct task_struct *p; int retval; +#ifdef CONFIG_CPU_ISOLATION_OPT + int dest_cpu; + cpumask_t allowed_mask; +#endif rcu_read_lock(); @@ -8596,6 +8675,16 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask) raw_spin_lock_irqsave(&p->pi_lock, flags); cpumask_and(mask, &p->cpus_mask, cpu_active_mask); + +#ifdef CONFIG_CPU_ISOLATION_OPT + /* The userspace tasks are forbidden to run on + * isolated CPUs. So exclude isolated CPUs from + * the getaffinity. + */ + if (!(p->flags & PF_KTHREAD)) + cpumask_andnot(mask, mask, cpu_isolated_mask); +#endif + raw_spin_unlock_irqrestore(&p->pi_lock, flags); out_unlock: @@ -9544,6 +9633,9 @@ static int __balance_push_cpu_stop(void *arg) struct rq *rq = this_rq(); struct rq_flags rf; int cpu; +#ifdef CONFIG_CPU_ISOLATION_OPT + bool allow_isolated = (p->flags & PF_KTHREAD); +#endif raw_spin_lock_irq(&p->pi_lock); rq_lock(rq, &rf); @@ -9551,7 +9643,11 @@ static int __balance_push_cpu_stop(void *arg) update_rq_clock(rq); if (task_rq(p) == rq && task_on_rq_queued(p)) { +#ifdef CONFIG_CPU_ISOLATION_OPT + cpu = select_fallback_rq(rq->cpu, p, allow_isolated); +#else cpu = select_fallback_rq(rq->cpu, p); +#endif rq = __migrate_task(rq, &rf, p, cpu); } @@ -9565,6 +9661,71 @@ static int __balance_push_cpu_stop(void *arg) static DEFINE_PER_CPU(struct cpu_stop_work, push_work); +static struct task_struct *__pick_migrate_task(struct rq *rq) +{ + const struct sched_class *class; + struct task_struct *next; + + for_each_class(class) { + next = class->pick_next_task(rq); + if (next) { + next->sched_class->put_prev_task(rq, next); + return next; + } + } + + /* The idle class should always have a runnable task */ + BUG(); +} + + +#ifdef CONFIG_CPU_ISOLATION_OPT +/* + * Remove a task from the runqueue and pretend that it's migrating. This + * should prevent migrations for the detached task and disallow further + * changes to tsk_cpus_allowed. + */ +static void +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) +{ + lockdep_assert_held(&rq->__lock); + + p->on_rq = TASK_ON_RQ_MIGRATING; + deactivate_task(rq, p, 0); + list_add(&p->se.group_node, tasks); +} + +static void attach_tasks_core(struct list_head *tasks, struct rq *rq) +{ + struct task_struct *p; + + lockdep_assert_held(&rq->__lock); + + while (!list_empty(tasks)) { + p = list_first_entry(tasks, struct task_struct, se.group_node); + list_del_init(&p->se.group_node); + + BUG_ON(task_rq(p) != rq); + activate_task(rq, p, 0); + p->on_rq = TASK_ON_RQ_QUEUED; + } +} + +#else + +static void +detach_one_task_core(struct task_struct *p, struct rq *rq, + struct list_head *tasks) +{ +} + +static void attach_tasks_core(struct list_head *tasks, struct rq *rq) +{ +} + +#endif /* CONFIG_CPU_ISOLATION_OPT */ + /* * Ensure we only run per-cpu kthreads once the CPU goes !active. * @@ -9677,9 +9838,381 @@ static inline void balance_push_set(int cpu, bool on) static inline void balance_hotplug_wait(void) { } - #endif /* CONFIG_HOTPLUG_CPU */ + +/* + * Migrate all tasks (not pinned if pinned argument say so) from the rq, + * sleeping tasks will be migrated by try_to_wake_up()->select_task_rq(). + * + * Called with rq->lock held even though we'er in stop_machine() and + * there's no concurrency possible, we hold the required locks anyway + * because of lock validation efforts. + */ +void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, + bool migrate_pinned_tasks) +{ + struct rq *rq = dead_rq; + struct task_struct *next, *stop = rq->stop; + struct rq_flags orf = *rf; + int dest_cpu; + unsigned int num_pinned_kthreads = 1; /* this thread */ + LIST_HEAD(tasks); + cpumask_t avail_cpus; + +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&avail_cpus, cpu_online_mask, cpu_isolated_mask); +#else + cpumask_copy(&avail_cpus, cpu_online_mask); +#endif + + /* + * Fudge the rq selection such that the below task selection loop + * doesn't get stuck on the currently eligible stop task. + * + * We're currently inside stop_machine() and the rq is either stuck + * in the stop_machine_cpu_stop() loop, or we're executing this code, + * either way we should never end up calling schedule() until we're + * done here. + */ + rq->stop = NULL; + + /* + * put_prev_task() and pick_next_task() sched + * class method both need to have an up-to-date + * value of rq->clock[_task] + */ + update_rq_clock(rq); + + for (;;) { + /* + * There's this thread running, bail when that's the only + * remaining thread. + */ + if (rq->nr_running == 1) + break; + + next = __pick_migrate_task(rq); + + if (!migrate_pinned_tasks && next->flags & PF_KTHREAD && + !cpumask_intersects(&avail_cpus, &next->cpus_mask)) { + detach_one_task_core(next, rq, &tasks); + num_pinned_kthreads += 1; + continue; + } + + /* + * Rules for changing task_struct::cpus_mask are holding + * both pi_lock and rq->lock, such that holding either + * stabilizes the mask. + * + * Drop rq->lock is not quite as disastrous as it usually is + * because !cpu_active at this point, which means load-balance + * will not interfere. Also, stop-machine. + */ + rq_unlock(rq, rf); + raw_spin_lock(&next->pi_lock); + rq_relock(rq, rf); + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + + /* + * Since we're inside stop-machine, _nothing_ should have + * changed the task, WARN if weird stuff happened, because in + * that case the above rq->lock drop is a fail too. + * However, during cpu isolation the load balancer might have + * interferred since we don't stop all CPUs. Ignore warning for + * this case. + */ + if (task_rq(next) != rq || !task_on_rq_queued(next)) { + WARN_ON(migrate_pinned_tasks); + raw_spin_unlock(&next->pi_lock); + continue; + } + + /* Find suitable destination for @next, with force if needed. */ +#ifdef CONFIG_CPU_ISOLATION_OPT + dest_cpu = select_fallback_rq(dead_rq->cpu, next, false); +#else + dest_cpu = select_fallback_rq(dead_rq->cpu, next); +#endif + rq = __migrate_task(rq, rf, next, dest_cpu); + if (rq != dead_rq) { + rq_unlock(rq, rf); + rq = dead_rq; + *rf = orf; + rq_relock(rq, rf); + if (!(rq->clock_update_flags & RQCF_UPDATED)) + update_rq_clock(rq); + } + raw_spin_unlock(&next->pi_lock); + } + + rq->stop = stop; + + if (num_pinned_kthreads > 1) + attach_tasks_core(&tasks, rq); +} + +#ifdef CONFIG_SCHED_EAS +static void clear_eas_migration_request(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + unsigned long flags; + + clear_reserved(cpu); + if (rq->push_task) { + struct task_struct *push_task = NULL; + + raw_spin_lock_irqsave(&rq->lock, flags); + if (rq->push_task) { + clear_reserved(rq->push_cpu); + push_task = rq->push_task; + rq->push_task = NULL; + } + rq->active_balance = 0; + raw_spin_unlock_irqrestore(&rq->lock, flags); + if (push_task) + put_task_struct(push_task); + } +} +#else +static inline void clear_eas_migration_request(int cpu) {} +#endif + +#ifdef CONFIG_CPU_ISOLATION_OPT +int do_isolation_work_cpu_stop(void *data) +{ + unsigned int cpu = smp_processor_id(); + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + + watchdog_disable(cpu); + + local_irq_disable(); + + irq_migrate_all_off_this_cpu(); + + flush_smp_call_function_queue(); + + /* Update our root-domain */ + rq_lock(rq, &rf); + + /* + * Temporarily mark the rq as offline. This will allow us to + * move tasks off the CPU. + */ + if (rq->rd) { + BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); + set_rq_offline(rq); + } + + migrate_tasks(rq, &rf, false); + + if (rq->rd) + set_rq_online(rq); + rq_unlock(rq, &rf); + + clear_eas_migration_request(cpu); + local_irq_enable(); + return 0; +} + +int do_unisolation_work_cpu_stop(void *data) +{ + watchdog_enable(smp_processor_id()); + return 0; +} + +static void sched_update_group_capacities(int cpu) +{ + struct sched_domain *sd; + + mutex_lock(&sched_domains_mutex); + rcu_read_lock(); + + for_each_domain(cpu, sd) { + int balance_cpu = group_balance_cpu(sd->groups); + + init_sched_groups_capacity(cpu, sd); + /* + * Need to ensure this is also called with balancing + * cpu. + */ + if (cpu != balance_cpu) + init_sched_groups_capacity(balance_cpu, sd); + } + + rcu_read_unlock(); + mutex_unlock(&sched_domains_mutex); +} + +static unsigned int cpu_isolation_vote[NR_CPUS]; + +int sched_isolate_count(const cpumask_t *mask, bool include_offline) +{ + cpumask_t count_mask = CPU_MASK_NONE; + + if (include_offline) { + //cpumask_complement(&count_mask, cpu_online_mask); + cpumask_or(&count_mask, &count_mask, cpu_isolated_mask); + cpumask_and(&count_mask, &count_mask, mask); + } else { + cpumask_and(&count_mask, mask, cpu_isolated_mask); + } + + return cpumask_weight(&count_mask); +} + +/* + * 1) CPU is isolated and cpu is offlined: + * Unisolate the core. + * 2) CPU is not isolated and CPU is offlined: + * No action taken. + * 3) CPU is offline and request to isolate + * Request ignored. + * 4) CPU is offline and isolated: + * Not a possible state. + * 5) CPU is online and request to isolate + * Normal case: Isolate the CPU + * 6) CPU is not isolated and comes back online + * Nothing to do + * + * Note: The client calling sched_isolate_cpu() is repsonsible for ONLY + * calling sched_unisolate_cpu() on a CPU that the client previously isolated. + * Client is also responsible for unisolating when a core goes offline + * (after CPU is marked offline). + */ + static void calc_load_migrate(struct rq *rq); +int sched_isolate_cpu(int cpu) +{ + struct rq *rq; + cpumask_t avail_cpus; + int ret_code = 0; + u64 start_time = 0; + + if (trace_sched_isolate_enabled()) + start_time = sched_clock(); + + cpu_maps_update_begin(); + + cpumask_andnot(&avail_cpus, cpu_online_mask, cpu_isolated_mask); + + if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu) || + !cpu_online(cpu) || cpu >= NR_CPUS) { + ret_code = -EINVAL; + goto out; + } + + rq = cpu_rq(cpu); + + if (++cpu_isolation_vote[cpu] > 1) + goto out; + + /* We cannot isolate ALL cpus in the system */ + if (cpumask_weight(&avail_cpus) == 1) { + --cpu_isolation_vote[cpu]; + ret_code = -EINVAL; + goto out; + } + + /* + * There is a race between watchdog being enabled by hotplug and + * core isolation disabling the watchdog. When a CPU is hotplugged in + * and the hotplug lock has been released the watchdog thread might + * not have run yet to enable the watchdog. + * We have to wait for the watchdog to be enabled before proceeding. + */ + if (!watchdog_configured(cpu)) { + msleep(20); + if (!watchdog_configured(cpu)) { + --cpu_isolation_vote[cpu]; + ret_code = -EBUSY; + goto out; + } + } + + set_cpu_isolated(cpu, true); + cpumask_clear_cpu(cpu, &avail_cpus); + + /* Migrate timers */ + smp_call_function_any(&avail_cpus, hrtimer_quiesce_cpu, &cpu, 1); + smp_call_function_any(&avail_cpus, timer_quiesce_cpu, &cpu, 1); + + watchdog_disable(cpu); + irq_lock_sparse(); + stop_cpus(cpumask_of(cpu), do_isolation_work_cpu_stop, 0); + irq_unlock_sparse(); + + calc_load_migrate(rq); + update_max_interval(); + sched_update_group_capacities(cpu); + +out: + cpu_maps_update_done(); + trace_sched_isolate(cpu, cpumask_bits(cpu_isolated_mask)[0], + start_time, 1); + return ret_code; +} + +/* + * Note: The client calling sched_isolate_cpu() is repsonsible for ONLY + * calling sched_unisolate_cpu() on a CPU that the client previously isolated. + * Client is also responsible for unisolating when a core goes offline + * (after CPU is marked offline). + */ +int sched_unisolate_cpu_unlocked(int cpu) +{ + int ret_code = 0; + u64 start_time = 0; + + if (cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu) + || cpu >= NR_CPUS) { + ret_code = -EINVAL; + goto out; + } + + if (trace_sched_isolate_enabled()) + start_time = sched_clock(); + + if (!cpu_isolation_vote[cpu]) { + ret_code = -EINVAL; + goto out; + } + + if (--cpu_isolation_vote[cpu]) + goto out; + + set_cpu_isolated(cpu, false); + update_max_interval(); + sched_update_group_capacities(cpu); + + if (cpu_online(cpu)) { + stop_cpus(cpumask_of(cpu), do_unisolation_work_cpu_stop, 0); + + /* Kick CPU to immediately do load balancing */ + if (!atomic_fetch_or(NOHZ_KICK_MASK, nohz_flags(cpu))) + smp_send_reschedule(cpu); + } + +out: + trace_sched_isolate(cpu, cpumask_bits(cpu_isolated_mask)[0], + start_time, 0); + return ret_code; +} + +int sched_unisolate_cpu(int cpu) +{ + int ret_code; + + cpu_maps_update_begin(); + ret_code = sched_unisolate_cpu_unlocked(cpu); + cpu_maps_update_done(); + return ret_code; +} + +#endif /* CONFIG_CPU_ISOLATION_OPT */ + void set_rq_online(struct rq *rq) { if (!rq->online) { diff --git a/kernel/sched/core_ctl.c b/kernel/sched/core_ctl.c index eef1d69211782dd890a4aab03788421adf64e80f..a9d5b98fd9529c0b1b9c51bb6756eb1aad96eecd 100755 --- a/kernel/sched/core_ctl.c +++ b/kernel/sched/core_ctl.c @@ -236,7 +236,7 @@ static struct attribute *default_attrs[] = { &global_state.attr, NULL }; - +ATTRIBUTE_GROUPS(default); #define to_cluster_data(k) container_of(k, struct cluster_data, kobj) #define to_attr(a) container_of(a, struct core_ctl_attr, attr) static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) @@ -271,7 +271,7 @@ static const struct sysfs_ops sysfs_ops = { static struct kobj_type ktype_core_ctl = { .sysfs_ops = &sysfs_ops, - .default_attrs = default_attrs, + .default_groups = default_groups, }; /* ==================== runqueue based core count =================== */ diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index 42c40cfdf83630be349949e5498bd16522405461..c8686fcdd0d2a7c4a620693d37b2b66ed12c5cd8 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -102,6 +102,9 @@ static inline int __cpupri_find(struct cpupri *cp, struct task_struct *p, if (lowest_mask) { cpumask_and(lowest_mask, &p->cpus_mask, vec->mask); cpumask_and(lowest_mask, lowest_mask, cpu_active_mask); +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(lowest_mask, lowest_mask, cpu_isolated_mask); +#endif /* * We have to ensure that we have at least one bit diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f722e118ea2bfc39878cd0aab0ece77b00d21f94..075e006a0df9dfa9efa3b16f72fa0cfeaa0997f6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -7136,6 +7136,9 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { struct rq *rq = cpu_rq(i); + if (cpu_isolated(i)) + continue; + if (!sched_core_cookie_match(rq, p)) continue; @@ -7297,6 +7300,10 @@ void __update_idle_core(struct rq *rq) */ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) { + +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(cpus, cpus, cpu_isolated_mask); +#endif bool idle = true; int cpu; @@ -7337,6 +7344,8 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t * Check if the CPU is in the LLC scheduling domain of @target. * Due to isolcpus, there is no guarantee that all the siblings are in the domain. */ + if (cpu_isolated(cpu)) + continue; if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) continue; if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) @@ -7438,6 +7447,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool } else { if (!--nr) return -1; + if (cpu_isolated(cpu)) + continue; idle_cpu = __select_idle_cpu(cpu, p); if ((unsigned int)idle_cpu < nr_cpumask_bits) break; @@ -7485,6 +7496,9 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) for_each_cpu_wrap(cpu, cpus, target) { unsigned long cpu_cap = capacity_of(cpu); + if (cpu_isolated(cpu)) + continue; + if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) continue; @@ -7557,15 +7571,15 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) lockdep_assert_irqs_disabled(); if ((available_idle_cpu(target) || sched_idle_cpu(target)) && - asym_fits_cpu(task_util, util_min, util_max, target)) + !cpu_isolated(target) && asym_fits_cpu(task_util, util_min, util_max, target)) return target; /* * If the previous CPU is cache affine and idle, don't be stupid: */ if (prev != target && cpus_share_cache(prev, target) && - (available_idle_cpu(prev) || sched_idle_cpu(prev)) && - asym_fits_cpu(task_util, util_min, util_max, prev)) + ((available_idle_cpu(prev) || sched_idle_cpu(prev)) && + !cpu_isolated(target) && asym_fits_cpu(task_util, util_min, util_max, prev))) return prev; /* @@ -9703,6 +9717,9 @@ void update_group_capacity(struct sched_domain *sd, int cpu) for_each_cpu(cpu, sched_group_span(sdg)) { unsigned long cpu_cap = capacity_of(cpu); + if (cpu_isolated(cpu)) + continue; + capacity += cpu_cap; min_capacity = min(cpu_cap, min_capacity); max_capacity = max(cpu_cap, max_capacity); @@ -9716,10 +9733,16 @@ void update_group_capacity(struct sched_domain *sd, int cpu) group = child->groups; do { struct sched_group_capacity *sgc = group->sgc; - - capacity += sgc->capacity; - min_capacity = min(sgc->min_capacity, min_capacity); - max_capacity = max(sgc->max_capacity, max_capacity); + __maybe_unused cpumask_t *cpus = + sched_group_span(group); + + if (!cpu_isolated(cpumask_first(cpus))) { + capacity += sgc->capacity; + min_capacity = min(sgc->min_capacity, + min_capacity); + max_capacity = max(sgc->max_capacity, + max_capacity); + } group = group->next; } while (group != child->groups); } @@ -10028,6 +10051,8 @@ static inline void update_sg_lb_stats(struct lb_env *env, for_each_cpu_and(i, sched_group_span(group), env->cpus) { struct rq *rq = cpu_rq(i); unsigned long load = cpu_load(rq); + if (cpu_isolated(i)) + continue; sgs->group_load += load; sgs->group_util += cpu_util_cfs(i); @@ -10077,6 +10102,15 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->group_weight = group->group_weight; + /* Isolated CPU has no weight */ + if (!group->group_weight) { + sgs->group_capacity = 0; + sgs->avg_load = 0; + sgs->group_type = group_has_spare; + sgs->group_weight = group->group_weight; + return; + } + /* Check if dst CPU is idle and preferred to this group */ if (!local_group && env->sd->flags & SD_ASYM_PACKING && env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running && @@ -10462,13 +10496,22 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) .avg_load = UINT_MAX, .group_type = group_overloaded, }; +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_t allowed_cpus; + cpumask_andnot(&allowed_cpus, p->cpus_ptr, cpu_isolated_mask); +#endif do { int local_group; /* Skip over this group if it has no CPUs allowed */ +#ifdef CONFIG_CPU_ISOLATION_OPT + if (!cpumask_intersects(sched_group_span(group), + &allowed_cpus)) +#else if (!cpumask_intersects(sched_group_span(group), p->cpus_ptr)) +#endif continue; /* Skip over this group if no cookie matched */ @@ -11154,6 +11197,9 @@ static struct rq *find_busiest_queue(struct lb_env *env, if (rt > env->fbq_type) continue; + if (cpu_isolated(i)) + continue; + nr_running = rq->cfs.h_nr_running; if (!nr_running) continue; @@ -11329,6 +11375,16 @@ static int need_active_balance(struct lb_env *env) return 0; } +#ifdef CONFIG_CPU_ISOLATION_OPT +int group_balance_cpu_not_isolated(struct sched_group *sg) +{ + cpumask_t cpus; + + cpumask_and(&cpus, sched_group_span(sg), group_balance_mask(sg)); + cpumask_andnot(&cpus, &cpus, cpu_isolated_mask); + return cpumask_first(&cpus); +} +#endif static int active_load_balance_cpu_stop(void *data); static int should_we_balance(struct lb_env *env) @@ -11360,7 +11416,7 @@ static int should_we_balance(struct lb_env *env) cpumask_copy(swb_cpus, group_balance_mask(sg)); /* Try to find first idle CPU */ for_each_cpu_and(cpu, swb_cpus, env->cpus) { - if (!idle_cpu(cpu)) + if (!idle_cpu(cpu) || cpu_isolated(cpu)) continue; /* @@ -11394,7 +11450,7 @@ static int should_we_balance(struct lb_env *env) return idle_smt == env->dst_cpu; /* Are we the first CPU of this group ? */ - return group_balance_cpu(sg) == env->dst_cpu; + return group_balance_cpu_not_isolated(sg) == env->dst_cpu; } /* @@ -11599,7 +11655,8 @@ static int load_balance(int this_cpu, struct rq *this_rq, * ->active_balance_work. Once set, it's cleared * only after active load balance is finished. */ - if (!busiest->active_balance) { + if (!busiest->active_balance && + !cpu_isolated(cpu_of(busiest))) { busiest->active_balance = 1; busiest->push_cpu = this_cpu; active_balance = 1; @@ -11798,7 +11855,17 @@ static DEFINE_SPINLOCK(balancing); */ void update_max_interval(void) { - max_load_balance_interval = HZ*num_online_cpus()/10; + unsigned int available_cpus; +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_t avail_mask; + + cpumask_andnot(&avail_mask, cpu_online_mask, cpu_isolated_mask); + available_cpus = cpumask_weight(&avail_mask); +#else + available_cpus = num_online_cpus(); +#endif + + max_load_balance_interval = HZ*available_cpus/10; } static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost) @@ -11937,6 +12004,9 @@ static inline int find_new_ilb(void) for_each_cpu_and(ilb, nohz.idle_cpus_mask, hk_mask) { + if (cpu_isolated(ilb)) + continue; + if (ilb == smp_processor_id()) continue; @@ -11994,6 +12064,7 @@ static void nohz_balancer_kick(struct rq *rq) struct sched_domain *sd; int nr_busy, i, cpu = rq->cpu; unsigned int flags = 0; + cpumask_t cpumask; if (unlikely(rq->idle_balance)) return; @@ -12008,8 +12079,15 @@ static void nohz_balancer_kick(struct rq *rq) * None are in tickless mode and hence no need for NOHZ idle load * balancing. */ +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&cpumask, nohz.idle_cpus_mask, cpu_isolated_mask); + if (cpumask_empty(&cpumask)) + return; +#else + cpumask_copy(&cpumask, nohz.idle_cpus_mask); if (likely(!atomic_read(&nohz.nr_cpus))) return; +#endif if (READ_ONCE(nohz.has_blocked) && time_after(now, READ_ONCE(nohz.next_blocked))) @@ -12048,7 +12126,7 @@ static void nohz_balancer_kick(struct rq *rq) * When balancing betwen cores, all the SMT siblings of the * preferred CPU must be idle. */ - for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { + for_each_cpu_and(i, sched_domain_span(sd), &cpumask) { if (sched_use_asym_prio(sd, i) && sched_asym_prefer(i, cpu)) { flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; @@ -12245,6 +12323,7 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) int this_cpu = this_rq->cpu; int balance_cpu; struct rq *rq; + cpumask_t cpus; SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); @@ -12269,11 +12348,17 @@ static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) */ smp_mb(); +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&cpus, nohz.idle_cpus_mask, cpu_isolated_mask); +#else + cpumask_copy(&cpus, nohz.idle_cpus_mask); +#endif + /* * Start with the next CPU after this_cpu so we will end with this_cpu and let a * chance for other idle cpu to pull load. */ - for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { + for_each_cpu_wrap(balance_cpu, &cpus, this_cpu+1) { if (!idle_cpu(balance_cpu)) continue; @@ -12428,6 +12513,9 @@ static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) struct sched_domain *sd; int pulled_task = 0; + if (cpu_isolated(this_cpu)) + return 0; + update_misfit_status(NULL, this_rq); /* @@ -12552,6 +12640,14 @@ static __latent_entropy void run_rebalance_domains(struct softirq_action *h) enum cpu_idle_type idle = this_rq->idle_balance ? CPU_IDLE : CPU_NOT_IDLE; + /* + * Since core isolation doesn't update nohz.idle_cpus_mask, there + * is a possibility this nohz kicked cpu could be isolated. Hence + * return if the cpu is isolated. + */ + if (cpu_isolated(this_rq->cpu)) + return; + /* * If this CPU has a pending nohz_balance_kick, then do the * balancing on behalf of the other idle CPUs whose ticks are @@ -12577,7 +12673,7 @@ void trigger_load_balance(struct rq *rq) * Don't need to rebalance while attached to NULL domain or * runqueue CPU is not active */ - if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) + if (unlikely(on_null_domain(rq)) || cpu_isolated(cpu_of(rq)) || !cpu_active(cpu_of(rq))) return; if (time_after_eq(jiffies, rq->next_balance)) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 758bb5d522d11aa721cba9d7149df2ff3bfab039..48b93b020e750f32caa11f31cde13238174dfd28 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -327,7 +327,8 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev) { /* Try to pull RT tasks here if we lower this rq's prio */ - return rq->online && rq->rt.highest_prio.curr > prev->prio; + return rq->online && rq->rt.highest_prio.curr > prev->prio && + !cpu_isolated(cpu_of(rq)); } static inline int rt_overloaded(struct rq *rq) @@ -2523,7 +2524,8 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p) * we may need to handle the pulling of RT tasks * now. */ - if (!task_on_rq_queued(p) || rq->rt.rt_nr_running) + if (!task_on_rq_queued(p) || rq->rt.rt_nr_running || + cpu_isolated(cpu_of(rq))) return; rt_queue_pull_task(rq); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index afef39e60e9e0ea3abe0bb916e12dc825a073ccf..484a34ddc8218b57cd9f3bab5f4e5163728c8b4d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -154,6 +154,10 @@ extern unsigned int sysctl_sched_child_runs_first; extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); +#ifdef CONFIG_SMP +extern void init_sched_groups_capacity(int cpu, struct sched_domain *sd); +#endif + extern void call_trace_sched_update_nr_running(struct rq *rq, int count); extern unsigned int sysctl_sched_rt_period; @@ -1780,6 +1784,14 @@ rq_lock(struct rq *rq, struct rq_flags *rf) rq_pin_lock(rq, rf); } +static inline void +rq_relock(struct rq *rq, struct rq_flags *rf) + __acquires(rq->lock) +{ + raw_spin_rq_lock(rq); + rq_repin_lock(rq, rf); +} + static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) __releases(rq->lock) @@ -3901,4 +3913,29 @@ static inline bool hmp_capable(void) } #endif /* CONFIG_SCHED_WALT */ +struct sched_avg_stats { + int nr; + int nr_misfit; + int nr_max; + int nr_scaled; +}; +#ifdef CONFIG_SCHED_RUNNING_AVG +extern void sched_get_nr_running_avg(struct sched_avg_stats *stats); +#else +static inline void sched_get_nr_running_avg(struct sched_avg_stats *stats) { } +#endif + +#ifdef CONFIG_CPU_ISOLATION_OPT +extern int group_balance_cpu_not_isolated(struct sched_group *sg); +#else +static inline int group_balance_cpu_not_isolated(struct sched_group *sg) +{ + return group_balance_cpu(sg); +} +#endif /* CONFIG_CPU_ISOLATION_OPT */ + +#ifdef CONFIG_HOTPLUG_CPU +extern void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf, + bool migrate_pinned_tasks); +#endif #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/sched_avg.c b/kernel/sched/sched_avg.c index d74579a1553db6f6545b0e02bcc7b21d9e5da312..53800486808779671355e0d9f753888d52279652 100755 --- a/kernel/sched/sched_avg.c +++ b/kernel/sched/sched_avg.c @@ -106,7 +106,7 @@ static inline void update_last_busy_time(int cpu, bool dequeue, if (prev_nr_run >= BUSY_NR_RUN && per_cpu(nr, cpu) < BUSY_NR_RUN) nr_run_trigger = true; - if (dequeue && (cpu_util(cpu) * BUSY_LOAD_FACTOR) > + if (dequeue && (cpu_util_cfs(cpu) * BUSY_LOAD_FACTOR) > capacity_orig_of(cpu)) load_trigger = true; @@ -161,7 +161,7 @@ unsigned int sched_get_cpu_util(int cpu) unsigned long capacity, flags; unsigned int busy; - raw_spin_lock_irqsave(&rq->lock, flags); + raw_spin_lock_irqsave(&rq->__lock, flags); util = rq->cfs.avg.util_avg; capacity = capacity_orig_of(cpu); @@ -173,7 +173,7 @@ unsigned int sched_get_cpu_util(int cpu) sched_ravg_window >> SCHED_CAPACITY_SHIFT); } #endif - raw_spin_unlock_irqrestore(&rq->lock, flags); + raw_spin_unlock_irqrestore(&rq->__lock, flags); util = (util >= capacity) ? capacity : util; busy = div64_ul((util * 100), capacity); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 423d08947962c8acd7ac304e6a7a995fb6a58d5f..c530731a755cc9fe1f5b0a4afafce45dfade79a8 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1271,9 +1271,12 @@ build_sched_groups(struct sched_domain *sd, int cpu) * group having more cpu_capacity will pickup more load compared to the * group having less cpu_capacity. */ -static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) +void init_sched_groups_capacity(int cpu, struct sched_domain *sd) { struct sched_group *sg = sd->groups; +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_t avail_mask; +#endif struct cpumask *mask = sched_domains_tmpmask2; WARN_ON(!sg); @@ -1281,8 +1284,13 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) do { int cpu, cores = 0, max_cpu = -1; +#ifdef CONFIG_CPU_ISOLATION_OPT + cpumask_andnot(&avail_mask, sched_group_span(sg), + cpu_isolated_mask); + sg->group_weight = cpumask_weight(&avail_mask); +#else sg->group_weight = cpumask_weight(sched_group_span(sg)); - +#endif cpumask_copy(mask, sched_group_span(sg)); for_each_cpu(cpu, mask) { cores++; diff --git a/kernel/smp.c b/kernel/smp.c index 695eb13a276d26302f5d413740fed2592502200e..1cebfa96c2860d23e1194c746a6bd72e68c288c6 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -1061,6 +1061,7 @@ void wake_up_all_idle_cpus(void) for_each_possible_cpu(cpu) { preempt_disable(); if (cpu != smp_processor_id() && cpu_online(cpu)) + if (!cpu_isolated(cpu)) wake_up_if_idle(cpu); preempt_enable(); } diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index cedb17ba158a9bf57f3510fbfc8141adbd12b74b..40b00eec565dd0729c4b852d633085127f6093fa 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -460,7 +460,11 @@ static int __stop_cpus(const struct cpumask *cpumask, * @cpumask were offline; otherwise, 0 if all executions of @fn * returned 0, any non zero return value if any returned non zero. */ +#ifdef CONFIG_CPU_ISOLATION_OPT +int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg) +#else static int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg) +#endif { int ret; diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index edb0f821dceaa1720ac94fc53f4002a1e5f7bdd3..ceda3c97d38e09852c51a00c99e4f6dbde6094d0 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1090,7 +1090,11 @@ static int enqueue_hrtimer(struct hrtimer *timer, base->cpu_base->active_bases |= 1 << base->index; /* Pairs with the lockless read in hrtimer_is_queued() */ +#ifdef CONFIG_CPU_ISOLATION_OPT + WRITE_ONCE(timer->state, (timer->state | HRTIMER_STATE_ENQUEUED)); +#else WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED); +#endif return timerqueue_add(&base->active, &timer->node); } @@ -1113,7 +1117,15 @@ static void __remove_hrtimer(struct hrtimer *timer, u8 state = timer->state; /* Pairs with the lockless read in hrtimer_is_queued() */ +#ifdef CONFIG_CPU_ISOLATION_OPT + /* + * We need to preserve PINNED state here, otherwise we may end up + * migrating pinned hrtimers as well. + */ + WRITE_ONCE(timer->state, newstate | (timer->state & HRTIMER_STATE_PINNED)); +#else WRITE_ONCE(timer->state, newstate); +#endif if (!(state & HRTIMER_STATE_ENQUEUED)) return; @@ -1167,6 +1179,10 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, reprogram &= !keep_local; __remove_hrtimer(timer, base, state, reprogram); +#ifdef CONFIG_CPU_ISOLATION_OPT + /* Make sure PINNED flag is cleared after removing hrtimer */ + timer->state &= ~HRTIMER_STATE_PINNED; +#endif return 1; } return 0; @@ -1259,6 +1275,12 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, new_base = base; } +#ifdef CONFIG_CPU_ISOLATION_OPT + timer->state &= ~HRTIMER_STATE_PINNED; + if (mode & HRTIMER_MODE_PINNED) + timer->state |= HRTIMER_STATE_PINNED; +#endif + first = enqueue_hrtimer(timer, new_base, mode); if (!force_local) return first; @@ -1613,9 +1635,13 @@ bool hrtimer_active(const struct hrtimer *timer) do { base = READ_ONCE(timer->base); seq = raw_read_seqcount_begin(&base->seq); - +#ifdef CONFIG_CPU_ISOLATION_OPT + if (((timer->state & ~HRTIMER_STATE_PINNED) != + HRTIMER_STATE_INACTIVE) || base->running == timer) +#else if (timer->state != HRTIMER_STATE_INACTIVE || base->running == timer) +#endif return true; } while (read_seqcount_retry(&base->seq, seq) || @@ -2191,6 +2217,117 @@ int hrtimers_prepare_cpu(unsigned int cpu) #ifdef CONFIG_HOTPLUG_CPU +#ifdef CONFIG_CPU_ISOLATION_OPT +static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, + struct hrtimer_clock_base *new_base, + bool remove_pinned) +{ + struct hrtimer *timer; + struct timerqueue_node *node; + struct timerqueue_head pinned; + int is_pinned; + bool is_hotplug = !cpu_online(old_base->cpu_base->cpu); + + timerqueue_init_head(&pinned); + + while ((node = timerqueue_getnext(&old_base->active))) { + timer = container_of(node, struct hrtimer, node); + if (is_hotplug) + BUG_ON(hrtimer_callback_running(timer)); + debug_deactivate(timer); + + /* + * Mark it as ENQUEUED not INACTIVE otherwise the + * timer could be seen as !active and just vanish away + * under us on another CPU + */ + __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); + + is_pinned = timer->state & HRTIMER_STATE_PINNED; + if (!remove_pinned && is_pinned) { + timerqueue_add(&pinned, &timer->node); + continue; + } + + timer->base = new_base; + /* + * Enqueue the timers on the new cpu. This does not + * reprogram the event device in case the timer + * expires before the earliest on this CPU, but we run + * hrtimer_interrupt after we migrated everything to + * sort out already expired timers and reprogram the + * event device. + */ + enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS); + } + + /* Re-queue pinned timers for non-hotplug usecase */ + while ((node = timerqueue_getnext(&pinned))) { + timer = container_of(node, struct hrtimer, node); + + timerqueue_del(&pinned, &timer->node); + enqueue_hrtimer(timer, old_base, HRTIMER_MODE_ABS); + } +} + +static void __migrate_hrtimers(unsigned int scpu, bool remove_pinned) +{ + struct hrtimer_cpu_base *old_base, *new_base; + unsigned long flags; + int i; + + local_irq_save(flags); + old_base = &per_cpu(hrtimer_bases, scpu); + new_base = this_cpu_ptr(&hrtimer_bases); + /* + * The caller is globally serialized and nobody else + * takes two locks at once, deadlock is not possible. + */ + raw_spin_lock(&new_base->lock); + raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); + + for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { + migrate_hrtimer_list(&old_base->clock_base[i], + &new_base->clock_base[i], remove_pinned); + } + + /* + * The migration might have changed the first expiring softirq + * timer on this CPU. Update it. + */ + hrtimer_update_softirq_timer(new_base, false); + + raw_spin_unlock(&old_base->lock); + raw_spin_unlock(&new_base->lock); + + /* Check, if we got expired work to do */ + __hrtimer_peek_ahead_timers(); + local_irq_restore(flags); +} + +int hrtimers_cpu_dying(unsigned int scpu) +{ + BUG_ON(cpu_online(scpu)); + tick_cancel_sched_timer(scpu); + + /* + * this BH disable ensures that raise_softirq_irqoff() does + * not wakeup ksoftirqd (and acquire the pi-lock) while + * holding the cpu_base lock + */ + local_bh_disable(); + __migrate_hrtimers(scpu, true); + local_bh_enable(); + return 0; +} + +void hrtimer_quiesce_cpu(void *cpup) +{ + __migrate_hrtimers(*(int *)cpup, false); +} + +#else + static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, struct hrtimer_clock_base *new_base) { @@ -2258,6 +2395,8 @@ int hrtimers_cpu_dying(unsigned int dying_cpu) return 0; } +#endif /* CONFIG_CPU_ISOLATION_OPT */ + #endif /* CONFIG_HOTPLUG_CPU */ void __init hrtimers_init(void) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 63a8ce7177dd49b76c7e0b82400d91bebe08f126..1bf77aaa2d59623525a05a3d388eae5aace4db9c 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2214,6 +2214,65 @@ signed long __sched schedule_timeout_idle(signed long timeout) EXPORT_SYMBOL(schedule_timeout_idle); #ifdef CONFIG_HOTPLUG_CPU + +#ifdef CONFIG_CPU_ISOLATION_OPT +static void migrate_timer_list(struct timer_base *new_base, + struct hlist_head *head, bool remove_pinned) +{ + struct timer_list *timer; + int cpu = new_base->cpu; + struct hlist_node *n; + int is_pinned; + + hlist_for_each_entry_safe(timer, n, head, entry) { + is_pinned = timer->flags & TIMER_PINNED; + if (!remove_pinned && is_pinned) + continue; + + detach_if_pending(timer, get_timer_base(timer->flags), false); + timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu; + internal_add_timer(new_base, timer); + } +} + +static void __migrate_timers(unsigned int cpu, bool remove_pinned) +{ + struct timer_base *old_base; + struct timer_base *new_base; + unsigned long flags; + int b, i; + + for (b = 0; b < NR_BASES; b++) { + old_base = per_cpu_ptr(&timer_bases[b], cpu); + new_base = get_cpu_ptr(&timer_bases[b]); + /* + * The caller is globally serialized and nobody else + * takes two locks at once, deadlock is not possible. + */ + raw_spin_lock_irqsave(&new_base->lock, flags); + raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); + + /* + * The current CPUs base clock might be stale. Update it + * before moving the timers over. + */ + forward_timer_base(new_base); + + if (!cpu_online(cpu)) + BUG_ON(old_base->running_timer); + + for (i = 0; i < WHEEL_SIZE; i++) + migrate_timer_list(new_base, old_base->vectors + i, + remove_pinned); + + raw_spin_unlock(&old_base->lock); + raw_spin_unlock_irqrestore(&new_base->lock, flags); + put_cpu_ptr(&timer_bases); + } +} + +#else + static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head) { struct timer_list *timer; @@ -2227,6 +2286,8 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h } } +#endif /* CONFIG_CPU_ISOLATION_OPT */ + int timers_prepare_cpu(unsigned int cpu) { struct timer_base *base; @@ -2243,6 +2304,20 @@ int timers_prepare_cpu(unsigned int cpu) return 0; } +#ifdef CONFIG_CPU_ISOLATION_OPT +int timers_dead_cpu(unsigned int cpu) +{ + BUG_ON(cpu_online(cpu)); + __migrate_timers(cpu, true); + return 0; +} + +void timer_quiesce_cpu(void *cpup) +{ + __migrate_timers(*(unsigned int *)cpup, false); +} + +#else int timers_dead_cpu(unsigned int cpu) { struct timer_base *old_base; @@ -2278,6 +2353,8 @@ int timers_dead_cpu(unsigned int cpu) return 0; } +#endif /* CONFIG_CPU_ISOLATION_OPT */ + #endif /* CONFIG_HOTPLUG_CPU */ static void __init init_timer_cpu(int cpu) diff --git a/kernel/watchdog.c b/kernel/watchdog.c index 5cd6d4e269157973acecd00afc91646720fee2be..c904872be06cc24993bd8cea0b08001eb5843e96 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -537,16 +538,20 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) return HRTIMER_RESTART; } -static void watchdog_enable(unsigned int cpu) +void watchdog_enable(unsigned int cpu) { struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); struct completion *done = this_cpu_ptr(&softlockup_completion); + unsigned int *enabled = this_cpu_ptr(&watchdog_en); WARN_ON_ONCE(cpu != smp_processor_id()); init_completion(done); complete(done); + if (*enabled) + return; + /* * Start the timer first to prevent the hardlockup watchdog triggering * before the timer has a chance to fire. @@ -561,11 +566,24 @@ static void watchdog_enable(unsigned int cpu) /* Enable the hardlockup detector */ if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED) watchdog_hardlockup_enable(cpu); + + /* + * Need to ensure above operations are observed by other CPUs before + * indicating that timer is enabled. This is to synchronize core + * isolation and hotplug. Core isolation will wait for this flag to be + * set. + */ + mb(); + *enabled = 1; } -static void watchdog_disable(unsigned int cpu) +void watchdog_disable(unsigned int cpu) { - struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer); + struct hrtimer *hrtimer = per_cpu_ptr(&watchdog_hrtimer, cpu); + unsigned int *enabled = per_cpu_ptr(&watchdog_en, cpu); + + if (!*enabled) + return; WARN_ON_ONCE(cpu != smp_processor_id()); @@ -576,7 +594,18 @@ static void watchdog_disable(unsigned int cpu) */ watchdog_hardlockup_disable(cpu); hrtimer_cancel(hrtimer); - wait_for_completion(this_cpu_ptr(&softlockup_completion)); + wait_for_completion(per_cpu_ptr(&softlockup_completion, cpu)); + + /* + * No need for barrier here since disabling the watchdog is + * synchronized with hotplug lock + */ + *enabled = 0; +} + +bool watchdog_configured(unsigned int cpu) +{ + return *per_cpu_ptr(&watchdog_en, cpu); } static int softlockup_stop_fn(void *data) diff --git a/mm/vmstat.c b/mm/vmstat.c index 00e81e99c6ee24e419446b8c476faea0b8ef75bd..2d972cf873bd2d200cc2873f92c815d34f070116 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1940,7 +1940,7 @@ int vmstat_refresh(struct ctl_table *table, int write, static void vmstat_update(struct work_struct *w) { - if (refresh_cpu_vm_stats(true)) { + if (refresh_cpu_vm_stats(true) && !cpu_isolated(smp_processor_id())) { /* * Counters were updated so we expect more updates * to occur in the future. Keep on running the @@ -2039,7 +2039,8 @@ static void vmstat_shepherd(struct work_struct *w) if (cpu_is_isolated(cpu)) continue; - if (!delayed_work_pending(dw) && need_update(cpu)) + if (!delayed_work_pending(dw) && need_update(cpu) && + !cpu_isolated(cpu)) queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0); cond_resched();