diff --git a/arch/riscv/include/riscv/csr.h b/arch/riscv/include/riscv/csr.h index b1796cca947153e996bb42881816b7b092936cd8..60972d4d405f45dea4fbdfe4f2045928f09c7353 100644 --- a/arch/riscv/include/riscv/csr.h +++ b/arch/riscv/include/riscv/csr.h @@ -1,48 +1,47 @@ #pragma once /* Status register flags */ -#define SR_SIE 0x00000002 /* Supervisor Interrupt Enable */ -#define SR_SPP 0x00000100 /* Previously Supervisor */ -#define SR_SUM 0x00040000 /* Supervisor User Memory Access */ - -#define SR_FS 0x00006000 /* Floating-point Status */ - -#define SR_VS 0x00000600 /* Vector Status */ - -#define SR_FS_VS (SR_FS | SR_VS) /* Vector and Floating-Point Unit */ - - -#define CSR_SSTATUS 0x100 -#define CSR_SIE 0x104 -#define CSR_STVEC 0x105 -#define CSR_SSCRATCH 0x140 -#define CSR_SEPC 0x141 -#define CSR_SCAUSE 0x142 -#define CSR_STVAL 0x143 -#define CSR_SIP 0x144 -#define CSR_SATP 0x180 - - -#define CSR_IE CSR_SIE -#define CSR_IP CSR_SIP -#define CSR_TVEC CSR_STVEC -#define CSR_SCRATCH CSR_SSCRATCH +#define SR_SIE 0x00000002 /* Supervisor Interrupt Enable */ +#define SR_SPP 0x00000100 /* Previously Supervisor */ +#define SR_SUM 0x00040000 /* Supervisor User Memory Access */ + +#define SR_FS 0x00006000 /* Floating-point Status */ + +#define SR_VS 0x00000600 /* Vector Status */ + +#define SR_FS_VS (SR_FS | SR_VS) /* Vector and Floating-Point Unit */ + +#define CSR_SSTATUS 0x100 +#define CSR_SIE 0x104 +#define CSR_STVEC 0x105 +#define CSR_SSCRATCH 0x140 +#define CSR_SEPC 0x141 +#define CSR_SCAUSE 0x142 +#define CSR_STVAL 0x143 +#define CSR_SIP 0x144 +#define CSR_SATP 0x180 + +#define CSR_IE CSR_SIE +#define CSR_IP CSR_SIP +#define CSR_TVEC CSR_STVEC +#define CSR_SCRATCH CSR_SSCRATCH #define CSR_STATUS CSR_SSTATUS -#define CSR_EPC CSR_SEPC -#define CSR_CAUSE CSR_SCAUSE -#define CSR_TVAL CSR_STVAL - -#define SR_IE SR_SIE +#define CSR_EPC CSR_SEPC +#define CSR_CAUSE CSR_SCAUSE +#define CSR_TVAL CSR_STVAL +#define SR_IE SR_SIE +#define IRQ_S_SOFT 1 #define IRQ_S_TIMER 5 #define IRQ_S_EXT 9 -#define MIP_STIP (1 << IRQ_S_TIMER) +#define MIP_STIP (1 << IRQ_S_TIMER) -#define SIP_STIP MIP_STIP /* timer interrupt */ +#define SIP_STIP MIP_STIP /* timer interrupt */ -#define RV_IRQ_TIMER IRQ_S_TIMER -#define RV_IRQ_EXT IRQ_S_EXT +#define RV_IRQ_TIMER IRQ_S_TIMER +#define RV_IRQ_EXT IRQ_S_EXT +#define RV_IRQ_SOFT IRQ_S_SOFT -#define SATP_MODE_39 0x8000000000000000UL +#define SATP_MODE_39 0x8000000000000000UL diff --git a/arch/riscv/include/riscv/ipi.h b/arch/riscv/include/riscv/ipi.h new file mode 100644 index 0000000000000000000000000000000000000000..e45e70f3d06086b395db5e2c14170fa7f434e4fe --- /dev/null +++ b/arch/riscv/include/riscv/ipi.h @@ -0,0 +1,25 @@ +#pragma once + +enum ipi_message_type +{ + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CPU_STOP, + IPI_CPU_CRASH_STOP, + IPI_IRQ_WORK, + IPI_TIMER, + IPI_CPU_BACKTRACE, + IPI_KGDB_ROUNDUP, + + IPI_MAX +}; + +/* + send_ipi_single - this is a 'weak' function, implete me in SMP +*/ +extern void send_ipi_single(int cpu, enum ipi_message_type op); + +/* + call me when IPI is received +*/ +void riscv_handle_ipi(enum ipi_message_type op); diff --git a/arch/riscv/include/riscv/sbi.h b/arch/riscv/include/riscv/sbi.h index c8f9582d1d4122cd58435bc9b97398626d8deb36..3447716da58bb1f5dca9695d88518bc7e86ab43a 100644 --- a/arch/riscv/include/riscv/sbi.h +++ b/arch/riscv/include/riscv/sbi.h @@ -68,6 +68,11 @@ enum sbi_ext_hsm_fid SBI_EXT_HSM_HART_SUSPEND, }; +enum sbi_ext_ipi_fid +{ + SBI_EXT_IPI_SEND_IPI = 0, +}; + struct sbiret __sbi_ecall(unsigned long arg0, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5, @@ -82,3 +87,4 @@ void sbi_set_timer(uint64_t stime_value); void sbi_console_putchar(int ch); int sbi_err_map_linux_errno(int err); +int sbi_send_ipi(unsigned int hartid); diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 65ba786461147777be77e74fa11c947436b2b5cd..02c01024454932482814adef15b8fe1aebc85566 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -4,7 +4,12 @@ __weak void irqchip_init(void) { } +__weak void sbi_ipi_init(void) +{ +} + void init_IRQ(void) { irqchip_init(); + sbi_ipi_init(); } diff --git a/arch/riscv/kernel/smp/boot.c b/arch/riscv/kernel/smp/boot.c index cd0bc6b3d5d3f146e8598e74b40caefb88a717d7..ce85e7c50d2473ebb93f4a7ae941807d3df2118f 100644 --- a/arch/riscv/kernel/smp/boot.c +++ b/arch/riscv/kernel/smp/boot.c @@ -28,6 +28,8 @@ void smp_callin(void) notify_cpu_starting(curr_cpuid); + set_cpu_online(curr_cpuid, true); + local_irq_enable(); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); } diff --git a/arch/riscv/kernel/smp/smp.c b/arch/riscv/kernel/smp/smp.c index 754868d678e3d450d8b84103afddde6a20355f13..8bf1c07c7babcc6063196943d3911bce0b7021f5 100644 --- a/arch/riscv/kernel/smp/smp.c +++ b/arch/riscv/kernel/smp/smp.c @@ -1,24 +1,12 @@ #include #include +#include #include +#include extern unsigned long boot_cpu_hartid; -enum ipi_message_type -{ - IPI_RESCHEDULE, - IPI_CALL_FUNC, - IPI_CPU_STOP, - IPI_CPU_CRASH_STOP, - IPI_IRQ_WORK, - IPI_TIMER, - IPI_CPU_BACKTRACE, - IPI_KGDB_ROUNDUP, - - IPI_MAX -}; - static unsigned int __cpuid_to_hartid_map[NR_CPUS] = { [0 ... NR_CPUS - 1] = INVALID_HARTID, }; @@ -42,3 +30,22 @@ unsigned raw_smp_processor_id(void) { return current_thread_info()->cpu; } + +void riscv_handle_ipi(enum ipi_message_type ipi) +{ + switch (ipi) + { + case IPI_RESCHEDULE: + scheduler_ipi(); + break; + } +} + +__weak void send_ipi_single(int cpu, enum ipi_message_type op) +{ +} + +void arch_smp_send_reschedule(int cpu) +{ + send_ipi_single(cpu, IPI_RESCHEDULE); +} diff --git a/arch/riscv/sbi/sbi.c b/arch/riscv/sbi/sbi.c index e4b687eb5cef68c7e4b69be30e2af54c6cc8d3ef..955cc02d0a8ac27f61c0fa4d1a88fbd1370d4f1a 100644 --- a/arch/riscv/sbi/sbi.c +++ b/arch/riscv/sbi/sbi.c @@ -6,6 +6,16 @@ static void __sbi_set_timer_v01(uint64_t stime_value) sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); } +static int __sbi_send_ipi_v02(unsigned int hartid) +{ + struct sbiret ret = {0}; + + ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, + 1, hartid, 0, 0, 0, 0); + + return ret.error; +} + void sbi_set_timer(uint64_t stime_value) { __sbi_set_timer_v01(stime_value); @@ -44,6 +54,11 @@ int sbi_err_map_linux_errno(int err) return ret; } +int sbi_send_ipi(unsigned int hartid) +{ + return __sbi_send_ipi_v02(hartid); +} + void sbi_init(void) { } diff --git a/bsp/qemu-riscv-virt64/drivers/timer-ce.c b/bsp/qemu-riscv-virt64/drivers/timer-ce.c index fffad7bb8c7e70d48c85e02253b29bb7cf14f156..0dff0ab86e861486de35bd6660cd50e7159e7954 100644 --- a/bsp/qemu-riscv-virt64/drivers/timer-ce.c +++ b/bsp/qemu-riscv-virt64/drivers/timer-ce.c @@ -37,9 +37,12 @@ static irqreturn_t riscv_timer_interrupt(int irq, void *dev_id) static int riscv_timer_starting_cpu(unsigned int cpu) { struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu); + + csr_clear(CSR_IE, SIP_STIP); + sbi_set_timer(get_cycles() + tick_cycles); clockevents_config_and_register(ce, CPUTIME_TIMER_FREQ, 100, 100000); - csr_set(CSR_IE, 1 << RV_IRQ_TIMER); + enable_percpu_irq(RV_IRQ_TIMER, 0); return 0; } @@ -52,9 +55,6 @@ static int riscv_timer_dying_cpu(unsigned int cpu) void timer_probe(void) { tick_cycles = CPUTIME_TIMER_FREQ / HZ; - csr_clear(CSR_IE, SIP_STIP); - - sbi_set_timer(get_cycles() + tick_cycles); request_percpu_irq(riscv_clock_event_irq, riscv_timer_interrupt, diff --git a/bsp/qemu-riscv-virt64/irq/irqchip.c b/bsp/qemu-riscv-virt64/irq/irqchip.c index 10854c995957b45d453c0a3f8c2bb6715e11107e..7e3a22d74852fb5125de9a03af71f57a0998b24d 100644 --- a/bsp/qemu-riscv-virt64/irq/irqchip.c +++ b/bsp/qemu-riscv-virt64/irq/irqchip.c @@ -43,6 +43,11 @@ int request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, return 0; } +void enable_percpu_irq(unsigned int irq, unsigned int type) +{ + csr_set(CSR_IE, 1 << irq); +} + static int plic_dying_cpu(unsigned int cpu) { if (plic_parent_irq) @@ -68,11 +73,10 @@ static int plic_starting_cpu(unsigned int cpu) static void plic_handle_irq(struct pt_regs *regs) { - int irq; + int irq = regs->cause & 0xff; - if ((regs->cause & 0xff) == 5) + if (irq < RV_IRQ_EXT) { - irq = 5; if (_act[irq].handler) { _act[irq].handler(irq, _act[irq].dev_id); diff --git a/bsp/qemu-riscv-virt64/irq/sbi-ipi.c b/bsp/qemu-riscv-virt64/irq/sbi-ipi.c new file mode 100644 index 0000000000000000000000000000000000000000..7faf18282e4b511c253729a24e2661b3b0707959 --- /dev/null +++ b/bsp/qemu-riscv-virt64/irq/sbi-ipi.c @@ -0,0 +1,39 @@ +#include +#include + +#include +#include +#include +#include + +static int sbi_ipi_starting_cpu(unsigned int cpu) +{ + enable_percpu_irq(RV_IRQ_SOFT, 0); + + return 0; +} + +static irqreturn_t ipi_handler(int irq, void *data) +{ + riscv_handle_ipi(IPI_RESCHEDULE); + + csr_clear(CSR_IP, (1 << RV_IRQ_SOFT)); + + return IRQ_HANDLED; +} + +void send_ipi_single(int cpu, enum ipi_message_type op) +{ + unsigned int hartid = cpuid_to_hartid_map(cpu); + + sbi_send_ipi(hartid); +} + +void sbi_ipi_init(void) +{ + request_percpu_irq(RV_IRQ_SOFT, ipi_handler, "ipi", 0); + + cpuhp_setup_state(CPUHP_AP_IRQ_IPI_STARTING, + "irqchip/sbi-ipi:starting", + sbi_ipi_starting_cpu, 0); +} diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index f9aebb50346ad43569660dd75a6c838114464d5a..bb4c9f3896164f2f5b4bf299ba786f1700da6d6e 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -75,6 +75,7 @@ enum cpuhp_state CPUHP_AP_RCUTREE_DYING, CPUHP_AP_CPU_PM_STARTING, CPUHP_AP_IRQ_CHIP_STARTING, + CPUHP_AP_IRQ_IPI_STARTING, CPUHP_AP_PERF_CPU_STARTING, diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 915bef9475a808bffc5cdf787bfc3b823e6abf3d..af2ec4904d7a5e05b4108ab71d962b13daf5c56f 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -20,3 +20,5 @@ extern int request_irq(unsigned int irq, irq_handler_t handler, unsigned long fl extern int request_percpu_irq(unsigned int irq, irq_handler_t handler, const char *devname, void *percpu_dev_id); + +extern void enable_percpu_irq(unsigned int irq, unsigned int type); diff --git a/include/linux/sched.h b/include/linux/sched.h index 3c0a82b8c99226ef4f36fd61394b44931120ab8e..60da9106c6caca24530c76566adc7454bea532d8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -165,3 +165,5 @@ extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match /* do_set_cpus_allowed() - consider using set_cpus_allowed_ptr() instead */ extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); + +void scheduler_ipi(void); diff --git a/include/linux/smp.h b/include/linux/smp.h index 7942c676f35a3fe4afeb17dab2b3ff4a1c47356c..b23ec3ee9883419e57c52279cfd758694948a6ca 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -40,6 +40,20 @@ extern unsigned int setup_max_cpus; */ extern void smp_prepare_cpus(unsigned int max_cpus); +/* + * sends a 'reschedule' event to another CPU: + */ +extern void arch_smp_send_reschedule(int cpu); + +/* + * scheduler_ipi() is inline so can't be passed as callback reason, but the + * callsite IP should be sufficient for root-causing IPIs sent from here. + */ +#define smp_send_reschedule(cpu) \ + ({ \ + arch_smp_send_reschedule(cpu); \ + }) + #else /* !smp */ #define raw_smp_processor_id() 0 diff --git a/kern/sched/_core_/init.c b/kern/sched/_core_/init.c index dc95f3ed79d220536e0357c375319f4d08c285f7..9356405e234dafc69dc9acd505c3127400bd7a70 100644 --- a/kern/sched/_core_/init.c +++ b/kern/sched/_core_/init.c @@ -13,10 +13,11 @@ #define for_each_class(class) \ for_class_range(class, __sched_class_highest(), __sched_class_lowest()) -static inline void init_rq(struct rq *rq) +static inline void init_rq(struct rq *rq, unsigned int cpu) { const struct sched_class *class; + rq->cpu = cpu; raw_spin_lock_init(&rq->__lock); INIT_LIST_HEAD(&rq->cfs_tasks); diff --git a/kern/sched/_core_/smp/resched_curr_cpu.c b/kern/sched/_core_/smp/resched_curr_cpu.c new file mode 100644 index 0000000000000000000000000000000000000000..4f694ee454a16610e0a62135886f5ef9d6521642 --- /dev/null +++ b/kern/sched/_core_/smp/resched_curr_cpu.c @@ -0,0 +1,27 @@ + +#ifdef CONFIG_SMP + +static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif) +{ + set_ti_thread_flag(ti, tif); + return true; +} + +static inline bool set_nr_if_polling(struct task_struct *p) +{ + return false; +} + +static void inline resched_curr_cpu(struct thread_info *cti, int tif, int cpu) +{ + if (set_nr_and_not_polling(cti, tif)) + { + if (tif == TIF_NEED_RESCHED) + smp_send_reschedule(cpu); + } +} +#else +static void inline resched_curr_cpu(struct thread_info *cti, int tif, int cpu) +{ +} +#endif diff --git a/kern/sched/core.c b/kern/sched/core.c index 146302ef67eda69ba81743237034fe8114f2740c..d0d7673bf64fde93ca258c92fcf3cba966347429 100644 --- a/kern/sched/core.c +++ b/kern/sched/core.c @@ -31,12 +31,13 @@ bool sched_smp_initialized = false; #include "_core_/pi.c" #include "_core_/wake_q.c" #include "_core_/task_state_match.c" -#include "_core_/smp/select_task_rq.c" +#include "_core_/smp/select_task_rq.c" #include "_core_/wake_up.c" #include "_core_/set_load_weight.c" #include "_core_/smp/set_rq_online.c" #include "_core_/smp/cpu_active.c" #include "_core_/wait_task_inactive.c" +#include "_core_/smp/resched_curr_cpu.c" /* * Constants for the sched_mode argument of __schedule(). @@ -142,13 +143,19 @@ static void __resched_curr(struct rq *rq, int tif) if (is_idle_task(curr) && tif == TIF_NEED_RESCHED_LAZY) tif = TIF_NEED_RESCHED; + if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED)) + return; + cpu = cpu_of(rq); if (cpu == smp_processor_id()) { set_ti_thread_flag(cti, tif); if (tif == TIF_NEED_RESCHED) set_preempt_need_resched(); - return; + } + else + { + resched_curr_cpu(cti, tif, cpu); } } @@ -363,7 +370,7 @@ void __init sched_init(void) rq = cpu_rq(i); - init_rq(rq); + init_rq(rq, i); } set_load_weight(&init_task, false); @@ -382,6 +389,11 @@ void __init sched_init_smp(void) sched_smp_initialized = true; } + +void scheduler_ipi(void) +{ + +} #else void __init sched_init_smp(void) {