diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index afde4d70ee1394304bfbd159759ce9d7ec8e9277..2002fe8d9ea399d3dba2852bb244d180aa6b0d83 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6682,6 +6682,7 @@ CONFIG_ARM_GIC_PHYTIUM_2500=y # CONFIG_AL_FIC is not set CONFIG_HISILICON_IRQ_MBIGEN=y CONFIG_IRQ_MBIGEN_ENABLE_SPI=y +CONFIG_VIRT_VTIMER_IRQ_BYPASS=y # CONFIG_XILINX_INTC is not set CONFIG_PARTITION_PERCPU=y CONFIG_QCOM_IRQ_COMBINER=y diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index c1fa64f6e5bed91c1fc1c74d736fdb621e5c88ae..589ee66e64cf9a9332343d376486efc5d865f0f0 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -112,6 +112,9 @@ static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu) { vcpu->arch.hcr_el2 &= ~HCR_TWE; if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) || +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + vcpu->kvm->arch.vgic.vtimer_irqbypass || +#endif vcpu->kvm->arch.vgic.nassgireq) vcpu->arch.hcr_el2 &= ~HCR_TWI; else diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 7f693efba3e1fb20bc2804d3cff203c1cc60dc97..cbe1d1f514afe4535d576b2a4a923205cff8e971 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -81,4 +81,9 @@ config ARCH_VCPU_STAT If unsure, say N. +config VIRT_VTIMER_IRQ_BYPASS + bool "MBIGEN enable vtimer irq bypass" + depends on KVM && ARM64 && ARCH_HISI && HISILICON_IRQ_MBIGEN + default n + endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index a1e24228aaaa764f3f96a3780e60a3003694b22b..43957fce50f6f9145a6edc26eb8080feeed4f57a 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -29,6 +29,21 @@ static unsigned int host_ptimer_irq; static u32 host_vtimer_irq_flags; static u32 host_ptimer_irq_flags; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +bool vtimer_irqbypass; + +static int __init early_vtimer_irqbypass(char *buf) +{ + return strtobool(buf, &vtimer_irqbypass); +} +early_param("kvm-arm.vtimer_irqbypass", early_vtimer_irqbypass); + +static inline bool vtimer_is_irqbypass(void) +{ + return !!vtimer_irqbypass && kvm_vgic_vtimer_irqbypass_support(); +} +#endif + static DEFINE_STATIC_KEY_FALSE(has_gic_active_state); static const u8 default_ppi[] = { @@ -653,6 +668,45 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo WARN_ON(r); } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void kvm_vtimer_mbigen_auto_clr_set(struct kvm_vcpu *vcpu, bool set) +{ + BUG_ON(!vtimer_is_irqbypass()); + + vtimer_mbigen_set_auto_clr(vcpu->cpu, set); +} + +static void kvm_vtimer_gic_auto_clr_set(struct kvm_vcpu *vcpu, bool set) +{ + BUG_ON(!vtimer_is_irqbypass()); + + vtimer_gic_set_auto_clr(vcpu->cpu, set); +} + +static void kvm_vtimer_mbigen_restore_stat(struct kvm_vcpu *vcpu) +{ + struct vtimer_mbigen_context *mbigen_ctx = vcpu_vtimer_mbigen(vcpu); + u16 vpeid = kvm_vgic_get_vcpu_vpeid(vcpu); + unsigned long flags; + + WARN_ON(!vtimer_is_irqbypass()); + + local_irq_save(flags); + + if (mbigen_ctx->loaded) + goto out; + + vtimer_mbigen_set_vector(vcpu->cpu, vpeid); + + if (mbigen_ctx->active) + vtimer_mbigen_set_active(vcpu->cpu, true); + + mbigen_ctx->loaded = true; +out: + local_irq_restore(flags); +} +#endif + static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx) { struct kvm_vcpu *vcpu = ctx->vcpu; @@ -834,20 +888,42 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu) get_timer_map(vcpu, &map); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + kvm_vtimer_mbigen_auto_clr_set(vcpu, false); + kvm_vtimer_mbigen_restore_stat(vcpu); + + goto skip_load_vtimer; + } +#endif if (static_branch_likely(&has_gic_active_state)) { if (vcpu_has_nv(vcpu)) kvm_timer_vcpu_load_nested_switch(vcpu, &map); kvm_timer_vcpu_load_gic(map.direct_vtimer); +#ifndef CONFIG_VIRT_VTIMER_IRQ_BYPASS if (map.direct_ptimer) kvm_timer_vcpu_load_gic(map.direct_ptimer); +#endif } else { kvm_timer_vcpu_load_nogic(vcpu); } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +skip_load_vtimer: + if (static_branch_likely(&has_gic_active_state) && map.direct_ptimer) + kvm_timer_vcpu_load_gic(map.direct_ptimer); +#endif kvm_timer_unblocking(vcpu); timer_restore_state(map.direct_vtimer); + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + kvm_vtimer_mbigen_auto_clr_set(vcpu, true); + kvm_vtimer_gic_auto_clr_set(vcpu, true); + } +#endif if (map.direct_ptimer) timer_restore_state(map.direct_ptimer); if (map.emul_vtimer) @@ -875,6 +951,31 @@ bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu) kvm_timer_should_fire(ptimer) != plevel; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void kvm_vtimer_mbigen_save_stat(struct kvm_vcpu *vcpu) +{ + struct vtimer_mbigen_context *mbigen_ctx = vcpu_vtimer_mbigen(vcpu); + unsigned long flags; + + WARN_ON(!vtimer_is_irqbypass()); + + local_irq_save(flags); + + if (!mbigen_ctx->loaded) + goto out; + + mbigen_ctx->active = vtimer_mbigen_get_active(vcpu->cpu); + + /* Clear active state in MBIGEN now that we've saved everything. */ + if (mbigen_ctx->active) + vtimer_mbigen_set_active(vcpu->cpu, false); + + mbigen_ctx->loaded = false; +out: + local_irq_restore(flags); +} +#endif + void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = vcpu_timer(vcpu); @@ -885,7 +986,21 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) get_timer_map(vcpu, &map); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + kvm_vtimer_mbigen_auto_clr_set(vcpu, false); + kvm_vtimer_gic_auto_clr_set(vcpu, false); + } +#endif + timer_save_state(map.direct_vtimer); + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + kvm_vtimer_mbigen_save_stat(vcpu); + kvm_vtimer_mbigen_auto_clr_set(vcpu, true); + } +#endif if (map.direct_ptimer) timer_save_state(map.direct_ptimer); @@ -965,6 +1080,16 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) } if (timer->enabled) { +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu)); + + if (irqchip_in_kernel(vcpu->kvm) && map.direct_ptimer) + kvm_vgic_reset_mapped_irq(vcpu, timer_irq(map.direct_ptimer)); + + goto skip_reset_vtimer; + } +#endif for (int i = 0; i < nr_timers(vcpu); i++) kvm_timer_update_irq(vcpu, false, vcpu_get_timer(vcpu, i)); @@ -978,6 +1103,10 @@ int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu) if (map.emul_vtimer) soft_timer_cancel(&map.emul_vtimer->hrtimer); + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +skip_reset_vtimer: +#endif if (map.emul_ptimer) soft_timer_cancel(&map.emul_ptimer->hrtimer); @@ -1036,6 +1165,12 @@ void kvm_timer_init_vm(struct kvm *kvm) void kvm_timer_cpu_up(void) { +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) { + enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags); + return; + } +#endif enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags); if (host_ptimer_irq) enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags); @@ -1043,6 +1178,10 @@ void kvm_timer_cpu_up(void) void kvm_timer_cpu_down(void) { +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) + return; +#endif disable_percpu_irq(host_vtimer_irq); if (host_ptimer_irq) disable_percpu_irq(host_ptimer_irq); @@ -1380,6 +1519,32 @@ int __init kvm_timer_hyp_init(bool has_gic) return -ENODEV; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* + * vtimer-irqbypass depends on: + * + * - HW support at mbigen level (vtimer_irqbypass_hw_support) + * - HW support at GIC level (kvm_vgic_vtimer_irqbypass_support) + * - in_kernel irqchip support + * - "kvm-arm.vtimer_irqbypass=1" + */ + vtimer_irqbypass &= vtimer_irqbypass_hw_support(info); + vtimer_irqbypass &= has_gic; + if (vtimer_is_irqbypass()) { + kvm_info("vtimer-irqbypass enabled\n"); + + /* + * If vtimer irqbypass is enabled, there's no need to use the + * vtimer forwarded irq inject. + */ + + if (info->physical_irq > 0) { + host_ptimer_irq = info->physical_irq; + kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags); + } + goto ptimer_irq_init; + } +#endif err = kvm_irq_init(info); if (err) return err; @@ -1407,6 +1572,9 @@ int __init kvm_timer_hyp_init(bool has_gic) kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +ptimer_irq_init: +#endif /* Now let's do the physical EL1 timer irq */ if (info->physical_irq > 0) { @@ -1506,15 +1674,85 @@ static bool kvm_arch_timer_get_input_level(int vintid) return false; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void vtimer_set_active_stat(struct kvm_vcpu *vcpu, int vintid, bool set) +{ + struct vtimer_mbigen_context *mbigen_ctx = vcpu_vtimer_mbigen(vcpu); + int hwirq = timer_irq(vcpu_vtimer(vcpu)); + + WARN_ON(!vtimer_is_irqbypass() || hwirq != vintid); + + if (!mbigen_ctx->loaded) + mbigen_ctx->active = set; + else + vtimer_mbigen_set_active(vcpu->cpu, set); +} + +static bool vtimer_get_active_stat(struct kvm_vcpu *vcpu, int vintid) +{ + struct vtimer_mbigen_context *mbigen_ctx = vcpu_vtimer_mbigen(vcpu); + int hwirq = timer_irq(vcpu_vtimer(vcpu)); + + WARN_ON(!vtimer_is_irqbypass() || hwirq != vintid); + + if (!mbigen_ctx->loaded) + return mbigen_ctx->active; + else + return vtimer_mbigen_get_active(vcpu->cpu); +} + +int kvm_vtimer_config(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu; + int ret = 0; + unsigned long c; + + if (!vtimer_is_irqbypass()) + return 0; + + if (!irqchip_in_kernel(kvm)) + return -EINVAL; + mutex_lock(&kvm->lock); + if (dist->vtimer_irqbypass) + goto out; + + kvm_for_each_vcpu(c, vcpu, kvm) { + struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu; + int intid; + + WARN_ON(timer->enabled); + + intid = timer_irq(vcpu_vtimer(vcpu)); + ret = kvm_vgic_config_vtimer_irqbypass(vcpu, intid, + vtimer_get_active_stat, + vtimer_set_active_stat); + if (ret) + goto out; + } + + dist->vtimer_irqbypass = true; + +out: + mutex_unlock(&kvm->lock); + return ret; +} +#endif + int kvm_timer_enable(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map; - int ret; + int ret = 0; if (timer->enabled) return 0; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (!irqchip_in_kernel(vcpu->kvm) && vtimer_is_irqbypass()) + return -EINVAL; +#endif + /* Without a VGIC we do not map virtual IRQs to physical IRQs */ if (!irqchip_in_kernel(vcpu->kvm)) goto no_vgic; @@ -1530,6 +1768,10 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) get_timer_map(vcpu, &map); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (vtimer_is_irqbypass()) + goto skip_map_vtimer; +#endif ret = kvm_vgic_map_phys_irq(vcpu, map.direct_vtimer->host_timer_irq, timer_irq(map.direct_vtimer), @@ -1537,6 +1779,9 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) if (ret) return ret; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +skip_map_vtimer: +#endif if (map.direct_ptimer) { ret = kvm_vgic_map_phys_irq(vcpu, map.direct_ptimer->host_timer_irq, diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 453528ba9cbb1bfcd0f2863a4a38efc485292d36..859689e80ec445869635521c8b3e70f33a5f66a3 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -667,6 +667,12 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) kvm_arm_vcpu_init_debug(vcpu); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + ret = kvm_vtimer_config(kvm); + if (ret) + return ret; +#endif + if (likely(irqchip_in_kernel(kvm))) { /* * Map the VGIC hardware resources before running a vcpu the diff --git a/arch/arm64/kvm/vgic/vgic-debug.c b/arch/arm64/kvm/vgic/vgic-debug.c index 07aa0437125a60b318f626abb556e35f635c11bc..20fed597531b09d6db7af08a578e7dd3ad29fb6c 100644 --- a/arch/arm64/kvm/vgic/vgic-debug.c +++ b/arch/arm64/kvm/vgic/vgic-debug.c @@ -193,7 +193,7 @@ static void print_irq_state(struct seq_file *s, struct vgic_irq *irq, print_header(s, irq, vcpu); pending = irq->pending_latch; - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { int err; err = irq_get_irqchip_state(irq->host_irq, diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index c1f1b1e1dd1ce2ab9d675b5a6c610071f3885015..f5d0520d137ebfcb58fbd6cba8613ab5c23c8357 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -229,6 +229,11 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) /* PPIs */ irq->config = VGIC_CONFIG_LEVEL; } + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* Needed? */ + irq->vtimer_info = NULL; +#endif } if (!irqchip_in_kernel(vcpu->kvm)) diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 188d2187eede935e43b31fcf4985205a1872d92f..8cbba361d65fc66501c28d90e747d4515818bb55 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -374,6 +374,21 @@ static int vgic_v3_uaccess_write_pending(struct kvm_vcpu *vcpu, irq->pending_latch = true; vgic_queue_irq_unlock(vcpu->kvm, irq, flags); } else { +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /** + * workaround: On reset, userspace clears pending status + * for all PPIs and SGIs by writing all 0's to + * GICR_ISPENDR0. The pending state of vtimer interrupt + * is somehow staying in redistributor and we have to + * explicitly clear it... + * + * P.S., irq->vtimer_info is NULL on restore. + */ + if (irq->vtimer_info) + WARN_ON_ONCE(irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + false)); +#endif irq->pending_latch = false; raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index ff558c05e990c728abd5361054cb04ef44083818..7aca296f9b3f8fab4d62768335fb015b94652c2c 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -78,7 +78,7 @@ void vgic_mmio_write_group(struct kvm_vcpu *vcpu, gpa_t addr, raw_spin_lock_irqsave(&irq->irq_lock, flags); irq->group = !!(val & BIT(i)); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { vgic_update_vsgi(irq); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); } else { @@ -125,7 +125,7 @@ void vgic_mmio_write_senable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { if (!irq->enabled) { struct irq_data *data; @@ -174,7 +174,7 @@ void vgic_mmio_write_cenable(struct kvm_vcpu *vcpu, struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw && vgic_irq_is_sgi(irq->intid) && irq->enabled) + if (vgic_direct_sgi_or_ppi(irq) && irq->enabled) disable_irq_nosync(irq->host_irq); irq->enabled = false; @@ -250,7 +250,7 @@ static unsigned long __read_pending(struct kvm_vcpu *vcpu, * for handling of ISPENDR and ICPENDR. */ raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { int err; val = false; @@ -320,7 +320,7 @@ void vgic_mmio_write_spending(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { /* HW SGI? Ask the GIC to inject it */ int err; err = irq_set_irqchip_state(irq->host_irq, @@ -413,7 +413,7 @@ void vgic_mmio_write_cpending(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) { + if (vgic_direct_sgi_or_ppi(irq)) { /* HW SGI? Ask the GIC to clear its pending bit */ int err; err = irq_set_irqchip_state(irq->host_irq, @@ -510,12 +510,23 @@ static unsigned long __vgic_mmio_read_active(struct kvm_vcpu *vcpu, /* Loop over all IRQs affected by this read */ for (i = 0; i < len * 8; i++) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, intid + i); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + struct vtimer_info *vtimer = irq->vtimer_info; + bool state = irq->active; + + if (vtimer) + state = vtimer->get_active_stat(vcpu, irq->intid); +#endif /* * Even for HW interrupts, don't evaluate the HW state as * all the guest is interested in is the virtual state. */ +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (state) +#else if (irq->active) +#endif value |= (1U << i); vgic_put_irq(vcpu->kvm, irq); @@ -575,6 +586,11 @@ static void vgic_mmio_change_active(struct kvm_vcpu *vcpu, struct vgic_irq *irq, * do here. */ irq->active = false; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + } else if (irq->vtimer_info) { + /* MMIO trap only */ + irq->vtimer_info->set_active_stat(vcpu, irq->intid, active); +#endif } else { u32 model = vcpu->kvm->arch.vgic.vgic_model; u8 active_source; @@ -718,7 +734,7 @@ void vgic_mmio_write_priority(struct kvm_vcpu *vcpu, raw_spin_lock_irqsave(&irq->irq_lock, flags); /* Narrow the priority range to what we actually support */ irq->priority = (val >> (i * 8)) & GENMASK(7, 8 - VGIC_PRI_BITS); - if (irq->hw && vgic_irq_is_sgi(irq->intid)) + if (vgic_direct_sgi_or_ppi(irq)) vgic_update_vsgi(irq); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 3dfc8b84e03e67868ff49cb72a97695a9222ef2b..c71fc1a70e24a4c520c57a1deade073e6fc5f37a 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -571,6 +571,10 @@ int vgic_v3_map_resources(struct kvm *kvm) if (kvm_vgic_global_state.has_gicv4_1) vgic_v4_configure_vsgis(kvm); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (kvm_vgic_vtimer_irqbypass_support()) + vgic_v4_configure_vtimer(kvm); +#endif return 0; } @@ -653,6 +657,12 @@ int vgic_v3_probe(const struct gic_kvm_info *info) kvm_info("GICv4%s support %sabled\n", kvm_vgic_global_state.has_gicv4_1 ? ".1" : "", gicv4_enable ? "en" : "dis"); + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + kvm_vgic_global_state.has_direct_vtimer = info->has_vtimer && gicv4_enable; + if (kvm_vgic_global_state.has_direct_vtimer) + kvm_info("vtimer-irqbypass support enabled at GIC level\n"); +#endif } kvm_vgic_global_state.vcpu_base = 0; diff --git a/arch/arm64/kvm/vgic/vgic-v4.c b/arch/arm64/kvm/vgic/vgic-v4.c index 339a55194b2c63e78a6c8083fe7acb34a3cfa5af..1d02c50593d9d85fa2f9587734d44540d3b32381 100644 --- a/arch/arm64/kvm/vgic/vgic-v4.c +++ b/arch/arm64/kvm/vgic/vgic-v4.c @@ -204,6 +204,65 @@ void vgic_v4_configure_vsgis(struct kvm *kvm) kvm_arm_resume_guest(kvm); } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void vgic_v4_enable_vtimer(struct kvm_vcpu *vcpu) +{ + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + struct vtimer_info *vtimer = &vgic_cpu->vtimer; + struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe; + struct vgic_irq *irq; + struct irq_desc *desc; + int ret; + + irq = vgic_get_irq(vcpu->kvm, vcpu, vtimer->intid); + irq->host_irq = irq_find_mapping(vpe->sgi_domain, vtimer->intid); + + /* Transfer the full irq state to the vPE */ + vgic_v4_sync_sgi_config(vpe, irq); + desc = irq_to_desc(irq->host_irq); + ret = irq_domain_activate_irq(irq_desc_get_irq_data(desc), + false); + if (!WARN_ON(ret)) { + /* Transfer pending state */ + ret = irq_set_irqchip_state(irq->host_irq, + IRQCHIP_STATE_PENDING, + irq->pending_latch); + WARN_ON(ret); + irq->pending_latch = false; + + /* Transfer active state */ + vtimer->set_active_stat(vcpu, irq->intid, irq->active); + irq->active = false; + } + + vgic_put_irq(vcpu->kvm, irq); +} + +/* Must be called with the kvm lock held */ +void vgic_v4_configure_vtimer(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu; + unsigned long i; + + if (!dist->vtimer_irqbypass) + return; + + kvm_for_each_vcpu(i, vcpu, kvm) + vgic_v4_enable_vtimer(vcpu); +} + +/** + * kvm_vgic_get_vcpu_vpeid - Get the VCPU's vpeid + * + * The vtimer mbigen needs the vcpu vpeid info which will resident. + */ +u16 kvm_vgic_get_vcpu_vpeid(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vpe_id; +} +#endif + /* * Must be called with GICv4.1 and the vPE unmapped, which * indicates the invalidation of any VPT caches associated diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 8be4c1ebdec27a3e13c12914af80227c8d0eb32b..6e1fa6f2918ffbd2b75f2d72ff6060bc84ccde4c 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -590,6 +590,32 @@ int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid) return ret; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +int kvm_vgic_config_vtimer_irqbypass(struct kvm_vcpu *vcpu, u32 vintid, + bool (*get_as)(struct kvm_vcpu *, int), + void (*set_as)(struct kvm_vcpu *, int, bool)) +{ + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + struct vtimer_info *vtimer = &vgic_cpu->vtimer; + struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid); + unsigned long flags; + + if (WARN_ON_ONCE(!irq || !kvm_vgic_vtimer_irqbypass_support())) + return -EINVAL; + + vtimer->intid = vintid; + vtimer->get_active_stat = get_as; + vtimer->set_active_stat = set_as; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + irq->vtimer_info = vtimer; + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + + return 0; +} +#endif + /** * kvm_vgic_set_owner - Set the owner of an interrupt for a VM * diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 0ab09b0d44404b7f6d7a3bee5689b0dccadaf1a1..8f8ea7ffc940a551c9a13419dc80f008af984d99 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -145,6 +145,18 @@ static inline int vgic_write_guest_lock(struct kvm *kvm, gpa_t gpa, return ret; } +static inline bool vgic_direct_sgi_or_ppi(struct vgic_irq *irq) +{ + bool direct_sgi = irq->hw && vgic_irq_is_sgi(irq->intid); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + bool direct_ppi = !!(irq->vtimer_info); + + return direct_sgi || direct_ppi; +#else + return direct_sgi; +#endif +} + /* * This struct provides an intermediate representation of the fields contained * in the GICH_VMCR and ICH_VMCR registers, such that code exporting the GIC @@ -341,5 +353,8 @@ void vgic_v4_teardown(struct kvm *kvm); void vgic_v4_configure_vsgis(struct kvm *kvm); void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val); int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +void vgic_v4_configure_vtimer(struct kvm *kvm); +#endif #endif diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 9950bf4030044c6327455fb44ad90828a726e70e..617b451f27418b1e1398deb529648f4dd026adcb 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -218,6 +218,9 @@ struct its_node { int numa_node; unsigned int msi_domain_flags; u32 pre_its_base; /* for Socionext Synquacer */ +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + u32 version; +#endif int vlpi_redist_offset; }; @@ -225,6 +228,25 @@ struct its_node { #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +#define is_vtimer_irqbypass(its) (!!((its)->version & GITS_VERSION_VTIMER)) + +/* Fetch it from gtdt->virtual_timer_interrupt. */ +#define is_vtimer_irq(irq) ((irq) == 27) + +static inline bool is_its_vsgi_cmd_valid(struct its_node *its, u8 hwirq) +{ + if (__get_intid_range(hwirq) == SGI_RANGE) + return true; + + /* For PPI range, only vtimer interrupt is supported atm. */ + if (is_vtimer_irq(hwirq) && is_vtimer_irqbypass(its)) + return true; + + return false; +} +#endif + #define ITS_ITT_ALIGN SZ_256 /* The maximum number of VPEID bits supported by VLPI commands */ @@ -756,6 +778,18 @@ static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi) its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void its_encode_sgi_intid_extension(struct its_cmd_block *cmd, u8 sgi) +{ + /* + * We reuse the VSGI command in this implementation to configure + * the vPPI or clear its pending state. The vINTID field has been + * therefore extended to [36:32]. + */ + its_mask_encode(&cmd->raw_cmd[0], sgi, 36, 32); +} +#endif + static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio) { its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); @@ -1155,7 +1189,14 @@ static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, its_encode_cmd(cmd, GITS_CMD_VSGI); its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (!is_vtimer_irqbypass(its)) + its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); + else + its_encode_sgi_intid_extension(cmd, desc->its_vsgi_cmd.sgi); +#else its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); +#endif its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); @@ -4423,6 +4464,14 @@ static void its_configure_sgi(struct irq_data *d, bool clear) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_cmd_desc desc; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + struct its_node *its = find_4_1_its(); + + if (!its || !is_its_vsgi_cmd_valid(its, d->hwirq)) { + pr_err("ITS: %s failed\n", __func__); + return; + } +#endif desc.its_vsgi_cmd.vpe = vpe; desc.its_vsgi_cmd.sgi = d->hwirq; @@ -4436,7 +4485,11 @@ static void its_configure_sgi(struct irq_data *d, bool clear) * destination VPE is mapped there. Since we map them eagerly at * activation time, we're pretty sure the first GICv4.1 ITS will do. */ +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + its_send_single_vcommand(its, its_build_vsgi_cmd, &desc); +#else its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc); +#endif } static void its_sgi_mask_irq(struct irq_data *d) @@ -4479,10 +4532,20 @@ static int its_sgi_set_irqchip_state(struct irq_data *d, struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_node *its = find_4_1_its(); u64 val; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + u64 offset = GITS_SGIR; + + if (__get_intid_range(d->hwirq) == PPI_RANGE) + offset = GITS_PPIR; +#endif val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + writeq_relaxed(val, its->sgir_base + offset - SZ_128K); +#else writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); +#endif } else { its_configure_sgi(d, true); } @@ -4503,6 +4566,20 @@ static int its_sgi_get_irqchip_state(struct irq_data *d, if (which != IRQCHIP_STATE_PENDING) return -EINVAL; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + enum gic_intid_range type; + + /* + * Plug the HiSilicon implementation details in comment! + * + * For vPPI, we re-use the GICR_VSGIR and GICR_VSGIPENDR in the + * implementation which allows reads to GICR_I{S,C}PENDR to be + * emulated. And note that the pending state of the vtimer + * interrupt is stored at bit[16] of GICR_VSGIPENDR. + */ + type = __get_intid_range(d->hwirq); + WARN_ON_ONCE(type == PPI_RANGE && !is_vtimer_irq(d->hwirq)); +#endif /* * Locking galore! We can race against two different events: * @@ -4538,7 +4615,14 @@ static int its_sgi_get_irqchip_state(struct irq_data *d, if (!count) return -ENXIO; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (is_vtimer_irq(d->hwirq)) + *val = !!(status & (1 << 16)); + else + *val = !!(status & (1 << d->hwirq)); +#else *val = !!(status & (1 << d->hwirq)); +#endif return 0; } @@ -4576,11 +4660,19 @@ static int its_sgi_irq_domain_alloc(struct irq_domain *domain, { struct its_vpe *vpe = args; int i; - +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* We may want 32 IRQs if vtimer irqbypass is supported. */ + WARN_ON(nr_irqs != 16 && nr_irqs != 32); +#else /* Yes, we do want 16 SGIs */ WARN_ON(nr_irqs != 16); +#endif +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + for (i = 0; i < nr_irqs; i++) { +#else for (i = 0; i < 16; i++) { +#endif vpe->sgi_config[i].priority = 0; vpe->sgi_config[i].enabled = false; vpe->sgi_config[i].group = false; @@ -5588,6 +5680,10 @@ static struct its_node __init *its_node_init(struct resource *res, its->numa_node = numa_node; its->fwnode_handle = handle; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (readl_relaxed(its_base + GITS_IIDR) == 0x00051736) + its->version = readl_relaxed(its_base + GITS_VERSION); +#endif return its; out_unmap: @@ -5872,6 +5968,9 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, struct its_node *its; bool has_v4 = false; bool has_v4_1 = false; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + bool has_vtimer_irqbypass = false; +#endif int err; gic_rdists = rdists; @@ -5895,12 +5994,23 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, list_for_each_entry(its, &its_nodes, entry) { has_v4 |= is_v4(its); has_v4_1 |= is_v4_1(its); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + has_vtimer_irqbypass |= is_vtimer_irqbypass(its); +#endif } /* Don't bother with inconsistent systems */ if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) rdists->has_rvpeid = false; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* vtimer irqbypass depends on rvpeid support */ + if (WARN_ON(!has_v4_1 && has_vtimer_irqbypass)) { + has_vtimer_irqbypass = false; + rdists->has_vtimer = false; + } +#endif + if (has_v4 & rdists->has_vlpis) { const struct irq_domain_ops *sgi_ops; @@ -5910,7 +6020,12 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, sgi_ops = NULL; if (its_init_vpe_domain() || +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops) || + vtimer_irqbypass_init(parent_domain, has_vtimer_irqbypass)) { +#else its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) { +#endif rdists->has_vlpis = false; pr_err("ITS: Disabling GICv4 support\n"); } diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 9a3d5acb01ca103295b48b228a12da9db4a75e14..73dfcbdd25663c5d67a1abf4b03b10fcac417763 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -140,16 +140,6 @@ static DEFINE_PER_CPU(bool, has_rss); /* Our default, arbitrary priority value. Linux only uses one anyway. */ #define DEFAULT_PMR_VALUE 0xf0 -enum gic_intid_range { - SGI_RANGE, - PPI_RANGE, - SPI_RANGE, - EPPI_RANGE, - ESPI_RANGE, - LPI_RANGE, - __INVALID_RANGE__ -}; - #ifdef CONFIG_ARM64 #include #include @@ -165,25 +155,12 @@ static inline bool has_v3_3_nmi(void) } #endif -static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) -{ - switch (hwirq) { - case 0 ... 15: - return SGI_RANGE; - case 16 ... 31: - return PPI_RANGE; - case 32 ... 1019: - return SPI_RANGE; - case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): - return EPPI_RANGE; - case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): - return ESPI_RANGE; - case 8192 ... GENMASK(23, 0): - return LPI_RANGE; - default: - return __INVALID_RANGE__; - } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +phys_addr_t get_gicr_paddr(int cpu) +{ + return (per_cpu_ptr(gic_data.rdists.rdist, cpu))->phys_base; } +#endif static enum gic_intid_range get_intid_range(struct irq_data *d) { @@ -1201,6 +1178,11 @@ static int __gic_update_rdist_properties(struct redist_region *region, gic_data.rdists.has_rvpeid = false; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* HiSilicon implement: if GICv4.1 is supported, vtimer irqbypass is supported */ + gic_data.rdists.has_vtimer = gic_data.rdists.has_rvpeid; +#endif + gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr); return 1; @@ -2166,6 +2148,9 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_data.rdists.has_vlpis = true; gic_data.rdists.has_direct_lpi = true; gic_data.rdists.has_vpend_valid_dirty = true; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + gic_data.rdists.has_vtimer = false; +#endif } if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { @@ -2345,6 +2330,9 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + gic_v3_kvm_info.has_vtimer = gic_data.rdists.has_vtimer; +#endif vgic_set_kvm_info(&gic_v3_kvm_info); } @@ -2685,6 +2673,9 @@ static void __init gic_acpi_setup_kvm_info(void) gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + gic_v3_kvm_info.has_vtimer = gic_data.rdists.has_vtimer; +#endif vgic_set_kvm_info(&gic_v3_kvm_info); } diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 94d56a03b1757f1336587898de72853bc65abf35..22a9a7f1739fda9cad313d02079805d51a18308e 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -86,6 +86,9 @@ static struct irq_domain *gic_domain; static const struct irq_domain_ops *vpe_domain_ops; static const struct irq_domain_ops *sgi_domain_ops; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static bool vtimer_irqbypass; +#endif #ifdef CONFIG_ARM64 #include @@ -110,6 +113,13 @@ static bool has_v4_1(void) return !!sgi_domain_ops; } +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static bool has_v4_1_vsgi_extend(void) +{ + return has_v4_1() && vtimer_irqbypass; +} +#endif + static bool has_v4_1_sgi(void) { return has_v4_1() && gic_cpuif_has_vsgi(); @@ -119,10 +129,18 @@ static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) { char *name; int sgi_base; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + int nr_irqs = 16; +#endif if (!has_v4_1_sgi()) return 0; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + if (has_v4_1_vsgi_extend()) + nr_irqs = 32; +#endif + name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); if (!name) goto err; @@ -134,18 +152,30 @@ static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) kfree(name); name = NULL; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, nr_irqs, +#else vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16, +#endif sgi_domain_ops, vpe); if (!vpe->sgi_domain) goto err; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + vpe->nr_irqs = nr_irqs; + sgi_base = irq_domain_alloc_irqs(vpe->sgi_domain, nr_irqs, NUMA_NO_NODE, vpe); +#else sgi_base = irq_domain_alloc_irqs(vpe->sgi_domain, 16, NUMA_NO_NODE, vpe); +#endif if (sgi_base <= 0) goto err; return 0; err: +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + vpe->nr_irqs = 0; +#endif if (vpe->sgi_domain) irq_domain_remove(vpe->sgi_domain); if (vpe->fwnode) @@ -211,7 +241,11 @@ static void its_free_sgi_irqs(struct its_vm *vm) if (WARN_ON(!irq)) continue; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + irq_domain_free_irqs(irq, vm->vpes[i]->nr_irqs); +#else irq_domain_free_irqs(irq, 16); +#endif irq_domain_remove(vm->vpes[i]->sgi_domain); irq_domain_free_fwnode(vm->vpes[i]->fwnode); } @@ -387,3 +421,16 @@ int its_init_v4(struct irq_domain *domain, pr_err("ITS: No GICv4 VPE domain allocated\n"); return -ENODEV; } + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +int vtimer_irqbypass_init(struct irq_domain *domain, + bool has_vtimer_irqbypass) +{ + if (domain) { + vtimer_irqbypass = has_vtimer_irqbypass; + return 0; + } + + return -ENODEV; +} +#endif diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 92a8bba7dd9432ff78bf37a25ecdcc5047848a27..d59dc4c8163dc31e8f887019c4317eccbde49611 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -58,6 +58,62 @@ #define REG_MBIGEN_SPI_TYPE_OFFSET 0x400 #endif +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +#include +#include +#include +#include +#include +#include + +#include +#include + +#define MBIGEN_CTLR_OFFSET 0x0 +#define MBIGEN_AFF3_MASK 0xff000000 +#define MBIGEN_AFF3_SHIFT 24 + +/** + * MBIX config register + * bit[25:24] mbi_type: + * - 0b10 support vtimer irqbypass + */ +#define MBIGEN_NODE_CFG_OFFSET 0x0004 +#define MBIGEN_TYPE_MASK 0x03000000 +#define MBIGEN_TYPE_SHIFT 24 +#define TYPE_VTIMER_ENABLED 0x02 + +#define VTIMER_MBIGEN_REG_WIDTH 4 +#define PPIS_PER_MBIGEN_NODE 32 +#define VTIMER_MBIGEN_REG_TYPE_OFFSET 0x1000 +#define VTIMER_MBIGEN_REG_SET_AUTO_CLR_OFFSET 0x1100 +#define VTIMER_MBIGEN_REG_CLR_AUTO_CLR_OFFSET 0x1110 +#define VTIMER_MBIGEN_REG_ATV_STAT_OFFSET 0x1120 +#define VTIMER_GIC_REG_SET_AUTO_CLR_OFFSET 0x1150 +#define VTIMER_GIC_REG_CLR_AUTO_CLR_OFFSET 0x1160 +#define VTIMER_MBIGEN_REG_VEC_OFFSET 0x1200 +#define VTIMER_MBIGEN_REG_ATV_CLR_OFFSET 0xa008 + +/** + * struct vtimer_mbigen_device - holds the information of vtimer mbigen device. + * + * @base: mapped address of this mbigen chip. + * @cpu_base : the base cpu_id attached to the mbigen chip. + * @cpu_num : the num of the cpus attached to the mbigen chip. + * @mpidr_aff3 : [socket_id : die_id] of the mbigen chip. + * @entry: list_head connecting this vtimer_mbigen to the full list. + * @vmgn_lock: spinlock for set type. + */ +struct vtimer_mbigen_device { + void __iomem *base; + int cpu_base; + int cpu_num; + int mpidr_aff3; + struct list_head entry; + spinlock_t vmgn_lock; +}; +#endif + /** * struct mbigen_device - holds the information of mbigen device. * @@ -67,8 +123,183 @@ struct mbigen_device { struct platform_device *pdev; void __iomem *base; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + struct vtimer_mbigen_device *vtimer_mbigen_chip; +#endif }; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static LIST_HEAD(vtimer_mgn_list); + +/** + * Due to the existence of hyper-threading technology, We need to get the + * absolute offset of a cpu relative to the base cpu. + */ +#define GICR_LENGTH 0x40000 +static inline int get_abs_offset(int cpu, int cpu_base) +{ + return ((get_gicr_paddr(cpu) - get_gicr_paddr(cpu_base)) / GICR_LENGTH); +} + +static struct vtimer_mbigen_device *get_vtimer_mbigen(int cpu_id) +{ + unsigned int mpidr_aff3; + struct vtimer_mbigen_device *chip; + + mpidr_aff3 = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu_id), 3); + + list_for_each_entry(chip, &vtimer_mgn_list, entry) { + if (chip->mpidr_aff3 == mpidr_aff3) + return chip; + } + + pr_debug("Failed to get vtimer mbigen of cpu%d!\n", cpu_id); + return NULL; +} + +void vtimer_mbigen_set_vector(int cpu_id, u16 vpeid) +{ + + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset, count = 100; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + addr = chip->base + VTIMER_MBIGEN_REG_VEC_OFFSET + + cpu_abs_offset * VTIMER_MBIGEN_REG_WIDTH; + + writel_relaxed(vpeid, addr); + + /* Make sure correct vpeid set */ + do { + if (readl_relaxed(addr) == vpeid) + break; + } while (count--); + + if (!count) + pr_err("Failed to set mbigen vector of CPU%d!\n", cpu_id); +} + +bool vtimer_mbigen_get_active(int cpu_id) +{ + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset; + u32 val; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return false; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + addr = chip->base + VTIMER_MBIGEN_REG_ATV_STAT_OFFSET + + (cpu_abs_offset / PPIS_PER_MBIGEN_NODE) * VTIMER_MBIGEN_REG_WIDTH; + + dsb(sy); + val = readl_relaxed(addr); + return (!!(val & (1 << (cpu_abs_offset % PPIS_PER_MBIGEN_NODE)))); +} + +void vtimer_mbigen_set_auto_clr(int cpu_id, bool set) +{ + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset; + u64 offset; + u32 val; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + offset = set ? VTIMER_MBIGEN_REG_SET_AUTO_CLR_OFFSET : + VTIMER_MBIGEN_REG_CLR_AUTO_CLR_OFFSET; + addr = chip->base + offset + + (cpu_abs_offset / PPIS_PER_MBIGEN_NODE) * VTIMER_MBIGEN_REG_WIDTH; + val = 1 << (cpu_abs_offset % PPIS_PER_MBIGEN_NODE); + + writel_relaxed(val, addr); + dsb(sy); +} + +void vtimer_gic_set_auto_clr(int cpu_id, bool set) +{ + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset; + u64 offset; + u32 val; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + offset = set ? VTIMER_GIC_REG_SET_AUTO_CLR_OFFSET : + VTIMER_GIC_REG_CLR_AUTO_CLR_OFFSET; + addr = chip->base + offset + + (cpu_abs_offset / PPIS_PER_MBIGEN_NODE) * VTIMER_MBIGEN_REG_WIDTH; + val = 1 << (cpu_abs_offset % PPIS_PER_MBIGEN_NODE); + + writel_relaxed(val, addr); + dsb(sy); +} + +void vtimer_mbigen_set_active(int cpu_id, bool set) +{ + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset; + u64 offset; + u32 val; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + offset = set ? VTIMER_MBIGEN_REG_ATV_STAT_OFFSET : + VTIMER_MBIGEN_REG_ATV_CLR_OFFSET; + addr = chip->base + offset + + (cpu_abs_offset / PPIS_PER_MBIGEN_NODE) * VTIMER_MBIGEN_REG_WIDTH; + val = 1 << (cpu_abs_offset % PPIS_PER_MBIGEN_NODE); + + writel_relaxed(val, addr); + dsb(sy); +} + +static int vtimer_mbigen_set_type(unsigned int cpu_id) +{ + struct vtimer_mbigen_device *chip; + void __iomem *addr; + int cpu_abs_offset; + u32 val, mask; + + chip = get_vtimer_mbigen(cpu_id); + if (!chip) + return -EINVAL; + + cpu_abs_offset = get_abs_offset(cpu_id, chip->cpu_base); + addr = chip->base + VTIMER_MBIGEN_REG_TYPE_OFFSET + + (cpu_abs_offset / PPIS_PER_MBIGEN_NODE) * VTIMER_MBIGEN_REG_WIDTH; + + mask = 1 << (cpu_abs_offset % PPIS_PER_MBIGEN_NODE); + + spin_lock(&chip->vmgn_lock); + val = readl_relaxed(addr); + val |= mask; + writel_relaxed(val, addr); + dsb(sy); + spin_unlock(&chip->vmgn_lock); + return 0; +} +#endif + static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) { unsigned int nid, pin; @@ -369,6 +600,239 @@ static inline int mbigen_acpi_create_domain(struct platform_device *pdev, } #endif +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static void vtimer_mbigen_set_kvm_info(void) +{ + struct arch_timer_kvm_info *info = arch_timer_get_kvm_info(); + + info->irqbypass_flag |= VT_EXPANDDEV_PROBED; +} + +static int vtimer_mbigen_chip_read_aff3(struct vtimer_mbigen_device *chip) +{ + void __iomem *base = chip->base; + void __iomem *addr = base + MBIGEN_CTLR_OFFSET; + u32 val = readl_relaxed(addr); + + return ((val & MBIGEN_AFF3_MASK) >> MBIGEN_AFF3_SHIFT); +} + +static int vtimer_mbigen_chip_match_cpu(struct vtimer_mbigen_device *chip) +{ + int cpu; + + for_each_possible_cpu(cpu) { + int mpidr_aff3 = MPIDR_AFFINITY_LEVEL(cpu_logical_map(cpu), 3); + + if (chip->mpidr_aff3 == mpidr_aff3) { + /* get the first cpu attached to the mbigen */ + if (chip->cpu_base == -1) { + /* Make sure cpu_base is attached to PIN0 */ + u64 mpidr = cpu_logical_map(cpu); + + if (!MPIDR_AFFINITY_LEVEL(mpidr, 2) && + !MPIDR_AFFINITY_LEVEL(mpidr, 1) && + !MPIDR_AFFINITY_LEVEL(mpidr, 0)) + chip->cpu_base = cpu; + } + } + } + + if (chip->cpu_base == -1) + return -EINVAL; + + return 0; +} + +static bool is_mbigen_vtimer_bypass_enabled(struct mbigen_device *mgn_chip) +{ + void __iomem *base = mgn_chip->base; + void __iomem *addr = base + MBIGEN_NODE_CFG_OFFSET; + u32 val = readl_relaxed(addr); + + return ((val & MBIGEN_TYPE_MASK) >> MBIGEN_TYPE_SHIFT) + == TYPE_VTIMER_ENABLED; +} + +/** + * MBIX_VPPI_ITS_TA: Indicates the address of the ITS corresponding + * to the mbigen. + */ +#define MBIX_VPPI_ITS_TA 0x0038 +static bool vtimer_mbigen_should_probe(struct mbigen_device *mgn_chip) +{ + unsigned int mpidr_aff3; + struct vtimer_mbigen_device *chip; + void __iomem *addr; + u32 val; + + /* find the valid mbigen */ + addr = mgn_chip->base + MBIX_VPPI_ITS_TA; + val = readl_relaxed(addr); + if (!val) + return false; + + addr = mgn_chip->base + MBIGEN_CTLR_OFFSET; + val = readl_relaxed(addr); + mpidr_aff3 = (val & MBIGEN_AFF3_MASK) >> MBIGEN_AFF3_SHIFT; + list_for_each_entry(chip, &vtimer_mgn_list, entry) { + if (chip->mpidr_aff3 == mpidr_aff3) + return false; + } + + return true; +} + +#define CHIP0_TA_MBIGEN_PHY_BASE 0x4604400000 +#define CHIP0_TA_MBIGEN_ITS_BASE 0x84028 +#define CHIP0_TA_PERI_PHY_BASE 0x4614002018 + +#define CHIP0_TB_MBIGEN_PHY_BASE 0xc604400000 +#define CHIP0_TB_PERI_PHY_BASE 0xc614002018 +#define CHIP0_TB_MBIGEN_ITS_BASE 0x4028 + +#define CHIP1_TA_MBIGEN_PHY_BASE 0x204604400000 +#define CHIP1_TA_PERI_PHY_BASE 0x204614002018 +#define CHIP1_TA_MBIGEN_ITS_BASE 0x2084028 + +#define CHIP1_TB_MBIGEN_PHY_BASE 0x20c604400000 +#define CHIP1_TB_MBIGEN_ITS_BASE 0x2004028 +#define CHIP1_TB_PERI_PHY_BASE 0x20c614002018 + +extern bool vtimer_irqbypass; + +static int vtimer_mbigen_set_regs(struct platform_device *pdev) +{ + struct mbigen_device *mgn_chip = platform_get_drvdata(pdev); + struct resource *res; + void __iomem *addr; + unsigned int mpidr_aff3; + u32 val; + struct vtimer_mbigen_device *chip; + + if (!vtimer_irqbypass) + return 0; + + addr = mgn_chip->base + MBIGEN_CTLR_OFFSET; + val = readl_relaxed(addr); + mpidr_aff3 = (val & MBIGEN_AFF3_MASK) >> MBIGEN_AFF3_SHIFT; + list_for_each_entry(chip, &vtimer_mgn_list, entry) { + if (chip->mpidr_aff3 == mpidr_aff3) + return 0; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!mgn_chip) + return -ENOMEM; + + if (res->start == CHIP0_TA_MBIGEN_PHY_BASE) { + addr = ioremap(CHIP0_TA_PERI_PHY_BASE, 4); + if (!addr) { + pr_err("Unable to map CHIP0-TA-PERI\n"); + return -ENOMEM; + } + + writel_relaxed(1, addr); + iounmap(addr); + + addr = mgn_chip->base + MBIX_VPPI_ITS_TA; + writel_relaxed(CHIP0_TA_MBIGEN_ITS_BASE, addr); + } + + if (res->start == CHIP0_TB_MBIGEN_PHY_BASE) { + addr = ioremap(CHIP0_TB_PERI_PHY_BASE, 4); + if (!addr) { + pr_err("Unable to map CHIP0-TB-PERI\n"); + return -ENOMEM; + } + + writel_relaxed(1, addr); + iounmap(addr); + + addr = mgn_chip->base + MBIX_VPPI_ITS_TA; + writel_relaxed(CHIP0_TB_MBIGEN_ITS_BASE, addr); + } + + if (res->start == CHIP1_TA_MBIGEN_PHY_BASE) { + addr = ioremap(CHIP1_TA_PERI_PHY_BASE, 4); + if (!addr) { + pr_err("Unable to map CHIP1-TA-PERI\n"); + return -ENOMEM; + } + + writel_relaxed(1, addr); + iounmap(addr); + + addr = mgn_chip->base + MBIX_VPPI_ITS_TA; + writel_relaxed(CHIP1_TA_MBIGEN_ITS_BASE, addr); + } + + if (res->start == CHIP1_TB_MBIGEN_PHY_BASE) { + addr = ioremap(CHIP1_TB_PERI_PHY_BASE, 4); + if (!addr) { + pr_err("Unable to map CHIP1-TB-PERI\n"); + return -ENOMEM; + } + + writel_relaxed(1, addr); + iounmap(addr); + + addr = mgn_chip->base + MBIX_VPPI_ITS_TA; + writel_relaxed(CHIP1_TB_MBIGEN_ITS_BASE, addr); + } + + return 0; +} + +static int vtimer_mbigen_device_probe(struct platform_device *pdev) +{ + struct mbigen_device *mgn_chip = platform_get_drvdata(pdev); + struct vtimer_mbigen_device *vtimer_mgn_chip; + int err; + + if (!vtimer_irqbypass) + return 0; + + err = vtimer_mbigen_set_regs(pdev); + if (err) + return err; + + if (!is_mbigen_vtimer_bypass_enabled(mgn_chip) || + !vtimer_mbigen_should_probe(mgn_chip)) + return 0; + + vtimer_mgn_chip = kzalloc(sizeof(*vtimer_mgn_chip), GFP_KERNEL); + if (!vtimer_mgn_chip) + return -ENOMEM; + + mgn_chip->vtimer_mbigen_chip = vtimer_mgn_chip; + vtimer_mgn_chip->base = mgn_chip->base; + vtimer_mgn_chip->mpidr_aff3 = vtimer_mbigen_chip_read_aff3(vtimer_mgn_chip); + vtimer_mgn_chip->cpu_base = -1; + err = vtimer_mbigen_chip_match_cpu(vtimer_mgn_chip); + if (err) { + dev_err(&pdev->dev, + "Fail to match vtimer mbigen device with cpu\n"); + goto out; + } + + spin_lock_init(&vtimer_mgn_chip->vmgn_lock); + list_add(&vtimer_mgn_chip->entry, &vtimer_mgn_list); + vtimer_mbigen_set_kvm_info(); + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "irqchip/mbigen-vtimer:online", + vtimer_mbigen_set_type, NULL); + + pr_info("vtimer mbigen device @%p probed success!\n", mgn_chip->base); + return 0; + +out: + kfree(vtimer_mgn_chip); + dev_err(&pdev->dev, "vtimer mbigen device @%p probed failed\n", + mgn_chip->base); + return err; +} +#endif + static int mbigen_device_probe(struct platform_device *pdev) { struct mbigen_device *mgn_chip; @@ -405,6 +869,15 @@ static int mbigen_device_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, mgn_chip); + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + err = vtimer_mbigen_device_probe(pdev); + + if (err) { + dev_err(&pdev->dev, "Failed to probe vtimer mbigen device\n"); + return err; + } +#endif return 0; } @@ -424,7 +897,22 @@ static struct platform_driver mbigen_platform_driver = { .probe = mbigen_device_probe, }; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static int __init mbigen_init(void) +{ + return platform_driver_register(&mbigen_platform_driver); +} + +static void __exit mbigen_exit(void) +{ + return platform_driver_unregister(&mbigen_platform_driver); +} + +arch_initcall(mbigen_init); +module_exit(mbigen_exit); +#else module_platform_driver(mbigen_platform_driver); +#endif MODULE_AUTHOR("Jun Ma "); MODULE_AUTHOR("Yun Wu "); diff --git a/include/clocksource/arm_arch_timer.h b/include/clocksource/arm_arch_timer.h index cbbc9a6dc571587db49b5e9b965f9e3610b83c55..2a82abe2bb7f3bfa5fd43a28e03d3b620cf555b3 100644 --- a/include/clocksource/arm_arch_timer.h +++ b/include/clocksource/arm_arch_timer.h @@ -67,6 +67,12 @@ struct arch_timer_kvm_info { struct timecounter timecounter; int virtual_irq; int physical_irq; + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/* vtimer expand device probed flag */ +#define VT_EXPANDDEV_PROBED (1 << 0) + unsigned long irqbypass_flag; +#endif }; struct arch_timer_mem_frame { @@ -109,4 +115,17 @@ static inline bool arch_timer_evtstrm_available(void) #endif +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +static inline bool vtimer_irqbypass_hw_support(struct arch_timer_kvm_info *info) +{ + return info->irqbypass_flag & VT_EXPANDDEV_PROBED; +} + +void vtimer_mbigen_set_vector(int cpu_id, u16 vpeid); +bool vtimer_mbigen_get_active(int cpu_id); +void vtimer_mbigen_set_auto_clr(int cpu_id, bool set); +void vtimer_gic_set_auto_clr(int cpu_id, bool set); +void vtimer_mbigen_set_active(int cpu_id, bool set); +#endif + #endif diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index e748bc957d83262233b3c8a655ed68b0ad8f34dd..fee20f66da52f02896a237450d16398464806015 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -75,6 +75,15 @@ struct arch_timer_context { u32 host_timer_irq; }; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +struct vtimer_mbigen_context { + /* Active state in vtimer mbigen */ + bool active; + + bool loaded; +}; +#endif + struct timer_map { struct arch_timer_context *direct_vtimer; struct arch_timer_context *direct_ptimer; @@ -92,10 +101,18 @@ struct arch_timer_cpu { /* Is the timer enabled */ bool enabled; + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* Info for vtimer mbigen device */ + struct vtimer_mbigen_context mbigen_ctx; +#endif }; int __init kvm_timer_hyp_init(bool has_gic); int kvm_timer_enable(struct kvm_vcpu *vcpu); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +int kvm_vtimer_config(struct kvm *kvm); +#endif int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu); void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu); void kvm_timer_sync_user(struct kvm_vcpu *vcpu); @@ -126,6 +143,9 @@ void kvm_timer_init_vhe(void); #define vcpu_hvtimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HVTIMER]) #define vcpu_hptimer(v) (&(v)->arch.timer_cpu.timers[TIMER_HPTIMER]) +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +#define vcpu_vtimer_mbigen(v) (&(v)->arch.timer_cpu.mbigen_ctx) +#endif #define arch_timer_ctx_index(ctx) ((ctx) - vcpu_timer((ctx)->vcpu)->timers) #define timer_vm_data(ctx) (&(ctx)->vcpu->kvm->arch.timer_data) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index f804210f9596a2261f2538fea2d23711dec8f121..2bc5c4a8e205daa1c72682cbfe3bc1087a297a27 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -52,6 +52,18 @@ struct shadow_dev { }; #endif +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/* Information about HiSilicon implementation of vtimer (GICv4.1-based) */ +struct vtimer_info { + u32 intid; + + bool (*get_active_stat)(struct kvm_vcpu *vcpu, int vintid); + void (*set_active_stat)(struct kvm_vcpu *vcpu, int vintid, bool active); +}; + +u16 kvm_vgic_get_vcpu_vpeid(struct kvm_vcpu *vcpu); +#endif + enum vgic_type { VGIC_V2, /* Good ol' GICv2 */ VGIC_V3, /* New fancy GICv3 */ @@ -94,6 +106,14 @@ struct vgic_global { /* Pseudo GICv3 from outer space */ bool no_hw_deactivation; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* + * Hardware (HiSilicon implementation) has vtimer interrupt + * direct injection support? + */ + bool has_direct_vtimer; +#endif + /* GIC system register CPU interface */ struct static_key_false gicv3_cpuif; @@ -172,6 +192,10 @@ struct vgic_irq { void *owner; /* Opaque pointer to reserve an interrupt for in-kernel devices. */ + +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + struct vtimer_info *vtimer_info; /* vtimer interrupt only */ +#endif }; static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq) @@ -274,6 +298,10 @@ struct vgic_dist { /* Wants SGIs without active state */ bool nassgireq; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* Indicate whether the vtimer irqbypass mode is used */ + bool vtimer_irqbypass; +#endif struct vgic_irq *spis; @@ -352,6 +380,12 @@ struct vgic_cpu { struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS]; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* Indicate whether the vtimer irqbypass mode is used */ + bool vtimer_irqbypass; + struct vtimer_info vtimer; +#endif + raw_spinlock_t ap_list_lock; /* Protects the ap_list */ /* @@ -423,6 +457,16 @@ void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid); void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/** + * kvm_vgic_vtimer_irqbypass_support - Get the vtimer irqbypass HW capability + */ +static inline bool kvm_vgic_vtimer_irqbypass_support(void) +{ + return kvm_vgic_global_state.has_direct_vtimer; +} +#endif + /** * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW * @@ -453,6 +497,11 @@ int kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int irq, int vgic_v4_load(struct kvm_vcpu *vcpu); void vgic_v4_commit(struct kvm_vcpu *vcpu); int vgic_v4_put(struct kvm_vcpu *vcpu); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +int kvm_vgic_config_vtimer_irqbypass(struct kvm_vcpu *vcpu, u32 vintid, + bool (*get_as)(struct kvm_vcpu *, int), + void (*set_as)(struct kvm_vcpu *, int, bool)); +#endif /* CPU HP callbacks */ void kvm_vgic_cpu_up(void); diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 3306456c135fe2cb7380017fa4a012e7ae62ed1d..7a91ec9e2afd1f299ab02ab5cc9d281d7a755f95 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -384,9 +384,18 @@ #define GITS_TRANSLATER 0x10040 #define GITS_SGIR 0x20020 +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/* HiSilicon IMP DEF register to set vPPI pending. */ +#define GITS_PPIR 0x200A8 +#endif #define GITS_SGIR_VPEID GENMASK_ULL(47, 32) +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/* Hackish... Extend it to [4:0] to support vPPI. */ +#define GITS_SGIR_VINTID GENMASK_ULL(4, 0) +#else #define GITS_SGIR_VINTID GENMASK_ULL(3, 0) +#endif #define GITS_CTLR_ENABLE (1U << 0) #define GITS_CTLR_ImDe (1U << 1) @@ -408,6 +417,19 @@ #define GITS_TYPER_VMAPP (1ULL << 40) #define GITS_TYPER_SVPET GENMASK_ULL(42, 41) +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +/* HiSilicon IMP DEF register */ +#define GITS_VERSION 0xC000 + +/** + * HiSilicon IMP DEF field which indicates if the vPPI direct injection + * is supported. + * - 0: not supported + * - 1: supported + */ +#define GITS_VERSION_VTIMER (1ULL << 12) +#endif + #define GITS_IIDR_REV_SHIFT 12 #define GITS_IIDR_REV_MASK (0xf << GITS_IIDR_REV_SHIFT) #define GITS_IIDR_REV(r) (((r) >> GITS_IIDR_REV_SHIFT) & 0xf) @@ -635,6 +657,9 @@ struct rdists { bool has_rvpeid; bool has_direct_lpi; bool has_vpend_valid_dirty; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + bool has_vtimer; +#endif }; struct irq_domain; @@ -645,6 +670,10 @@ int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain); int mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +phys_addr_t get_gicr_paddr(int cpu); +#endif + static inline bool gic_enable_sre(void) { u32 val; @@ -660,6 +689,36 @@ static inline bool gic_enable_sre(void) return !!(val & ICC_SRE_EL1_SRE); } +enum gic_intid_range { + SGI_RANGE, + PPI_RANGE, + SPI_RANGE, + EPPI_RANGE, + ESPI_RANGE, + LPI_RANGE, + __INVALID_RANGE__ +}; + +static inline enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq) +{ + switch (hwirq) { + case 0 ... 15: + return SGI_RANGE; + case 16 ... 31: + return PPI_RANGE; + case 32 ... 1019: + return SPI_RANGE; + case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63): + return EPPI_RANGE; + case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023): + return ESPI_RANGE; + case 8192 ... GENMASK(23, 0): + return LPI_RANGE; + default: + return __INVALID_RANGE__; + } +} + #endif #endif diff --git a/include/linux/irqchip/arm-gic-v4.h b/include/linux/irqchip/arm-gic-v4.h index 2c63375bbd43f41d3c5019ab4f4f3583196b522f..90b23d8ebab9ee4231d2b680262931e2fd5bc3e5 100644 --- a/include/linux/irqchip/arm-gic-v4.h +++ b/include/linux/irqchip/arm-gic-v4.h @@ -57,7 +57,12 @@ struct its_vpe { u8 priority; bool enabled; bool group; - } sgi_config[16]; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + } sgi_config[32]; + int nr_irqs; +#else + } sgi_config[16]; +#endif atomic_t vmapp_count; }; }; @@ -144,6 +149,10 @@ struct irq_domain_ops; int its_init_v4(struct irq_domain *domain, const struct irq_domain_ops *vpe_ops, const struct irq_domain_ops *sgi_ops); +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS +int vtimer_irqbypass_init(struct irq_domain *domain, + bool has_vtimer_irqbypass); +#endif bool gic_cpuif_has_vsgi(void); diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index a75b2c7de69d09262946ab4aade39b46c5be6098..ac362587d2fb9e581f45435d96af1c823d556925 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h @@ -32,6 +32,10 @@ struct gic_kvm_info { bool has_v4; /* rvpeid support */ bool has_v4_1; +#ifdef CONFIG_VIRT_VTIMER_IRQ_BYPASS + /* vtimer irqbypass support */ + bool has_vtimer; +#endif /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; };