From 3ed7dcc4a8ccf443d125e7908d8293b562c68d4b Mon Sep 17 00:00:00 2001
From: Keqian Zhu <zhukeqian1@huawei.com>
Date: Fri, 10 Apr 2020 13:15:35 +0800
Subject: [PATCH] intc/gicv3: Add pre-sizing capability to GICv3

Currently GICv3 supports fixed smp_cpus CPUs, and all CPUs are
present always. Now we want to pre-sizing GICv3 to support max_cpus
CPUs and not all of them are present always, so some sizing codes
should be concerned.

GIC irqs, GICR and GICC are pre-created for all possible CPUs at
start, but only smp_cpus CPUs are realize and irqs of smp_cpus CPUs
are connected.

Other code changes are mainly for arm_gicv3, and we do little about
kvm_arm_gicv3 becasue KVM will deal with the sizing information properly.

Signed-off-by: Keqian Zhu <zhukeqian1@huawei.com>
Signed-off-by: Salil Mehta <salil.mehta@huawei.com>
---
 hw/arm/virt.c              | 17 +++++++++++----
 hw/intc/arm_gicv3.c        | 43 +++++++++++++++++++++++++-------------
 hw/intc/arm_gicv3_common.c | 22 +++++++++++++++++--
 hw/intc/arm_gicv3_cpuif.c  |  4 ++++
 hw/intc/arm_gicv3_kvm.c    | 28 ++++++++++++++++++++++++-
 include/hw/arm/virt.h      |  3 ++-
 6 files changed, 95 insertions(+), 22 deletions(-)

diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 0af0a996a1..b1224fb1e4 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -827,14 +827,19 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
     SysBusDevice *gicbusdev;
     const char *gictype;
     int type = vms->gic_version, i;
+    /* The max number of CPUs suppored by GIC */
+    unsigned int num_cpus = ms->smp.cpus;
+    /* The number of CPUs present before boot */
     unsigned int smp_cpus = ms->smp.cpus;
     uint32_t nb_redist_regions = 0;
 
+    assert(num_cpus >= smp_cpus);
+
     gictype = (type == 3) ? gicv3_class_name() : gic_class_name();
 
     vms->gic = qdev_new(gictype);
     qdev_prop_set_uint32(vms->gic, "revision", type);
-    qdev_prop_set_uint32(vms->gic, "num-cpu", smp_cpus);
+    qdev_prop_set_uint32(vms->gic, "num-cpu", num_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
      */
@@ -846,7 +851,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
     if (type == 3) {
         uint32_t redist0_capacity =
                     vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE;
-        uint32_t redist0_count = MIN(smp_cpus, redist0_capacity);
+        uint32_t redist0_count = MIN(num_cpus, redist0_capacity);
 
         nb_redist_regions = virt_gicv3_redist_region_count(vms);
 
@@ -867,7 +872,7 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
                     vms->memmap[VIRT_HIGH_GIC_REDIST2].size / GICV3_REDIST_SIZE;
 
             qdev_prop_set_uint32(vms->gic, "redist-region-count[1]",
-                MIN(smp_cpus - redist0_count, redist1_capacity));
+                MIN(num_cpus - redist0_count, redist1_capacity));
         }
     } else {
         if (!kvm_irqchip_in_kernel()) {
@@ -894,7 +899,11 @@ static void create_gic(VirtMachineState *vms, MemoryRegion *mem)
 
     /* Wire the outputs from each CPU's generic timer and the GICv3
      * maintenance interrupt signal to the appropriate GIC PPI inputs,
-     * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's inputs.
+     * and the GIC's IRQ/FIQ/VIRQ/VFIQ interrupt outputs to the CPU's
+     * inputs.
+     *
+     * The irqs of remaining CPUs (if we has) will be connected during
+     * hotplugging.
      */
     for (i = 0; i < smp_cpus; i++) {
         connect_gic_cpu_irqs(vms, i);
diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c
index 9591cfbcc0..864d4e4034 100644
--- a/hw/intc/arm_gicv3.c
+++ b/hw/intc/arm_gicv3.c
@@ -19,6 +19,7 @@
 #include "qapi/error.h"
 #include "qemu/module.h"
 #include "hw/intc/arm_gicv3.h"
+#include "hw/core/cpu.h"
 #include "gicv3_internal.h"
 
 static bool irqbetter(GICv3CPUState *cs, int irq, uint8_t prio)
@@ -217,7 +218,9 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len)
     assert(len > 0);
 
     for (i = 0; i < s->num_cpu; i++) {
-        s->cpu[i].seenbetter = false;
+        if (qemu_get_cpu(i)) {
+            s->cpu[i].seenbetter = false;
+        }
     }
 
     /* Find the highest priority pending interrupt in this range. */
@@ -259,16 +262,18 @@ static void gicv3_update_noirqset(GICv3State *s, int start, int len)
      * now be the new best one).
      */
     for (i = 0; i < s->num_cpu; i++) {
-        GICv3CPUState *cs = &s->cpu[i];
+        if (qemu_get_cpu(i)) {
+            GICv3CPUState *cs = &s->cpu[i];
 
-        if (cs->seenbetter) {
-            cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
-        }
+            if (cs->seenbetter) {
+                cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
+            }
 
-        if (!cs->seenbetter && cs->hppi.prio != 0xff &&
-            cs->hppi.irq >= start && cs->hppi.irq < start + len) {
-            gicv3_full_update_noirqset(s);
-            break;
+            if (!cs->seenbetter && cs->hppi.prio != 0xff &&
+                cs->hppi.irq >= start && cs->hppi.irq < start + len) {
+                gicv3_full_update_noirqset(s);
+                break;
+            }
         }
     }
 }
@@ -279,7 +284,9 @@ void gicv3_update(GICv3State *s, int start, int len)
 
     gicv3_update_noirqset(s, start, len);
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpuif_update(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpuif_update(&s->cpu[i]);
+        }
     }
 }
 
@@ -291,7 +298,9 @@ void gicv3_full_update_noirqset(GICv3State *s)
     int i;
 
     for (i = 0; i < s->num_cpu; i++) {
-        s->cpu[i].hppi.prio = 0xff;
+        if (qemu_get_cpu(i)) {
+            s->cpu[i].hppi.prio = 0xff;
+        }
     }
 
     /* Note that we can guarantee that these functions will not
@@ -302,7 +311,9 @@ void gicv3_full_update_noirqset(GICv3State *s)
     gicv3_update_noirqset(s, GIC_INTERNAL, s->num_irq - GIC_INTERNAL);
 
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_redist_update_noirqset(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_redist_update_noirqset(&s->cpu[i]);
+        }
     }
 }
 
@@ -315,7 +326,9 @@ void gicv3_full_update(GICv3State *s)
 
     gicv3_full_update_noirqset(s);
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpuif_update(&s->cpu[i]);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpuif_update(&s->cpu[i]);
+        }
     }
 }
 
@@ -406,7 +419,9 @@ static void arm_gic_realize(DeviceState *dev, Error **errp)
     gicv3_init_irqs_and_mmio(s, gicv3_set_irq, gic_ops);
 
     for (i = 0; i < s->num_cpu; i++) {
-        gicv3_cpu_realize(s, i);
+        if (qemu_get_cpu(i)) {
+            gicv3_cpu_realize(s, i);
+        }
     }
 }
 
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
index f8ef6817a4..a4976b2ba0 100644
--- a/hw/intc/arm_gicv3_common.c
+++ b/hw/intc/arm_gicv3_common.c
@@ -24,12 +24,14 @@
 #include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qemu/module.h"
+#include "qemu/error-report.h"
 #include "hw/core/cpu.h"
 #include "hw/intc/arm_gicv3_common.h"
 #include "hw/qdev-properties.h"
 #include "migration/vmstate.h"
 #include "gicv3_internal.h"
 #include "hw/arm/linux-boot-if.h"
+#include "hw/boards.h"
 #include "sysemu/kvm.h"
 
 
@@ -377,9 +379,14 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
     for (i = 0; i < s->num_cpu; i++) {
         CPUState *cpu = qemu_get_cpu(i);
 
+        MachineState *ms = MACHINE(qdev_get_machine());
+        MachineClass *mc = MACHINE_GET_CLASS(ms);
+        const CPUArchIdList *possible_cpus = NULL;
         uint64_t cpu_affid;
 
-        arm_gicv3_common_cpu_realize(s, i);
+        if (cpu) {
+            arm_gicv3_common_cpu_realize(s, i);
+        }
 
         /* Pre-construct the GICR_TYPER:
          * For our implementation:
@@ -393,7 +400,18 @@ static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
          *  VLPIS == 0 (virtual LPIs not supported)
          *  PLPIS == 0 (physical LPIs not supported)
          */
-        cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL);
+        if (cpu) {
+            cpu_affid = object_property_get_uint(OBJECT(cpu), "mp-affinity", NULL);
+        } else {
+            if (!mc->possible_cpu_arch_ids) {
+                error_report("MachineClass must implement possible_cpu_arch_ids "
+                             "hook to support pre-sizing GICv3");
+                exit(1);
+            }
+
+            possible_cpus = mc->possible_cpu_arch_ids(ms);
+            cpu_affid = possible_cpus->cpus[i].arch_id;
+        }
 
         /* The CPU mp-affinity property is in MPIDR register format; squash
          * the affinity bytes into 32 bits as the GICR_TYPER has them.
diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c
index 70809bcddd..274a40a40c 100644
--- a/hw/intc/arm_gicv3_cpuif.c
+++ b/hw/intc/arm_gicv3_cpuif.c
@@ -1676,6 +1676,10 @@ static void icc_generate_sgi(CPUARMState *env, GICv3CPUState *cs,
                                  aff, targetlist);
 
     for (i = 0; i < s->num_cpu; i++) {
+        if (!qemu_get_cpu(i)) {
+            continue;
+        }
+
         GICv3CPUState *ocs = &s->cpu[i];
 
         if (irm) {
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
index 95271e754b..2e2b08e31f 100644
--- a/hw/intc/arm_gicv3_kvm.c
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -342,6 +342,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
         for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
             GICv3CPUState *c = &s->cpu[ncpu];
 
+            if (!qemu_get_cpu(ncpu)) {
+                continue;
+            }
+
             reg64 = c->gicr_propbaser;
             regl = (uint32_t)reg64;
             kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, true);
@@ -361,6 +365,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
         GICv3CPUState *c = &s->cpu[ncpu];
 
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         reg = c->gicr_ctlr;
         kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, true);
 
@@ -457,6 +465,10 @@ static void kvm_arm_gicv3_put(GICv3State *s)
         GICv3CPUState *c = &s->cpu[ncpu];
         int num_pri_bits;
 
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         kvm_gicc_access(s, ICC_SRE_EL1, ncpu, &c->icc_sre_el1, true);
         kvm_gicc_access(s, ICC_CTLR_EL1, ncpu,
                         &c->icc_ctlr_el1[GICV3_NS], true);
@@ -524,6 +536,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)
     /* Redistributor state (one per CPU) */
 
     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         GICv3CPUState *c = &s->cpu[ncpu];
 
         kvm_gicr_access(s, GICR_CTLR, ncpu, &reg, false);
@@ -559,6 +575,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)
 
     if (redist_typer & GICR_TYPER_PLPIS) {
         for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+            if (!qemu_get_cpu(ncpu)) {
+                continue;
+            }
+
             GICv3CPUState *c = &s->cpu[ncpu];
 
             kvm_gicr_access(s, GICR_PROPBASER, ncpu, &regl, false);
@@ -612,6 +632,10 @@ static void kvm_arm_gicv3_get(GICv3State *s)
      */
 
     for (ncpu = 0; ncpu < s->num_cpu; ncpu++) {
+        if (!qemu_get_cpu(ncpu)) {
+            continue;
+        }
+
         GICv3CPUState *c = &s->cpu[ncpu];
         int num_pri_bits;
 
@@ -805,7 +829,9 @@ static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
     gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL);
 
     for (i = 0; i < s->num_cpu; i++) {
-        kvm_arm_gicv3_cpu_realize(s, i);
+        if (qemu_get_cpu(i)) {
+            kvm_arm_gicv3_cpu_realize(s, i);
+        }
     }
 
     /* Try to create the device via the device control API */
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
index 2a838620d8..947d41f767 100644
--- a/include/hw/arm/virt.h
+++ b/include/hw/arm/virt.h
@@ -196,8 +196,9 @@ static inline int virt_gicv3_redist_region_count(VirtMachineState *vms)
                 vms->memmap[VIRT_GIC_REDIST].size / GICV3_REDIST_SIZE;
 
     assert(vms->gic_version == VIRT_GIC_VERSION_3);
+    GICv3State *s = ARM_GICV3_COMMON(vms->gic);
 
-    return MACHINE(vms)->smp.cpus > redist0_capacity ? 2 : 1;
+    return s->num_cpu > redist0_capacity ? 2 : 1;
 }
 
 #endif /* QEMU_ARM_VIRT_H */
-- 
2.27.0