From 3ed5fb028744fa38c3cf0cdff6076c50af4615f5 Mon Sep 17 00:00:00 2001 From: "Xinle.Guo" Date: Mon, 17 Jan 2022 09:38:13 +0800 Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to kata container Because stratovirt supports both microVM and standardVM architectures, adapts two sandbox types for kata container. Besides basic features, it also support virtio-fs, hotplug VFIO, support more devices. Signed-off-by: Xinle.Guo --- kata-containers.spec | 8 +- ...-standard-virtual-machine-sandbox-ty.patch | 771 ++++++++++++++++++ series.conf | 1 + 3 files changed, 779 insertions(+), 1 deletion(-) create mode 100644 patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch diff --git a/kata-containers.spec b/kata-containers.spec index 6596d5b..0b4d1e9 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} %define VERSION 2.1.0 -%define RELEASE 22 +%define RELEASE 23 Name: kata-containers Version: %{VERSION} @@ -108,6 +108,12 @@ strip %{buildroot}/usr/bin/containerd-shim-kata-v2 %doc %changelog +* Mon Jan 17 2022 Xinle.Guo - 2.1.0-23 +- Type:feature +- ID:NA +- SUG:NA +- DESC:add the stratovirt standardVM sandbox type to kata container + * Thur Jan 13 2022 Xinle.Guo - 2.1.0-22 - Type:feature - ID:NA diff --git a/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch b/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch new file mode 100644 index 0000000..6de9b6a --- /dev/null +++ b/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch @@ -0,0 +1,771 @@ +From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001 +From: "Xinle.Guo" +Date: Fri, 14 Jan 2022 17:07:37 +0800 +Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to + kata container + +Because stratovirt supports both microVM and standardVM +architectures, adapts two sandbox types for kata container. +Besides basic features, it also support virtio-fs, hotplug +VFIO, support more devices. + +Signed-off-by: Xinle.Guo +--- + src/runtime/pkg/katautils/config.go | 1 + + src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++--- + 2 files changed, 438 insertions(+), 45 deletions(-) + +diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go +index e523ed3..b04cdee 100644 +--- a/src/runtime/pkg/katautils/config.go ++++ b/src/runtime/pkg/katautils/config.go +@@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, + BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, + EnableIOThreads: h.EnableIOThreads, ++ PCIeRootPort: h.PCIeRootPort, + DisableVhostNet: h.DisableVhostNet, + EnableVhostUserStore: h.EnableVhostUserStore, + VhostUserStorePath: h.vhostUserStorePath(), +diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go +index 4fcfb94..ffe8965 100644 +--- a/src/runtime/virtcontainers/stratovirt.go ++++ b/src/runtime/virtcontainers/stratovirt.go +@@ -14,14 +14,14 @@ import ( + "time" + + govmmQemu "github.com/kata-containers/govmm/qemu" +- "github.com/pkg/errors" +- "github.com/sirupsen/logrus" +- + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" ++ vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ++ "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel" + otelLabel "go.opentelemetry.io/otel/label" + otelTrace "go.opentelemetry.io/otel/trace" +@@ -44,6 +44,11 @@ const ( + const ( + WaitSandboxTimeoutSecs = 15 + MachineTypeMicrovm = "microvm" ++ MachineTypeQ35 = "q35" ++ MachineTypeVirt = "virt" ++ RootPortPrefix = "pcie" ++ Q35PFlashCode = "/usr/share/edk2/ovmf/OVMF_CODE.fd" ++ VirtPFlashCode = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw" + MmioBus VirtioDriver = "mmio" + PciBus VirtioDriver = "pci" + ) +@@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver { + } + } + ++type rootPortDevice struct { ++ id string ++ port string ++ bus string ++ slot int ++ plugged bool ++ addedDev string ++} ++ ++func (r rootPortDevice) isVaild() bool { ++ if r.id == "" || r.port == "" { ++ return false ++ } ++ return true ++} ++ ++func (r rootPortDevice) getParams(config *vmConfig) []string { ++ if !r.isVaild() { ++ return nil ++ } ++ ++ var params []string ++ var devParams []Param ++ devParams = append(devParams, Param{"id", r.id}) ++ devParams = append(devParams, Param{"port", r.port}) ++ if r.bus == "" { ++ r.bus = "pcie.0" ++ } ++ devParams = append(devParams, Param{"bus", r.bus}) ++ devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)}) ++ ++ driver := "pcie-root-port" ++ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) ++ return params ++} ++ + type blkDevice struct { + id string + filePath string +@@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string { + return params + } + ++type virtioFs struct { ++ driver VirtioDriver ++ backend string ++ charID string ++ charDev string ++ tag string ++ deviceID string ++ bus string ++ addr string ++} ++ ++var virtiofsDriver = map[VirtioDriver]string{ ++ MmioBus: "vhost-user-fs-device", ++ PciBus: "vhost-user-fs-pci", ++} ++ ++func (v virtioFs) isVaild() bool { ++ if v.charID == "" || v.charDev == "" || v.deviceID == "" { ++ return false ++ } ++ return true ++} ++ ++func (v virtioFs) getParams(config *vmConfig) []string { ++ if !v.isVaild() { ++ return nil ++ } ++ ++ var params []string ++ var charParams []Param ++ var fsParams []Param ++ ++ charParams = append(charParams, Param{"id", v.charID}) ++ charParams = append(charParams, Param{"path", config.fsSockPath}) ++ ++ v.driver = v.driver.getDriver(config) ++ driver := virtiofsDriver[v.driver] ++ fsParams = append(fsParams, Param{"chardev", v.charDev}) ++ fsParams = append(fsParams, Param{"tag", v.tag}) ++ fsParams = append(fsParams, Param{"id", v.deviceID}) ++ if v.bus != "" { ++ fsParams = append(fsParams, Param{"bus", v.bus}) ++ fsParams = append(fsParams, Param{"addr", v.addr}) ++ } ++ ++ params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ","))) ++ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ","))) ++ return params ++} ++ + type vhostVsock struct { + driver VirtioDriver + id string +@@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) { + for _, d := range c.devices { + *params = append(*params, d.getParams(c)...) + } ++ ++ if c.machineType == MachineTypeMicrovm { ++ return ++ } ++ // Add flag to unplug devices from their root port faster. ++ *params = append(*params, "-global", "pcie-root-port.fast-unplug=1") + } + + func (c *vmConfig) appendPFlash(params *[]string) { +@@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) { + type State struct { + mmioBlkSlots [mmioBlkCount]bool + mmioNetSlots [mmioNetCount]bool ++ // The list of RootPorts that can be hot-added. ++ rootPort []rootPortDevice + pid int + virtiofsPid int + } +@@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str + return strings.Join(params, " "), nil + } + ++func (s *stratovirt) getPFlash(machineType string) ([]string, error) { ++ if s.config.PFlash != nil { ++ return s.config.PFlash, nil ++ } ++ ++ var PFlash []string ++ switch machineType { ++ case MachineTypeQ35: ++ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode)) ++ case MachineTypeVirt: ++ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode)) ++ case MachineTypeMicrovm: ++ return nil, nil ++ default: ++ return nil, fmt.Errorf("failed to match machine type %s", machineType) ++ } ++ ++ return PFlash, nil ++} ++ + func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket { + socketPath := filepath.Join(vmPath, apiSocket) + +@@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev { + } + } + ++ // Create root port for all devices that need to be hot-added. ++ if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 { ++ devices = s.appendRootPort(ctx, devices) ++ } ++ ++ return devices ++} ++ ++func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev { ++ number := s.config.PCIeRootPort ++ ++ for i := uint32(1); i < number+1; i++ { ++ addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i)) ++ if err != nil { ++ return devices ++ } ++ ++ rp := rootPortDevice{ ++ id: fmt.Sprintf("%s.%d", RootPortPrefix, i), ++ port: fmt.Sprintf("%d", i), ++ bus: defaultBridgeBus, ++ slot: int(addr), ++ addedDev: "", ++ } ++ s.state.rootPort = append(s.state.rootPort, rp) ++ devices = append(devices, rp) ++ } ++ + return devices + } + +@@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end + return devices + } + ++func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev { ++ if s.config.SharedFS != config.VirtioFS { ++ return devices ++ } ++ ++ var bus string ++ var addr uint32 ++ var err error ++ name := "virtio_fs" ++ ++ if s.vmConfig.machineType != MachineTypeMicrovm { ++ bus = "pcie.0" ++ addr, err = s.vmConfig.rootBus.AddDevice(ctx, name) ++ if err != nil { ++ return devices ++ } ++ } ++ ++ devices = append(devices, virtioFs{ ++ backend: "socket", ++ // Virtio-fs must be bound to unique charDev, it uses the same name. ++ charID: name, ++ charDev: name, ++ tag: volume.MountTag, ++ deviceID: "virtio-fs0", ++ bus: bus, ++ addr: fmt.Sprintf("%d", addr), ++ }) ++ ++ return devices ++} ++ + func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error { + span, _ := s.trace(s.ctx, "setStratoVirtUp") + defer span.End() +@@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) + return err + } + +- var PFlash []string ++ PFlash, err := s.getPFlash(machineType) ++ if err != nil { ++ return err ++ } + + vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id) + qmpSocket := s.createQMPSocket(vmPath) +@@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error { + return nil + } + +-func (s *stratovirt) hypervisorConfig() HypervisorConfig { +- return s.config ++// Virtio fs daemon is a shared file system that lets VM access a directory ++// tree on the host. ++func (s *stratovirt) setupVirtioFs() error { ++ if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS { ++ return nil ++ } ++ ++ if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) { ++ return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon) ++ } ++ ++ args := []string{ ++ "-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"), ++ "-source", getSharePath(s.id)} ++ if len(s.config.VirtioFSExtraArgs) != 0 { ++ args = append(args, s.config.VirtioFSExtraArgs...) ++ } ++ ++ cmd := exec.Command(s.config.VirtioFSDaemon, args...) ++ s.Logger().Info("Virtiofsd start with cmd: ", cmd) ++ ++ if err := cmd.Start(); err != nil { ++ return fmt.Errorf("failed to strat virtiofsd: %v", cmd) ++ } ++ ++ s.state.virtiofsPid = cmd.Process.Pid ++ return nil + } + + // Get StratoVirt binary path. +@@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) { + return path, nil + } + ++func (s *stratovirt) hypervisorConfig() HypervisorConfig { ++ return s.config ++} ++ + func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { + var span otelTrace.Span + span, _ = s.trace(ctx, "createSandbox") +@@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { + } + }() + ++ if err = s.setupVirtioFs(); err != nil { ++ return err ++ } ++ + var params []string + s.createBaseParams(s.vmConfig, ¶ms) + +@@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType + s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v) + case config.BlockDrive: + s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices) ++ case types.Volume: ++ s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v) + default: + s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type") + } + return nil + } + +-func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { ++func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) { + Name = filepath.Base(strings.ToLower(Name)) +- + if strings.HasPrefix(Name, "eth") { + idxStr := strings.TrimPrefix(Name, "eth") + if idxStr == Name { +- return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ return 0, fmt.Errorf("could not parse idx from name %q", Name) + } + + idx, err := strconv.Atoi(idxStr) + if err != nil { +- return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) ++ return 0, fmt.Errorf("could not convert to int from str %q", idxStr) + } + + if !isPut && s.state.mmioNetSlots[idx] { +- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) ++ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr) + } + s.state.mmioNetSlots[idx] = !isPut + +@@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { + } else if strings.HasPrefix(Name, "vd") { + charStr := strings.TrimPrefix(Name, "vd") + if charStr == Name { +- return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ return 0, fmt.Errorf("could not parse idx from name %q", Name) + } + + char := []rune(charStr) + idx := int(char[0] - 'a') + + if !isPut && s.state.mmioBlkSlots[idx] { +- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) ++ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr) + } + s.state.mmioBlkSlots[idx] = !isPut + + return idx, nil + } + +- return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) ++ return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name) ++} ++ ++func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) { ++ rootports := &s.state.rootPort ++ if len(*rootports) == 0 { ++ return "", 0, fmt.Errorf("failed to get available address from bridges") ++ } ++ ++ for i, rootport := range *rootports { ++ if !isPut && !rootport.plugged && rootport.addedDev == "" { ++ (*rootports)[i].plugged = true ++ (*rootports)[i].addedDev = Name ++ return rootport.id, rootport.slot, nil ++ } else if isPut && rootport.plugged && rootport.addedDev == Name { ++ (*rootports)[i].plugged = false ++ (*rootports)[i].addedDev = "" ++ return rootport.id, rootport.slot, nil ++ } ++ } ++ ++ return "", 0, fmt.Errorf("no more bridge slots available") ++} ++ ++func (s *stratovirt) getDevSlot(Name string) (string, int, error) { ++ if s.config.HypervisorMachineType == MachineTypeMicrovm { ++ slot, err := s.setupMmioSlot(Name, false) ++ if err != nil { ++ return "", 0, err ++ } ++ ++ return "", slot, nil ++ } ++ ++ bus, slot, err := s.setupPciSlot(Name, false) ++ if err != nil { ++ return "pcie.0", 0, err ++ } ++ ++ return bus, slot, nil ++} ++ ++func (s *stratovirt) delDevSlot(Name string) error { ++ if s.vmConfig.machineType == MachineTypeMicrovm { ++ if _, err := s.setupMmioSlot(Name, true); err != nil { ++ return err ++ } ++ ++ return nil ++ } ++ ++ if _, _, err := s.setupPciSlot(Name, true); err != nil { ++ return err ++ } ++ ++ return nil + } + +-func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) { +- err = s.qmpSetup() ++func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error { ++ err := s.qmpSetup() + if err != nil { + return err + } +@@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera + return fmt.Errorf("Endpoint is not supported") + } + ++ defer func() { ++ if err != nil { ++ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } ++ } ++ }() ++ + switch op { + case addDevice: + var ( +@@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera + VhostFdNames = append(VhostFdNames, fdName) + } + +- slot, err := s.getDevSlot(endpoint.Name(), false) ++ bus, slot, err := s.getDevSlot(endpoint.Name()) + if err != nil { +- return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) ++ return fmt.Errorf("could not get unused slot for %q", endpoint.Name()) + } + + if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { + if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { +- s.getDevSlot(endpoint.Name(), true) + return err + } + } else { + if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { +- s.getDevSlot(endpoint.Name(), true) + return err + } + } +- if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { +- s.getDevSlot(endpoint.Name(), true) ++ ++ // The slot of net device that hotplugged to the root port ++ // must be zero. ++ devAddr := "0x0.0x0" ++ if s.vmConfig.machineType == MachineTypeMicrovm { ++ devAddr = fmt.Sprintf("%d", slot) ++ } else { ++ bridgeSlot, err := vcTypes.PciSlotFromInt(slot) ++ if err != nil { ++ return err ++ } ++ ++ devSlot, err := vcTypes.PciSlotFromString("0") ++ if err != nil { ++ return err ++ } ++ ++ pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot) ++ if err != nil { ++ return err ++ } ++ endpoint.SetPciPath(pciPath) ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil { + return err + } + case removeDevice: ++ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } + if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { + return err + } +@@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera + return err + } + default: +- return fmt.Errorf("Operation is not supported") ++ return fmt.Errorf("operation is not supported") + } + + return nil + } + +-func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { +- var filePath string +- err = s.qmpSetup() ++func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error { ++ err := s.qmpSetup() + if err != nil { + return err + } + ++ driver := "virtio-blk-pci" ++ if s.vmConfig.machineType == MachineTypeMicrovm { ++ driver = "virtio-blk-mmio" ++ } ++ ++ defer func() { ++ if err != nil { ++ s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID) ++ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } ++ } ++ }() ++ + switch op { + case addDevice: +- driver := "virtio-blk-mmio" ++ filePath := drive.File + if s.vmConfig.useOzone { + filePath, err = s.updateOzoneRes(drive.File, true) +- if err != nil { +- return fmt.Errorf("Failed to update ozone resources") +- } +- } else { +- filePath = drive.File +- } +- slot, err := s.getDevSlot(drive.VirtPath, false) +- if err != nil { +- return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) + } + + if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil { +- s.getDevSlot(drive.VirtPath, true) + return err + } + +- if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { +- s.getDevSlot(drive.VirtPath, true) ++ bus, slot, err := s.getDevSlot(drive.VirtPath) ++ if err != nil { ++ return err ++ } ++ ++ // The slot of block device that hotplugged to the root port ++ // must be zero. ++ devAddr := "0x0.0x0" ++ if s.vmConfig.machineType == MachineTypeMicrovm { ++ devAddr = fmt.Sprintf("%d", slot) ++ } else { ++ bridgeSlot, err := vcTypes.PciSlotFromInt(slot) ++ if err != nil { ++ return err ++ } ++ ++ devSlot, err := vcTypes.PciSlotFromString("0") ++ if err != nil { ++ return err ++ } ++ ++ drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) ++ if err != nil { ++ return err ++ } ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil { + return err + } + case removeDevice: + if s.vmConfig.useOzone { + s.updateOzoneRes(drive.File, false) + } ++ ++ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } + if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { + return err + } + if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { + return err + } ++ default: ++ return fmt.Errorf("operation is not supported %d", op) ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error { ++ err := s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ defer func() { ++ if err != nil { ++ if errDel := s.delDevSlot(device.ID); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } ++ } ++ }() + +- s.getDevSlot(drive.VirtPath, true) ++ switch op { ++ case addDevice: ++ var bus string ++ // The slot of block device that hotplugged to the root port ++ // must be zero. ++ devAddr := "0x0.0x0" ++ // The vfio device BDF format is 0000:1a:00.3 ++ device.BDF = "0000:" + device.BDF ++ ++ bus, _, err = s.getDevSlot(device.ID) ++ if err != nil { ++ return err ++ } ++ ++ if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil { ++ return err ++ } ++ case removeDevice: ++ if errDel := s.delDevSlot(device.ID); errDel != nil { ++ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil { ++ return err ++ } + default: +- return fmt.Errorf("Operation is not supported") ++ return fmt.Errorf("operation is not supported %d", op) + } + + return nil +@@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, + case netDev: + return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice) + case blockDev: +- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) ++ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice) ++ case vfioDev: ++ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice) + default: +- return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) ++ return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType) + } + } + +@@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ + case netDev: + return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice) + case blockDev: +- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) ++ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice) ++ case vfioDev: ++ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice) + default: +- return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) ++ return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType) + } + } + +@@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities { + var caps types.Capabilities + caps.SetBlockDeviceHotplugSupport() + ++ if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS { ++ caps.SetFsSharingSupport() ++ } ++ + return caps + } + +@@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int { + + pids = append(pids, s.state.pid) + ++ if s.state.virtiofsPid != 0 { ++ pids = append(pids, s.state.virtiofsPid) ++ } + return pids + } + +@@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool { + func (s *stratovirt) save() (p persistapi.HypervisorState) { + pids := s.getPids() + p.Pid = pids[0] ++ p.VirtiofsdPid = s.state.virtiofsPid + p.Type = string(StratovirtHypervisor) + return + } + + func (s *stratovirt) load(p persistapi.HypervisorState) { + s.state.pid = p.Pid ++ s.state.virtiofsPid = p.VirtiofsdPid + } + + func (s *stratovirt) setSandbox(sandbox *Sandbox) { +-- +2.20.1.windows.1 + diff --git a/series.conf b/series.conf index 24ab7ef..62d731b 100644 --- a/series.conf +++ b/series.conf @@ -23,3 +23,4 @@ 0023-stratovirt-update-configuration-toml-file.patch 0024-stratovirt-add-struct-vmConfig-and-methods-to-get-al.patch 0025-stratovirt-refactor-hypervisor-type-stratovirt-and-i.patch +0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch -- Gitee