diff --git a/kata-containers.spec b/kata-containers.spec index 6d5ac8d10c1cec717e56e1ccbd0801180f93c7dd..ec9168e1f239e656aab9d452ef94b05bb2250ce3 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} %define VERSION 2.1.0 -%define RELEASE 1 +%define RELEASE 2 Name: kata-containers Version: %{VERSION} @@ -87,9 +87,9 @@ install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildro install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/ install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/ install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml +install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-stratovirt.toml %{buildroot}/usr/share/defaults/kata-containers/configuration-stratovirt.toml install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/ mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/ -install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml strip %{buildroot}/usr/bin/kata* strip %{buildroot}/usr/bin/containerd-shim-kata-v2 @@ -107,6 +107,12 @@ strip %{buildroot}/usr/bin/containerd-shim-kata-v2 %doc %changelog +* Wed Aug 20 2021 yangfeiyu - 2.1.0-2 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:support with stratovirt and isulad + * Wed Aug 18 2021 yangfeiyu - 2.1.0-1 - Type:enhancement - ID:NA diff --git a/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch b/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch new file mode 100644 index 0000000000000000000000000000000000000000..d63805358c9027d94d18eb391cb7ceb110d3e823 --- /dev/null +++ b/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch @@ -0,0 +1,837 @@ +From f56d66f196bee808526e86df2c3c063a887c6fef Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Sat, 7 Aug 2021 10:39:11 +0800 +Subject: [PATCH 1/6] runtime: add support of new sandbox hypervisor type + StratoVirt. + +Signed-off-by: Wei Gao +--- + src/runtime/pkg/katautils/config.go | 96 +++- + src/runtime/virtcontainers/hypervisor.go | 12 + + src/runtime/virtcontainers/stratovirt.go | 642 +++++++++++++++++++++++ + 3 files changed, 749 insertions(+), 1 deletion(-) + create mode 100644 src/runtime/virtcontainers/stratovirt.go + +diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go +index 6114aa39..f94ac4fd 100644 +--- a/src/runtime/pkg/katautils/config.go ++++ b/src/runtime/pkg/katautils/config.go +@@ -50,6 +50,7 @@ const ( + clhHypervisorTableType = "clh" + qemuHypervisorTableType = "qemu" + acrnHypervisorTableType = "acrn" ++ stratovirtHypervisorTable = "stratovirt" + + // the maximum amount of PCI bridges that can be cold plugged in a VM + maxPCIBridges uint32 = 5 +@@ -870,6 +871,96 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + }, nil + } + ++func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { ++ hypervisor, err := h.path() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ kernel, err := h.kernel() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ initrd, image, err := h.getInitrdAndImage() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ if image != "" && initrd != "" { ++ return vc.HypervisorConfig{}, ++ errors.New("having both an image and an initrd defined in the configuration file is not supported") ++ } ++ ++ if image == "" && initrd == "" { ++ return vc.HypervisorConfig{}, ++ errors.New("either image or initrd must be defined in the configuration file") ++ } ++ ++ kernelParams := h.kernelParams() ++ machineType := h.machineType() ++ ++ blockDriver, err := h.blockDeviceDriver() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ sharedFS, err := h.sharedFS() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" { ++ return vc.HypervisorConfig{}, ++ errors.New("cannot enable virtio-fs without daemon path in configuration file") ++ } ++ ++ if vSock, err := utils.SupportsVsocks(); !vSock { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ return vc.HypervisorConfig{ ++ HypervisorPath: hypervisor, ++ HypervisorPathList: h.HypervisorPathList, ++ KernelPath: kernel, ++ InitrdPath: initrd, ++ ImagePath: image, ++ KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), ++ HypervisorMachineType: machineType, ++ NumVCPUs: h.defaultVCPUs(), ++ DefaultMaxVCPUs: h.defaultMaxVCPUs(), ++ MemorySize: h.defaultMemSz(), ++ MemSlots: h.defaultMemSlots(), ++ MemOffset: h.defaultMemOffset(), ++ EntropySource: h.GetEntropySource(), ++ EntropySourceList: h.EntropySourceList, ++ DefaultBridges: h.defaultBridges(), ++ DisableBlockDeviceUse: h.DisableBlockDeviceUse, ++ SharedFS: sharedFS, ++ VirtioFSDaemon: h.VirtioFSDaemon, ++ VirtioFSDaemonList: h.VirtioFSDaemonList, ++ VirtioFSCacheSize: h.VirtioFSCacheSize, ++ VirtioFSCache: h.defaultVirtioFSCache(), ++ VirtioFSExtraArgs: h.VirtioFSExtraArgs, ++ FileBackedMemRootDir: h.FileBackedMemRootDir, ++ FileBackedMemRootList: h.FileBackedMemRootList, ++ Mlock: !h.Swap, ++ Debug: h.Debug, ++ DisableNestingChecks: h.DisableNestingChecks, ++ BlockDeviceDriver: blockDriver, ++ BlockDeviceCacheSet: h.BlockDeviceCacheSet, ++ BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, ++ BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, ++ EnableIOThreads: h.EnableIOThreads, ++ DisableVhostNet: h.DisableVhostNet, ++ EnableVhostUserStore: h.EnableVhostUserStore, ++ VhostUserStorePath: h.vhostUserStorePath(), ++ VhostUserStorePathList: h.VhostUserStorePathList, ++ GuestHookPath: h.guestHookPath(), ++ EnableAnnotations: h.EnableAnnotations, ++ }, nil ++} ++ + func newFactoryConfig(f factory) (oci.FactoryConfig, error) { + if f.TemplatePath == "" { + f.TemplatePath = defaultTemplatePath +@@ -903,6 +994,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi + case clhHypervisorTableType: + config.HypervisorType = vc.ClhHypervisor + hConfig, err = newClhHypervisorConfig(hypervisor) ++ case stratovirtHypervisorTable: ++ config.HypervisorType = vc.StratovirtHypervisor ++ hConfig, err = newStratovirtHypervisorConfig(hypervisor) + } + + if err != nil { +@@ -1287,7 +1381,7 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error { + memSizeMB := int64(config.MemorySize) + + if memSizeMB == 0 { +- return errors.New("VM memory cannot be zero") ++ return errors.New(fmt.Sprintf("The VM memory cannot be zero, %s", config.ImagePath)) + } + + mb := int64(1024 * 1024) +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index 767215b6..615baa80 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -44,6 +44,9 @@ const ( + // ClhHypervisor is the ICH hypervisor. + ClhHypervisor HypervisorType = "clh" + ++ // StratovirtHypervisor is the StratoVirt hypervisor ++ StratovirtHypervisor HypervisorType = "stratovirt" ++ + // MockHypervisor is a mock hypervisor for testing purposes + MockHypervisor HypervisorType = "mock" + ) +@@ -159,6 +162,9 @@ func (hType *HypervisorType) Set(value string) error { + case "clh": + *hType = ClhHypervisor + return nil ++ case "stratovirt": ++ *hType = StratovirtHypervisor ++ return nil + case "mock": + *hType = MockHypervisor + return nil +@@ -178,6 +184,8 @@ func (hType *HypervisorType) String() string { + return string(AcrnHypervisor) + case ClhHypervisor: + return string(ClhHypervisor) ++ case StratovirtHypervisor: ++ return string(StratovirtHypervisor) + case MockHypervisor: + return string(MockHypervisor) + default: +@@ -207,6 +215,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { + return &cloudHypervisor{ + store: store, + }, nil ++ case StratovirtHypervisor: ++ return &stratovirt{ ++ store: store, ++ }, nil + case MockHypervisor: + return &mockHypervisor{}, nil + default: +diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go +new file mode 100644 +index 00000000..4fec96d3 +--- /dev/null ++++ b/src/runtime/virtcontainers/stratovirt.go +@@ -0,0 +1,642 @@ ++package virtcontainers ++ ++import ( ++ "context" ++ "fmt" ++ "os" ++ "os/exec" ++ "path/filepath" ++ "strconv" ++ "strings" ++ "syscall" ++ "time" ++ ++ govmmQemu "github.com/kata-containers/govmm/qemu" ++ "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" ++ ++ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" ++ persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" ++ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ++ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ++ "go.opentelemetry.io/otel" ++ otelLabel "go.opentelemetry.io/otel/label" ++ otelTrace "go.opentelemetry.io/otel/trace" ++) ++ ++const defaultDummyMac = "22:33:44:aa:bb:" ++const mmioBlkCount = 4 ++const mmioNetCount = 2 ++const randomDevice = "/dev/urandom" ++ ++type stratovirtDev struct { ++ dev interface{} ++ devType deviceType ++} ++ ++type stratovirt struct { ++ id string ++ ctx context.Context ++ sandbox *Sandbox ++ store persistapi.PersistDriver ++ config HypervisorConfig ++ pid int ++ consolePath string ++ socketPath string ++ qmpMonitorCh qmpChannel ++ devices []stratovirtDev ++ HotpluggedVCPUs []CPUDevice ++ mmioBlkSlots [mmioBlkCount]bool ++ mmioNetSlots [mmioNetCount]bool ++} ++ ++func (s *stratovirt) Logger() *logrus.Entry { ++ return virtLog.WithField("subsystem", "stratovirt") ++} ++ ++func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, context.Context) { ++ if parent == nil { ++ s.Logger().WithField("type", "bug").Error("trace called before context set") ++ parent = context.Background() ++ } ++ ++ tracer := otel.Tracer("kata") ++ ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id))) ++ ++ return span, ctx ++} ++ ++func (s *stratovirt) getKernelCmdLine(useImage bool) string { ++ var params []string ++ ++ if useImage { ++ params = append(params, "root=/dev/vda") ++ } ++ ++ params = append(params, "pci=off") ++ params = append(params, "reboot=k") ++ params = append(params, "panic=1") ++ params = append(params, "iommu=off") ++ params = append(params, "acpi=off") ++ params = append(params, "quiet") ++ params = append(params, "agent.use_vsock=true") ++ params = append(params, "random.trust_cpu=on") ++ params = append(params, "rw") ++ params = append(params, SerializeParams(s.config.KernelParams, "=")...) ++ ++ return strings.Join(params, " ") ++} ++ ++func (s *stratovirt) hypervisorConfig() HypervisorConfig { ++ return s.config ++} ++ ++func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { ++ s.ctx = ctx ++ ++ var span otelTrace.Span ++ span, _ = s.trace(ctx, "createSandbox") ++ defer span.End() ++ ++ s.id = id ++ s.config = *hypervisorConfig ++ ++ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") ++ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") ++ s.qmpMonitorCh = qmpChannel{ ++ ctx: s.ctx, ++ path: s.socketPath, ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) waitSandBoxStarted(timeout int) error { ++ timeStart := time.Now() ++ for { ++ err := s.qmpSetup() ++ if err == nil { ++ break ++ } ++ ++ if int(time.Since(timeStart).Seconds()) > timeout { ++ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err) ++ } ++ ++ time.Sleep(time.Duration(50) * time.Millisecond) ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil { ++ s.Logger().WithError(err).Error(qmpCapErrMsg) ++ return err ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { ++ span, _ := s.trace(ctx, "startSandbox") ++ defer span.End() ++ ++ var params []string ++ var use_image bool ++ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) ++ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) ++ ++ if kernelPath, err := s.config.KernelAssetPath(); err == nil { ++ params = append(params, "-kernel", kernelPath) ++ } ++ ++ initrdPath, err := s.config.InitrdAssetPath() ++ if err != nil { ++ return err ++ } ++ ++ if initrdPath == "" { ++ imagePath, err := s.config.ImageAssetPath() ++ if err != nil { ++ return err ++ } ++ use_image = true ++ s.mmioBlkSlots[0] = true ++ params = append(params, "-device", "virtio-blk-device,drive=rootfs") ++ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath)) ++ } else { ++ use_image = false ++ params = append(params, "-initrd", initrdPath) ++ } ++ ++ params = append(params, "-append", s.getKernelCmdLine(use_image)) ++ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) ++ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) ++ params = append(params, "-device", "virtio-serial-device") ++ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") ++ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) ++ params = append(params, "-device", "virtio-rng-device,rng=objrng0") ++ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) ++ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) ++ ++ // add devices to cmdline ++ for _, d := range s.devices { ++ switch v := d.dev.(type) { ++ case Endpoint: ++ name := v.Name() ++ mac := v.HardwareAddr() ++ tapName := v.NetworkPair().TapInterface.TAPIface.Name ++ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) ++ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) ++ case config.BlockDrive: ++ id := v.ID ++ path := v.File ++ params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id)) ++ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) ++ case types.VSock: ++ v.VhostFd.Close() ++ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) ++ default: ++ s.Logger().Error("Adding device type is unsupported") ++ } ++ } ++ ++ // daemonize ++ params = append(params, "-daemonize") ++ ++ // append logfile only on debug ++ if s.config.Debug { ++ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) ++ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) ++ } ++ ++ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) ++ err = os.MkdirAll(dir, DirMode) ++ if err != nil { ++ return err ++ } ++ defer func() { ++ if err != nil { ++ if err := os.RemoveAll(dir); err != nil { ++ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) ++ } ++ } ++ }() ++ ++ binPath, err := s.config.HypervisorAssetPath() ++ if err != nil { ++ s.Logger().WithField("Fail to get hypervisor bin path", err).Error() ++ return err ++ } ++ ++ cmd := exec.CommandContext(s.ctx, binPath, params...) ++ s.Logger().Info("StratoVirt start with params: ", cmd) ++ ++ if err := cmd.Start(); err != nil { ++ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() ++ return err ++ } ++ s.pid = cmd.Process.Pid ++ ++ if err = s.waitSandBoxStarted(timeout); err != nil { ++ return err ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { ++ span, _ := s.trace(ctx, "stopSandbox") ++ defer span.End() ++ ++ defer func() { ++ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) ++ link, _ := filepath.EvalSymlinks(dir) ++ ++ if err := os.RemoveAll(dir); err != nil { ++ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir) ++ } ++ ++ if link != dir && link != "" { ++ if err := os.RemoveAll(link); err != nil { ++ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link) ++ } ++ } ++ }() ++ ++ if !force { ++ err := s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) ++ if err != nil { ++ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT") ++ return err ++ } ++ } else { ++ if s.pid > 0 { ++ syscall.Kill(s.pid, syscall.SIGKILL) ++ } ++ } ++ return nil ++} ++ ++func (s *stratovirt) pauseSandbox(ctx context.Context) error { ++ return nil ++} ++ ++func (s *stratovirt) saveSandbox() error { ++ return nil ++} ++ ++func (s *stratovirt) resumeSandbox(ctx context.Context) error { ++ return nil ++} ++ ++func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error { ++ span, _ := s.trace(ctx, "addDevice") ++ defer span.End() ++ ++ dev := stratovirtDev{ ++ dev: devInfo, ++ devType: devType, ++ } ++ s.devices = append(s.devices, dev) ++ ++ return nil ++} ++ ++func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { ++ Name = filepath.Base(strings.ToLower(Name)) ++ ++ if strings.HasPrefix(Name, "eth") { ++ idxStr := strings.TrimPrefix(Name, "eth") ++ if idxStr == Name { ++ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ } ++ ++ idx, err := strconv.Atoi(idxStr) ++ if err != nil { ++ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) ++ } ++ ++ if !isPut && s.mmioNetSlots[idx] { ++ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) ++ } ++ s.mmioNetSlots[idx] = !isPut ++ ++ return idx, nil ++ } else if strings.HasPrefix(Name, "vd") { ++ charStr := strings.TrimPrefix(Name, "vd") ++ if charStr == Name { ++ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ } ++ ++ char := []rune(charStr) ++ idx := int(char[0] - 'a') ++ ++ if !isPut && s.mmioBlkSlots[idx] { ++ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) ++ } ++ s.mmioBlkSlots[idx] = !isPut ++ ++ return idx, nil ++ } ++ ++ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) ++} ++ ++func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { ++ err = s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ var tap TapInterface ++ ++ switch endpoint.Type() { ++ case VethEndpointType: ++ drive := endpoint.(*VethEndpoint) ++ tap = drive.NetPair.TapInterface ++ case TapEndpointType: ++ drive := endpoint.(*TapEndpoint) ++ tap = drive.TapInterface ++ default: ++ return fmt.Errorf("Endpoint is not supported") ++ } ++ ++ switch op { ++ case addDevice: ++ var ( ++ VMFdNames []string ++ VhostFdNames []string ++ ) ++ for i, VMFd := range tap.VMFds { ++ fdName := fmt.Sprintf("fd%d", i) ++ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil { ++ return err ++ } ++ VMFdNames = append(VMFdNames, fdName) ++ } ++ for i, VhostFd := range tap.VhostFds { ++ fdName := fmt.Sprintf("vhostfd%d", i) ++ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil { ++ return err ++ } ++ VhostFd.Close() ++ VhostFdNames = append(VhostFdNames, fdName) ++ } ++ ++ slot, err := s.getDevSlot(endpoint.Name(), false) ++ if err != nil { ++ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) ++ } ++ ++ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ } else { ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ case removeDevice: ++ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { ++ return err ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { ++ return err ++ } ++ default: ++ return fmt.Errorf("Operation is not supported") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { ++ err = s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ switch op { ++ case addDevice: ++ driver := "virtio-blk-pci" ++ slot, err := s.getDevSlot(drive.VirtPath, false) ++ if err != nil { ++ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil { ++ s.getDevSlot(drive.VirtPath, true) ++ return err ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { ++ s.getDevSlot(drive.VirtPath, true) ++ return err ++ } ++ case removeDevice: ++ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { ++ return err ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { ++ return err ++ } ++ ++ s.getDevSlot(drive.VirtPath, true) ++ default: ++ return fmt.Errorf("Operation is not supported") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) { ++ span, _ := s.trace(ctx, "hotplugAddDevice") ++ defer span.End() ++ ++ switch devType { ++ case netDev: ++ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) ++ case blockDev: ++ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) ++ default: ++ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) ++ } ++} ++ ++func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) { ++ span, _ := s.trace(ctx, "hotplugRemoveDevice") ++ defer span.End() ++ ++ switch devType { ++ case netDev: ++ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) ++ case blockDev: ++ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) ++ default: ++ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) ++ } ++} ++ ++func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { ++ return 0, memoryDevice{}, nil ++} ++ ++func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { ++ return 0, 0, nil ++} ++ ++func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) { ++ span, _ := s.trace(ctx, "getSandboxConsole") ++ defer span.End() ++ ++ var consolePath string ++ if s.config.Debug { ++ consolePath = s.consolePath ++ } else { ++ consolePath = "" ++ } ++ consoleURL, err := utils.BuildSocketPath(consolePath) ++ if err != nil { ++ return consoleProtoUnix, "", err ++ } ++ return consoleProtoUnix, consoleURL, nil ++ ++} ++ ++func (s *stratovirt) disconnect(ctx context.Context) { ++ span, _ := s.trace(ctx, "disconnect") ++ defer span.End() ++ ++ s.qmpTeardown() ++} ++ ++func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities { ++ span, _ := s.trace(ctx, "capabilities") ++ defer span.End() ++ ++ var caps types.Capabilities ++ caps.SetBlockDeviceHotplugSupport() ++ ++ return caps ++} ++ ++func (s *stratovirt) qmpTeardown() { ++ if s.qmpMonitorCh.qmp != nil { ++ s.qmpMonitorCh.qmp.Shutdown() ++ <-s.qmpMonitorCh.disconn ++ s.qmpMonitorCh.qmp = nil ++ s.qmpMonitorCh.disconn = nil ++ } ++} ++ ++func (s *stratovirt) qmpSetup() error { ++ s.qmpTeardown() ++ ++ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} ++ ++ // Auto-closed by QMPStart(). ++ disconnectCh := make(chan struct{}) ++ ++ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh) ++ if err != nil { ++ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance") ++ return err ++ } ++ ++ s.qmpMonitorCh.qmp = qmp ++ s.qmpMonitorCh.disconn = disconnectCh ++ ++ return nil ++} ++ ++func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) { ++ span, _ := s.trace(ctx, "getThreadIDs") ++ defer span.End() ++ ++ tid := vcpuThreadIDs{} ++ if err := s.qmpSetup(); err != nil { ++ return tid, err ++ } ++ ++ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx) ++ if err != nil { ++ s.Logger().WithError(err).Error("Failed to query cpu infos") ++ return tid, err ++ } ++ ++ tid.vcpus = make(map[int]int, len(cpuInfos)) ++ for _, i := range cpuInfos { ++ if i.ThreadID > 0 { ++ tid.vcpus[i.CPU] = i.ThreadID ++ } ++ } ++ return tid, nil ++} ++ ++func (s *stratovirt) cleanup(ctx context.Context) error { ++ span, _ := s.trace(ctx, "cleanup") ++ defer span.End() ++ ++ s.qmpTeardown() ++ ++ return nil ++} ++ ++func (s *stratovirt) getPids() []int { ++ return []int{s.pid} ++} ++ ++func (s *stratovirt) getVirtioFsPid() *int { ++ return nil ++} ++ ++func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { ++ return errors.New("stratovirt is not supported by VM cache") ++} ++ ++func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) { ++ return nil, errors.New("stratovirt is not supported by VM cache") ++} ++ ++func (s *stratovirt) check() error { ++ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil { ++ return errors.Wrapf(err, "Failed to ping StratoVirt process") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) generateSocket(id string) (interface{}, error) { ++ return generateVMSocket(id, s.store.RunVMStoragePath()) ++} ++ ++func (s *stratovirt) isRateLimiterBuiltin() bool { ++ return true ++} ++ ++func (s *stratovirt) save() (p persistapi.HypervisorState) { ++ pids := s.getPids() ++ p.Pid = pids[0] ++ p.Type = string(StratovirtHypervisor) ++ return ++} ++ ++func (s *stratovirt) load(p persistapi.HypervisorState) { ++ s.pid = p.Pid ++ ++ return ++} ++ ++func (s *stratovirt) setSandbox(sandbox *Sandbox) { ++ s.sandbox = sandbox ++ return ++} +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch b/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch new file mode 100644 index 0000000000000000000000000000000000000000..3eb330027ae83911c489612079f7b05c0aafb973 --- /dev/null +++ b/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch @@ -0,0 +1,174 @@ +From 1ffd95187a61582e858dd37c0ab434d3159a0f52 Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Mon, 9 Aug 2021 14:26:35 +0800 +Subject: [PATCH 2/6] agent: add support of new sandbox hypervisor kind + StratoVirt. + +1. add new grpc interface `UpdateInterfaceHwAddrByName`. +2. comment out rescan_pci temporarily. + +Signed-off-by: Wei Gao +--- + src/agent/protocols/protos/agent.proto | 5 +++ + src/agent/src/netlink.rs | 31 ++++++++++++++++ + src/agent/src/rpc.rs | 51 +++++++++++++++++++++++++- + 3 files changed, 85 insertions(+), 2 deletions(-) + +diff --git a/src/agent/protocols/protos/agent.proto b/src/agent/protocols/protos/agent.proto +index 6cbf5a28..e00f5c63 100644 +--- a/src/agent/protocols/protos/agent.proto ++++ b/src/agent/protocols/protos/agent.proto +@@ -46,6 +46,7 @@ service AgentService { + + // networking + rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface); ++ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface); + rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); + rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); + rpc ListRoutes(ListRoutesRequest) returns (Routes); +@@ -308,6 +309,10 @@ message UpdateInterfaceRequest { + types.Interface interface = 1; + } + ++message UpdateInterfaceHwAddrByNameRequest { ++ types.Interface interface = 1; ++} ++ + message UpdateRoutesRequest { + Routes routes = 1; + } +diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs +index 3ab6dbaa..82632d1b 100644 +--- a/src/agent/src/netlink.rs ++++ b/src/agent/src/netlink.rs +@@ -104,6 +104,29 @@ impl Handle { + Ok(()) + } + ++ pub async fn update_interface_hw_addr_by_name(&mut self, iface: &Interface) -> Result<()> { ++ let link = self.find_link(LinkFilter::Name(&iface.name)).await?; ++ ++ // Delete all addresses associated with the link ++ let addresses = self ++ .list_addresses(AddressFilter::LinkIndex(link.index())) ++ .await?; ++ self.delete_addresses(addresses).await?; ++ ++ if iface.IPAddresses.len() == 0 { ++ self.enable_link(link.index(), false).await?; ++ } ++ ++ // Update hardware mac address ++ let mac_addr = parse_mac_address(iface.get_hwAddr()) ++ .with_context(|| format!("Failed to parse MAC address: {}", iface.get_hwAddr()))?; ++ self.link_set_hw_addr(link.index(), mac_addr) ++ .await ++ .with_context(|| format!("Could not set {:?} to {}", mac_addr, link.name()))?; ++ ++ Ok(()) ++ } ++ + pub async fn handle_localhost(&self) -> Result<()> { + let link = self.find_link(LinkFilter::Name("lo")).await?; + self.enable_link(link.index(), true).await?; +@@ -216,6 +239,14 @@ impl Handle { + Ok(()) + } + ++ async fn link_set_hw_addr(&self, link_index: u32, hw_addr: [u8; 6]) -> Result<()> { ++ let link_req = self.handle.link().set(link_index); ++ let set_req = link_req.address(hw_addr.to_vec()); ++ set_req.execute().await?; ++ ++ Ok(()) ++ } ++ + async fn query_routes( + &self, + ip_version: Option, +diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs +index 92025af3..2cc1c983 100644 +--- a/src/agent/src/rpc.rs ++++ b/src/agent/src/rpc.rs +@@ -40,7 +40,7 @@ use nix::sys::stat; + use nix::unistd::{self, Pid}; + use rustjail::process::ProcessOperations; + +-use crate::device::{add_devices, rescan_pci_bus, update_device_cgroup}; ++use crate::device::{add_devices, update_device_cgroup}; + use crate::linux_abi::*; + use crate::metrics::get_metrics; + use crate::mount::{add_storages, remove_mounts, BareMount, STORAGE_HANDLER_LIST}; +@@ -123,7 +123,9 @@ impl AgentService { + + // re-scan PCI bus + // looking for hidden devices +- rescan_pci_bus().context("Could not rescan PCI bus")?; ++ // FIXME: Comment out this code temporarily, because once the PCIBus is scanned, ++ // the device hot-plug event is lost ++ // rescan_pci_bus().context("Could not rescan PCI bus")?; + + // Some devices need some extra processing (the ones invoked with + // --device for instance), and that's what this call is doing. It +@@ -797,6 +799,34 @@ impl protocols::agent_ttrpc::AgentService for AgentService { + Ok(interface) + } + ++ async fn update_interface_hw_addr_by_name( ++ &self, ++ _ctx: &TtrpcContext, ++ req: protocols::agent::UpdateInterfaceHwAddrByNameRequest, ++ ) -> ttrpc::Result { ++ let interface = req.interface.into_option().ok_or_else(|| { ++ ttrpc_error( ++ ttrpc::Code::INVALID_ARGUMENT, ++ "empty update interface request".to_string(), ++ ) ++ })?; ++ ++ self.sandbox ++ .lock() ++ .await ++ .rtnl ++ .update_interface_hw_addr_by_name(&interface) ++ .await ++ .map_err(|e| { ++ ttrpc_error( ++ ttrpc::Code::INTERNAL, ++ format!("update interface hw addr: {:?}", e), ++ ) ++ })?; ++ ++ Ok(interface) ++ } ++ + async fn update_routes( + &self, + _ctx: &TtrpcContext, +@@ -1670,6 +1700,23 @@ mod tests { + assert!(result.is_err(), "expected update interface to fail"); + } + ++ #[tokio::test] ++ async fn test_update_interface_hw_addr_by_name() { ++ let logger = slog::Logger::root(slog::Discard, o!()); ++ let sandbox = Sandbox::new(&logger).unwrap(); ++ ++ let agent_service = Box::new(AgentService { ++ sandbox: Arc::new(Mutex::new(sandbox)), ++ }); ++ ++ let req = protocols::agent::UpdateInterfaceHwAddrByNameRequest::default(); ++ let ctx = mk_ttrpc_context(); ++ ++ let result = agent_service.update_interface_hw_addr_by_name(&ctx, req).await; ++ ++ assert!(result.is_err(), "expected update interface to fail"); ++ } ++ + #[tokio::test] + async fn test_update_routes() { + let logger = slog::Logger::root(slog::Discard, o!()); +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch b/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch new file mode 100644 index 0000000000000000000000000000000000000000..2419f0bb319fd20c6aafd59e3e8bcf914a645aa7 --- /dev/null +++ b/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch @@ -0,0 +1,2346 @@ +From cfc3ca2b200cd406cb47e0709b31e307bed9c6c9 Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Mon, 9 Aug 2021 14:36:15 +0800 +Subject: [PATCH 3/6] runtime: implement "updateInterfaceHwAddrByName" + interface for agent. + +1. add "updateInterfaceHwAddrByName" grpc interface. +2. update protocol pb.go files. + +Signed-off-by: Wei Gao +--- + src/runtime/virtcontainers/agent.go | 3 + + src/runtime/virtcontainers/kata_agent.go | 88 +- + src/runtime/virtcontainers/mock_agent.go | 5 + + .../pkg/agent/protocols/grpc/agent.pb.go | 936 +++++++++--------- + .../pkg/agent/protocols/grpc/health.pb.go | 15 +- + .../pkg/agent/protocols/grpc/oci.pb.go | 169 +--- + .../pkg/agent/protocols/types.pb.go | 20 +- + src/runtime/virtcontainers/pkg/mock/mock.go | 4 + + src/runtime/virtcontainers/sandbox.go | 5 + + src/runtime/virtcontainers/stratovirt.go | 6 +- + 10 files changed, 607 insertions(+), 644 deletions(-) + +diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go +index b8576c35..19a6c398 100644 +--- a/src/runtime/virtcontainers/agent.go ++++ b/src/runtime/virtcontainers/agent.go +@@ -171,6 +171,9 @@ type agent interface { + // updateInterface will tell the agent to update a nic for an existed Sandbox. + updateInterface(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) + ++ // updateInterfaceHwAddrByName will tell the agent to update a nic for an existed Sandbox by nic name. ++ updateInterfaceHwAddrByName(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) ++ + // listInterfaces will tell the agent to list interfaces of an existed Sandbox + listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) + +diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go +index 12756f0b..13d31658 100644 +--- a/src/runtime/virtcontainers/kata_agent.go ++++ b/src/runtime/virtcontainers/kata_agent.go +@@ -106,37 +106,38 @@ const ( + ) + + const ( +- grpcCheckRequest = "grpc.CheckRequest" +- grpcExecProcessRequest = "grpc.ExecProcessRequest" +- grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" +- grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" +- grpcCreateContainerRequest = "grpc.CreateContainerRequest" +- grpcStartContainerRequest = "grpc.StartContainerRequest" +- grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" +- grpcSignalProcessRequest = "grpc.SignalProcessRequest" +- grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" +- grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" +- grpcListInterfacesRequest = "grpc.ListInterfacesRequest" +- grpcListRoutesRequest = "grpc.ListRoutesRequest" +- grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" +- grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" +- grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" +- grpcWaitProcessRequest = "grpc.WaitProcessRequest" +- grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" +- grpcWriteStreamRequest = "grpc.WriteStreamRequest" +- grpcCloseStdinRequest = "grpc.CloseStdinRequest" +- grpcStatsContainerRequest = "grpc.StatsContainerRequest" +- grpcPauseContainerRequest = "grpc.PauseContainerRequest" +- grpcResumeContainerRequest = "grpc.ResumeContainerRequest" +- grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" +- grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" +- grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" +- grpcCopyFileRequest = "grpc.CopyFileRequest" +- grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" +- grpcStartTracingRequest = "grpc.StartTracingRequest" +- grpcStopTracingRequest = "grpc.StopTracingRequest" +- grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" +- grpcGetMetricsRequest = "grpc.GetMetricsRequest" ++ grpcCheckRequest = "grpc.CheckRequest" ++ grpcExecProcessRequest = "grpc.ExecProcessRequest" ++ grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" ++ grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" ++ grpcCreateContainerRequest = "grpc.CreateContainerRequest" ++ grpcStartContainerRequest = "grpc.StartContainerRequest" ++ grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" ++ grpcSignalProcessRequest = "grpc.SignalProcessRequest" ++ grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" ++ grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" ++ grpcUpdateInterfaceHwAddrByNameRequest = "grpc.UpdateInterfaceHwAddrByNameRequest" ++ grpcListInterfacesRequest = "grpc.ListInterfacesRequest" ++ grpcListRoutesRequest = "grpc.ListRoutesRequest" ++ grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" ++ grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" ++ grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" ++ grpcWaitProcessRequest = "grpc.WaitProcessRequest" ++ grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" ++ grpcWriteStreamRequest = "grpc.WriteStreamRequest" ++ grpcCloseStdinRequest = "grpc.CloseStdinRequest" ++ grpcStatsContainerRequest = "grpc.StatsContainerRequest" ++ grpcPauseContainerRequest = "grpc.PauseContainerRequest" ++ grpcResumeContainerRequest = "grpc.ResumeContainerRequest" ++ grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" ++ grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" ++ grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" ++ grpcCopyFileRequest = "grpc.CopyFileRequest" ++ grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" ++ grpcStartTracingRequest = "grpc.StartTracingRequest" ++ grpcStopTracingRequest = "grpc.StopTracingRequest" ++ grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" ++ grpcGetMetricsRequest = "grpc.GetMetricsRequest" + ) + + // newKataAgent returns an agent from an agent type. +@@ -668,6 +669,25 @@ func (k *kataAgent) updateInterfaces(ctx context.Context, interfaces []*pbTypes. + return nil + } + ++func (k *kataAgent) updateInterfaceHwAddrByName(ctx context.Context, ifc *pbTypes.Interface) (*pbTypes.Interface, error) { ++ ifcReq := &grpc.UpdateInterfaceHwAddrByNameRequest{ ++ Interface: ifc, ++ } ++ _, err := k.sendReq(ctx, ifcReq) ++ ++ return nil, err ++} ++ ++func (k *kataAgent) updateInterfacesHwAddrByName(ctx context.Context, interfaces []*pbTypes.Interface) error { ++ for _, ifc := range interfaces { ++ if _, err := k.updateInterfaceHwAddrByName(ctx, ifc); err != nil { ++ return err ++ } ++ } ++ ++ return nil ++} ++ + func (k *kataAgent) updateRoutes(ctx context.Context, routes []*pbTypes.Route) ([]*pbTypes.Route, error) { + if routes != nil { + routesReq := &grpc.UpdateRoutesRequest{ +@@ -819,6 +839,9 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error { + if err != nil { + return err + } ++ if err = k.updateInterfacesHwAddrByName(ctx, interfaces); err != nil { ++ return err ++ } + if err = k.updateInterfaces(ctx, interfaces); err != nil { + return err + } +@@ -1952,6 +1975,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { + k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest)) + } ++ k.reqHandlers[grpcUpdateInterfaceHwAddrByNameRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { ++ return k.client.AgentServiceClient.UpdateInterfaceHwAddrByName(ctx, req.(*grpc.UpdateInterfaceHwAddrByNameRequest)) ++ } + k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { + return k.client.AgentServiceClient.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest)) + } +diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go +index f12093b3..dae8cc5c 100644 +--- a/src/runtime/virtcontainers/mock_agent.go ++++ b/src/runtime/virtcontainers/mock_agent.go +@@ -106,6 +106,11 @@ func (n *mockAgent) updateInterface(ctx context.Context, inf *pbTypes.Interface) + return nil, nil + } + ++// updateInterfaceHwAddrByName is the Noop agent Interface HwAddr update implementation. It does nothing. ++func (n *mockAgent) updateInterfaceHwAddrByName(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) { ++ return nil, nil ++} ++ + // listInterfaces is the Noop agent Interfaces list implementation. It does nothing. + func (n *mockAgent) listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) { + return nil, nil +diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +index 3ee3bff7..a5c1e655 100644 +--- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go ++++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go +@@ -1488,6 +1488,45 @@ func (m *UpdateInterfaceRequest) XXX_DiscardUnknown() { + + var xxx_messageInfo_UpdateInterfaceRequest proto.InternalMessageInfo + ++type UpdateInterfaceHwAddrByNameRequest struct { ++ Interface *protocols.Interface `protobuf:"bytes,1,opt,name=interface,proto3" json:"interface,omitempty"` ++ XXX_NoUnkeyedLiteral struct{} `json:"-"` ++ XXX_unrecognized []byte `json:"-"` ++ XXX_sizecache int32 `json:"-"` ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } ++func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} ++func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { ++ return fileDescriptor_c1460208c38ccf5e, []int{35} ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Unmarshal(b []byte) error { ++ return m.Unmarshal(b) ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { ++ if deterministic { ++ return xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.Marshal(b, m, deterministic) ++ } else { ++ b = b[:cap(b)] ++ n, err := m.MarshalToSizedBuffer(b) ++ if err != nil { ++ return nil, err ++ } ++ return b[:n], nil ++ } ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Merge(src proto.Message) { ++ xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.Merge(m, src) ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Size() int { ++ return m.Size() ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) XXX_DiscardUnknown() { ++ xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.DiscardUnknown(m) ++} ++ ++var xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest proto.InternalMessageInfo ++ + type UpdateRoutesRequest struct { + Routes *Routes `protobuf:"bytes,1,opt,name=routes,proto3" json:"routes,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` +@@ -1498,7 +1537,7 @@ type UpdateRoutesRequest struct { + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } + func (*UpdateRoutesRequest) ProtoMessage() {} + func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{35} ++ return fileDescriptor_c1460208c38ccf5e, []int{36} + } + func (m *UpdateRoutesRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1536,7 +1575,7 @@ type ListInterfacesRequest struct { + func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } + func (*ListInterfacesRequest) ProtoMessage() {} + func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{36} ++ return fileDescriptor_c1460208c38ccf5e, []int{37} + } + func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1574,7 +1613,7 @@ type ListRoutesRequest struct { + func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } + func (*ListRoutesRequest) ProtoMessage() {} + func (*ListRoutesRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{37} ++ return fileDescriptor_c1460208c38ccf5e, []int{38} + } + func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1613,7 +1652,7 @@ type ARPNeighbors struct { + func (m *ARPNeighbors) Reset() { *m = ARPNeighbors{} } + func (*ARPNeighbors) ProtoMessage() {} + func (*ARPNeighbors) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{38} ++ return fileDescriptor_c1460208c38ccf5e, []int{39} + } + func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1652,7 +1691,7 @@ type AddARPNeighborsRequest struct { + func (m *AddARPNeighborsRequest) Reset() { *m = AddARPNeighborsRequest{} } + func (*AddARPNeighborsRequest) ProtoMessage() {} + func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{39} ++ return fileDescriptor_c1460208c38ccf5e, []int{40} + } + func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1698,7 +1737,7 @@ type OnlineCPUMemRequest struct { + func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } + func (*OnlineCPUMemRequest) ProtoMessage() {} + func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{40} ++ return fileDescriptor_c1460208c38ccf5e, []int{41} + } + func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1738,7 +1777,7 @@ type ReseedRandomDevRequest struct { + func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } + func (*ReseedRandomDevRequest) ProtoMessage() {} + func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{41} ++ return fileDescriptor_c1460208c38ccf5e, []int{42} + } + func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1788,7 +1827,7 @@ type AgentDetails struct { + func (m *AgentDetails) Reset() { *m = AgentDetails{} } + func (*AgentDetails) ProtoMessage() {} + func (*AgentDetails) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{42} ++ return fileDescriptor_c1460208c38ccf5e, []int{43} + } + func (m *AgentDetails) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1834,7 +1873,7 @@ type GuestDetailsRequest struct { + func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } + func (*GuestDetailsRequest) ProtoMessage() {} + func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{43} ++ return fileDescriptor_c1460208c38ccf5e, []int{44} + } + func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1876,7 +1915,7 @@ type GuestDetailsResponse struct { + func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } + func (*GuestDetailsResponse) ProtoMessage() {} + func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{44} ++ return fileDescriptor_c1460208c38ccf5e, []int{45} + } + func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1917,7 +1956,7 @@ type MemHotplugByProbeRequest struct { + func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } + func (*MemHotplugByProbeRequest) ProtoMessage() {} + func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{45} ++ return fileDescriptor_c1460208c38ccf5e, []int{46} + } + func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -1959,7 +1998,7 @@ type SetGuestDateTimeRequest struct { + func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } + func (*SetGuestDateTimeRequest) ProtoMessage() {} + func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{46} ++ return fileDescriptor_c1460208c38ccf5e, []int{47} + } + func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2024,7 +2063,7 @@ type Storage struct { + func (m *Storage) Reset() { *m = Storage{} } + func (*Storage) ProtoMessage() {} + func (*Storage) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{47} ++ return fileDescriptor_c1460208c38ccf5e, []int{48} + } + func (m *Storage) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2095,7 +2134,7 @@ type Device struct { + func (m *Device) Reset() { *m = Device{} } + func (*Device) ProtoMessage() {} + func (*Device) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{48} ++ return fileDescriptor_c1460208c38ccf5e, []int{49} + } + func (m *Device) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2136,7 +2175,7 @@ type StringUser struct { + func (m *StringUser) Reset() { *m = StringUser{} } + func (*StringUser) ProtoMessage() {} + func (*StringUser) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{49} ++ return fileDescriptor_c1460208c38ccf5e, []int{50} + } + func (m *StringUser) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2193,7 +2232,7 @@ type CopyFileRequest struct { + func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } + func (*CopyFileRequest) ProtoMessage() {} + func (*CopyFileRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{50} ++ return fileDescriptor_c1460208c38ccf5e, []int{51} + } + func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2231,7 +2270,7 @@ type StartTracingRequest struct { + func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } + func (*StartTracingRequest) ProtoMessage() {} + func (*StartTracingRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{51} ++ return fileDescriptor_c1460208c38ccf5e, []int{52} + } + func (m *StartTracingRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2269,7 +2308,7 @@ type StopTracingRequest struct { + func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } + func (*StopTracingRequest) ProtoMessage() {} + func (*StopTracingRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{52} ++ return fileDescriptor_c1460208c38ccf5e, []int{53} + } + func (m *StopTracingRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2307,7 +2346,7 @@ type GetOOMEventRequest struct { + func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } + func (*GetOOMEventRequest) ProtoMessage() {} + func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{53} ++ return fileDescriptor_c1460208c38ccf5e, []int{54} + } + func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2346,7 +2385,7 @@ type OOMEvent struct { + func (m *OOMEvent) Reset() { *m = OOMEvent{} } + func (*OOMEvent) ProtoMessage() {} + func (*OOMEvent) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{54} ++ return fileDescriptor_c1460208c38ccf5e, []int{55} + } + func (m *OOMEvent) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2384,7 +2423,7 @@ type GetMetricsRequest struct { + func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } + func (*GetMetricsRequest) ProtoMessage() {} + func (*GetMetricsRequest) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{55} ++ return fileDescriptor_c1460208c38ccf5e, []int{56} + } + func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2423,7 +2462,7 @@ type Metrics struct { + func (m *Metrics) Reset() { *m = Metrics{} } + func (*Metrics) ProtoMessage() {} + func (*Metrics) Descriptor() ([]byte, []int) { +- return fileDescriptor_c1460208c38ccf5e, []int{56} ++ return fileDescriptor_c1460208c38ccf5e, []int{57} + } + func (m *Metrics) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +@@ -2490,6 +2529,7 @@ func init() { + proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") + proto.RegisterType((*Routes)(nil), "grpc.Routes") + proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") ++ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") + proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") + proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") + proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") +@@ -2519,193 +2559,195 @@ func init() { + } + + var fileDescriptor_c1460208c38ccf5e = []byte{ +- // 2971 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, +- 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, +- 0xb5, 0x8c, 0xf1, 0xc8, 0x5e, 0x3b, 0x58, 0x3f, 0xc2, 0x6c, 0xac, 0x1e, 0x96, 0x64, 0x5b, 0x96, +- 0xdc, 0x5a, 0x85, 0x09, 0x08, 0xe8, 0x68, 0x75, 0x97, 0x66, 0xca, 0x9a, 0xee, 0x6a, 0x57, 0x57, +- 0x6b, 0x35, 0x26, 0x82, 0x80, 0x0b, 0xdc, 0x38, 0x72, 0xe3, 0x07, 0x08, 0x6e, 0x1c, 0xb9, 0x72, +- 0x70, 0x70, 0xe2, 0xc8, 0x89, 0xc0, 0xfb, 0x09, 0x7c, 0x01, 0x51, 0xaf, 0x7e, 0xcc, 0x43, 0x06, +- 0x85, 0x22, 0xb8, 0x4c, 0x54, 0x66, 0x65, 0xe5, 0xab, 0xaa, 0xb2, 0x32, 0xb3, 0x07, 0x3e, 0x1f, +- 0x60, 0x36, 0x8c, 0x4f, 0xfb, 0x2e, 0xf1, 0x37, 0xce, 0x1d, 0xe6, 0xbc, 0xe1, 0x92, 0x80, 0x39, +- 0x38, 0x40, 0x34, 0x9a, 0x82, 0x23, 0xea, 0x6e, 0x38, 0x03, 0x14, 0xb0, 0x8d, 0x90, 0x12, 0x46, +- 0x5c, 0x32, 0x8a, 0xe4, 0x28, 0x92, 0xe8, 0xbe, 0x00, 0x8c, 0xf2, 0x80, 0x86, 0x6e, 0xaf, 0x46, +- 0x5c, 0x2c, 0x11, 0xbd, 0x3a, 0x1b, 0x87, 0x28, 0x52, 0xc0, 0xf3, 0x03, 0x42, 0x06, 0x23, 0x24, +- 0x17, 0x9e, 0xc6, 0x67, 0x1b, 0xc8, 0x0f, 0xd9, 0x58, 0x4e, 0x9a, 0x7f, 0x28, 0xc2, 0xea, 0x16, +- 0x45, 0x0e, 0x43, 0x5b, 0x5a, 0xac, 0x85, 0xbe, 0x8a, 0x51, 0xc4, 0x8c, 0x97, 0xa0, 0x91, 0xa8, +- 0x62, 0x63, 0xaf, 0x5b, 0xb8, 0x5b, 0x58, 0xaf, 0x59, 0xf5, 0x04, 0xb7, 0xef, 0x19, 0xb7, 0xa1, +- 0x82, 0x2e, 0x91, 0xcb, 0x67, 0x8b, 0x62, 0x76, 0x91, 0x83, 0xfb, 0x9e, 0xf1, 0x16, 0xd4, 0x23, +- 0x46, 0x71, 0x30, 0xb0, 0xe3, 0x08, 0xd1, 0x6e, 0xe9, 0x6e, 0x61, 0xbd, 0xfe, 0x60, 0xa9, 0xcf, +- 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64, 0x6c, 0xdc, 0x87, 0x8a, 0x87, +- 0x2e, 0xb0, 0x8b, 0xa2, 0x6e, 0xf9, 0x6e, 0x69, 0xbd, 0xfe, 0xa0, 0x21, 0xc9, 0xb7, 0x05, 0xd2, +- 0xd2, 0x93, 0xc6, 0x6b, 0x50, 0x8d, 0x18, 0xa1, 0xce, 0x00, 0x45, 0xdd, 0x05, 0x41, 0xd8, 0xd4, +- 0x7c, 0x05, 0xd6, 0x4a, 0xa6, 0x8d, 0x17, 0xa0, 0x74, 0xb8, 0xb5, 0xdf, 0x5d, 0x14, 0xd2, 0x41, +- 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, 0x07, 0xcd, 0xc8, 0x09, 0xbc, 0x53, 0x72, 0x69, 0x87, +- 0xd8, 0x0b, 0xa2, 0x6e, 0xe5, 0x6e, 0x61, 0xbd, 0x6a, 0x35, 0x14, 0xf2, 0x88, 0xe3, 0xcc, 0xf7, +- 0xe1, 0xd6, 0x31, 0x73, 0x28, 0xbb, 0x86, 0x77, 0xcc, 0x13, 0x58, 0xb5, 0x90, 0x4f, 0x2e, 0xae, +- 0xe5, 0xda, 0x2e, 0x54, 0x18, 0xf6, 0x11, 0x89, 0x99, 0x70, 0x6d, 0xd3, 0xd2, 0xa0, 0xf9, 0xa7, +- 0x02, 0x18, 0x3b, 0x97, 0xc8, 0x3d, 0xa2, 0xc4, 0x45, 0x51, 0xf4, 0x7f, 0xda, 0xae, 0x57, 0xa1, +- 0x12, 0x4a, 0x05, 0xba, 0x65, 0x41, 0xae, 0x76, 0x41, 0x6b, 0xa5, 0x67, 0xcd, 0x2f, 0x61, 0xe5, +- 0x18, 0x0f, 0x02, 0x67, 0x74, 0x83, 0xfa, 0xae, 0xc2, 0x62, 0x24, 0x78, 0x0a, 0x55, 0x9b, 0x96, +- 0x82, 0xcc, 0x23, 0x30, 0xbe, 0x70, 0x30, 0xbb, 0x39, 0x49, 0xe6, 0x1b, 0xb0, 0x9c, 0xe3, 0x18, +- 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3, 0x05, 0x4b, 0x41, 0x26, 0x81, +- 0xd5, 0x93, 0xd0, 0xbb, 0xe6, 0x6d, 0x7a, 0x00, 0x35, 0x8a, 0x22, 0x12, 0x53, 0x7e, 0x07, 0x8a, +- 0xc2, 0xa9, 0x2b, 0xd2, 0xa9, 0x9f, 0xe2, 0x20, 0xbe, 0xb4, 0xf4, 0x9c, 0x95, 0x92, 0xa9, 0xf3, +- 0xc9, 0xa2, 0xeb, 0x9c, 0xcf, 0xf7, 0xe1, 0xd6, 0x91, 0x13, 0x47, 0xd7, 0xd1, 0xd5, 0xfc, 0x80, +- 0x9f, 0xed, 0x28, 0xf6, 0xaf, 0xb5, 0xf8, 0x8f, 0x05, 0xa8, 0x6e, 0x85, 0xf1, 0x49, 0xe4, 0x0c, +- 0x90, 0xf1, 0x3d, 0xa8, 0x33, 0xc2, 0x9c, 0x91, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, +- 0x49, 0xf0, 0x12, 0x34, 0x42, 0x44, 0xdd, 0x30, 0x56, 0x14, 0xc5, 0xbb, 0xa5, 0xf5, 0xb2, 0x55, +- 0x97, 0x38, 0x49, 0xd2, 0x87, 0x65, 0x31, 0x67, 0xe3, 0xc0, 0x3e, 0x47, 0x34, 0x40, 0x23, 0x9f, +- 0x78, 0x48, 0x1c, 0x8e, 0xb2, 0xd5, 0x11, 0x53, 0xfb, 0xc1, 0x27, 0xc9, 0x84, 0xf1, 0x7d, 0xe8, +- 0x24, 0xf4, 0xfc, 0xc4, 0x0b, 0xea, 0xb2, 0xa0, 0x6e, 0x2b, 0xea, 0x13, 0x85, 0x36, 0x7f, 0x09, +- 0xad, 0x27, 0x43, 0x4a, 0x18, 0x1b, 0xe1, 0x60, 0xb0, 0xed, 0x30, 0x87, 0x5f, 0xcd, 0x10, 0x51, +- 0x4c, 0xbc, 0x48, 0x69, 0xab, 0x41, 0xe3, 0x75, 0xe8, 0x30, 0x49, 0x8b, 0x3c, 0x5b, 0xd3, 0x14, +- 0x05, 0xcd, 0x52, 0x32, 0x71, 0xa4, 0x88, 0x5f, 0x81, 0x56, 0x4a, 0xcc, 0x2f, 0xb7, 0xd2, 0xb7, +- 0x99, 0x60, 0x9f, 0x60, 0x1f, 0x99, 0x17, 0xc2, 0x57, 0x62, 0x93, 0x8d, 0xd7, 0xa1, 0x96, 0xfa, +- 0xa1, 0x20, 0x4e, 0x48, 0x4b, 0x9e, 0x10, 0xed, 0x4e, 0xab, 0x9a, 0x38, 0xe5, 0x43, 0x68, 0xb3, +- 0x44, 0x71, 0xdb, 0x73, 0x98, 0x93, 0x3f, 0x54, 0x79, 0xab, 0xac, 0x16, 0xcb, 0xc1, 0xe6, 0x07, +- 0x50, 0x3b, 0xc2, 0x5e, 0x24, 0x05, 0x77, 0xa1, 0xe2, 0xc6, 0x94, 0xa2, 0x80, 0x69, 0x93, 0x15, +- 0x68, 0xac, 0xc0, 0xc2, 0x08, 0xfb, 0x98, 0x29, 0x33, 0x25, 0x60, 0x12, 0x80, 0x03, 0xe4, 0x13, +- 0x3a, 0x16, 0x0e, 0x5b, 0x81, 0x85, 0xec, 0xe6, 0x4a, 0xc0, 0x78, 0x1e, 0x6a, 0xbe, 0x73, 0x99, +- 0x6c, 0x2a, 0x9f, 0xa9, 0xfa, 0xce, 0xa5, 0x54, 0xbe, 0x0b, 0x95, 0x33, 0x07, 0x8f, 0xdc, 0x80, +- 0x29, 0xaf, 0x68, 0x30, 0x15, 0x58, 0xce, 0x0a, 0xfc, 0x6b, 0x11, 0xea, 0x52, 0xa2, 0x54, 0x78, +- 0x05, 0x16, 0x5c, 0xc7, 0x1d, 0x26, 0x22, 0x05, 0x60, 0xdc, 0xd7, 0x8a, 0x14, 0xb3, 0x11, 0x2e, +- 0xd5, 0x54, 0xab, 0xb6, 0x01, 0x10, 0x3d, 0x75, 0x42, 0xa5, 0x5b, 0x69, 0x0e, 0x71, 0x8d, 0xd3, +- 0x48, 0x75, 0xdf, 0x86, 0x86, 0x3c, 0x77, 0x6a, 0x49, 0x79, 0xce, 0x92, 0xba, 0xa4, 0x92, 0x8b, +- 0xee, 0x41, 0x33, 0x8e, 0x90, 0x3d, 0xc4, 0x88, 0x3a, 0xd4, 0x1d, 0x8e, 0xbb, 0x0b, 0xf2, 0x01, +- 0x8a, 0x23, 0xb4, 0xa7, 0x71, 0xc6, 0x03, 0x58, 0xe0, 0xb1, 0x25, 0xea, 0x2e, 0x8a, 0xb7, 0xee, +- 0x85, 0x2c, 0x4b, 0x61, 0x6a, 0x5f, 0xfc, 0xee, 0x04, 0x8c, 0x8e, 0x2d, 0x49, 0xda, 0x7b, 0x17, +- 0x20, 0x45, 0x1a, 0x4b, 0x50, 0x3a, 0x47, 0x63, 0x75, 0x0f, 0xf9, 0x90, 0x3b, 0xe7, 0xc2, 0x19, +- 0xc5, 0xda, 0xeb, 0x12, 0x78, 0xbf, 0xf8, 0x6e, 0xc1, 0x74, 0xa1, 0xbd, 0x39, 0x3a, 0xc7, 0x24, +- 0xb3, 0x7c, 0x05, 0x16, 0x7c, 0xe7, 0x4b, 0x42, 0xb5, 0x27, 0x05, 0x20, 0xb0, 0x38, 0x20, 0x54, +- 0xb3, 0x10, 0x80, 0xd1, 0x82, 0x22, 0x09, 0x85, 0xbf, 0x6a, 0x56, 0x91, 0x84, 0xa9, 0xa0, 0x72, +- 0x46, 0x90, 0xf9, 0xcf, 0x32, 0x40, 0x2a, 0xc5, 0xb0, 0xa0, 0x87, 0x89, 0x1d, 0x21, 0xca, 0xdf, +- 0x77, 0xfb, 0x74, 0xcc, 0x50, 0x64, 0x53, 0xe4, 0xc6, 0x34, 0xc2, 0x17, 0x7c, 0xff, 0xb8, 0xd9, +- 0xb7, 0xa4, 0xd9, 0x13, 0xba, 0x59, 0xb7, 0x31, 0x39, 0x96, 0xeb, 0x36, 0xf9, 0x32, 0x4b, 0xaf, +- 0x32, 0xf6, 0xe1, 0x56, 0xca, 0xd3, 0xcb, 0xb0, 0x2b, 0x5e, 0xc5, 0x6e, 0x39, 0x61, 0xe7, 0xa5, +- 0xac, 0x76, 0x60, 0x19, 0x13, 0xfb, 0xab, 0x18, 0xc5, 0x39, 0x46, 0xa5, 0xab, 0x18, 0x75, 0x30, +- 0xf9, 0x5c, 0x2c, 0x48, 0xd9, 0x1c, 0xc1, 0x9d, 0x8c, 0x95, 0xfc, 0xba, 0x67, 0x98, 0x95, 0xaf, +- 0x62, 0xb6, 0x9a, 0x68, 0xc5, 0xe3, 0x41, 0xca, 0xf1, 0x63, 0x58, 0xc5, 0xc4, 0x7e, 0xea, 0x60, +- 0x36, 0xc9, 0x6e, 0xe1, 0x3b, 0x8c, 0xe4, 0x2f, 0x5a, 0x9e, 0x97, 0x34, 0xd2, 0x47, 0x74, 0x90, +- 0x33, 0x72, 0xf1, 0x3b, 0x8c, 0x3c, 0x10, 0x0b, 0x52, 0x36, 0x8f, 0xa1, 0x83, 0xc9, 0xa4, 0x36, +- 0x95, 0xab, 0x98, 0xb4, 0x31, 0xc9, 0x6b, 0xb2, 0x09, 0x9d, 0x08, 0xb9, 0x8c, 0xd0, 0xec, 0x21, +- 0xa8, 0x5e, 0xc5, 0x62, 0x49, 0xd1, 0x27, 0x3c, 0xcc, 0x9f, 0x42, 0x63, 0x2f, 0x1e, 0x20, 0x36, +- 0x3a, 0x4d, 0x82, 0xc1, 0x8d, 0xc5, 0x1f, 0xf3, 0xdf, 0x45, 0xa8, 0x6f, 0x0d, 0x28, 0x89, 0xc3, +- 0x5c, 0x4c, 0x96, 0x97, 0x74, 0x32, 0x26, 0x0b, 0x12, 0x11, 0x93, 0x25, 0xf1, 0x3b, 0xd0, 0xf0, +- 0xc5, 0xd5, 0x55, 0xf4, 0x32, 0x0e, 0x75, 0xa6, 0x2e, 0xb5, 0x55, 0xf7, 0x33, 0xc1, 0xac, 0x0f, +- 0x10, 0x62, 0x2f, 0x52, 0x6b, 0x64, 0x38, 0x6a, 0xab, 0x74, 0x4b, 0x87, 0x68, 0xab, 0x16, 0x26, +- 0xd1, 0xfa, 0x2d, 0xa8, 0x9f, 0x72, 0x27, 0xa9, 0x05, 0xb9, 0x60, 0x94, 0x7a, 0xcf, 0x82, 0xd3, +- 0xf4, 0x12, 0xee, 0x41, 0x73, 0x28, 0x5d, 0xa6, 0x16, 0xc9, 0x33, 0x74, 0x4f, 0x59, 0x92, 0xda, +- 0xdb, 0xcf, 0x7a, 0x56, 0x6e, 0x40, 0x63, 0x98, 0x41, 0xf5, 0x8e, 0xa1, 0x33, 0x45, 0x32, 0x23, +- 0x06, 0xad, 0x67, 0x63, 0x50, 0xfd, 0x81, 0x21, 0x05, 0x65, 0x57, 0x66, 0xe3, 0xd2, 0xef, 0x8a, +- 0xd0, 0xf8, 0x0c, 0xb1, 0xa7, 0x84, 0x9e, 0x4b, 0x7d, 0x0d, 0x28, 0x07, 0x8e, 0x8f, 0x14, 0x47, +- 0x31, 0x36, 0xee, 0x40, 0x95, 0x5e, 0xca, 0x00, 0xa2, 0xf6, 0xb3, 0x42, 0x2f, 0x45, 0x60, 0x30, +- 0x5e, 0x04, 0xa0, 0x97, 0x76, 0xe8, 0xb8, 0xe7, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xe5, 0x91, +- 0x44, 0xf0, 0xa3, 0x40, 0x2f, 0x6d, 0x44, 0x29, 0xa1, 0x91, 0x8a, 0x55, 0x55, 0x7a, 0xb9, 0x23, +- 0x60, 0xb5, 0xd6, 0xa3, 0x24, 0x0c, 0x91, 0x27, 0x62, 0xb4, 0x58, 0xbb, 0x2d, 0x11, 0x5c, 0x2a, +- 0xd3, 0x52, 0x17, 0xa5, 0x54, 0x96, 0x4a, 0x65, 0xa9, 0xd4, 0x8a, 0x5c, 0xc9, 0xb2, 0x52, 0x59, +- 0x22, 0xb5, 0x2a, 0xa5, 0xb2, 0x8c, 0x54, 0x96, 0x4a, 0xad, 0xe9, 0xb5, 0x4a, 0xaa, 0xf9, 0xdb, +- 0x02, 0xac, 0x4e, 0x26, 0x7e, 0x2a, 0x37, 0x7d, 0x07, 0x1a, 0xae, 0xd8, 0xaf, 0xdc, 0x99, 0xec, +- 0x4c, 0xed, 0xa4, 0x55, 0x77, 0x33, 0xc7, 0xf8, 0x21, 0x34, 0x03, 0xe9, 0xe0, 0xe4, 0x68, 0x96, +- 0xd2, 0x7d, 0xc9, 0xfa, 0xde, 0x6a, 0x04, 0x19, 0xc8, 0xf4, 0xc0, 0xf8, 0x82, 0x62, 0x86, 0x8e, +- 0x19, 0x45, 0x8e, 0x7f, 0x13, 0xd9, 0xbd, 0x01, 0x65, 0x91, 0xad, 0xf0, 0x6d, 0x6a, 0x58, 0x62, +- 0x6c, 0xbe, 0x0a, 0xcb, 0x39, 0x29, 0xca, 0xd6, 0x25, 0x28, 0x8d, 0x50, 0x20, 0xb8, 0x37, 0x2d, +- 0x3e, 0x34, 0x1d, 0xe8, 0x58, 0xc8, 0xf1, 0x6e, 0x4e, 0x1b, 0x25, 0xa2, 0x94, 0x8a, 0x58, 0x07, +- 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x64, 0xb4, 0x3e, 0x84, 0xce, 0xd6, 0x88, 0x44, 0xe8, +- 0x98, 0x79, 0x38, 0xb8, 0x89, 0x72, 0xe4, 0x17, 0xb0, 0xfc, 0x84, 0x8d, 0xbf, 0xe0, 0xcc, 0x22, +- 0xfc, 0x35, 0xba, 0x21, 0xfb, 0x28, 0x79, 0xaa, 0xed, 0xa3, 0xe4, 0x29, 0x2f, 0x6e, 0x5c, 0x32, +- 0x8a, 0xfd, 0x40, 0x5c, 0x85, 0xa6, 0xa5, 0x20, 0x73, 0x13, 0x1a, 0x32, 0x87, 0x3e, 0x20, 0x5e, +- 0x3c, 0x42, 0x33, 0xef, 0xe0, 0x1a, 0x40, 0xe8, 0x50, 0xc7, 0x47, 0x0c, 0x51, 0x79, 0x86, 0x6a, +- 0x56, 0x06, 0x63, 0xfe, 0xbe, 0x08, 0x2b, 0xb2, 0xdf, 0x70, 0x2c, 0xcb, 0x6c, 0x6d, 0x42, 0x0f, +- 0xaa, 0x43, 0x12, 0xb1, 0x0c, 0xc3, 0x04, 0xe6, 0x2a, 0xf2, 0xfa, 0x5c, 0x72, 0xe3, 0xc3, 0x5c, +- 0x13, 0xa0, 0x74, 0x75, 0x13, 0x60, 0xaa, 0xcc, 0x2f, 0x4f, 0x97, 0xf9, 0xfc, 0xb6, 0x69, 0x22, +- 0x2c, 0xef, 0x78, 0xcd, 0xaa, 0x29, 0xcc, 0xbe, 0x67, 0xdc, 0x87, 0xf6, 0x80, 0x6b, 0x69, 0x0f, +- 0x09, 0x39, 0xb7, 0x43, 0x87, 0x0d, 0xc5, 0x55, 0xaf, 0x59, 0x4d, 0x81, 0xde, 0x23, 0xe4, 0xfc, +- 0xc8, 0x61, 0x43, 0xe3, 0x3d, 0x68, 0xa9, 0x34, 0xd0, 0x17, 0x2e, 0x8a, 0xd4, 0xe3, 0xa7, 0x6e, +- 0x51, 0xd6, 0x7b, 0x56, 0xf3, 0x3c, 0x03, 0x45, 0xe6, 0x6d, 0xb8, 0xb5, 0x8d, 0x22, 0x46, 0xc9, +- 0x38, 0xef, 0x18, 0xf3, 0x47, 0x00, 0xfb, 0x01, 0x43, 0xf4, 0xcc, 0x71, 0x51, 0x64, 0xbc, 0x99, +- 0x85, 0x54, 0x72, 0xb4, 0xd4, 0x97, 0xed, 0x9e, 0x64, 0xc2, 0xca, 0xd0, 0x98, 0x7d, 0x58, 0xb4, +- 0x48, 0xcc, 0xc3, 0xd1, 0xcb, 0x7a, 0xa4, 0xd6, 0x35, 0xd4, 0x3a, 0x81, 0xb4, 0xd4, 0x9c, 0xb9, +- 0xa7, 0x4b, 0xd8, 0x94, 0x9d, 0xda, 0xa2, 0x3e, 0xd4, 0xb0, 0xc6, 0xa9, 0xa8, 0x32, 0x2d, 0x3a, +- 0x25, 0x31, 0x3f, 0x80, 0x65, 0xc9, 0x49, 0x72, 0xd6, 0x6c, 0x5e, 0x86, 0x45, 0xaa, 0xd5, 0x28, +- 0xa4, 0x7d, 0x1e, 0x45, 0xa4, 0xe6, 0xb8, 0x3f, 0x3e, 0xc5, 0x11, 0x4b, 0x0d, 0xd1, 0xfe, 0x58, +- 0x86, 0x0e, 0x9f, 0xc8, 0xf1, 0x34, 0x3f, 0x82, 0xc6, 0x63, 0xeb, 0xe8, 0x33, 0x84, 0x07, 0xc3, +- 0x53, 0x1e, 0x3d, 0x7f, 0x98, 0x87, 0x95, 0xc1, 0x86, 0xd2, 0x36, 0x33, 0x65, 0xe5, 0xe8, 0xcc, +- 0x8f, 0x61, 0xf5, 0xb1, 0xe7, 0x65, 0x51, 0x5a, 0xeb, 0x37, 0xa1, 0x16, 0x64, 0xd8, 0x65, 0xde, +- 0xac, 0x1c, 0x75, 0x4a, 0x64, 0xfe, 0x0c, 0x96, 0x0f, 0x83, 0x11, 0x0e, 0xd0, 0xd6, 0xd1, 0xc9, +- 0x01, 0x4a, 0x62, 0x91, 0x01, 0x65, 0x9e, 0xb3, 0x09, 0x1e, 0x55, 0x4b, 0x8c, 0xf9, 0xe5, 0x0c, +- 0x4e, 0x6d, 0x37, 0x8c, 0x23, 0xd5, 0xec, 0x59, 0x0c, 0x4e, 0xb7, 0xc2, 0x38, 0xe2, 0x8f, 0x0b, +- 0x4f, 0x2e, 0x48, 0x30, 0x1a, 0x8b, 0x1b, 0x5a, 0xb5, 0x2a, 0x6e, 0x18, 0x1f, 0x06, 0xa3, 0xb1, +- 0xf9, 0x03, 0x51, 0x81, 0x23, 0xe4, 0x59, 0x4e, 0xe0, 0x11, 0x7f, 0x1b, 0x5d, 0x64, 0x24, 0x24, +- 0xd5, 0x9e, 0x8e, 0x44, 0xdf, 0x14, 0xa0, 0xf1, 0x78, 0x80, 0x02, 0xb6, 0x8d, 0x98, 0x83, 0x47, +- 0xa2, 0xa2, 0xbb, 0x40, 0x34, 0xc2, 0x24, 0x50, 0xd7, 0x4d, 0x83, 0xbc, 0x20, 0xc7, 0x01, 0x66, +- 0xb6, 0xe7, 0x20, 0x9f, 0x04, 0x82, 0x4b, 0xd5, 0x02, 0x8e, 0xda, 0x16, 0x18, 0xe3, 0x55, 0x68, +- 0xcb, 0x66, 0x9c, 0x3d, 0x74, 0x02, 0x6f, 0xc4, 0x2f, 0x7a, 0x49, 0x5c, 0xcd, 0x96, 0x44, 0xef, +- 0x29, 0xac, 0xf1, 0x1a, 0x2c, 0xa9, 0x6b, 0x98, 0x52, 0x96, 0x05, 0x65, 0x5b, 0xe1, 0x73, 0xa4, +- 0x71, 0x18, 0x12, 0xca, 0x22, 0x3b, 0x42, 0xae, 0x4b, 0xfc, 0x50, 0x95, 0x43, 0x6d, 0x8d, 0x3f, +- 0x96, 0x68, 0x73, 0x00, 0xcb, 0xbb, 0xdc, 0x4e, 0x65, 0x49, 0x7a, 0xac, 0x5a, 0x3e, 0xf2, 0xed, +- 0xd3, 0x11, 0x71, 0xcf, 0x6d, 0x1e, 0x1c, 0x95, 0x87, 0x79, 0xc2, 0xb5, 0xc9, 0x91, 0xc7, 0xf8, +- 0x6b, 0x51, 0xf9, 0x73, 0xaa, 0x21, 0x61, 0xe1, 0x28, 0x1e, 0xd8, 0x21, 0x25, 0xa7, 0x48, 0x99, +- 0xd8, 0xf6, 0x91, 0xbf, 0x27, 0xf1, 0x47, 0x1c, 0x6d, 0xfe, 0xa5, 0x00, 0x2b, 0x79, 0x49, 0x2a, +- 0xd4, 0x6f, 0xc0, 0x4a, 0x5e, 0x94, 0x7a, 0xfe, 0x65, 0x7a, 0xd9, 0xc9, 0x0a, 0x94, 0x89, 0xc0, +- 0x43, 0x68, 0x8a, 0x7e, 0xad, 0xed, 0x49, 0x4e, 0xf9, 0xa4, 0x27, 0xbb, 0x2f, 0x56, 0xc3, 0xc9, +- 0xee, 0xd2, 0x7b, 0x70, 0x47, 0x99, 0x6f, 0x4f, 0xab, 0x2d, 0x0f, 0xc4, 0xaa, 0x22, 0x38, 0x98, +- 0xd0, 0xfe, 0x53, 0xe8, 0xa6, 0xa8, 0xcd, 0xb1, 0x40, 0xa6, 0x87, 0x79, 0x79, 0xc2, 0xd8, 0xc7, +- 0x9e, 0x47, 0xc5, 0x2d, 0x29, 0x5b, 0xb3, 0xa6, 0xcc, 0x47, 0x70, 0xfb, 0x18, 0x31, 0xe9, 0x0d, +- 0x87, 0xa9, 0x4a, 0x44, 0x32, 0x5b, 0x82, 0xd2, 0x31, 0x72, 0x85, 0xf1, 0x25, 0x8b, 0x0f, 0xf9, +- 0x01, 0x3c, 0x89, 0x90, 0x2b, 0xac, 0x2c, 0x59, 0x62, 0x6c, 0xfe, 0xb9, 0x00, 0x15, 0x15, 0x9c, +- 0xf9, 0x03, 0xe3, 0x51, 0x7c, 0x81, 0xa8, 0x3a, 0x7a, 0x0a, 0x32, 0x5e, 0x81, 0x96, 0x1c, 0xd9, +- 0x24, 0x64, 0x98, 0x24, 0x21, 0xbf, 0x29, 0xb1, 0x87, 0x12, 0x29, 0x9a, 0x6f, 0xa2, 0xfd, 0xa5, +- 0x2a, 0x4d, 0x05, 0x71, 0xfc, 0x59, 0xc4, 0x6f, 0xb8, 0x08, 0xf1, 0x35, 0x4b, 0x41, 0xfc, 0xa8, +- 0x6b, 0x7e, 0x0b, 0x82, 0x9f, 0x06, 0xf9, 0x51, 0xf7, 0x49, 0x1c, 0x30, 0x3b, 0x24, 0x38, 0x60, +- 0x2a, 0xa6, 0x83, 0x40, 0x1d, 0x71, 0x8c, 0xf9, 0x9b, 0x02, 0x2c, 0xca, 0x06, 0x34, 0xaf, 0x6d, +- 0x93, 0x97, 0xb5, 0x88, 0x45, 0x96, 0x22, 0x64, 0xc9, 0xd7, 0x54, 0x8c, 0xf9, 0x3d, 0xbe, 0xf0, +- 0xe5, 0xfb, 0xa0, 0x54, 0xbb, 0xf0, 0xc5, 0xc3, 0xf0, 0x0a, 0xb4, 0xd2, 0x07, 0x5a, 0xcc, 0x4b, +- 0x15, 0x9b, 0x09, 0x56, 0x90, 0xcd, 0xd5, 0xd4, 0xfc, 0x31, 0x2f, 0xe9, 0x93, 0xe6, 0xeb, 0x12, +- 0x94, 0xe2, 0x44, 0x19, 0x3e, 0xe4, 0x98, 0x41, 0xf2, 0xb4, 0xf3, 0xa1, 0x71, 0x1f, 0x5a, 0x8e, +- 0xe7, 0x61, 0xbe, 0xdc, 0x19, 0xed, 0x62, 0x2f, 0xb9, 0xa4, 0x79, 0xac, 0xf9, 0xb7, 0x02, 0xb4, +- 0xb7, 0x48, 0x38, 0xfe, 0x08, 0x8f, 0x50, 0x26, 0x82, 0x08, 0x25, 0xd5, 0xcb, 0xce, 0xc7, 0x3c, +- 0x5b, 0x3d, 0xc3, 0x23, 0x24, 0xaf, 0x96, 0xdc, 0xd9, 0x2a, 0x47, 0x88, 0x6b, 0xa5, 0x27, 0x93, +- 0xb6, 0x5b, 0x53, 0x4e, 0x1e, 0x10, 0x4f, 0xe4, 0xe5, 0x1e, 0xa6, 0x76, 0xd2, 0x64, 0x6b, 0x5a, +- 0x15, 0x0f, 0x53, 0x31, 0xa5, 0x0c, 0x59, 0x10, 0x4d, 0xd4, 0xac, 0x21, 0x8b, 0x12, 0xc3, 0x0d, +- 0x59, 0x85, 0x45, 0x72, 0x76, 0x16, 0x21, 0x26, 0x32, 0xe8, 0x92, 0xa5, 0xa0, 0x24, 0xcc, 0x55, +- 0x33, 0x61, 0xee, 0x16, 0x2c, 0x8b, 0x76, 0xfd, 0x13, 0xea, 0xb8, 0x38, 0x18, 0xe8, 0xe7, 0x61, +- 0x05, 0x8c, 0x63, 0x46, 0xc2, 0x69, 0xec, 0x2e, 0x62, 0x87, 0x87, 0x07, 0x3b, 0x17, 0x28, 0x60, +- 0x1a, 0xfb, 0x06, 0x54, 0x35, 0xea, 0xbf, 0xe9, 0x65, 0x2e, 0x43, 0x67, 0x17, 0xb1, 0x03, 0xc4, +- 0x28, 0x76, 0x93, 0xe7, 0xe8, 0x1e, 0x54, 0x14, 0x86, 0x6f, 0xa9, 0x2f, 0x87, 0x3a, 0xce, 0x2a, +- 0xf0, 0xc1, 0xaf, 0x3b, 0x2a, 0x24, 0xab, 0xea, 0xde, 0xd8, 0x85, 0xf6, 0xc4, 0xa7, 0x18, 0x43, +- 0xb5, 0x7b, 0x66, 0x7f, 0xa1, 0xe9, 0xad, 0xf6, 0xe5, 0xa7, 0x9d, 0xbe, 0xfe, 0xb4, 0xd3, 0xdf, +- 0xf1, 0x43, 0x36, 0x36, 0x76, 0xa0, 0x95, 0xff, 0x68, 0x61, 0x3c, 0xaf, 0xb3, 0xa3, 0x19, 0x9f, +- 0x32, 0xe6, 0xb2, 0xd9, 0x85, 0xf6, 0xc4, 0xf7, 0x0b, 0xad, 0xcf, 0xec, 0xcf, 0x1a, 0x73, 0x19, +- 0x3d, 0x82, 0x7a, 0xe6, 0x83, 0x85, 0xd1, 0x95, 0x4c, 0xa6, 0xbf, 0x61, 0xcc, 0x65, 0xb0, 0x05, +- 0xcd, 0xdc, 0x37, 0x04, 0xa3, 0xa7, 0xec, 0x99, 0xf1, 0x61, 0x61, 0x2e, 0x93, 0x4d, 0xa8, 0x67, +- 0x5a, 0xf9, 0x5a, 0x8b, 0xe9, 0xef, 0x05, 0xbd, 0x3b, 0x33, 0x66, 0x54, 0xe4, 0xdf, 0x85, 0xf6, +- 0x44, 0x7f, 0x5f, 0xbb, 0x64, 0x76, 0xdb, 0x7f, 0xae, 0x32, 0x9f, 0x88, 0x2d, 0xca, 0x94, 0x6f, +- 0x99, 0x2d, 0x9a, 0xee, 0xe6, 0xf7, 0x5e, 0x98, 0x3d, 0xa9, 0xb4, 0xda, 0x81, 0x56, 0xbe, 0x91, +- 0xaf, 0x99, 0xcd, 0x6c, 0xef, 0x5f, 0xbd, 0xdf, 0xb9, 0x9e, 0x7e, 0xba, 0xdf, 0xb3, 0x5a, 0xfd, +- 0x73, 0x19, 0x3d, 0x06, 0x50, 0xc5, 0x9a, 0x87, 0x83, 0xc4, 0xd1, 0x53, 0x45, 0x62, 0xe2, 0xe8, +- 0x19, 0x85, 0xdd, 0x23, 0x00, 0x59, 0x63, 0x79, 0x24, 0x66, 0xc6, 0x6d, 0xad, 0xc6, 0x44, 0x61, +- 0xd7, 0xeb, 0x4e, 0x4f, 0x4c, 0x31, 0x40, 0x94, 0x5e, 0x87, 0xc1, 0x87, 0x00, 0x69, 0xed, 0xa6, +- 0x19, 0x4c, 0x55, 0x73, 0x57, 0xf8, 0xa0, 0x91, 0xad, 0xd4, 0x0c, 0x65, 0xeb, 0x8c, 0xea, 0xed, +- 0x0a, 0x16, 0xed, 0x89, 0x4c, 0x3c, 0x7f, 0xd8, 0x26, 0x13, 0xf4, 0xde, 0x54, 0x36, 0x6e, 0x3c, +- 0x84, 0x46, 0x36, 0x05, 0xd7, 0x5a, 0xcc, 0x48, 0xcb, 0x7b, 0xb9, 0x34, 0xdc, 0x78, 0x04, 0xad, +- 0x7c, 0xfa, 0xad, 0x8f, 0xd4, 0xcc, 0xa4, 0xbc, 0xa7, 0x9a, 0x4b, 0x19, 0xf2, 0xb7, 0x01, 0xd2, +- 0x34, 0x5d, 0xbb, 0x6f, 0x2a, 0x71, 0x9f, 0x90, 0xba, 0x0b, 0xed, 0x89, 0xf4, 0x5b, 0x5b, 0x3c, +- 0x3b, 0x2b, 0xbf, 0xca, 0xfb, 0xd9, 0x77, 0x40, 0xdb, 0x3d, 0xe3, 0x6d, 0xb8, 0x2a, 0x68, 0x65, +- 0xde, 0x0c, 0x7d, 0x8a, 0xa7, 0x9f, 0x91, 0xb9, 0x0c, 0xde, 0x01, 0x48, 0x5f, 0x06, 0xed, 0x81, +- 0xa9, 0xb7, 0xa2, 0xd7, 0xd4, 0xcd, 0x3f, 0x49, 0xb7, 0x05, 0xcd, 0x5c, 0x7d, 0xac, 0x43, 0xdd, +- 0xac, 0xa2, 0xf9, 0xaa, 0x07, 0x20, 0x5f, 0x4c, 0xea, 0xdd, 0x9b, 0x59, 0x62, 0x5e, 0xe5, 0xc5, +- 0x6c, 0x05, 0xa3, 0xbd, 0x38, 0xa3, 0xaa, 0xf9, 0x8e, 0x98, 0x92, 0xad, 0x52, 0x32, 0x31, 0x65, +- 0x46, 0xf1, 0x32, 0x97, 0xd1, 0x1e, 0xb4, 0x77, 0x75, 0x02, 0xaa, 0x92, 0x63, 0xa5, 0xce, 0x8c, +- 0x62, 0xa0, 0xd7, 0x9b, 0x35, 0xa5, 0x2e, 0xf6, 0x27, 0xd0, 0x99, 0x4a, 0x8c, 0x8d, 0xb5, 0xa4, +- 0x05, 0x3b, 0x33, 0x63, 0x9e, 0xab, 0xd6, 0x3e, 0x2c, 0x4d, 0xe6, 0xc5, 0xc6, 0x8b, 0xea, 0xa8, +- 0xcc, 0xce, 0x97, 0xe7, 0xb2, 0x7a, 0x0f, 0xaa, 0x3a, 0x0f, 0x33, 0x54, 0xab, 0x7b, 0x22, 0x2f, +- 0x9b, 0xbb, 0xf4, 0x21, 0xd4, 0x33, 0x99, 0x8c, 0x3e, 0xab, 0xd3, 0xc9, 0x4d, 0x4f, 0x75, 0xa6, +- 0x35, 0x7a, 0xf3, 0xf2, 0x9b, 0x6f, 0xd7, 0x9e, 0xfb, 0xc7, 0xb7, 0x6b, 0xcf, 0xfd, 0xea, 0xd9, +- 0x5a, 0xe1, 0x9b, 0x67, 0x6b, 0x85, 0xbf, 0x3f, 0x5b, 0x2b, 0xfc, 0xeb, 0xd9, 0x5a, 0xe1, 0x27, +- 0x3f, 0xff, 0x1f, 0xff, 0xa6, 0x42, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x5c, 0x60, 0xca, 0x32, 0x53, +- 0xe1, 0xf9, 0x60, 0xea, 0x1f, 0x2c, 0x5c, 0x85, 0xd3, 0x45, 0x01, 0xbf, 0xfd, 0x9f, 0x00, 0x00, +- 0x00, 0xff, 0xff, 0x23, 0xa0, 0xd8, 0x6f, 0x0f, 0x23, 0x00, 0x00, ++ // 3007 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x73, 0x1b, 0xc7, ++ 0xd1, 0xc6, 0x83, 0x04, 0xd0, 0x78, 0x11, 0x4b, 0x8a, 0x82, 0x20, 0x9b, 0x9f, 0xbc, 0xb2, 0x65, ++ 0xfa, 0xf3, 0x67, 0xc8, 0x96, 0x5d, 0x9f, 0xfc, 0x28, 0x7f, 0x2a, 0x91, 0xa2, 0x49, 0xda, 0xa6, ++ 0x45, 0x2f, 0xa5, 0xf2, 0x97, 0xa4, 0x92, 0xad, 0xe5, 0xee, 0x08, 0x18, 0x13, 0xbb, 0xb3, 0x9e, ++ 0x9d, 0xa5, 0x08, 0xa7, 0x2a, 0x95, 0x53, 0x72, 0xcb, 0x31, 0xb7, 0xfc, 0x81, 0x54, 0x6e, 0x39, ++ 0xe6, 0x9a, 0x83, 0x2b, 0xa7, 0x1c, 0x73, 0x48, 0xa5, 0x62, 0xfd, 0x84, 0xfc, 0x82, 0xd4, 0xbc, ++ 0xf6, 0x01, 0x2c, 0xe0, 0x84, 0xa5, 0xaa, 0x5c, 0x50, 0xdb, 0x3d, 0x3d, 0xfd, 0x9a, 0xe9, 0x9e, ++ 0xee, 0x19, 0xc0, 0x17, 0x23, 0xcc, 0xc6, 0xf1, 0xe9, 0xd0, 0x25, 0xfe, 0xed, 0x33, 0x87, 0x39, ++ 0x6f, 0xba, 0x24, 0x60, 0x0e, 0x0e, 0x10, 0x8d, 0xe6, 0xe0, 0x88, 0xba, 0xb7, 0x9d, 0x11, 0x0a, ++ 0xd8, 0xed, 0x90, 0x12, 0x46, 0x5c, 0x32, 0x89, 0xe4, 0x57, 0x24, 0xd1, 0x43, 0x01, 0x18, 0xd5, ++ 0x11, 0x0d, 0xdd, 0x41, 0x83, 0xb8, 0x58, 0x22, 0x06, 0x4d, 0x36, 0x0d, 0x51, 0xa4, 0x80, 0xeb, ++ 0x23, 0x42, 0x46, 0x13, 0x24, 0x27, 0x9e, 0xc6, 0x4f, 0x6e, 0x23, 0x3f, 0x64, 0x53, 0x39, 0x68, ++ 0xfe, 0xa6, 0x0c, 0x9b, 0xbb, 0x14, 0x39, 0x0c, 0xed, 0x6a, 0xb1, 0x16, 0xfa, 0x3a, 0x46, 0x11, ++ 0x33, 0x5e, 0x86, 0x56, 0xa2, 0x8a, 0x8d, 0xbd, 0x7e, 0xe9, 0x46, 0x69, 0xbb, 0x61, 0x35, 0x13, ++ 0xdc, 0xa1, 0x67, 0x5c, 0x85, 0x1a, 0xba, 0x40, 0x2e, 0x1f, 0x2d, 0x8b, 0xd1, 0x55, 0x0e, 0x1e, ++ 0x7a, 0xc6, 0xdb, 0xd0, 0x8c, 0x18, 0xc5, 0xc1, 0xc8, 0x8e, 0x23, 0x44, 0xfb, 0x95, 0x1b, 0xa5, ++ 0xed, 0xe6, 0x9d, 0xb5, 0x21, 0xd7, 0x73, 0x78, 0x22, 0x06, 0x1e, 0x47, 0x88, 0x5a, 0x10, 0x25, ++ 0xdf, 0xc6, 0x2d, 0xa8, 0x79, 0xe8, 0x1c, 0xbb, 0x28, 0xea, 0x57, 0x6f, 0x54, 0xb6, 0x9b, 0x77, ++ 0x5a, 0x92, 0xfc, 0x81, 0x40, 0x5a, 0x7a, 0xd0, 0x78, 0x1d, 0xea, 0x11, 0x23, 0xd4, 0x19, 0xa1, ++ 0xa8, 0xbf, 0x22, 0x08, 0xdb, 0x9a, 0xaf, 0xc0, 0x5a, 0xc9, 0xb0, 0xf1, 0x22, 0x54, 0x1e, 0xee, ++ 0x1e, 0xf6, 0x57, 0x85, 0x74, 0x50, 0x54, 0x21, 0x72, 0x2d, 0x8e, 0x36, 0x6e, 0x42, 0x3b, 0x72, ++ 0x02, 0xef, 0x94, 0x5c, 0xd8, 0x21, 0xf6, 0x82, 0xa8, 0x5f, 0xbb, 0x51, 0xda, 0xae, 0x5b, 0x2d, ++ 0x85, 0x3c, 0xe6, 0x38, 0xf3, 0x03, 0xb8, 0x72, 0xc2, 0x1c, 0xca, 0x2e, 0xe1, 0x1d, 0xf3, 0x31, ++ 0x6c, 0x5a, 0xc8, 0x27, 0xe7, 0x97, 0x72, 0x6d, 0x1f, 0x6a, 0x0c, 0xfb, 0x88, 0xc4, 0x4c, 0xb8, ++ 0xb6, 0x6d, 0x69, 0xd0, 0xfc, 0x5d, 0x09, 0x8c, 0xbd, 0x0b, 0xe4, 0x1e, 0x53, 0xe2, 0xa2, 0x28, ++ 0xfa, 0x0f, 0x2d, 0xd7, 0x6b, 0x50, 0x0b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, 0xab, 0xa0, 0xb5, ++ 0xd2, 0xa3, 0xe6, 0x57, 0xb0, 0x71, 0x82, 0x47, 0x81, 0x33, 0x79, 0x8e, 0xfa, 0x6e, 0xc2, 0x6a, ++ 0x24, 0x78, 0x0a, 0x55, 0xdb, 0x96, 0x82, 0xcc, 0x63, 0x30, 0xbe, 0x74, 0x30, 0x7b, 0x7e, 0x92, ++ 0xcc, 0x37, 0x61, 0x3d, 0xc7, 0x31, 0x0a, 0x49, 0x10, 0x21, 0xa1, 0x00, 0x73, 0x58, 0x1c, 0x09, ++ 0x66, 0x2b, 0x96, 0x82, 0x4c, 0x02, 0x9b, 0x8f, 0x43, 0xef, 0x92, 0xd1, 0x74, 0x07, 0x1a, 0x14, ++ 0x45, 0x24, 0xa6, 0x3c, 0x06, 0xca, 0xc2, 0xa9, 0x1b, 0xd2, 0xa9, 0x9f, 0xe1, 0x20, 0xbe, 0xb0, ++ 0xf4, 0x98, 0x95, 0x92, 0xa9, 0xfd, 0xc9, 0xa2, 0xcb, 0xec, 0xcf, 0x0f, 0xe0, 0xca, 0xb1, 0x13, ++ 0x47, 0x97, 0xd1, 0xd5, 0xfc, 0x90, 0xef, 0xed, 0x28, 0xf6, 0x2f, 0x35, 0xf9, 0xb7, 0x25, 0xa8, ++ 0xef, 0x86, 0xf1, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0xbf, 0xa0, 0xc9, 0x08, 0x73, 0x26, 0x76, 0xcc, ++ 0x41, 0x41, 0x5e, 0xb5, 0x40, 0xa0, 0x24, 0xc1, 0xcb, 0xd0, 0x0a, 0x11, 0x75, 0xc3, 0x58, 0x51, ++ 0x94, 0x6f, 0x54, 0xb6, 0xab, 0x56, 0x53, 0xe2, 0x24, 0xc9, 0x10, 0xd6, 0xc5, 0x98, 0x8d, 0x03, ++ 0xfb, 0x0c, 0xd1, 0x00, 0x4d, 0x7c, 0xe2, 0x21, 0xb1, 0x39, 0xaa, 0x56, 0x4f, 0x0c, 0x1d, 0x06, ++ 0x9f, 0x26, 0x03, 0xc6, 0x7f, 0x43, 0x2f, 0xa1, 0xe7, 0x3b, 0x5e, 0x50, 0x57, 0x05, 0x75, 0x57, ++ 0x51, 0x3f, 0x56, 0x68, 0xf3, 0x67, 0xd0, 0x79, 0x34, 0xa6, 0x84, 0xb1, 0x09, 0x0e, 0x46, 0x0f, ++ 0x1c, 0xe6, 0xf0, 0xd0, 0x0c, 0x11, 0xc5, 0xc4, 0x8b, 0x94, 0xb6, 0x1a, 0x34, 0xde, 0x80, 0x1e, ++ 0x93, 0xb4, 0xc8, 0xb3, 0x35, 0x4d, 0x59, 0xd0, 0xac, 0x25, 0x03, 0xc7, 0x8a, 0xf8, 0x55, 0xe8, ++ 0xa4, 0xc4, 0x3c, 0xb8, 0x95, 0xbe, 0xed, 0x04, 0xfb, 0x08, 0xfb, 0xc8, 0x3c, 0x17, 0xbe, 0x12, ++ 0x8b, 0x6c, 0xbc, 0x01, 0x8d, 0xd4, 0x0f, 0x25, 0xb1, 0x43, 0x3a, 0x72, 0x87, 0x68, 0x77, 0x5a, ++ 0xf5, 0xc4, 0x29, 0x1f, 0x41, 0x97, 0x25, 0x8a, 0xdb, 0x9e, 0xc3, 0x9c, 0xfc, 0xa6, 0xca, 0x5b, ++ 0x65, 0x75, 0x58, 0x0e, 0x36, 0x3f, 0x84, 0xc6, 0x31, 0xf6, 0x22, 0x29, 0xb8, 0x0f, 0x35, 0x37, ++ 0xa6, 0x14, 0x05, 0x4c, 0x9b, 0xac, 0x40, 0x63, 0x03, 0x56, 0x26, 0xd8, 0xc7, 0x4c, 0x99, 0x29, ++ 0x01, 0x93, 0x00, 0x1c, 0x21, 0x9f, 0xd0, 0xa9, 0x70, 0xd8, 0x06, 0xac, 0x64, 0x17, 0x57, 0x02, ++ 0xc6, 0x75, 0x68, 0xf8, 0xce, 0x45, 0xb2, 0xa8, 0x7c, 0xa4, 0xee, 0x3b, 0x17, 0x52, 0xf9, 0x3e, ++ 0xd4, 0x9e, 0x38, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x6a, 0x56, 0xe0, 0x1f, ++ 0xcb, 0xd0, 0x94, 0x12, 0xa5, 0xc2, 0x1b, 0xb0, 0xe2, 0x3a, 0xee, 0x38, 0x11, 0x29, 0x00, 0xe3, ++ 0x96, 0x56, 0xa4, 0x9c, 0xcd, 0x70, 0xa9, 0xa6, 0x5a, 0xb5, 0xdb, 0x00, 0xd1, 0x53, 0x27, 0x54, ++ 0xba, 0x55, 0x16, 0x10, 0x37, 0x38, 0x8d, 0x54, 0xf7, 0x1d, 0x68, 0xc9, 0x7d, 0xa7, 0xa6, 0x54, ++ 0x17, 0x4c, 0x69, 0x4a, 0x2a, 0x39, 0xe9, 0x26, 0xb4, 0xe3, 0x08, 0xd9, 0x63, 0x8c, 0xa8, 0x43, ++ 0xdd, 0xf1, 0xb4, 0xbf, 0x22, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0xdc, 0x81, 0x15, 0x9e, ++ 0x5b, 0xa2, 0xfe, 0xaa, 0x38, 0xeb, 0x5e, 0xcc, 0xb2, 0x14, 0xa6, 0x0e, 0xc5, 0xef, 0x5e, 0xc0, ++ 0xe8, 0xd4, 0x92, 0xa4, 0x83, 0xf7, 0x00, 0x52, 0xa4, 0xb1, 0x06, 0x95, 0x33, 0x34, 0x55, 0x71, ++ 0xc8, 0x3f, 0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0xca, 0xef, 0x95, 0x4c, ++ 0x17, 0xba, 0x3b, 0x93, 0x33, 0x4c, 0x32, 0xd3, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, ++ 0x52, 0x00, 0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x1d, 0x28, 0x93, 0x50, 0xf8, 0xab, ++ 0x61, 0x95, 0x49, 0x98, 0x0a, 0xaa, 0x66, 0x04, 0x99, 0x7f, 0xab, 0x02, 0xa4, 0x52, 0x0c, 0x0b, ++ 0x06, 0x98, 0xd8, 0x11, 0xa2, 0xfc, 0x7c, 0xb7, 0x4f, 0xa7, 0x0c, 0x45, 0x36, 0x45, 0x6e, 0x4c, ++ 0x23, 0x7c, 0xce, 0xd7, 0x8f, 0x9b, 0x7d, 0x45, 0x9a, 0x3d, 0xa3, 0x9b, 0x75, 0x15, 0x93, 0x13, ++ 0x39, 0x6f, 0x87, 0x4f, 0xb3, 0xf4, 0x2c, 0xe3, 0x10, 0xae, 0xa4, 0x3c, 0xbd, 0x0c, 0xbb, 0xf2, ++ 0x32, 0x76, 0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, ++ 0x31, 0xaa, 0x2c, 0x63, 0xd4, 0xc3, 0xe4, 0x0b, 0x31, 0x21, 0x65, 0x73, 0x0c, 0xd7, 0x32, 0x56, ++ 0xf2, 0x70, 0xcf, 0x30, 0xab, 0x2e, 0x63, 0xb6, 0x99, 0x68, 0xc5, 0xf3, 0x41, 0xca, 0xf1, 0x13, ++ 0xd8, 0xc4, 0xc4, 0x7e, 0xea, 0x60, 0x36, 0xcb, 0x6e, 0xe5, 0x7b, 0x8c, 0xe4, 0x27, 0x5a, 0x9e, ++ 0x97, 0x34, 0xd2, 0x47, 0x74, 0x94, 0x33, 0x72, 0xf5, 0x7b, 0x8c, 0x3c, 0x12, 0x13, 0x52, 0x36, ++ 0xf7, 0xa1, 0x87, 0xc9, 0xac, 0x36, 0xb5, 0x65, 0x4c, 0xba, 0x98, 0xe4, 0x35, 0xd9, 0x81, 0x5e, ++ 0x84, 0x5c, 0x46, 0x68, 0x76, 0x13, 0xd4, 0x97, 0xb1, 0x58, 0x53, 0xf4, 0x09, 0x0f, 0xf3, 0x47, ++ 0xd0, 0x3a, 0x88, 0x47, 0x88, 0x4d, 0x4e, 0x93, 0x64, 0xf0, 0xdc, 0xf2, 0x8f, 0xf9, 0x8f, 0x32, ++ 0x34, 0x77, 0x47, 0x94, 0xc4, 0x61, 0x2e, 0x27, 0xcb, 0x20, 0x9d, 0xcd, 0xc9, 0x82, 0x44, 0xe4, ++ 0x64, 0x49, 0xfc, 0x2e, 0xb4, 0x7c, 0x11, 0xba, 0x8a, 0x5e, 0xe6, 0xa1, 0xde, 0x5c, 0x50, 0x5b, ++ 0x4d, 0x3f, 0x93, 0xcc, 0x86, 0x00, 0x21, 0xf6, 0x22, 0x35, 0x47, 0xa6, 0xa3, 0xae, 0x2a, 0xb7, ++ 0x74, 0x8a, 0xb6, 0x1a, 0x61, 0x92, 0xad, 0xdf, 0x86, 0xe6, 0x29, 0x77, 0x92, 0x9a, 0x90, 0x4b, ++ 0x46, 0xa9, 0xf7, 0x2c, 0x38, 0x4d, 0x83, 0xf0, 0x00, 0xda, 0x63, 0xe9, 0x32, 0x35, 0x49, 0xee, ++ 0xa1, 0x9b, 0xca, 0x92, 0xd4, 0xde, 0x61, 0xd6, 0xb3, 0x72, 0x01, 0x5a, 0xe3, 0x0c, 0x6a, 0x70, ++ 0x02, 0xbd, 0x39, 0x92, 0x82, 0x1c, 0xb4, 0x9d, 0xcd, 0x41, 0xcd, 0x3b, 0x86, 0x14, 0x94, 0x9d, ++ 0x99, 0xcd, 0x4b, 0xbf, 0x2a, 0x43, 0xeb, 0x73, 0xc4, 0x9e, 0x12, 0x7a, 0x26, 0xf5, 0x35, 0xa0, ++ 0x1a, 0x38, 0x3e, 0x52, 0x1c, 0xc5, 0xb7, 0x71, 0x0d, 0xea, 0xf4, 0x42, 0x26, 0x10, 0xb5, 0x9e, ++ 0x35, 0x7a, 0x21, 0x12, 0x83, 0xf1, 0x12, 0x00, 0xbd, 0xb0, 0x43, 0xc7, 0x3d, 0x43, 0xca, 0x83, ++ 0x55, 0xab, 0x41, 0x2f, 0x8e, 0x25, 0x82, 0x6f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, ++ 0xae, 0xaa, 0xd3, 0x8b, 0x3d, 0x01, 0xab, 0xb9, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, ++ 0xdc, 0x07, 0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, ++ 0xc9, 0x99, 0x2c, 0x2b, 0x95, 0x25, 0x52, 0xeb, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x86, ++ 0x9e, 0xab, 0xa4, 0x9a, 0xbf, 0x2c, 0xc1, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe5, ++ 0x8a, 0xf5, 0xca, 0xed, 0xc9, 0xde, 0xdc, 0x4a, 0x5a, 0x4d, 0x37, 0xb3, 0x8d, 0xef, 0x42, 0x3b, ++ 0x90, 0x0e, 0x4e, 0xb6, 0x66, 0x25, 0x5d, 0x97, 0xac, 0xef, 0xad, 0x56, 0x90, 0x81, 0x4c, 0x0f, ++ 0x8c, 0x2f, 0x29, 0x66, 0xe8, 0x84, 0x51, 0xe4, 0xf8, 0xcf, 0xa3, 0xba, 0x37, 0xa0, 0x2a, 0xaa, ++ 0x15, 0xbe, 0x4c, 0x2d, 0x4b, 0x7c, 0x9b, 0xaf, 0xc1, 0x7a, 0x4e, 0x8a, 0xb2, 0x75, 0x0d, 0x2a, ++ 0x13, 0x14, 0x08, 0xee, 0x6d, 0x8b, 0x7f, 0x9a, 0x0e, 0xf4, 0x2c, 0xe4, 0x78, 0xcf, 0x4f, 0x1b, ++ 0x25, 0xa2, 0x92, 0x8a, 0xd8, 0x06, 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x65, 0xb4, 0x7e, ++ 0x08, 0xbd, 0xdd, 0x09, 0x89, 0xd0, 0x09, 0xf3, 0x70, 0xf0, 0x3c, 0xda, 0x91, 0x9f, 0xc2, 0xfa, ++ 0x23, 0x36, 0xfd, 0x92, 0x33, 0x8b, 0xf0, 0x37, 0xe8, 0x39, 0xd9, 0x47, 0xc9, 0x53, 0x6d, 0x1f, ++ 0x25, 0x4f, 0x79, 0x73, 0xe3, 0x92, 0x49, 0xec, 0x07, 0x22, 0x14, 0xda, 0x96, 0x82, 0xcc, 0x1d, ++ 0x68, 0xc9, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0x15, 0xc6, 0xe0, 0x16, 0x40, 0xe8, 0x50, 0xc7, ++ 0x47, 0x0c, 0x51, 0xb9, 0x87, 0x1a, 0x56, 0x06, 0x63, 0xfe, 0xba, 0x0c, 0x1b, 0xf2, 0xbe, 0xe1, ++ 0x44, 0xb6, 0xd9, 0xda, 0x84, 0x01, 0xd4, 0xc7, 0x24, 0x62, 0x19, 0x86, 0x09, 0xcc, 0x55, 0xe4, ++ 0xfd, 0xb9, 0xe4, 0xc6, 0x3f, 0x73, 0x97, 0x00, 0x95, 0xe5, 0x97, 0x00, 0x73, 0x6d, 0x7e, 0x75, ++ 0xbe, 0xcd, 0xe7, 0xd1, 0xa6, 0x89, 0xb0, 0x8c, 0xf1, 0x86, 0xd5, 0x50, 0x98, 0x43, 0xcf, 0xb8, ++ 0x05, 0xdd, 0x11, 0xd7, 0xd2, 0x1e, 0x13, 0x72, 0x66, 0x87, 0x0e, 0x1b, 0x8b, 0x50, 0x6f, 0x58, ++ 0x6d, 0x81, 0x3e, 0x20, 0xe4, 0xec, 0xd8, 0x61, 0x63, 0xe3, 0x7d, 0xe8, 0xa8, 0x32, 0xd0, 0x17, ++ 0x2e, 0x8a, 0xd4, 0xe1, 0xa7, 0xa2, 0x28, 0xeb, 0x3d, 0xab, 0x7d, 0x96, 0x81, 0x22, 0xf3, 0x2a, ++ 0x5c, 0x79, 0x80, 0x22, 0x46, 0xc9, 0x34, 0xef, 0x18, 0xf3, 0xff, 0x00, 0x0e, 0x03, 0x86, 0xe8, ++ 0x13, 0xc7, 0x45, 0x91, 0xf1, 0x56, 0x16, 0x52, 0xc5, 0xd1, 0xda, 0x50, 0x5e, 0xf7, 0x24, 0x03, ++ 0x56, 0x86, 0xc6, 0x1c, 0xc2, 0xaa, 0x45, 0x62, 0x9e, 0x8e, 0x5e, 0xd1, 0x5f, 0x6a, 0x5e, 0x4b, ++ 0xcd, 0x13, 0x48, 0x4b, 0x8d, 0x99, 0x07, 0xba, 0x85, 0x4d, 0xd9, 0xa9, 0x25, 0x1a, 0x42, 0x03, ++ 0x6b, 0x9c, 0xca, 0x2a, 0xf3, 0xa2, 0x53, 0x12, 0xf3, 0x11, 0x98, 0x33, 0x9c, 0x0e, 0x9e, 0xde, ++ 0xf7, 0x3c, 0xba, 0x33, 0xfd, 0xdc, 0xf1, 0x2f, 0xcd, 0xf5, 0x43, 0x58, 0x97, 0x5c, 0xa5, 0xbe, ++ 0x9a, 0xcd, 0x2b, 0xb0, 0x4a, 0xb5, 0x71, 0xa5, 0xf4, 0xf6, 0x48, 0x11, 0xa9, 0x31, 0xee, 0xe5, ++ 0xcf, 0x70, 0xc4, 0x52, 0xf7, 0x68, 0x2f, 0xaf, 0x43, 0x8f, 0x0f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, ++ 0xeb, 0xbe, 0x75, 0xfc, 0x39, 0xc2, 0xa3, 0xf1, 0x29, 0xcf, 0xc9, 0xff, 0x9b, 0x87, 0x95, 0x1b, ++ 0x0d, 0xa5, 0x6d, 0x66, 0xc8, 0xca, 0xd1, 0x99, 0x9f, 0xc0, 0xe6, 0x7d, 0xcf, 0xcb, 0xa2, 0xb4, ++ 0xd6, 0x6f, 0x41, 0x23, 0xc8, 0xb0, 0xcb, 0x9c, 0x84, 0x39, 0xea, 0x94, 0xc8, 0xfc, 0x31, 0xac, ++ 0x3f, 0x0c, 0x26, 0x38, 0x40, 0xbb, 0xc7, 0x8f, 0x8f, 0x50, 0x92, 0xe1, 0x0c, 0xa8, 0xf2, 0x4a, ++ 0x50, 0xf0, 0xa8, 0x5b, 0xe2, 0x9b, 0x87, 0x7c, 0x70, 0x6a, 0xbb, 0x61, 0x1c, 0xa9, 0x2b, 0xa4, ++ 0xd5, 0xe0, 0x74, 0x37, 0x8c, 0x23, 0x7e, 0x64, 0xf1, 0x92, 0x85, 0x04, 0x93, 0xa9, 0x88, 0xfb, ++ 0xba, 0x55, 0x73, 0xc3, 0xf8, 0x61, 0x30, 0x99, 0x9a, 0xff, 0x23, 0xfa, 0x7a, 0x84, 0x3c, 0xcb, ++ 0x09, 0x3c, 0xe2, 0x3f, 0x40, 0xe7, 0x19, 0x09, 0x49, 0x0f, 0xa9, 0xf3, 0xdb, 0xb7, 0x25, 0x68, ++ 0xdd, 0x1f, 0xa1, 0x80, 0x3d, 0x40, 0xcc, 0xc1, 0x13, 0xd1, 0x27, 0x9e, 0x23, 0x1a, 0x61, 0x12, ++ 0xa8, 0x20, 0xd6, 0x20, 0x6f, 0xf3, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, ++ 0xb7, 0x80, 0xa3, 0x1e, 0x08, 0x8c, 0xf1, 0x1a, 0x74, 0xe5, 0x15, 0x9f, 0x3d, 0x76, 0x02, 0x6f, ++ 0xc2, 0xd3, 0x47, 0x45, 0x04, 0x7c, 0x47, 0xa2, 0x0f, 0x14, 0xd6, 0x78, 0x1d, 0xd6, 0x54, 0x70, ++ 0xa7, 0x94, 0x55, 0x41, 0xd9, 0x55, 0xf8, 0x1c, 0x69, 0x1c, 0x86, 0x84, 0xb2, 0xc8, 0x8e, 0x90, ++ 0xeb, 0x12, 0x3f, 0x54, 0x4d, 0x56, 0x57, 0xe3, 0x4f, 0x24, 0xda, 0x1c, 0xc1, 0xfa, 0x3e, 0xb7, ++ 0x53, 0x59, 0x92, 0x6e, 0xab, 0x8e, 0x8f, 0x7c, 0xfb, 0x74, 0x42, 0xdc, 0x33, 0x9b, 0xa7, 0x5c, ++ 0xe5, 0x61, 0x5e, 0xc6, 0xed, 0x70, 0xe4, 0x09, 0xfe, 0x46, 0xdc, 0x27, 0x70, 0xaa, 0x31, 0x61, ++ 0xe1, 0x24, 0x1e, 0xd9, 0x21, 0x25, 0xa7, 0x48, 0x99, 0xd8, 0xf5, 0x91, 0x7f, 0x20, 0xf1, 0xc7, ++ 0x1c, 0x6d, 0xfe, 0xa1, 0x04, 0x1b, 0x79, 0x49, 0xea, 0x00, 0xb9, 0x0d, 0x1b, 0x79, 0x51, 0xaa, ++ 0xa8, 0x90, 0x45, 0x6b, 0x2f, 0x2b, 0x50, 0x96, 0x17, 0x77, 0xa1, 0x2d, 0x6e, 0x81, 0x6d, 0x4f, ++ 0x72, 0xca, 0x97, 0x52, 0xd9, 0x75, 0xb1, 0x5a, 0x4e, 0x76, 0x95, 0xde, 0x87, 0x6b, 0xca, 0x7c, ++ 0x7b, 0x5e, 0x6d, 0xb9, 0x21, 0x36, 0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x3f, 0x45, 0xed, ++ 0x4c, 0x05, 0x32, 0xdd, 0xcc, 0xeb, 0x33, 0xc6, 0xf2, 0x68, 0x17, 0x51, 0x52, 0xb5, 0x8a, 0x86, ++ 0xcc, 0x7b, 0x70, 0xf5, 0x04, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xfe, 0x46, 0x32, 0x5b, 0x83, 0xca, ++ 0x09, 0x72, 0x85, 0xf1, 0x15, 0x8b, 0x7f, 0xf2, 0x0d, 0xf8, 0x38, 0x42, 0xae, 0xb0, 0xb2, 0x62, ++ 0x89, 0x6f, 0xf3, 0xf7, 0x25, 0xa8, 0xa9, 0x94, 0xcf, 0x8f, 0x2d, 0x8f, 0xe2, 0x73, 0x44, 0xd5, ++ 0xd6, 0x53, 0x90, 0xf1, 0x2a, 0x74, 0xe4, 0x97, 0x4d, 0x42, 0x86, 0x49, 0x72, 0x90, 0xb4, 0x25, ++ 0xf6, 0xa1, 0x44, 0x8a, 0x2b, 0x3d, 0x71, 0xa9, 0xa6, 0xfa, 0x57, 0x05, 0x71, 0xfc, 0x93, 0x88, ++ 0x47, 0xb8, 0x38, 0x38, 0x1a, 0x96, 0x82, 0xf8, 0x56, 0xd7, 0xfc, 0x56, 0x04, 0x3f, 0x0d, 0xf2, ++ 0xad, 0xee, 0x93, 0x38, 0x60, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0x49, 0x01, 0x02, 0x75, 0xcc, 0x31, ++ 0xe6, 0x2f, 0x4a, 0xb0, 0x2a, 0xaf, 0xb5, 0x79, 0xc7, 0x9c, 0x9c, 0xd7, 0x65, 0x2c, 0x6a, 0x1f, ++ 0x21, 0x4b, 0x9e, 0xd1, 0xe2, 0x9b, 0xc7, 0xf1, 0xb9, 0x2f, 0x4f, 0x1d, 0xa5, 0xda, 0xb9, 0x2f, ++ 0x8e, 0x9b, 0x57, 0xa1, 0x93, 0x1e, 0xfb, 0x62, 0x5c, 0xaa, 0xd8, 0x4e, 0xb0, 0x82, 0x6c, 0xa1, ++ 0xa6, 0xe6, 0xff, 0x03, 0xa4, 0xd7, 0xbb, 0xdc, 0xe5, 0x71, 0xa2, 0x0c, 0xff, 0xe4, 0x98, 0x51, ++ 0x52, 0x30, 0xf0, 0x4f, 0xe3, 0x16, 0x74, 0x1c, 0xcf, 0xc3, 0x7c, 0xba, 0x33, 0xd9, 0xc7, 0x5e, ++ 0x12, 0xa4, 0x79, 0xac, 0xf9, 0xa7, 0x12, 0x74, 0x77, 0x49, 0x38, 0xfd, 0x18, 0x4f, 0x50, 0x26, ++ 0x83, 0x08, 0x25, 0x55, 0xbd, 0xc0, 0xbf, 0x79, 0x0d, 0xfc, 0x04, 0x4f, 0x90, 0x0c, 0x2d, 0xb9, ++ 0xb2, 0x75, 0x8e, 0x10, 0x61, 0xa5, 0x07, 0x93, 0xcb, 0xbc, 0xb6, 0x1c, 0x3c, 0x22, 0x9e, 0xa8, ++ 0xf6, 0x3d, 0x4c, 0xed, 0xe4, 0xea, 0xae, 0x6d, 0xd5, 0x3c, 0x4c, 0xc5, 0x90, 0x32, 0x64, 0x45, ++ 0x5c, 0xcd, 0x66, 0x0d, 0x59, 0x95, 0x18, 0x6e, 0xc8, 0x26, 0xac, 0x92, 0x27, 0x4f, 0x22, 0xc4, ++ 0x44, 0x5d, 0x5e, 0xb1, 0x14, 0x94, 0xa4, 0xb9, 0x7a, 0x26, 0xcd, 0x5d, 0x81, 0x75, 0xf1, 0x08, ++ 0xf0, 0x88, 0x3a, 0x2e, 0x0e, 0x46, 0xfa, 0x78, 0xd8, 0x00, 0xe3, 0x84, 0x91, 0x70, 0x1e, 0xbb, ++ 0x8f, 0xd8, 0xc3, 0x87, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0x1a, 0xfb, 0x26, 0xd4, 0x35, 0xea, 0x5f, ++ 0xb9, 0x21, 0x5d, 0x87, 0xde, 0x3e, 0x62, 0x47, 0x88, 0x51, 0xec, 0x26, 0xc7, 0xd1, 0x4d, 0xa8, ++ 0x29, 0x0c, 0x5f, 0x52, 0x5f, 0x7e, 0xea, 0x3c, 0xab, 0xc0, 0x3b, 0x7f, 0xed, 0xa9, 0x94, 0xac, ++ 0xee, 0x0c, 0x8c, 0x7d, 0xe8, 0xce, 0x3c, 0xf0, 0x18, 0xea, 0x12, 0xa9, 0xf8, 0xdd, 0x67, 0xb0, ++ 0x39, 0x94, 0x0f, 0x46, 0x43, 0xfd, 0x60, 0x34, 0xdc, 0xf3, 0x43, 0x36, 0x35, 0xf6, 0xa0, 0x93, ++ 0x7f, 0x0a, 0x31, 0xae, 0xeb, 0x9a, 0xab, 0xe0, 0x81, 0x64, 0x21, 0x9b, 0x7d, 0xe8, 0xce, 0xbc, ++ 0x8a, 0x68, 0x7d, 0x8a, 0x1f, 0x4b, 0x16, 0x32, 0xba, 0x07, 0xcd, 0xcc, 0x33, 0x88, 0xd1, 0x97, ++ 0x4c, 0xe6, 0x5f, 0x46, 0x16, 0x32, 0xd8, 0x85, 0x76, 0xee, 0x65, 0xc2, 0x18, 0x28, 0x7b, 0x0a, ++ 0x9e, 0x2b, 0x16, 0x32, 0xd9, 0x81, 0x66, 0xe6, 0x81, 0x40, 0x6b, 0x31, 0xff, 0x0a, 0x31, 0xb8, ++ 0x56, 0x30, 0xa2, 0x32, 0xff, 0x3e, 0x74, 0x67, 0x5e, 0x0d, 0xb4, 0x4b, 0x8a, 0x1f, 0x13, 0x16, ++ 0x2a, 0xf3, 0xa9, 0x58, 0xa2, 0x4c, 0x53, 0x98, 0x59, 0xa2, 0xf9, 0x37, 0x82, 0xc1, 0x8b, 0xc5, ++ 0x83, 0x4a, 0xab, 0x3d, 0xe8, 0xe4, 0x9f, 0x07, 0x34, 0xb3, 0xc2, 0x47, 0x83, 0xe5, 0xeb, 0x9d, ++ 0x7b, 0x29, 0x48, 0xd7, 0xbb, 0xe8, 0x01, 0x61, 0x21, 0xa3, 0xfb, 0x00, 0xaa, 0x05, 0xf4, 0x70, ++ 0x90, 0x38, 0x7a, 0xae, 0xf5, 0x4c, 0x1c, 0x5d, 0xd0, 0x2e, 0xde, 0x03, 0x90, 0x9d, 0x9b, 0x47, ++ 0x62, 0x66, 0x5c, 0xd5, 0x6a, 0xcc, 0xb4, 0x8b, 0x83, 0xfe, 0xfc, 0xc0, 0x1c, 0x03, 0x44, 0xe9, ++ 0x65, 0x18, 0x7c, 0x04, 0x90, 0x76, 0x84, 0x9a, 0xc1, 0x5c, 0x8f, 0xb8, 0xc4, 0x07, 0xad, 0x6c, ++ 0xff, 0x67, 0x28, 0x5b, 0x0b, 0x7a, 0xc2, 0x25, 0x2c, 0xba, 0x33, 0x55, 0x79, 0x7e, 0xb3, 0xcd, ++ 0x96, 0xfd, 0x83, 0xb9, 0x6a, 0xdc, 0xf8, 0x01, 0x5c, 0x5f, 0x52, 0xd8, 0x1b, 0xdb, 0x85, 0xec, ++ 0x0a, 0x6a, 0xff, 0x02, 0xd6, 0x77, 0xa1, 0x95, 0xad, 0xee, 0xb5, 0x81, 0x05, 0x15, 0xff, 0x20, ++ 0x57, 0xe1, 0x1b, 0xf7, 0xa0, 0x93, 0xaf, 0xec, 0xf5, 0x6e, 0x2d, 0xac, 0xf7, 0x07, 0xea, 0x36, ++ 0x2c, 0x43, 0xfe, 0x0e, 0x40, 0xda, 0x01, 0xe8, 0x95, 0x99, 0xeb, 0x09, 0x66, 0xa4, 0xee, 0x43, ++ 0x77, 0xa6, 0xb2, 0xd7, 0xce, 0x2c, 0x2e, 0xf8, 0x97, 0x2d, 0x6c, 0xf6, 0x88, 0xd1, 0x76, 0x17, ++ 0x1c, 0x3b, 0xcb, 0xf2, 0x61, 0xe6, 0x38, 0xd2, 0x01, 0x32, 0x7f, 0x42, 0x2d, 0x64, 0xf0, 0x2e, ++ 0x40, 0x7a, 0xe8, 0x68, 0x0f, 0xcc, 0x1d, 0x43, 0x83, 0xb6, 0xbe, 0xad, 0x94, 0x74, 0xbb, 0xd0, ++ 0xce, 0x35, 0xf4, 0x3a, 0x8b, 0x16, 0x75, 0xf9, 0xcb, 0xce, 0x96, 0x7c, 0xf7, 0xab, 0x57, 0xaf, ++ 0xb0, 0x27, 0x5e, 0xe6, 0xc5, 0x6c, 0x73, 0xa4, 0xbd, 0x58, 0xd0, 0x30, 0x7d, 0x4f, 0xba, 0xca, ++ 0x36, 0x40, 0x99, 0x74, 0x55, 0xd0, 0x17, 0x2d, 0x64, 0x74, 0x00, 0xdd, 0x7d, 0x5d, 0xdb, 0xaa, ++ 0xba, 0x5b, 0xa9, 0x53, 0xd0, 0x67, 0x0c, 0x06, 0x45, 0x43, 0x2a, 0x67, 0x7c, 0x0a, 0xbd, 0xb9, ++ 0x9a, 0xdb, 0xd8, 0x4a, 0xee, 0x8c, 0x0b, 0x8b, 0xf1, 0x85, 0x6a, 0x1d, 0xc2, 0xda, 0x6c, 0xc9, ++ 0x6d, 0xbc, 0xa4, 0xb6, 0x4a, 0x71, 0x29, 0xbe, 0x90, 0xd5, 0xfb, 0x50, 0xd7, 0x25, 0x9e, 0xa1, ++ 0xee, 0xe6, 0x67, 0x4a, 0xbe, 0x85, 0x53, 0xef, 0x42, 0x33, 0x53, 0x24, 0xe9, 0xbd, 0x3a, 0x5f, ++ 0x37, 0x0d, 0xd4, 0x55, 0xba, 0x46, 0xef, 0x5c, 0x7c, 0xfb, 0xdd, 0xd6, 0x0b, 0x7f, 0xf9, 0x6e, ++ 0xeb, 0x85, 0x9f, 0x3f, 0xdb, 0x2a, 0x7d, 0xfb, 0x6c, 0xab, 0xf4, 0xe7, 0x67, 0x5b, 0xa5, 0xbf, ++ 0x3f, 0xdb, 0x2a, 0xfd, 0xf0, 0x27, 0xff, 0xe6, 0xff, 0x6a, 0x68, 0x1c, 0x30, 0xec, 0xa3, 0xdb, ++ 0xe7, 0x98, 0xb2, 0xcc, 0x50, 0x78, 0x36, 0x9a, 0xfb, 0xcb, 0x0d, 0x57, 0xe1, 0x74, 0x55, 0xc0, ++ 0xef, 0xfc, 0x33, 0x00, 0x00, 0xff, 0xff, 0x45, 0x71, 0x2f, 0x90, 0xc0, 0x23, 0x00, 0x00, + } + + func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { +@@ -4544,6 +4586,45 @@ func (m *UpdateInterfaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) + return len(dAtA) - i, nil + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalToSizedBuffer(dAtA[:size]) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { ++ size := m.Size() ++ return m.MarshalToSizedBuffer(dAtA[:size]) ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { ++ i := len(dAtA) ++ _ = i ++ var l int ++ _ = l ++ if m.XXX_unrecognized != nil { ++ i -= len(m.XXX_unrecognized) ++ copy(dAtA[i:], m.XXX_unrecognized) ++ } ++ if m.Interface != nil { ++ { ++ size, err := m.Interface.MarshalToSizedBuffer(dAtA[:i]) ++ if err != nil { ++ return 0, err ++ } ++ i -= size ++ i = encodeVarintAgent(dAtA, i, uint64(size)) ++ } ++ i-- ++ dAtA[i] = 0xa ++ } ++ return len(dAtA) - i, nil ++} ++ + func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) +@@ -5001,20 +5082,20 @@ func (m *MemHotplugByProbeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.MemHotplugProbeAddr) > 0 { +- dAtA24 := make([]byte, len(m.MemHotplugProbeAddr)*10) +- var j23 int ++ dAtA25 := make([]byte, len(m.MemHotplugProbeAddr)*10) ++ var j24 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { +- dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) ++ dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 +- j23++ ++ j24++ + } +- dAtA24[j23] = uint8(num) +- j23++ ++ dAtA25[j24] = uint8(num) ++ j24++ + } +- i -= j23 +- copy(dAtA[i:], dAtA24[:j23]) +- i = encodeVarintAgent(dAtA, i, uint64(j23)) ++ i -= j24 ++ copy(dAtA[i:], dAtA25[:j24]) ++ i = encodeVarintAgent(dAtA, i, uint64(j24)) + i-- + dAtA[i] = 0xa + } +@@ -6348,6 +6429,22 @@ func (m *UpdateInterfaceRequest) Size() (n int) { + return n + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { ++ if m == nil { ++ return 0 ++ } ++ var l int ++ _ = l ++ if m.Interface != nil { ++ l = m.Interface.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.XXX_unrecognized != nil { ++ n += len(m.XXX_unrecognized) ++ } ++ return n ++} ++ + func (m *UpdateRoutesRequest) Size() (n int) { + if m == nil { + return 0 +@@ -7009,7 +7106,7 @@ func (this *MemoryStats) String() string { + return "nil" + } + keysForStats := make([]string, 0, len(this.Stats)) +- for k := range this.Stats { ++ for k, _ := range this.Stats { + keysForStats = append(keysForStats, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForStats) +@@ -7120,7 +7217,7 @@ func (this *CgroupStats) String() string { + return "nil" + } + keysForHugetlbStats := make([]string, 0, len(this.HugetlbStats)) +- for k := range this.HugetlbStats { ++ for k, _ := range this.HugetlbStats { + keysForHugetlbStats = append(keysForHugetlbStats, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForHugetlbStats) +@@ -7342,6 +7439,17 @@ func (this *UpdateInterfaceRequest) String() string { + }, "") + return s + } ++func (this *UpdateInterfaceHwAddrByNameRequest) String() string { ++ if this == nil { ++ return "nil" ++ } ++ s := strings.Join([]string{`&UpdateInterfaceHwAddrByNameRequest{`, ++ `Interface:` + strings.Replace(fmt.Sprintf("%v", this.Interface), "Interface", "protocols.Interface", 1) + `,`, ++ `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, ++ `}`, ++ }, "") ++ return s ++} + func (this *UpdateRoutesRequest) String() string { + if this == nil { + return "nil" +@@ -7637,6 +7745,7 @@ type AgentServiceService interface { + CloseStdin(ctx context.Context, req *CloseStdinRequest) (*types.Empty, error) + TtyWinResize(ctx context.Context, req *TtyWinResizeRequest) (*types.Empty, error) + UpdateInterface(ctx context.Context, req *UpdateInterfaceRequest) (*protocols.Interface, error) ++ UpdateInterfaceHwAddrByName(ctx context.Context, req *UpdateInterfaceHwAddrByNameRequest) (*protocols.Interface, error) + UpdateRoutes(ctx context.Context, req *UpdateRoutesRequest) (*Routes, error) + ListInterfaces(ctx context.Context, req *ListInterfacesRequest) (*Interfaces, error) + ListRoutes(ctx context.Context, req *ListRoutesRequest) (*Routes, error) +@@ -7769,6 +7878,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag + } + return svc.UpdateInterface(ctx, &req) + }, ++ "UpdateInterfaceHwAddrByName": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { ++ var req UpdateInterfaceHwAddrByNameRequest ++ if err := unmarshal(&req); err != nil { ++ return nil, err ++ } ++ return svc.UpdateInterfaceHwAddrByName(ctx, &req) ++ }, + "UpdateRoutes": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req UpdateRoutesRequest + if err := unmarshal(&req); err != nil { +@@ -8022,6 +8138,14 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, req *UpdateInt + return &resp, nil + } + ++func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, req *UpdateInterfaceHwAddrByNameRequest) (*protocols.Interface, error) { ++ var resp protocols.Interface ++ if err := c.client.Call(ctx, "grpc.AgentService", "UpdateInterfaceHwAddrByName", req, &resp); err != nil { ++ return nil, err ++ } ++ return &resp, nil ++} ++ + func (c *agentServiceClient) UpdateRoutes(ctx context.Context, req *UpdateRoutesRequest) (*Routes, error) { + var resp Routes + if err := c.client.Call(ctx, "grpc.AgentService", "UpdateRoutes", req, &resp); err != nil { +@@ -8408,10 +8532,7 @@ func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -8494,10 +8615,7 @@ func (m *StartContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -8599,10 +8717,7 @@ func (m *RemoveContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -8789,10 +8904,7 @@ func (m *ExecProcessRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -8926,10 +9038,7 @@ func (m *SignalProcessRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9044,10 +9153,7 @@ func (m *WaitProcessRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9117,10 +9223,7 @@ func (m *WaitProcessResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9239,10 +9342,7 @@ func (m *UpdateContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9325,10 +9425,7 @@ func (m *StatsContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9411,10 +9508,7 @@ func (m *PauseContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9497,10 +9591,7 @@ func (m *ResumeContainerRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9684,10 +9775,7 @@ func (m *CpuUsage) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9795,10 +9883,7 @@ func (m *ThrottlingData) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -9921,10 +10006,7 @@ func (m *CpuStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -10013,10 +10095,7 @@ func (m *PidsStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -10143,10 +10222,7 @@ func (m *MemoryData) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -10440,7 +10516,7 @@ func (m *MemoryStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > postIndex { +@@ -10457,10 +10533,7 @@ func (m *MemoryStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -10600,10 +10673,7 @@ func (m *BlkioStatsEntry) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -10926,10 +10996,7 @@ func (m *BlkioStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11037,10 +11104,7 @@ func (m *HugetlbStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11347,7 +11411,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > postIndex { +@@ -11364,10 +11428,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11602,10 +11663,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11726,10 +11784,7 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11878,10 +11933,7 @@ func (m *WriteStreamRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -11951,10 +12003,7 @@ func (m *WriteStreamResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12088,10 +12137,7 @@ func (m *ReadStreamRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12176,10 +12222,7 @@ func (m *ReadStreamResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12294,10 +12337,7 @@ func (m *CloseStdinRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12450,10 +12490,7 @@ func (m *TtyWinResizeRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12568,10 +12605,7 @@ func (m *KernelModule) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12838,10 +12872,7 @@ func (m *CreateSandboxRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12892,10 +12923,7 @@ func (m *DestroySandboxRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -12980,10 +13008,7 @@ func (m *Interfaces) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13068,10 +13093,7 @@ func (m *Routes) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13158,10 +13180,94 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= uint64(b&0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= int(b&0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex < 0 { + return ErrInvalidLengthAgent + } +- if (iNdEx + skippy) < 0 { ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Interface == nil { ++ m.Interface = &protocols.Interface{} ++ } ++ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13248,10 +13354,7 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13302,10 +13405,7 @@ func (m *ListInterfacesRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13356,10 +13456,7 @@ func (m *ListRoutesRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13444,10 +13541,7 @@ func (m *ARPNeighbors) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13534,10 +13628,7 @@ func (m *AddARPNeighborsRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13647,10 +13738,7 @@ func (m *OnlineCPUMemRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13735,10 +13823,7 @@ func (m *ReseedRandomDevRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -13925,10 +14010,7 @@ func (m *AgentDetails) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14019,10 +14101,7 @@ func (m *GuestDetailsRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14148,10 +14227,7 @@ func (m *GuestDetailsResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14278,10 +14354,7 @@ func (m *MemHotplugByProbeRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14370,10 +14443,7 @@ func (m *SetGuestDateTimeRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14616,10 +14686,7 @@ func (m *Storage) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14830,10 +14897,7 @@ func (m *Device) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -14980,10 +15044,7 @@ func (m *StringUser) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15214,10 +15275,7 @@ func (m *CopyFileRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15268,10 +15326,7 @@ func (m *StartTracingRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15322,10 +15377,7 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15376,10 +15428,7 @@ func (m *GetOOMEventRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15462,10 +15511,7 @@ func (m *OOMEvent) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15516,10 +15562,7 @@ func (m *GetMetricsRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +@@ -15602,10 +15645,7 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthAgent +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthAgent + } + if (iNdEx + skippy) > l { +diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go +index 931605fb..7a46e51a 100644 +--- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go ++++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go +@@ -732,10 +732,7 @@ func (m *CheckRequest) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthHealth +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHealth + } + if (iNdEx + skippy) > l { +@@ -805,10 +802,7 @@ func (m *HealthCheckResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthHealth +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHealth + } + if (iNdEx + skippy) > l { +@@ -923,10 +917,7 @@ func (m *VersionCheckResponse) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthHealth +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthHealth + } + if (iNdEx + skippy) > l { +diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go +index 4a7cd2d0..2dc8fd75 100644 +--- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go ++++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go +@@ -6797,7 +6797,7 @@ func (this *Spec) String() string { + } + repeatedStringForMounts += "}" + keysForAnnotations := make([]string, 0, len(this.Annotations)) +- for k := range this.Annotations { ++ for k, _ := range this.Annotations { + keysForAnnotations = append(keysForAnnotations, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations) +@@ -6996,7 +6996,7 @@ func (this *Linux) String() string { + } + repeatedStringForDevices += "}" + keysForSysctl := make([]string, 0, len(this.Sysctl)) +- for k := range this.Sysctl { ++ for k, _ := range this.Sysctl { + keysForSysctl = append(keysForSysctl, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForSysctl) +@@ -7717,10 +7717,7 @@ func (m *Spec) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > postIndex { +@@ -7845,10 +7842,7 @@ func (m *Spec) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -8257,10 +8251,7 @@ func (m *Process) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -8349,10 +8340,7 @@ func (m *Box) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -8549,10 +8537,7 @@ func (m *User) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -8763,10 +8748,7 @@ func (m *LinuxCapabilities) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -8887,10 +8869,7 @@ func (m *POSIXRlimit) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -9069,10 +9048,7 @@ func (m *Mount) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -9175,10 +9151,7 @@ func (m *Root) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -9331,10 +9304,7 @@ func (m *Hooks) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -9500,10 +9470,7 @@ func (m *Hook) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -9732,10 +9699,7 @@ func (m *Linux) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > postIndex { +@@ -10088,10 +10052,7 @@ func (m *Linux) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -10174,10 +10135,7 @@ func (m *Windows) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -10260,10 +10218,7 @@ func (m *Solaris) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -10371,10 +10326,7 @@ func (m *LinuxIDMapping) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -10489,10 +10441,7 @@ func (m *LinuxNamespace) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -10702,10 +10651,7 @@ func (m *LinuxDevice) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11004,10 +10950,7 @@ func (m *LinuxResources) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11192,10 +11135,7 @@ func (m *LinuxMemory) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11405,10 +11345,7 @@ func (m *LinuxCPU) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11535,10 +11472,7 @@ func (m *LinuxWeightDevice) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11646,10 +11580,7 @@ func (m *LinuxThrottleDevice) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11908,10 +11839,7 @@ func (m *LinuxBlockIO) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -11981,10 +11909,7 @@ func (m *LinuxPids) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12157,10 +12082,7 @@ func (m *LinuxDeviceCgroup) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12264,10 +12186,7 @@ func (m *LinuxNetwork) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12369,10 +12288,7 @@ func (m *LinuxHugepageLimit) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12474,10 +12390,7 @@ func (m *LinuxInterfacePriority) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12658,10 +12571,7 @@ func (m *LinuxSeccomp) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12801,10 +12711,7 @@ func (m *LinuxSeccompArg) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -12973,10 +12880,7 @@ func (m *LinuxSyscall) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +@@ -13059,10 +12963,7 @@ func (m *LinuxIntelRdt) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthOci +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOci + } + if (iNdEx + skippy) > l { +diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go +index 497cb55a..263afa8f 100644 +--- a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go ++++ b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go +@@ -865,10 +865,7 @@ func (m *IPAddress) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthTypes +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { +@@ -1151,10 +1148,7 @@ func (m *Interface) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthTypes +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { +@@ -1352,10 +1346,7 @@ func (m *Route) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthTypes +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { +@@ -1544,10 +1535,7 @@ func (m *ARPNeighbor) Unmarshal(dAtA []byte) error { + if err != nil { + return err + } +- if skippy < 0 { +- return ErrInvalidLengthTypes +- } +- if (iNdEx + skippy) < 0 { ++ if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { +diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go +index bec736d6..0cf4a253 100644 +--- a/src/runtime/virtcontainers/pkg/mock/mock.go ++++ b/src/runtime/virtcontainers/pkg/mock/mock.go +@@ -147,6 +147,10 @@ func (p *HybridVSockTTRPCMockImp) UpdateInterface(ctx context.Context, req *pb.U + return &aTypes.Interface{}, nil + } + ++func (p *HybridVSockTTRPCMockImp) UpdateInterfaceHwAddrByName(ctx context.Context, req *pb.UpdateInterfaceHwAddrByNameRequest) (*aTypes.Interface, error) { ++ return &aTypes.Interface{}, nil ++} ++ + func (p *HybridVSockTTRPCMockImp) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { + return &pb.Routes{}, nil + } +diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go +index dc4d3bf7..408b23ed 100644 +--- a/src/runtime/virtcontainers/sandbox.go ++++ b/src/runtime/virtcontainers/sandbox.go +@@ -855,6 +855,11 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb + + // Add network for vm + inf.PciPath = endpoint.PciPath().String() ++ ++ // Update MAC address for network ++ if _, err := s.agent.updateInterfaceHwAddrByName(ctx, inf); err != nil { ++ return nil, err ++ } + return s.agent.updateInterface(ctx, inf) + } + +diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go +index 4fec96d3..0f473e31 100644 +--- a/src/runtime/virtcontainers/stratovirt.go ++++ b/src/runtime/virtcontainers/stratovirt.go +@@ -345,7 +345,7 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { + return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) + } + +-func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { ++func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) { + err = s.qmpSetup() + if err != nil { + return err +@@ -464,7 +464,7 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, + + switch devType { + case netDev: +- return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) ++ return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice) + case blockDev: + return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) + default: +@@ -478,7 +478,7 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ + + switch devType { + case netDev: +- return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) ++ return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice) + case blockDev: + return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) + default: +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0004-configuration-add-configuration-generator-for-hyperv.patch b/patches/0004-configuration-add-configuration-generator-for-hyperv.patch new file mode 100644 index 0000000000000000000000000000000000000000..4279ed77daa14acb3e4f826429235451a0d7b043 --- /dev/null +++ b/patches/0004-configuration-add-configuration-generator-for-hyperv.patch @@ -0,0 +1,542 @@ +From 950c0db14a9a9baccefd83e87893d7f40c2bd13d Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Mon, 9 Aug 2021 14:47:19 +0800 +Subject: [PATCH 4/6] configuration: add configuration generator for hypervisor + type stratovirt. + +Signed-off-by: Wei Gao +--- + src/runtime/.gitignore | 1 + + src/runtime/Makefile | 40 +- + src/runtime/arch/amd64-options.mk | 3 + + src/runtime/arch/arm64-options.mk | 3 + + .../config/configuration-stratovirt.toml.in | 356 ++++++++++++++++++ + 5 files changed, 402 insertions(+), 1 deletion(-) + create mode 100644 src/runtime/cli/config/configuration-stratovirt.toml.in + +diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore +index 52b9e4e5..0a630a07 100644 +--- a/src/runtime/.gitignore ++++ b/src/runtime/.gitignore +@@ -10,6 +10,7 @@ coverage.html + /cli/config/configuration-fc.toml + /cli/config/configuration-qemu.toml + /cli/config/configuration-clh.toml ++/cli/config/configuration-stratovirt.toml + /cli/config-generated.go + /cli/containerd-shim-kata-v2/config-generated.go + /cli/coverage.html +diff --git a/src/runtime/Makefile b/src/runtime/Makefile +index 4a69c05c..ea2cd296 100644 +--- a/src/runtime/Makefile ++++ b/src/runtime/Makefile +@@ -74,6 +74,7 @@ QEMUBINDIR := $(PREFIXDEPS)/bin + CLHBINDIR := $(PREFIXDEPS)/bin + FCBINDIR := $(PREFIXDEPS)/bin + ACRNBINDIR := $(PREFIXDEPS)/bin ++STRATOVIRTBINDIR := $(PREFIXDEPS)/bin + SYSCONFDIR := /etc + LOCALSTATEDIR := /var + +@@ -93,6 +94,7 @@ GENERATED_VARS = \ + CONFIG_QEMU_IN \ + CONFIG_CLH_IN \ + CONFIG_FC_IN \ ++ CONFIG_STRATOVIRT_IN \ + $(USER_VARS) + SCRIPTS += $(COLLECT_SCRIPT) + SCRIPTS_DIR := $(BINDIR) +@@ -116,12 +118,13 @@ HYPERVISOR_ACRN = acrn + HYPERVISOR_FC = firecracker + HYPERVISOR_QEMU = qemu + HYPERVISOR_CLH = cloud-hypervisor ++HYPERVISOR_STRATOVIRT = stratovirt + + # Determines which hypervisor is specified in $(CONFIG_FILE). + DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU) + + # List of hypervisors this build system can generate configuration for. +-HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) ++HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_STRATOVIRT) + + QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) + QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"] +@@ -141,6 +144,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"] + ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) + ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] + ++STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) ++STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] ++ + NETMONCMD := $(BIN_PREFIX)-netmon + NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) + +@@ -267,6 +273,29 @@ ifneq (,$(CLHCMD)) + KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) + endif + ++ifneq (,$(STRATOVIRTCMD)) ++ KNOWN_HYPERVISORS += $(HYPERVISOR_STRATOVIRT) ++ ++ CONFIG_FILE_STRATOVIRT = configuration-stratovirt.toml ++ CONFIG_STRATOVIRT = $(CLI_DIR)/config/$(CONFIG_FILE_STRATOVIRT) ++ CONFIG_STRATOVIRT_IN = $(CONFIG_STRATOVIRT).in ++ ++ CONFIG_PATH_STRATOVIRT = $(abspath $(CONFDIR)/$(CONFIG_FILE_STRATOVIRT)) ++ CONFIG_PATHS += $(CONFIG_PATH_STRATOVIRT) ++ ++ SYSCONFIG_STRATOVIRT = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_STRATOVIRT)) ++ SYSCONFIG_PATHS += $(SYSCONFIG_STRATOVIRT) ++ ++ CONFIGS += $(CONFIG_STRATOVIRT) ++ ++ # stratovirt-specific options (all should be suffixed by "_STRATOVIRT") ++ DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio ++ DEFNETWORKMODEL_STRATOVIRT := none ++ KENRELTYPE_STRATOVIRT = uncompressed ++ KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT)) ++ KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT) ++endif ++ + ifneq (,$(FCCMD)) + KNOWN_HYPERVISORS += $(HYPERVISOR_FC) + +@@ -363,6 +392,7 @@ USER_VARS += BINDIR + USER_VARS += CONFIG_ACRN_IN + USER_VARS += CONFIG_CLH_IN + USER_VARS += CONFIG_FC_IN ++USER_VARS += CONFIG_STRATOVIRT_IN + USER_VARS += CONFIG_PATH + USER_VARS += CONFIG_QEMU_IN + USER_VARS += DESTDIR +@@ -382,6 +412,8 @@ USER_VARS += FCPATH + USER_VARS += FCVALIDHYPERVISORPATHS + USER_VARS += FCJAILERPATH + USER_VARS += FCVALIDJAILERPATHS ++USER_VARS += STRATOVIRTPATH ++USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS + USER_VARS += SYSCONFIG + USER_VARS += IMAGENAME + USER_VARS += IMAGEPATH +@@ -395,6 +427,7 @@ USER_VARS += KERNELPATH_ACRN + USER_VARS += KERNELPATH + USER_VARS += KERNELPATH_CLH + USER_VARS += KERNELPATH_FC ++USER_VARS += KERNELPATH_STRATOVIRT + USER_VARS += KERNELVIRTIOFSPATH + USER_VARS += FIRMWAREPATH + USER_VARS += MACHINEACCELERATORS +@@ -434,12 +467,14 @@ USER_VARS += DEFNETWORKMODEL_ACRN + USER_VARS += DEFNETWORKMODEL_CLH + USER_VARS += DEFNETWORKMODEL_FC + USER_VARS += DEFNETWORKMODEL_QEMU ++USER_VARS += DEFNETWORKMODEL_STRATOVIRT + USER_VARS += DEFDISABLEGUESTSECCOMP + USER_VARS += DEFAULTEXPFEATURES + USER_VARS += DEFDISABLEBLOCK + USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN + USER_VARS += DEFBLOCKSTORAGEDRIVER_FC + USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU ++USER_VARS += DEFBLOCKSTORAGEDRIVER_STRATOVIRT + USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS + USER_VARS += DEFVIRTIOFSDAEMON + USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS +@@ -773,6 +808,9 @@ ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) + endif + ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) + @printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH)) ++endif ++ifneq (,$(findstring $(HYPERVISOR_STRATOVIRT),$(KNOWN_HYPERVISORS))) ++ @printf "\t$(HYPERVISOR_STRATOVIRT) hypervisor path (STRATOVIRTPATH) : %s\n" $(abspath $(STRATOVIRTPATH)) + endif + @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) + @printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) +diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk +index 83af8cc0..ff2af9e6 100644 +--- a/src/runtime/arch/amd64-options.mk ++++ b/src/runtime/arch/amd64-options.mk +@@ -23,3 +23,6 @@ ACRNCTLCMD := acrnctl + + # cloud-hypervisor binary name + CLHCMD := cloud-hypervisor ++ ++# stratovirt binary name ++STRATOVIRTCMD := stratovirt +\ No newline at end of file +diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk +index ad5ef5d4..2ad3f657 100644 +--- a/src/runtime/arch/arm64-options.mk ++++ b/src/runtime/arch/arm64-options.mk +@@ -19,3 +19,6 @@ FCJAILERCMD := jailer + + # cloud-hypervisor binary name + CLHCMD := cloud-hypervisor ++ ++# stratovirt binary name ++STRATOVIRTCMD := stratovirt +\ No newline at end of file +diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in +new file mode 100644 +index 00000000..5c83c3c9 +--- /dev/null ++++ b/src/runtime/cli/config/configuration-stratovirt.toml.in +@@ -0,0 +1,356 @@ ++# Copyright (c) 2017-2019 Intel Corporation ++# ++# SPDX-License-Identifier: Apache-2.0 ++# ++ ++# XXX: WARNING: this file is auto-generated. ++# XXX: ++# XXX: Source file: "@CONFIG_STRATOVIRT_IN@" ++# XXX: Project: ++# XXX: Name: @PROJECT_NAME@ ++# XXX: Type: @PROJECT_TYPE@ ++ ++[hypervisor.stratovirt] ++path = "@STRATOVIRTPATH@" ++kernel = "@KERNELPATH_STRATOVIRT@" ++image = "@IMAGEPATH@" ++ ++# List of valid annotation names for the hypervisor ++# Each member of the list is a regular expression, which is the base name ++# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" ++enable_annotations = @DEFENABLEANNOTATIONS@ ++ ++# List of valid annotations values for the hypervisor ++# Each member of the list is a path pattern as described by glob(3). ++# The default if not set is empty (all annotations rejected.) ++# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@ ++valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ ++ ++# Optional space-separated list of options to pass to the guest kernel. ++# For example, use `kernel_params = "vsyscall=emulate"` if you are having ++# trouble running pre-2.15 glibc. ++# ++# WARNING: - any parameter specified here will take priority over the default ++# parameter value of the same name used to start the virtual machine. ++# Do not set values here unless you understand the impact of doing so as you ++# may stop the virtual machine from booting. ++# To see the list of default parameters, enable hypervisor debug, create a ++# container and look for 'default-kernel-parameters' log entries. ++kernel_params = "@KERNELPARAMS@" ++ ++# Default number of vCPUs per SB/VM: ++# unspecified or 0 --> will be set to @DEFVCPUS@ ++# < 0 --> will be set to the actual number of physical cores ++# > 0 <= number of physical cores --> will be set to the specified number ++# > number of physical cores --> will be set to the actual number of physical cores ++default_vcpus = 1 ++ ++# Default maximum number of vCPUs per SB/VM: ++# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number ++# of vCPUs supported by KVM if that number is exceeded ++# > 0 <= number of physical cores --> will be set to the specified number ++# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number ++# of vCPUs supported by KVM if that number is exceeded ++# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when ++# the actual number of physical cores is greater than it. ++# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU ++# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs ++# can be added to a SB/VM, but the memory footprint will be big. Another example, with ++# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of ++# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, ++# unless you know what are you doing. ++# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. ++default_maxvcpus = @DEFMAXVCPUS@ ++ ++# Bridges can be used to hot plug devices. ++# Limitations: ++# * Currently only pci bridges are supported ++# * Until 30 devices per bridge can be hot plugged. ++# * Until 5 PCI bridges can be cold plugged per VM. ++# This limitation could be a bug in the kernel ++# Default number of bridges per SB/VM: ++# unspecified or 0 --> will be set to @DEFBRIDGES@ ++# > 1 <= 5 --> will be set to the specified number ++# > 5 --> will be set to 5 ++default_bridges = @DEFBRIDGES@ ++ ++# Default memory size in MiB for SB/VM. ++# If unspecified then it will be set @DEFMEMSZ@ MiB. ++default_memory = @DEFMEMSZ@ ++# ++# Default memory slots per SB/VM. ++# If unspecified then it will be set @DEFMEMSLOTS@. ++# This is will determine the times that memory will be hotadded to sandbox/VM. ++# memory_slots = @DEFMEMSLOTS@ ++ ++# The size in MiB will be plused to max memory of hypervisor. ++# It is the memory address space for the NVDIMM devie. ++# If set block storage driver (block_device_driver) to "nvdimm", ++# should set memory_offset to the size of block device. ++# Default 0 ++# memory_offset = 0 ++ ++# Disable block device from being used for a container's rootfs. ++# In case of a storage driver like devicemapper where a container's ++# root file system is backed by a block device, the block device is passed ++# directly to the hypervisor for performance reasons. ++# This flag prevents the block device from being passed to the hypervisor, ++# 9pfs is used instead to pass the rootfs. ++disable_block_device_use = @DEFDISABLEBLOCK@ ++ ++# Block storage driver to be used for the hypervisor in case the container ++# rootfs is backed by a block device. This is virtio-scsi, virtio-blk ++# or nvdimm. ++block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" ++ ++# Specifies cache-related options will be set to block devices or not. ++# Default false ++#block_device_cache_set = true ++ ++# Specifies cache-related options for block devices. ++# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. ++# Default false ++# block_device_cache_direct = true ++ ++# Specifies cache-related options for block devices. ++# Denotes whether flush requests for the device are ignored. ++# Default false ++# block_device_cache_noflush = true ++ ++# Enable pre allocation of VM RAM, default false ++# Enabling this will result in lower container density ++# as all of the memory will be allocated and locked ++# This is useful when you want to reserve all the memory ++# upfront or in the cases where you want memory latencies ++# to be very predictable ++# Default false ++# enable_mem_prealloc = true ++ ++# Enable huge pages for VM RAM, default false ++# Enabling this will result in the VM memory ++# being allocated using huge pages. ++# This is useful when you want to use vhost-user network ++# stacks within the container. This will automatically ++# result in memory pre allocation ++# enable_hugepages = true ++ ++# Enable vIOMMU, default false ++# Enabling this will result in the VM having a vIOMMU device ++# This will also add the following options to the kernel's ++# command line: intel_iommu=on,iommu=pt ++# enable_iommu = true ++ ++# Enable swap of vm memory. Default false. ++# The behaviour is undefined if mem_prealloc is also set to true ++# enable_swap = true ++ ++# This option changes the default hypervisor and kernel parameters ++# to enable debug output where available. ++# ++# Default false ++# enable_debug = true ++ ++# Disable the customizations done in the runtime when it detects ++# that it is running on top a VMM. This will result in the runtime ++# behaving as it would when running on bare metal. ++# ++# disable_nesting_checks = true ++ ++# This is the msize used for 9p shares. It is the number of bytes ++# used for 9p packet payload. ++# msize_9p = ++ ++# VFIO devices are hotplugged on a bridge by default. ++# Enable hotplugging on root bus. This may be required for devices with ++# a large PCI bar, as this is a current limitation with hotplugging on ++# a bridge. ++# Default false ++# hotplug_vfio_on_root_bus = true ++ ++# ++# Default entropy source. ++# The path to a host source of entropy (including a real hardware RNG) ++# /dev/urandom and /dev/random are two main options. ++# Be aware that /dev/random is a blocking source of entropy. If the host ++# runs out of entropy, the VMs boot time will increase leading to get startup ++# timeouts. ++# The source of entropy /dev/urandom is non-blocking and provides a ++# generally acceptable source of entropy. It should work well for pretty much ++# all practical purposes. ++# entropy_source= "" ++ ++# List of valid annotations values for entropy_source ++# The default if not set is empty (all annotations rejected.) ++# Your distribution recommends: @DEFVALIDENTROPYSOURCES@ ++valid_entropy_sources = @DEFVALIDENTROPYSOURCES@ ++ ++# Path to OCI hook binaries in the *guest rootfs*. ++# This does not affect host-side hooks which must instead be added to ++# the OCI spec passed to the runtime. ++# ++# You can create a rootfs with hooks by customizing the osbuilder scripts: ++# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder ++# ++# Hooks must be stored in a subdirectory of guest_hook_path according to their ++# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". ++# The agent will scan these directories for executable files and add them, in ++# lexicographical order, to the lifecycle of the guest container. ++# Hooks are executed in the runtime namespace of the guest. See the official documentation: ++# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks ++# Warnings will be logged if any error is encountered will scanning for hooks, ++# but it will not abort container execution. ++# guest_hook_path = "/usr/share/oci/hooks" ++ ++[factory] ++# VM templating support. Once enabled, new VMs are created from template ++# using vm cloning. They will share the same initial kernel, initramfs and ++# agent memory by mapping it readonly. It helps speeding up new container ++# creation and saves a lot of memory if there are many kata containers running ++# on the same host. ++# ++# When disabled, new VMs are created from scratch. ++# ++# Note: Requires "initrd=" to be set ("image=" is not supported). ++# ++# Default false ++#enable_template = true ++ ++[agent.@PROJECT_TYPE@] ++# If enabled, make the agent display debug-level messages. ++# (default: disabled) ++#enable_debug = true ++ ++# Enable agent tracing. ++# ++# If enabled, the default trace mode is "dynamic" and the ++# default trace type is "isolated". The trace mode and type are set ++# explicity with the `trace_type=` and `trace_mode=` options. ++# ++# Notes: ++# ++# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly ++# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` ++# will NOT activate agent tracing. ++# ++# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for ++# full details. ++# ++# (default: disabled) ++#enable_tracing = true ++# ++#trace_mode = "dynamic" ++#trace_type = "isolated" ++ ++# Comma separated list of kernel modules and their parameters. ++# These modules will be loaded in the guest kernel using modprobe(8). ++# The following example can be used to load two kernel modules with parameters ++# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] ++# The first word is considered as the module name and the rest as its parameters. ++# Container will not be started when: ++# * A kernel module is specified and the modprobe command is not installed in the guest ++# or it fails loading the module. ++# * The module is not available in the guest or it doesn't met the guest kernel ++# requirements, like architecture and version. ++# ++kernel_modules=[] ++ ++# Enable debug console. ++ ++# If enabled, user can connect guest OS running inside hypervisor ++# through "kata-runtime exec " command ++ ++#debug_console_enabled = true ++ ++# Agent connection dialing timeout value in seconds ++# (default: 30) ++#dial_timeout = 30 ++ ++[netmon] ++# If enabled, the network monitoring process gets started when the ++# sandbox is created. This allows for the detection of some additional ++# network being added to the existing network namespace, after the ++# sandbox has been created. ++# (default: disabled) ++#enable_netmon = true ++ ++# Specify the path to the netmon binary. ++path = "@NETMONPATH@" ++ ++# If enabled, netmon messages will be sent to the system log ++# (default: disabled) ++#enable_debug = true ++ ++[runtime] ++# If enabled, the runtime will log additional debug messages to the ++# system log ++# (default: disabled) ++#enable_debug = true ++# ++# Internetworking model ++# Determines how the VM should be connected to the ++# the container network interface ++# Options: ++# ++# - macvtap ++# Used when the Container network interface can be bridged using ++# macvtap. ++# ++# - none ++# Used when customize network. Only creates a tap device. No veth pair. ++# ++# - tcfilter ++# Uses tc filter rules to redirect traffic from the network interface ++# provided by plugin to a tap interface connected to the VM. ++# ++internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@" ++ ++# disable guest seccomp ++# Determines whether container seccomp profiles are passed to the virtual ++# machine and applied by the kata agent. If set to true, seccomp is not applied ++# within the guest ++# (default: true) ++disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ ++ ++# If enabled, the runtime will create opentracing.io traces and spans. ++# (See https://www.jaegertracing.io/docs/getting-started). ++# (default: disabled) ++#enable_tracing = true ++ ++# Set the full url to the Jaeger HTTP Thrift collector. ++# The default if not set will be "http://localhost:14268/api/traces" ++#jaeger_endpoint = "" ++ ++# Sets the username to be used if basic auth is required for Jaeger. ++#jaeger_user = "" ++ ++# Sets the password to be used if basic auth is required for Jaeger. ++#jaeger_password = "" ++ ++# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. ++# This option may have some potential impacts to your host. It should only be used when you know what you're doing. ++# `disable_new_netns` conflicts with `enable_netmon` ++# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only ++# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge ++# (like OVS) directly. ++# If you are using docker, `disable_new_netns` only works with `docker run --net=none` ++# (default: false) ++#disable_new_netns = true ++ ++# if enable, the runtime will add all the kata processes inside one dedicated cgroup. ++# The container cgroups in the host are not created, just one single cgroup per sandbox. ++# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. ++# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. ++# The sandbox cgroup is constrained if there is no container type annotation. ++# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType ++sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ ++ ++# Enabled experimental feature list, format: ["a", "b"]. ++# Experimental features are features not stable enough for production, ++# they may break compatibility, and are prepared for a big version bump. ++# Supported experimental features: ++# (default: []) ++experimental=@DEFAULTEXPFEATURES@ ++ ++# If enabled, user can run pprof tools with shim v2 process through kata-monitor. ++# (default: false) ++# enable_pprof = true +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch b/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch new file mode 100644 index 0000000000000000000000000000000000000000..a8727ba7f3ee415d9bddc0b0a35ad9b0f7483811 --- /dev/null +++ b/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch @@ -0,0 +1,655 @@ +From 45c8e108497eb93d69afd38e6281b837e65cf3ec Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Mon, 9 Aug 2021 14:55:41 +0800 +Subject: [PATCH 5/6] runtime: add the secure component "ozone" support for + hypervisor type stratovirt. + +Signed-off-by: Wei Gao +--- + src/runtime/Makefile | 4 + + src/runtime/arch/amd64-options.mk | 4 +- + src/runtime/arch/arm64-options.mk | 4 +- + .../config/configuration-stratovirt.toml.in | 10 + + .../pkg/katautils/config-settings.go.in | 1 + + src/runtime/pkg/katautils/config.go | 18 + + src/runtime/virtcontainers/hypervisor.go | 3 + + src/runtime/virtcontainers/persist.go | 1 + + .../virtcontainers/persist/api/config.go | 3 + + src/runtime/virtcontainers/stratovirt.go | 309 ++++++++++++++---- + 10 files changed, 292 insertions(+), 65 deletions(-) + +diff --git a/src/runtime/Makefile b/src/runtime/Makefile +index ea2cd296..745bcc10 100644 +--- a/src/runtime/Makefile ++++ b/src/runtime/Makefile +@@ -146,6 +146,8 @@ ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] + + STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) + STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] ++STRATOVIRTOZONEPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTOZONECMD) ++STRATOVIRTVALIDOZONEPATHS = [\"$(STRATOVIRTOZONEPATH)\"] + + NETMONCMD := $(BIN_PREFIX)-netmon + NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) +@@ -414,6 +416,8 @@ USER_VARS += FCJAILERPATH + USER_VARS += FCVALIDJAILERPATHS + USER_VARS += STRATOVIRTPATH + USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS ++USER_VARS += STRATOVIRTOZONEPATH ++USER_VARS += STRATOVIRTVALIDOZONEPATHS + USER_VARS += SYSCONFIG + USER_VARS += IMAGENAME + USER_VARS += IMAGEPATH +diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk +index ff2af9e6..4c6c329a 100644 +--- a/src/runtime/arch/amd64-options.mk ++++ b/src/runtime/arch/amd64-options.mk +@@ -25,4 +25,6 @@ ACRNCTLCMD := acrnctl + CLHCMD := cloud-hypervisor + + # stratovirt binary name +-STRATOVIRTCMD := stratovirt +\ No newline at end of file ++STRATOVIRTCMD := stratovirt ++# stratovirt's ozone binary name ++STRATOVIRTOZONECMD := ozone +\ No newline at end of file +diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk +index 2ad3f657..5dfa2c80 100644 +--- a/src/runtime/arch/arm64-options.mk ++++ b/src/runtime/arch/arm64-options.mk +@@ -21,4 +21,6 @@ FCJAILERCMD := jailer + CLHCMD := cloud-hypervisor + + # stratovirt binary name +-STRATOVIRTCMD := stratovirt +\ No newline at end of file ++STRATOVIRTCMD := stratovirt ++# stratovirt's ozone binary name ++STRATOVIRTOZONECMD := ozone +\ No newline at end of file +diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in +index 5c83c3c9..b557b71f 100644 +--- a/src/runtime/cli/config/configuration-stratovirt.toml.in ++++ b/src/runtime/cli/config/configuration-stratovirt.toml.in +@@ -26,6 +26,16 @@ enable_annotations = @DEFENABLEANNOTATIONS@ + # Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@ + valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ + ++# Path for the ozone specific to stratovirt ++# If the ozone path is set, stratovirt will be launched in ++# ozone secure environment. It is disabled by default. ++# ozone_path = "@STRATOVIRTOZONEPATH@" ++ ++# List of valid ozone path values for the hypervisor ++# Each member of the list can be a regular expression ++# The default if not set is empty (all annotations rejected.) ++# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@ ++ + # Optional space-separated list of options to pass to the guest kernel. + # For example, use `kernel_params = "vsyscall=emulate"` if you are having + # trouble running pre-2.15 glibc. +diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in +index 7cd9138b..c168c608 100644 +--- a/src/runtime/pkg/katautils/config-settings.go.in ++++ b/src/runtime/pkg/katautils/config-settings.go.in +@@ -17,6 +17,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" + var defaultFirmwarePath = "" + var defaultMachineAccelerators = "" + var defaultCPUFeatures = "" ++var defaultOzonePath = "/usr/bin/ozone" + var systemdUnitName = "kata-containers.target" + + const defaultKernelParams = "" +diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go +index f94ac4fd..828c2a43 100644 +--- a/src/runtime/pkg/katautils/config.go ++++ b/src/runtime/pkg/katautils/config.go +@@ -92,6 +92,7 @@ type hypervisor struct { + FileBackedMemRootDir string `toml:"file_mem_backend"` + GuestHookPath string `toml:"guest_hook_path"` + GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` ++ OzonePath string `toml:"ozone_path"` + HypervisorPathList []string `toml:"valid_hypervisor_paths"` + JailerPathList []string `toml:"valid_jailer_paths"` + CtlPathList []string `toml:"valid_ctlpaths"` +@@ -452,6 +453,16 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) + return + } + ++func (h hypervisor) ozonePath() (string, error) { ++ p := h.OzonePath ++ ++ if h.OzonePath == "" { ++ return "", nil ++ } ++ ++ return ResolvePath(p) ++} ++ + func (h hypervisor) getRxRateLimiterCfg() uint64 { + return h.RxRateLimiterMaxRate + } +@@ -877,6 +888,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + return vc.HypervisorConfig{}, err + } + ++ ozone, err := h.ozonePath() ++ if err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ + kernel, err := h.kernel() + if err != nil { + return vc.HypervisorConfig{}, err +@@ -925,6 +941,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + KernelPath: kernel, + InitrdPath: initrd, + ImagePath: image, ++ OzonePath: ozone, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + HypervisorMachineType: machineType, + NumVCPUs: h.defaultVCPUs(), +@@ -1155,6 +1172,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { + RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate, + TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate, + SGXEPCSize: defaultSGXEPCSize, ++ OzonePath: defaultOzonePath, + } + } + +diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go +index 615baa80..04e14b4e 100644 +--- a/src/runtime/virtcontainers/hypervisor.go ++++ b/src/runtime/virtcontainers/hypervisor.go +@@ -302,6 +302,9 @@ type HypervisorConfig struct { + // JailerPathList is the list of jailer paths names allowed in annotations + JailerPathList []string + ++ // OzonePath is the ozone executable host path. ++ OzonePath string ++ + // BlockDeviceDriver specifies the driver to be used for block device + // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver + BlockDeviceDriver string +diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go +index 203495e8..ae499c97 100644 +--- a/src/runtime/virtcontainers/persist.go ++++ b/src/runtime/virtcontainers/persist.go +@@ -219,6 +219,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList, + JailerPath: sconfig.HypervisorConfig.JailerPath, + JailerPathList: sconfig.HypervisorConfig.JailerPathList, ++ OzonePath: sconfig.HypervisorConfig.OzonePath, + BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, + HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType, + MemoryPath: sconfig.HypervisorConfig.MemoryPath, +diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go +index 3bd5567d..88903723 100644 +--- a/src/runtime/virtcontainers/persist/api/config.go ++++ b/src/runtime/virtcontainers/persist/api/config.go +@@ -76,6 +76,9 @@ type HypervisorConfig struct { + // JailerPathList is the list of jailer paths names allowed in annotations + JailerPathList []string + ++ // OzonePath is the ozone executable host path. ++ OzonePath string ++ + // BlockDeviceDriver specifies the driver to be used for block device + // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver + BlockDeviceDriver string +diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go +index 0f473e31..47daa817 100644 +--- a/src/runtime/virtcontainers/stratovirt.go ++++ b/src/runtime/virtcontainers/stratovirt.go +@@ -3,6 +3,7 @@ package virtcontainers + import ( + "context" + "fmt" ++ "io/ioutil" + "os" + "os/exec" + "path/filepath" +@@ -24,10 +25,15 @@ import ( + otelTrace "go.opentelemetry.io/otel/trace" + ) + +-const defaultDummyMac = "22:33:44:aa:bb:" +-const mmioBlkCount = 4 +-const mmioNetCount = 2 +-const randomDevice = "/dev/urandom" ++const ( ++ apiSocket = "qmp.socket" ++ debugSocket = "console.socket" ++ ozoneBaseDir = "/srv/ozone/stratovirt" ++ defaultDummyMac = "22:33:44:aa:bb:" ++ mmioBlkCount = 4 ++ mmioNetCount = 2 ++ randomDevice = "/dev/urandom" ++) + + type stratovirtDev struct { + dev interface{} +@@ -40,10 +46,19 @@ type stratovirt struct { + sandbox *Sandbox + store persistapi.PersistDriver + config HypervisorConfig ++ rootfsPath string ++ kernelPath string + pid int + consolePath string + socketPath string ++ netNSPath string + qmpMonitorCh qmpChannel ++ ozoneRoot string ++ ozoneRes []string ++ useOzone bool ++ useImage bool ++ pidfile string ++ logfile string + devices []stratovirtDev + HotpluggedVCPUs []CPUDevice + mmioBlkSlots [mmioBlkCount]bool +@@ -66,10 +81,10 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, + return span, ctx + } + +-func (s *stratovirt) getKernelCmdLine(useImage bool) string { ++func (s *stratovirt) getKernelCmdLine() string { + var params []string + +- if useImage { ++ if s.useImage { + params = append(params, "root=/dev/vda") + } + +@@ -100,14 +115,49 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net + + s.id = id + s.config = *hypervisorConfig +- +- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") +- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") ++ if s.config.OzonePath == "" { ++ s.useOzone = false ++ s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") ++ s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") ++ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket) ++ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket) ++ } else { ++ s.useOzone = true ++ s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id) ++ s.pidfile = filepath.Join(s.ozoneRoot, "pid") ++ s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log") ++ s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) ++ s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) ++ } ++ s.netNSPath = networkNS.NetNsPath + s.qmpMonitorCh = qmpChannel{ + ctx: s.ctx, + path: s.socketPath, + } + ++ if kernelPath, err := s.config.KernelAssetPath(); err == nil { ++ s.kernelPath = kernelPath ++ s.ozoneRes = append(s.ozoneRes, s.kernelPath) ++ } ++ ++ initrdPath, err := s.config.InitrdAssetPath() ++ if err != nil { ++ return err ++ } ++ ++ if initrdPath == "" { ++ imagePath, err := s.config.ImageAssetPath() ++ if err != nil { ++ return err ++ } ++ s.useImage = true ++ s.rootfsPath = imagePath ++ } else { ++ s.useImage = false ++ s.rootfsPath = initrdPath ++ } ++ s.ozoneRes = append(s.ozoneRes, s.rootfsPath) ++ + return nil + } + +@@ -134,48 +184,43 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error { + return nil + } + +-func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { +- span, _ := s.trace(ctx, "startSandbox") +- defer span.End() +- ++func (s *stratovirt) createbaseParams() []string { + var params []string +- var use_image bool ++ + params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) +- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) ++ params = append(params, "-append", s.getKernelCmdLine()) ++ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) ++ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) ++ params = append(params, "-device", "virtio-serial-device") ++ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") ++ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) ++ params = append(params, "-device", "virtio-rng-device,rng=objrng0") + +- if kernelPath, err := s.config.KernelAssetPath(); err == nil { +- params = append(params, "-kernel", kernelPath) +- } ++ // daemonize ++ params = append(params, "-daemonize") + +- initrdPath, err := s.config.InitrdAssetPath() +- if err != nil { +- return err ++ return params ++} ++ ++func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { ++ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket)) ++ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket)) ++ params = append(params, "-kernel", filepath.Base(s.kernelPath)) ++ params = append(params, "-pidfile", filepath.Base(s.pidfile)) ++ ++ // append logfile only on debug ++ if s.config.Debug { ++ params = append(params, "-D", filepath.Base(s.logfile)) + } + +- if initrdPath == "" { +- imagePath, err := s.config.ImageAssetPath() +- if err != nil { +- return err +- } +- use_image = true ++ if s.useImage { + s.mmioBlkSlots[0] = true + params = append(params, "-device", "virtio-blk-device,drive=rootfs") +- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath)) ++ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath))) + } else { +- use_image = false +- params = append(params, "-initrd", initrdPath) ++ params = append(params, "-initrd", filepath.Base(s.rootfsPath)) + } + +- params = append(params, "-append", s.getKernelCmdLine(use_image)) +- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) +- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) +- params = append(params, "-device", "virtio-serial-device") +- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") +- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) +- params = append(params, "-device", "virtio-rng-device,rng=objrng0") +- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) +- params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) +- + // add devices to cmdline + for _, d := range s.devices { + switch v := d.dev.(type) { +@@ -188,8 +233,9 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { + case config.BlockDrive: + id := v.ID + path := v.File +- params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id)) +- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) ++ s.ozoneRes = append(s.ozoneRes, path) ++ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) ++ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path))) + case types.VSock: + v.VhostFd.Close() + params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) +@@ -198,42 +244,125 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { + } + } + +- // daemonize +- params = append(params, "-daemonize") ++ return params, nil ++} ++ ++func (s *stratovirt) createParams(params []string) ([]string, error) { ++ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) ++ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) ++ params = append(params, "-kernel", s.kernelPath) ++ params = append(params, "-pidfile", s.pidfile) + + // append logfile only on debug + if s.config.Debug { +- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) +- params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) ++ params = append(params, "-D", s.logfile) ++ } ++ ++ if s.useImage { ++ s.mmioBlkSlots[0] = true ++ params = append(params, "-device", "virtio-blk-device,drive=rootfs") ++ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath)) ++ } else { ++ params = append(params, "-initrd", s.rootfsPath) + } + +- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) +- err = os.MkdirAll(dir, DirMode) ++ // add devices to cmdline ++ for _, d := range s.devices { ++ switch v := d.dev.(type) { ++ case Endpoint: ++ name := v.Name() ++ mac := v.HardwareAddr() ++ tapName := v.NetworkPair().TapInterface.TAPIface.Name ++ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) ++ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) ++ case config.BlockDrive: ++ id := v.ID ++ path := v.File ++ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) ++ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) ++ case types.VSock: ++ v.VhostFd.Close() ++ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) ++ default: ++ s.Logger().Error("Adding device type is unsupported") ++ } ++ } ++ ++ return params, nil ++} ++ ++func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { ++ span, _ := s.trace(ctx, "startSandbox") ++ defer span.End() ++ ++ var err error ++ var cmd *exec.Cmd ++ ++ params := s.createbaseParams() ++ ++ stratovirtBinPath, err := s.config.HypervisorAssetPath() + if err != nil { + return err + } +- defer func() { ++ ++ if s.useOzone { ++ var ozoneParams []string ++ extend_params, err := s.createOzoneParams(params) + if err != nil { +- if err := os.RemoveAll(dir); err != nil { +- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) ++ return err ++ } ++ ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath) ++ ozoneParams = append(ozoneParams, "-name", s.id) ++ ozoneParams = append(ozoneParams, "-gid", "0") ++ ozoneParams = append(ozoneParams, "-uid", "0") ++ if s.netNSPath != "" { ++ ozoneParams = append(ozoneParams, "-netns", s.netNSPath) ++ } ++ ++ ozoneParams = append(ozoneParams, "-source") ++ ozoneParams = append(ozoneParams, s.ozoneRes...) ++ ++ defer func() { ++ if err != nil { ++ ozoneParams = append(ozoneParams, "-clean-resource") ++ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) ++ if err := cmd.Run(); err != nil { ++ s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot) ++ } + } ++ }() ++ ++ ozoneParams = append(ozoneParams, "--") ++ ozoneParams = append(ozoneParams, extend_params...) ++ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) ++ s.Logger().Info("StratoVirt/Ozone start with params: ", cmd) ++ } else { ++ params, err = s.createParams(params) ++ if err != nil { ++ return err + } +- }() + +- binPath, err := s.config.HypervisorAssetPath() +- if err != nil { +- s.Logger().WithField("Fail to get hypervisor bin path", err).Error() +- return err +- } ++ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) ++ err = os.MkdirAll(dir, DirMode) ++ if err != nil { ++ return err ++ } ++ defer func() { ++ if err != nil { ++ if err := os.RemoveAll(dir); err != nil { ++ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) ++ } ++ } ++ }() + +- cmd := exec.CommandContext(s.ctx, binPath, params...) +- s.Logger().Info("StratoVirt start with params: ", cmd) ++ cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...) ++ s.Logger().Info("StratoVirt start with params: ", cmd) ++ } + + if err := cmd.Start(); err != nil { + s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() + return err + } +- s.pid = cmd.Process.Pid + + if err = s.waitSandBoxStarted(timeout); err != nil { + return err +@@ -420,6 +549,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera + } + + func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { ++ var filePath string + err = s.qmpSetup() + if err != nil { + return err +@@ -427,13 +557,18 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err + + switch op { + case addDevice: +- driver := "virtio-blk-pci" ++ driver := "virtio-blk-mmio" ++ if s.useOzone { ++ filePath, err = s.updateOzoneRes(drive.File, true) ++ } else { ++ filePath = drive.File ++ } + slot, err := s.getDevSlot(drive.VirtPath, false) + if err != nil { + return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) + } + +- if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil { ++ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil { + s.getDevSlot(drive.VirtPath, true) + return err + } +@@ -443,6 +578,9 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err + return err + } + case removeDevice: ++ if s.useOzone { ++ s.updateOzoneRes(drive.File, false) ++ } + if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { + return err + } +@@ -582,17 +720,62 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) { + return tid, nil + } + ++func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { ++ dst := filepath.Join(s.ozoneRoot, filepath.Base(src)) ++ if add { ++ if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil { ++ s.Logger().WithField("bindMount failed", err).Error() ++ return "", err ++ } ++ } else { ++ syscall.Unmount(dst, syscall.MNT_DETACH) ++ } ++ return filepath.Base(src), nil ++} ++ ++func (s *stratovirt) cleanOzoneRes() { ++ s.updateOzoneRes(s.rootfsPath, false) ++ s.updateOzoneRes(s.kernelPath, false) ++ ++ if err := os.RemoveAll(s.ozoneRoot); err != nil { ++ s.Logger().WithField("cleanupOzone failed", err).Error() ++ } ++} ++ + func (s *stratovirt) cleanup(ctx context.Context) error { + span, _ := s.trace(ctx, "cleanup") + defer span.End() + + s.qmpTeardown() ++ if s.useOzone { ++ s.cleanOzoneRes() ++ } + + return nil + } + + func (s *stratovirt) getPids() []int { +- return []int{s.pid} ++ var pids []int ++ if s.pid != 0 { ++ pids = append(pids, s.pid) ++ } else { ++ pid, err := ioutil.ReadFile(s.pidfile) ++ if err != nil { ++ s.Logger().WithError(err).Error("Read pid file failed.") ++ return []int{0} ++ } ++ ++ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t ")) ++ if err != nil { ++ s.Logger().WithError(err).Error("Get pid from pid file failed.") ++ return []int{0} ++ } ++ ++ pids = append(pids, p) ++ s.pid = p ++ } ++ ++ return pids + } + + func (s *stratovirt) getVirtioFsPid() *int { +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch b/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch new file mode 100644 index 0000000000000000000000000000000000000000..4a297eab2d9efd7e513cb2c13c2a2d5d464b5224 --- /dev/null +++ b/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch @@ -0,0 +1,295 @@ +From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001 +From: Wei Gao +Date: Mon, 9 Aug 2021 14:57:06 +0800 +Subject: [PATCH 6/6] factory: add the template factory support for hypervisor + type stratovirt. + +Signed-off-by: Wei Gao +--- + src/runtime/pkg/katautils/config.go | 2 +- + .../factory/template/template.go | 21 +++-- + src/runtime/virtcontainers/kata_agent.go | 7 +- + src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++-- + src/runtime/virtcontainers/vm.go | 28 ++++-- + 5 files changed, 125 insertions(+), 22 deletions(-) + +diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go +index 828c2a43..718677b4 100644 +--- a/src/runtime/pkg/katautils/config.go ++++ b/src/runtime/pkg/katautils/config.go +@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { + // checkFactoryConfig ensures the VM factory configuration is valid. + func checkFactoryConfig(config oci.RuntimeConfig) error { + if config.FactoryConfig.Template { +- if config.HypervisorConfig.InitrdPath == "" { ++ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) { + return errors.New("Factory option enable_template requires an initrd image") + } + } +diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go +index 66070126..02497097 100644 +--- a/src/runtime/virtcontainers/factory/template/template.go ++++ b/src/runtime/virtcontainers/factory/template/template.go +@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error { + if err != nil { + return err + } +- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) +- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) +- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { +- t.close() +- return err ++ ++ // If use hypervisor stratovirt, no need to create template path with ramdisk. ++ if t.config.HypervisorType != vc.StratovirtHypervisor { ++ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) ++ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) ++ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { ++ t.close() ++ return err ++ } + } + f, err := os.Create(t.statePath + "/memory") + if err != nil { +@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error { + } + defer vm.Stop(ctx) + +- if err = vm.Disconnect(ctx); err != nil { +- return err ++ // Create template on hypervisor stratovirt, don't have connection with agent. ++ if config.HypervisorType != vc.StratovirtHypervisor { ++ if err = vm.Disconnect(ctx); err != nil { ++ return err ++ } + } + + // Sleep a bit to let the agent grpc server clean up +diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go +index 13d31658..bc882c70 100644 +--- a/src/runtime/virtcontainers/kata_agent.go ++++ b/src/runtime/virtcontainers/kata_agent.go +@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox, + // TODO: remove dependency on shared fs path. shared fs is just one kind of storage source. + // we should not always use shared fs path for all kinds of storage. Instead, all storage + // should be bind mounted to a tmpfs path for containers to use. +- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { +- return nil, err ++ // If boot from template on stratovirt, no need to mkdir mount path. ++ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) { ++ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { ++ return nil, err ++ } + } + return rootfs, nil + } +diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go +index 47daa817..e9b2ba85 100644 +--- a/src/runtime/virtcontainers/stratovirt.go ++++ b/src/runtime/virtcontainers/stratovirt.go +@@ -48,6 +48,7 @@ type stratovirt struct { + config HypervisorConfig + rootfsPath string + kernelPath string ++ templatePath string + pid int + consolePath string + socketPath string +@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net + + s.id = id + s.config = *hypervisorConfig +- if s.config.OzonePath == "" { ++ if (s.config.OzonePath == "") || s.config.BootToBeTemplate { + s.useOzone = false + s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") + s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") +@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net + s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) + s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) + } ++ ++ if s.config.VMid != "" && s.useOzone { ++ // Make sure the symlinks do not exist ++ os.RemoveAll(s.ozoneRoot) ++ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) ++ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil { ++ return err ++ } ++ } ++ ++ if s.config.BootFromTemplate || s.config.BootToBeTemplate { ++ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1) ++ } ++ + s.netNSPath = networkNS.NetNsPath + s.qmpMonitorCh = qmpChannel{ + ctx: s.ctx, +@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { + params = append(params, "-initrd", filepath.Base(s.rootfsPath)) + } + ++ // handle boot from template ++ if s.config.BootFromTemplate { ++ s.ozoneRes = append(s.ozoneRes, s.templatePath) ++ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath))) ++ } ++ + // add devices to cmdline + for _, d := range s.devices { + switch v := d.dev.(type) { +@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) { + params = append(params, "-initrd", s.rootfsPath) + } + ++ // handle boot from template ++ if s.config.BootFromTemplate { ++ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath)) ++ } ++ + // add devices to cmdline + for _, d := range s.devices { + switch v := d.dev.(type) { +@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { + } + + func (s *stratovirt) pauseSandbox(ctx context.Context) error { +- return nil ++ span, _ := s.trace(ctx, "pauseSandbox") ++ defer span.End() ++ ++ return s.togglePauseSandbox(ctx, true) + } + + func (s *stratovirt) saveSandbox() error { ++ s.Logger().Info("save sandbox") ++ ++ err := s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from. ++ // We would want to bypass shared memory when saving VM to local file through migrate. ++ if s.config.BootToBeTemplate { ++ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath)) ++ if err != nil { ++ s.Logger().WithError(err).Error("exec migration") ++ return err ++ } ++ } ++ + return nil + } + + func (s *stratovirt) resumeSandbox(ctx context.Context) error { ++ span, _ := s.trace(ctx, "resumeSandbox") ++ defer span.End() ++ ++ return s.togglePauseSandbox(ctx, false) ++} ++ ++func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error { ++ span, _ := s.trace(ctx, "togglePauseSandbox") ++ defer span.End() ++ ++ err := s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ if pause { ++ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx) ++ } else { ++ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx) ++ } ++ + return nil + } + +@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { + } + + func (s *stratovirt) cleanOzoneRes() { +- s.updateOzoneRes(s.rootfsPath, false) +- s.updateOzoneRes(s.kernelPath, false) ++ // Umount all resource in ozoneRoot ++ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil { ++ for _, file := range dir { ++ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH) ++ } ++ } + + if err := os.RemoveAll(s.ozoneRoot); err != nil { +- s.Logger().WithField("cleanupOzone failed", err).Error() ++ s.Logger().WithField("cleanup Ozone failed", err).Error() ++ } ++ ++ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean. ++ if s.config.VMid != "" { ++ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) ++ if err := os.RemoveAll(ozoneVmRoot); err != nil { ++ s.Logger().WithField("cleanup Ozone failed", err).Error() ++ } + } + } + +diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go +index e6f02b6e..c4f9df73 100644 +--- a/src/runtime/virtcontainers/vm.go ++++ b/src/runtime/virtcontainers/vm.go +@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { + }() + + // 4. check agent aliveness +- // VMs booted from template are paused, do not check +- if !config.HypervisorConfig.BootFromTemplate { ++ // On hypervisor StratoVirt, VMs booted from template are running, check agent ++ // On other hypervisors, VMs booted from template are paused, do not check ++ if config.HypervisorType == StratovirtHypervisor { ++ if !config.HypervisorConfig.BootToBeTemplate { ++ virtLog.WithField("vm", id).Info("check agent status") ++ err = agent.check(ctx) ++ } ++ } else if !config.HypervisorConfig.BootFromTemplate { + virtLog.WithField("vm", id).Info("check agent status") + err = agent.check(ctx) +- if err != nil { +- return nil, err +- } ++ } ++ if err != nil { ++ return nil, err + } + + return &VM{ +@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error { + // - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/) + + vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath()) +- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id) + sbSharePath := getMountPath(s.id) +- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id) ++ var vmSockDir string ++ var sbSockDir string ++ if v.hypervisor.hypervisorConfig().OzonePath != "" { ++ vmSockDir = filepath.Join(ozoneBaseDir, v.id) ++ sbSockDir = filepath.Join(ozoneBaseDir, s.id) ++ } else { ++ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id) ++ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id) ++ } + + v.logger().WithFields(logrus.Fields{ + "vmSharePath": vmSharePath, +@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error { + + s.hypervisor = v.hypervisor + s.config.HypervisorConfig.VMid = v.id ++ s.config.HypervisorConfig.BootFromTemplate = true + + return nil + } +-- +2.21.1 (Apple Git-122.3) + diff --git a/patches/0007-kata-containers-support-with-iSulad.patch b/patches/0007-kata-containers-support-with-iSulad.patch new file mode 100644 index 0000000000000000000000000000000000000000..c9c1d1a0a61acf1c800c65572d335f443f64cd21 --- /dev/null +++ b/patches/0007-kata-containers-support-with-iSulad.patch @@ -0,0 +1,224 @@ +From d4605dafaa9c326a5cf24c28d0c1efe6c9997f49 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Sat, 21 Aug 2021 17:08:46 +0800 +Subject: [PATCH] kata-containers: support with iSulad + +reason: support with iSulad + +Signed-off-by: holyfei +--- + src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +- + src/runtime/containerd-shim-v2/container.go | 9 +++ + src/runtime/containerd-shim-v2/service.go | 55 +++++++++++++++++++ + src/runtime/containerd-shim-v2/start.go | 10 ++++ + .../containerd/runtime/v2/shim/shim.go | 8 ++- + 5 files changed, 81 insertions(+), 3 deletions(-) + +diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs +index 7f41cb4..6c3bb32 100644 +--- a/src/agent/rustjail/src/cgroups/fs/mod.rs ++++ b/src/agent/rustjail/src/cgroups/fs/mod.rs +@@ -369,7 +369,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool + if let Some(swappiness) = memory.swappiness { + if (0..=100).contains(&swappiness) { + mem_controller.set_swappiness(swappiness as u64)?; +- } else { ++ } else if swappiness != -1 { + return Err(anyhow!( + "invalid value:{}. valid memory swappiness range is 0-100", + swappiness +diff --git a/src/runtime/containerd-shim-v2/container.go b/src/runtime/containerd-shim-v2/container.go +index faea0e2..d563888 100644 +--- a/src/runtime/containerd-shim-v2/container.go ++++ b/src/runtime/containerd-shim-v2/container.go +@@ -7,10 +7,13 @@ package containerdshim + + import ( + "io" ++ "os" ++ "path" + "time" + + "github.com/containerd/containerd/api/types/task" + "github.com/containerd/containerd/errdefs" ++ cdshim "github.com/containerd/containerd/runtime/v2/shim" + taskAPI "github.com/containerd/containerd/runtime/v2/task" + "github.com/opencontainers/runtime-spec/specs-go" + +@@ -37,6 +40,8 @@ type container struct { + status task.Status + terminal bool + mounted bool ++ exitFifo string ++ exitFd *os.File + } + + func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) { +@@ -49,6 +54,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con + spec = &specs.Spec{} + } + ++ dir := os.Getenv(cdshim.ExitFifoDir) ++ exitFifo := path.Join(dir, r.ID, exitFifoName) ++ + c := &container{ + s: s, + spec: spec, +@@ -65,6 +73,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con + exitCh: make(chan uint32, 1), + stdinCloser: make(chan struct{}), + mounted: mounted, ++ exitFifo: exitFifo, + } + return c, nil + } +diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go +index 1003f8e..e13283c 100644 +--- a/src/runtime/containerd-shim-v2/service.go ++++ b/src/runtime/containerd-shim-v2/service.go +@@ -6,13 +6,16 @@ + package containerdshim + + import ( ++ "bytes" + "context" ++ "encoding/binary" + "io/ioutil" + "os" + sysexec "os/exec" + "sync" + "syscall" + "time" ++ "unsafe" + + eventstypes "github.com/containerd/containerd/api/events" + "github.com/containerd/containerd/api/types/task" +@@ -51,6 +54,8 @@ const ( + // A time span used to wait for publish a containerd event, + // once it costs a longer time than timeOut, it will be canceld. + timeOut = 5 * time.Second ++ ++ exitFifoName = "exit_fifo" + ) + + var ( +@@ -1019,6 +1024,10 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI. + func (s *service) processExits() { + for e := range s.ec { + s.checkProcesses(e) ++ ++ if os.Getenv(cdshim.ExitFifoDir) != "" { ++ s.closeExitFifo(e) ++ } + } + } + +@@ -1070,3 +1079,49 @@ func (s *service) getContainerStatus(containerID string) (task.Status, error) { + + return status, nil + } ++ ++func isBigEndian() (ret bool) { ++ i := int(0x1) ++ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i)) ++ return bs[0] == 0 ++} ++ ++func (s *service) closeExitFifo(e exit) { ++ if e.execid != "" { ++ // not a container, no need to close exit fifo ++ return ++ } ++ ++ var ret uint32 ++ var nativeEndian binary.ByteOrder ++ ++ s.mu.Lock() ++ c, err := s.getContainer(e.id) ++ s.mu.Unlock() ++ ++ if err != nil { ++ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id) ++ return ++ } ++ ++ ret = <-c.exitCh ++ // refill the exitCh with the container process's exit code in case ++ // there were other waits on this process. ++ c.exitCh <- ret ++ ++ if isBigEndian() { ++ nativeEndian = binary.BigEndian ++ } else { ++ nativeEndian = binary.LittleEndian ++ } ++ ++ bytesBuffer := bytes.NewBuffer([]byte{}) ++ binary.Write(bytesBuffer, nativeEndian, &ret) ++ ++ _, err = c.exitFd.Write(bytesBuffer.Bytes()) ++ if err != nil { ++ logrus.WithError(err).Error("write exit fifo failed") ++ } ++ ++ c.exitFd.Close() ++} +diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go +index 72420e4..e89dc48 100644 +--- a/src/runtime/containerd-shim-v2/start.go ++++ b/src/runtime/containerd-shim-v2/start.go +@@ -8,8 +8,11 @@ package containerdshim + import ( + "context" + "fmt" ++ "golang.org/x/sys/unix" ++ "os" + + "github.com/containerd/containerd/api/types/task" ++ cdshim "github.com/containerd/containerd/runtime/v2/shim" + "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" + ) + +@@ -59,6 +62,13 @@ func startContainer(ctx context.Context, s *service, c *container) error { + + c.status = task.StatusRunning + ++ if os.Getenv(cdshim.ExitFifoDir) != "" { ++ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) ++ if err != nil { ++ return err ++ } ++ } ++ + stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id) + if err != nil { + return err +diff --git a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go +index d60d496..946c386 100644 +--- a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go ++++ b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go +@@ -84,6 +84,8 @@ var ( + action string + ) + ++var ExitFifoDir = "EXIT_FIFO_DIR" ++ + func parseFlags() { + flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") + flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") +@@ -198,8 +200,10 @@ func run(id string, initFunc Init, config Config) error { + } + return nil + default: +- if err := setLogger(ctx, idFlag); err != nil { +- return err ++ if os.Getenv("EXIT_FIFO_DIR") == "" { ++ if err := setLogger(ctx, idFlag); err != nil { ++ return err ++ } + } + client := NewShimClient(ctx, service, signals) + return client.Serve() +-- +2.23.0 + diff --git a/series.conf b/series.conf index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..eabe1382d3bdfcac6f1776998f207f36bb989e7b 100644 --- a/series.conf +++ b/series.conf @@ -0,0 +1,7 @@ +0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch +0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch +0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch +0004-configuration-add-configuration-generator-for-hyperv.patch +0005-runtime-add-the-secure-component-ozone-support-for-h.patch +0006-factory-add-the-template-factory-support-for-hypervi.patch +0007-kata-containers-support-with-iSulad.patch