From fd0c8f9738ef50fe34d1f87f05fb984cb6657952 Mon Sep 17 00:00:00 2001 From: Vanient Date: Mon, 12 Sep 2022 16:46:47 +0800 Subject: [PATCH] kata:sync bugfix patches, runtime 0078-0096 agent 0021-0024 runtime: 0078-kata-containers-check-file-size-before-creating-cont.patch 0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch 0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch 0081-kata-runtime-cut-too-long-message-in-grpc-log.patch 0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch 0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch 0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch 0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch 0086-kata-runtime-add-checkCPUSet-before-create-container.patch 0087-kata-runtime-force-delete-the-sandbox-and-container.patch 0088-kata-runtime-check-sandbox-healthy-state-before-call.patch 0089-kata-add-support-for-update-iface.patch 0090-kata-set-sandbox-or-container-status-to-unhealthy.patch 0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch 0092-qemu-add-arm64-to-support-list-of-dimm.patch 0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch 0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch 0095-kata-runtime-fix-qemu-process-resource-resi.patch 0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch agent: 0021-kata-agent-fix-sync-clock-not-work-problem.patch 0022-kata-agent-delete-container-id-from-sandbox-struct.patch 0023-kata-agent-modify-log-level.patch 0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch Signed-off-by: Vanient --- ...gent-fix-sync-clock-not-work-problem.patch | 39 ++++ ...ete-container-id-from-sandbox-struct.patch | 32 +++ .../0023-kata-agent-modify-log-level.patch | 44 ++++ ...gent.debug_console-not-work-when-bui.patch | 33 +++ agent/series.conf | 4 + gen-version.sh | 19 ++ kata-containers.spec | 8 +- ...check-file-size-before-creating-cont.patch | 91 ++++++++ ...-qemu-SCSIBus-info-not-saved-into-pe.patch | 54 +++++ ...-the-block-device-not-removed-in-dev.patch | 53 +++++ ...ime-cut-too-long-message-in-grpc-log.patch | 51 ++++ ...nge-sandbox-state-to-unhealthy-when-.patch | 60 +++++ ...-removeMountBlockDevices-for-contain.patch | 59 +++++ ...-validInterface-func-cause-crash-pro.patch | 34 +++ ...-kata-netmon-does-not-exit-when-cont.patch | 70 ++++++ ...-checkCPUSet-before-create-container.patch | 68 ++++++ ...rce-delete-the-sandbox-and-container.patch | 55 +++++ ...ck-sandbox-healthy-state-before-call.patch | 219 ++++++++++++++++++ ...89-kata-add-support-for-update-iface.patch | 31 +++ ...box-or-container-status-to-unhealthy.patch | 115 +++++++++ ...-sandbox-file-lock-while-call-GetSan.patch | 51 ++++ ...mu-add-arm64-to-support-list-of-dimm.patch | 32 +++ ...d-timeout-for-grpcWaitProcessRequest.patch | 40 ++++ ...-update-iface-clean-NIC-cause-route-.patch | 115 +++++++++ ...ntime-fix-qemu-process-resource-resi.patch | 95 ++++++++ ...Move-from-query-cpus-to-query-cpus-f.patch | 46 ++++ runtime/series.conf | 19 ++ 27 files changed, 1536 insertions(+), 1 deletion(-) create mode 100644 agent/patches/0021-kata-agent-fix-sync-clock-not-work-problem.patch create mode 100644 agent/patches/0022-kata-agent-delete-container-id-from-sandbox-struct.patch create mode 100644 agent/patches/0023-kata-agent-modify-log-level.patch create mode 100644 agent/patches/0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch create mode 100755 gen-version.sh create mode 100644 runtime/patches/0078-kata-containers-check-file-size-before-creating-cont.patch create mode 100644 runtime/patches/0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch create mode 100644 runtime/patches/0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch create mode 100644 runtime/patches/0081-kata-runtime-cut-too-long-message-in-grpc-log.patch create mode 100644 runtime/patches/0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch create mode 100644 runtime/patches/0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch create mode 100644 runtime/patches/0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch create mode 100644 runtime/patches/0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch create mode 100644 runtime/patches/0086-kata-runtime-add-checkCPUSet-before-create-container.patch create mode 100644 runtime/patches/0087-kata-runtime-force-delete-the-sandbox-and-container.patch create mode 100644 runtime/patches/0088-kata-runtime-check-sandbox-healthy-state-before-call.patch create mode 100644 runtime/patches/0089-kata-add-support-for-update-iface.patch create mode 100644 runtime/patches/0090-kata-set-sandbox-or-container-status-to-unhealthy.patch create mode 100644 runtime/patches/0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch create mode 100644 runtime/patches/0092-qemu-add-arm64-to-support-list-of-dimm.patch create mode 100644 runtime/patches/0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch create mode 100644 runtime/patches/0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch create mode 100644 runtime/patches/0095-kata-runtime-fix-qemu-process-resource-resi.patch create mode 100644 runtime/patches/0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch diff --git a/agent/patches/0021-kata-agent-fix-sync-clock-not-work-problem.patch b/agent/patches/0021-kata-agent-fix-sync-clock-not-work-problem.patch new file mode 100644 index 0000000..028edf5 --- /dev/null +++ b/agent/patches/0021-kata-agent-fix-sync-clock-not-work-problem.patch @@ -0,0 +1,39 @@ +From 5cba0d988eec5a200a28fdc71ad404bba5014bf3 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 14 Dec 2020 20:05:11 -0500 +Subject: [PATCH] kata-agent: fix sync clock not work problem + +reason: SyncClock goroutine is not started in the kata-agent, +so add SyncClock function back into kata-agent to make sync +clock func avaliable. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Signed-off-by: jiangpengfei +--- + agent.go | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/agent.go b/agent.go +index 50afd7a..c3d6524 100644 +--- a/agent.go ++++ b/agent.go +@@ -1199,6 +1199,15 @@ func (s *sandbox) startGRPC() { + return + } + ++ // accept the sync clock stream first ++ syncClockStream, err := l.Accept() ++ if err != nil { ++ agentLog.WithError(err).Warn("Failed to accpet the sync clock stream") ++ return ++ } ++ ++ go SyncClock(syncClockStream) ++ + // l is closed when Serve() returns + servErr = grpcServer.Serve(l) + if servErr != nil { +-- +1.8.3.1 + diff --git a/agent/patches/0022-kata-agent-delete-container-id-from-sandbox-struct.patch b/agent/patches/0022-kata-agent-delete-container-id-from-sandbox-struct.patch new file mode 100644 index 0000000..a0a2313 --- /dev/null +++ b/agent/patches/0022-kata-agent-delete-container-id-from-sandbox-struct.patch @@ -0,0 +1,32 @@ +From 4aac7e5067ba0d8ace8ea592c34ad615851d67b3 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Tue, 23 Feb 2021 21:39:05 +0800 +Subject: [PATCH] kata-agent: delete container id from sandbox struct + +reason: delete container id from sandbox struct, when remove +the container, the id should be also deleted + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: Id925fdc822d93d8ac13be8b9bb8343cac2e0a514 +Signed-off-by: yangfeiyu +--- + grpc.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/grpc.go b/grpc.go +index f2cfcc1..2f606b5 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -1274,6 +1274,7 @@ func (a *agentGRPC) RemoveContainer(ctx context.Context, req *pb.RemoveContainer + + if timeout == 0 { + if err := ctr.removeContainer(); err != nil { ++ delete(a.sandbox.containers, ctr.id) + return emptyResp, err + } + +-- +2.23.0 + diff --git a/agent/patches/0023-kata-agent-modify-log-level.patch b/agent/patches/0023-kata-agent-modify-log-level.patch new file mode 100644 index 0000000..4dd5a61 --- /dev/null +++ b/agent/patches/0023-kata-agent-modify-log-level.patch @@ -0,0 +1,44 @@ +From 62678eceb1f202bdc476452f50af602fd9fde09e Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Wed, 2 Jun 2021 11:01:50 +0800 +Subject: [PATCH] kata-agent: modify log level + +reason: modify log level + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: I8139f94430858ff956970d9b932b7d3d3a3ab3dd +Signed-off-by: yangfeiyu +--- + agent.go | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + mode change 100755 => 100644 agent.go + +diff --git a/agent.go b/agent.go +old mode 100755 +new mode 100644 +index c3d6524..2ddad83 +--- a/agent.go ++++ b/agent.go +@@ -1091,7 +1091,7 @@ func makeUnaryInterceptor() grpc.UnaryServerInterceptor { + + agentLog.WithFields(logrus.Fields{ + "request": grpcCall, +- "req": message.String()}).Debug("new request") ++ "req": message.String()}).Info("new request") + start = time.Now() + } + +@@ -1109,7 +1109,7 @@ func makeUnaryInterceptor() grpc.UnaryServerInterceptor { + "request": info.FullMethod, + "duration": elapsed.String(), + "resp": message.String()}) +- logger.Debug("request end") ++ logger.Info("request end") + } + + // Handle the following scenarios: +-- +2.23.0 + diff --git a/agent/patches/0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch b/agent/patches/0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch new file mode 100644 index 0000000..bdafc17 --- /dev/null +++ b/agent/patches/0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch @@ -0,0 +1,33 @@ +From 22e00bc2d63212924c51261d2228699abd269657 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 2 Jun 2021 15:23:20 -0400 +Subject: [PATCH] kata-agent: fix agent.debug_console not work when built with + golang 1.15 + +reason: fix agent.debug_console not work when built with golang 1.15 + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Signed-off-by: jiangpengfei +--- + agent.go | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/agent.go b/agent.go +index c3d6524..34d67c8 100644 +--- a/agent.go ++++ b/agent.go +@@ -1459,9 +1459,6 @@ func setupDebugConsole(ctx context.Context, debugConsolePath string) error { + cmd.SysProcAttr = &syscall.SysProcAttr{ + // Create Session + Setsid: true, +- // Set Controlling terminal to Ctty +- Setctty: true, +- Ctty: int(f.Fd()), + } + + go func() { +-- +1.8.3.1 + diff --git a/agent/series.conf b/agent/series.conf index 0c2b848..a3f6137 100644 --- a/agent/series.conf +++ b/agent/series.conf @@ -17,3 +17,7 @@ 0018-kata-agent-update-nic-in-guest.patch 0019-kata-agent-modify-make-flags.patch 0020-kata-agent-add-linkmode-to-resolve-build-error.patch +0021-kata-agent-fix-sync-clock-not-work-problem.patch +0022-kata-agent-delete-container-id-from-sandbox-struct.patch +0023-kata-agent-modify-log-level.patch +0024-kata-agent-fix-agent.debug_console-not-work-when-bui.patch diff --git a/gen-version.sh b/gen-version.sh new file mode 100755 index 0000000..48bc9d9 --- /dev/null +++ b/gen-version.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. +# Description: This script uses to update docker-ce component's version and release +# Create: 2022-09-12 + +old_version=`head -n 5 kata-containers.spec|grep RELEASE|awk '{print $NF}'` +let new_version=$old_version+1 +sed -i -e "s/RELEASE $old_version/RELEASE $new_version/g" ./kata-containers.spec + +author=$(git config user.name) +email=$(git config user.email) +version=$(head -5 kata-containers.spec | grep VERSION | awk '{print $NF}') +release=$(head -5 kata-containers.spec | grep RELEASE | awk '{print $NF}') +new_all=$version-$release +new_changelog=$(cat << EOF +* $(LC_ALL="C" date '+%a %b %d %Y') $author<$email> - $new_all\n- Type:\n- CVE:\n- SUG:\n- DESC:\n +EOF +) +sed -i -e "/\%changelog/a$new_changelog" kata-containers.spec diff --git a/kata-containers.spec b/kata-containers.spec index 6e4dcaa..026bfcc 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} %define VERSION v1.11.1 -%define RELEASE 13 +%define RELEASE 14 Name: kata-containers Version: %{VERSION} @@ -92,6 +92,12 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u %changelog +* Mon Sep 12 2022 Vanient - v1.11.1-14 +- Type:bugfix +- CVE:NA +- SUG:NA +- DESC:sync bugfix patches, runtime 0078-0096 agent 0021-0024 + * Fri Dec 17 2021 Lichang Zhao - 1.11.1-13 +- Type:bugfix +- ID:NA diff --git a/runtime/patches/0078-kata-containers-check-file-size-before-creating-cont.patch b/runtime/patches/0078-kata-containers-check-file-size-before-creating-cont.patch new file mode 100644 index 0000000..03dca41 --- /dev/null +++ b/runtime/patches/0078-kata-containers-check-file-size-before-creating-cont.patch @@ -0,0 +1,91 @@ +From e0fdf20e84cf8c31eab33c562cefd943a6656215 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Mon, 21 Feb 2022 09:58:04 +0800 +Subject: [PATCH] kata-runtime: check file size before creating container and + doing network operation + +reason: check file size before creating container and doing network operation + +Signed-off-by: holyfei +--- + cli/network.go | 9 +++++++++ + pkg/katautils/config.go | 9 +++++++++ + virtcontainers/utils/utils.go | 3 +++ + 3 files changed, 21 insertions(+) + +diff --git a/cli/network.go b/cli/network.go +index 7dce052..9d3a6dc 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -8,11 +8,13 @@ package main + import ( + "context" + "encoding/json" ++ "errors" + "fmt" + "os" + + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" + ) +@@ -261,6 +263,13 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + if input == "-" { + f = os.Stdin + } else { ++ st, err := os.Lstat(input) ++ if err != nil { ++ return err ++ } ++ if st.Size() > utils.MaxFileSize { ++ return errors.New("network file too big") ++ } + f, err = os.Open(input) + if err != nil { + return err +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index fd7f5eb..b0d8f71 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -10,6 +10,7 @@ import ( + "errors" + "fmt" + "io/ioutil" ++ "os" + "path/filepath" + "strings" + +@@ -1291,6 +1292,14 @@ func decodeConfig(configPath string) (tomlConfig, string, error) { + return tomlConf, "", fmt.Errorf("Cannot find usable config file (%v)", err) + } + ++ st, err := os.Lstat(resolved) ++ if err != nil { ++ return tomlConf, resolved, err ++ } ++ if st.Size() > utils.MaxFileSize { ++ return tomlConf, resolved, errors.New("config file too big") ++ } ++ + configData, err := ioutil.ReadFile(resolved) + if err != nil { + return tomlConf, resolved, err +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index d4dad40..04b6bce 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -41,6 +41,9 @@ const ( + // Max support memory size in the Kata VM + MaxMemorySizeInMB = 512 * 1024 + MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift ++ ++ // Max file size for config and network json file ++ MaxFileSize = 1 * 1024 * 1024 + ) + + // MaxSocketPathLen is the effective maximum Unix domain socket length. +-- +2.27.0 + diff --git a/runtime/patches/0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch b/runtime/patches/0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch new file mode 100644 index 0000000..0e059ee --- /dev/null +++ b/runtime/patches/0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch @@ -0,0 +1,54 @@ +From 3f47ddd1b793b6fe92cba13737cd7b0c14a19edc Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Sun, 15 Nov 2020 21:48:14 +0800 +Subject: [PATCH 2/3] kata-runtime: fix qemu SCSIBus info not saved into + persist.json + +reason: SCSIBus has a map object to save the slots info which +indicates which slot is useful. However, the SCSIBus object is +not saved into persist.json file, which causes fetchSandbox will +create a new one instead of restoring it from persist.json file. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Signed-off-by: holyfei +--- + virtcontainers/persist/api/hypervisor.go | 1 + + virtcontainers/qemu.go | 2 ++ + 2 files changed, 3 insertions(+) + +diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go +index fd61b3c2..213453ac 100644 +--- a/virtcontainers/persist/api/hypervisor.go ++++ b/virtcontainers/persist/api/hypervisor.go +@@ -39,6 +39,7 @@ type HypervisorState struct { + // Refs: virtcontainers/qemu.go:QemuState + Bridges []Bridge + PCIeRootPortsPool *types.PCIeRootPortPool ++ ScsiBus *types.SCSIBus + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + HotpluggedMemory int +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index be6e33b9..e5610e89 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -2342,6 +2342,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) { + s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus + s.PCIeRootPort = q.state.PCIeRootPort + s.PCIeRootPortsPool = q.state.PCIeRootPortsPool ++ s.ScsiBus = q.state.ScsiBus + + for _, bridge := range q.arch.getBridges() { + s.Bridges = append(s.Bridges, persistapi.Bridge{ +@@ -2367,6 +2368,7 @@ func (q *qemu) load(s persistapi.HypervisorState) { + q.state.VirtiofsdPid = s.VirtiofsdPid + q.state.PCIeRootPort = s.PCIeRootPort + q.state.PCIeRootPortsPool = s.PCIeRootPortsPool ++ q.state.ScsiBus = s.ScsiBus + + for _, bridge := range s.Bridges { + q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr)) +-- +2.23.0 + diff --git a/runtime/patches/0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch b/runtime/patches/0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch new file mode 100644 index 0000000..0f111f2 --- /dev/null +++ b/runtime/patches/0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch @@ -0,0 +1,53 @@ +From b0097d60789a6531e07f123e0f297fd4d9f817a6 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Sun, 15 Nov 2020 22:00:43 +0800 +Subject: [PATCH 3/3] kata-runtime: fix the block device not removed in + devManager + +reason: In the case of hotpluging the block device number exceed +the number of max capacity of sandbox, kata-runtime need to rollback +to remove the device info in the devManager, otherwise the created +device info will write into persist.json file, which leads to problem. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Signed-off-by: holyfei +--- + virtcontainers/container.go | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 1b703825..29a7fb52 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1462,6 +1462,7 @@ func (c *Container) plugDevice(devicePath string) error { + } + + if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { ++ var err error + b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{ + HostPath: devicePath, + ContainerPath: filepath.Join(kataGuestSharedDir(), c.id), +@@ -1473,10 +1474,18 @@ func (c *Container) plugDevice(devicePath string) error { + return fmt.Errorf("device manager failed to create rootfs device for %q: %v", devicePath, err) + } + ++ defer func() { ++ if err != nil { ++ if newErr := c.sandbox.devManager.RemoveDevice(b.DeviceID()); newErr != nil { ++ c.Logger().WithError(newErr).Error("fail rollback to remove block device") ++ } ++ } ++ }() ++ + c.state.BlockDeviceID = b.DeviceID() + + // attach rootfs device +- if err := c.sandbox.devManager.AttachDevice(b.DeviceID(), c.sandbox); err != nil { ++ if err = c.sandbox.devManager.AttachDevice(b.DeviceID(), c.sandbox); err != nil { + return err + } + } +-- +2.23.0 + diff --git a/runtime/patches/0081-kata-runtime-cut-too-long-message-in-grpc-log.patch b/runtime/patches/0081-kata-runtime-cut-too-long-message-in-grpc-log.patch new file mode 100644 index 0000000..c19df5e --- /dev/null +++ b/runtime/patches/0081-kata-runtime-cut-too-long-message-in-grpc-log.patch @@ -0,0 +1,51 @@ +From a5c0594b1874d1693c97475136a1850a9cf6ee73 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 18 Nov 2020 19:51:40 -0500 +Subject: [PATCH] kata-runtime: cut too long message in grpc log + +reason: cut too long message in grpc log to avoid logrus prints +"too long message" error + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Change-Id: I221e78ef142c80e6b3241d10ece235fcd36a6104 +Signed-off-by: jiangpengfei +--- + virtcontainers/kata_agent.go | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 66f6ef6..75985e2 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -91,6 +91,7 @@ var ( + localDirOptions = []string{"mode=0777"} + maxHostnameLen = 64 + GuestDNSFile = "/etc/resolv.conf" ++ maxLogLength = 1024 + ) + + const ( +@@ -2200,12 +2201,17 @@ func (k *kataAgent) sendReq(request interface{}) (interface{}, error) { + if msgName == "" || handler == nil { + return nil, errors.New("Invalid request type") + } +- message := request.(proto.Message) ++ message := request.(proto.Message).String() ++ // if message is too long, we just print top 1024 char in log ++ if len(message) > maxLogLength { ++ message = message[:maxLogLength] ++ } ++ + ctx, cancel := k.getReqContext(msgName) + if cancel != nil { + defer cancel() + } +- k.Logger().WithField("name", msgName).WithField("req", message.String()).Debug("sending request") ++ k.Logger().WithField("name", msgName).WithField("req", message).Debug("sending request") + + return handler(ctx, request) + } +-- +1.8.3.1 + diff --git a/runtime/patches/0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch b/runtime/patches/0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch new file mode 100644 index 0000000..d1762ea --- /dev/null +++ b/runtime/patches/0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch @@ -0,0 +1,60 @@ +From a4776b4a5cfd80d983b26dc8991bdcf1cfb9ad4a Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 10 Dec 2020 18:54:09 -0500 +Subject: [PATCH] kata-runtime: change sandbox state to unhealthy when cmd is + kill or delete + +reason: If qemu or kata-proxy process's state is D and execute command +isn't kill or delete, we should not change sandbox state to unhealthy, +because D state maybe change to normal soon. So we change the sandbox +state to unhealthy only when execute command is kill or delete. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Change-Id: I21ac55aecae9bf83b6f877832c5976698bc9dbf7 +Signed-off-by: jiangpengfei +--- + virtcontainers/api.go | 20 ++++++++++---------- + 1 file changed, 10 insertions(+), 10 deletions(-) + +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 38c8235..0a6ba59 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -1117,16 +1117,6 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force + // procesUnhealthySandbox only change sandbox state to unhealthy + // when caller is kata-runtime kill or kata-runtime delete + func processUnhealthySandbox(sandbox *Sandbox, container *Container) error { +- // Set all containers state to unhealthy +- if err := sandbox.setContainersState(types.StateUnhealthy); err != nil { +- container.Logger().WithError(err).Warn("set all containers state to unhealthy fail") +- } +- +- // Set sandbox state to unhealthy +- if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil { +- container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail") +- } +- + forceDelete := false + + // If process is kata-runtime kill or kata-runtime delete, +@@ -1138,6 +1128,16 @@ func processUnhealthySandbox(sandbox *Sandbox, container *Container) error { + } + + if forceDelete { ++ // Set all containers state to unhealthy ++ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil { ++ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail") ++ } ++ ++ // Set sandbox state to unhealthy ++ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil { ++ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail") ++ } ++ + // force stop podSandbox type container's kata-shim process + if err := stopShim(container.process.Pid); err != nil { + container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim") +-- +1.8.3.1 + diff --git a/runtime/patches/0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch b/runtime/patches/0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch new file mode 100644 index 0000000..38160ca --- /dev/null +++ b/runtime/patches/0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch @@ -0,0 +1,59 @@ +From 490879dfd4d886bda2127d94967039771ff14d15 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Thu, 10 Dec 2020 17:21:59 +0800 +Subject: [PATCH] kata-runtime: add removeMountBlockDevices for container + +reason: when create container with -v, the device is created +and stored in sandbox device manager, if create container failed +in next steps, the rollback operation will not deal with container +mount device(mount blockdevice) + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Signed-off-by: yangfeiyu +--- + virtcontainers/container.go | 21 +++++++++++++++++++++ + 1 file changed, 21 insertions(+) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index e40e313..601860c 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -909,6 +909,11 @@ func (c *Container) rollbackFailingContainerCreation() { + if err := c.detachDevices(); err != nil { + c.Logger().WithError(err).Error("rollback failed detachDevices()") + } ++ ++ if errs := c.removeMountBlockDevices(); len(errs) > 0 { ++ c.Logger().Error("rollback failed removeMountBlockDevices()") ++ } ++ + if err := c.removeDrive(); err != nil { + c.Logger().WithError(err).Error("rollback failed removeDrive()") + } +@@ -1494,6 +1499,22 @@ func (c *Container) isDriveUsed() bool { + return !(c.state.Fstype == "") + } + ++func (c *Container) removeMountBlockDevices() []error { ++ var result []error ++ for _, mount := range c.mounts { ++ if len(mount.BlockDeviceID) > 0 { ++ if err := c.sandbox.devManager.RemoveDevice(mount.BlockDeviceID); err != nil { ++ result = append(result, err) ++ c.Logger().WithFields(logrus.Fields{ ++ "container": c.id, ++ "block-device": mount.BlockDeviceID, ++ }).WithError(err).Error("remove block device failed") ++ } ++ } ++ } ++ return result ++} ++ + func (c *Container) removeDrive() (err error) { + if c.isDriveUsed() { + c.Logger().Info("unplugging block device") +-- +2.23.0 + diff --git a/runtime/patches/0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch b/runtime/patches/0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch new file mode 100644 index 0000000..f92664c --- /dev/null +++ b/runtime/patches/0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch @@ -0,0 +1,34 @@ +From 6148852131b216dd1def5a5c7fee6863b3f88269 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 15 Dec 2020 12:29:21 -0500 +Subject: [PATCH] kata-runtime: fix validInterface func cause crash problem + +reason: fix validInterface func crach problem which is found +by the fuzz testcase. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers +Change-Id: I41d65d6c4bf44175c0ff74e6600447cf2ad6e422 +Signed-off-by: jiangpengfei +--- + virtcontainers/network.go | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index 15eb790..89c5ce3 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -1532,6 +1532,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { + if len(inf.IPAddresses) != 1 { + return fmt.Errorf("only one IP address is supported currently") + } ++ ++ if inf.IPAddresses[0] == nil { ++ return fmt.Errorf("input IP address info should not be null") ++ } + _, err := verifyIP(inf.IPAddresses[0].Address) + if err != nil { + return err +-- +1.8.3.1 + diff --git a/runtime/patches/0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch b/runtime/patches/0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch new file mode 100644 index 0000000..ebd7375 --- /dev/null +++ b/runtime/patches/0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch @@ -0,0 +1,70 @@ +From a1f1c9de04dd3cd2003f06c26b415c4e30fa3a53 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Wed, 16 Dec 2020 10:36:39 +0800 +Subject: [PATCH] kata-runtime: fix kata-netmon does not exit when container is + stopped + +reason: start a container and stop it, the netmon process will not exit, +we should store the netmon info to disk and read it when fetching the sandbox, +then the remove netmon operation will be executed. + +Change-Id: I2d8fd02747eaf208445444bc0c43a7d221ba1715 +Conflict:NA +Reference:https://gitee.com/holyfei/kata-runtime +Signed-off-by: yangfeiyu +--- + virtcontainers/persist.go | 11 ++++++++++- + virtcontainers/persist/api/config.go | 7 +++++++ + 2 files changed, 17 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go +index efa4506..57b5336 100644 +--- a/virtcontainers/persist.go ++++ b/virtcontainers/persist.go +@@ -192,8 +192,12 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, + EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI, + InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), ++ NetmonConfig: persistapi.NetmonConfig{ ++ Path: sconfig.NetworkConfig.NetmonConfig.Path, ++ Debug: sconfig.NetworkConfig.NetmonConfig.Debug, ++ Enable: sconfig.NetworkConfig.NetmonConfig.Enable, ++ }, + }, +- + ShmSize: sconfig.ShmSize, + SharePidNs: sconfig.SharePidNs, + Stateful: sconfig.Stateful, +@@ -484,6 +488,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { + DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, + EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI, + InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), ++ NetmonConfig: NetmonConfig{ ++ Path: savedConf.NetworkConfig.NetmonConfig.Path, ++ Debug: savedConf.NetworkConfig.NetmonConfig.Debug, ++ Enable: savedConf.NetworkConfig.NetmonConfig.Enable, ++ }, + }, + + ShmSize: savedConf.ShmSize, +diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go +index 28204fc..26f6cf4 100644 +--- a/virtcontainers/persist/api/config.go ++++ b/virtcontainers/persist/api/config.go +@@ -215,6 +215,13 @@ type NetworkConfig struct { + DisableNewNetNs bool + EnableCompatOldCNI bool + InterworkingModel int ++ NetmonConfig NetmonConfig ++} ++ ++type NetmonConfig struct { ++ Path string ++ Debug bool ++ Enable bool + } + + type ContainerConfig struct { +-- +2.23.0 + diff --git a/runtime/patches/0086-kata-runtime-add-checkCPUSet-before-create-container.patch b/runtime/patches/0086-kata-runtime-add-checkCPUSet-before-create-container.patch new file mode 100644 index 0000000..af8ea87 --- /dev/null +++ b/runtime/patches/0086-kata-runtime-add-checkCPUSet-before-create-container.patch @@ -0,0 +1,68 @@ +From 2a8e2726902ec344bc8c23d8bd7eb2336d236890 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 17 Dec 2020 17:54:45 -0500 +Subject: [PATCH] kata-runtime: add checkCPUSet before create container + +reason: add checkCPUSet before create container to make sure +guest has the avaliable vcpus. + +Change-Id: Idc34f7c18c6d3ffca2d1d015f298348679464bd2 +Conflict:NA +Reference:https://gitee.com/src-openeuler/kata-runtime +Signed-off-by: jiangpengfei +--- + virtcontainers/container.go | 27 +++++++++++++++++++++++++++ + 1 file changed, 27 insertions(+) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 601860c..724b58c 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -13,6 +13,8 @@ import ( + "io" + "os" + "path/filepath" ++ "strconv" ++ "strings" + "syscall" + "time" + +@@ -941,6 +943,13 @@ func (c *Container) checkBlockDeviceSupport() bool { + // createContainer creates and start a container inside a Sandbox. It has to be + // called only when a new container, not known by the sandbox, has to be created. + func (c *Container) create() (err error) { ++ if c.config.Resources.CPU != nil { ++ hypervisorCPUs := c.sandbox.config.HypervisorConfig.NumVCPUs ++ if err := checkCPUSet(int(hypervisorCPUs), c.config.Resources.CPU.Cpus); err != nil { ++ return err ++ } ++ } ++ + // In case the container creation fails, the following takes care + // of rolling back all the actions previously performed. + defer func() { +@@ -1723,3 +1732,21 @@ func (c *Container) forceKillContainer() { + c.Logger().WithError(err).Warn("force kill container: remove container drive failed") + } + } ++ ++// checkCPUSet returns nil if the value of cpuset-cpus is smaller than sandbox_cpu ++func checkCPUSet(sandboxCPU int, cpus string) error { ++ cpuMax := 0 ++ c := strings.Split(strings.Replace(cpus, "-", ",", -1), ",") ++ for _, cpu := range c { ++ if tmp, _ := strconv.Atoi(cpu); tmp > cpuMax { ++ cpuMax = tmp ++ } ++ } ++ ++ if cpuMax >= sandboxCPU { ++ err := fmt.Errorf("the value of cpuset %d should be smaller than sandbox cpu number %d", cpuMax, sandboxCPU) ++ return err ++ } ++ ++ return nil ++} +-- +1.8.3.1 + diff --git a/runtime/patches/0087-kata-runtime-force-delete-the-sandbox-and-container.patch b/runtime/patches/0087-kata-runtime-force-delete-the-sandbox-and-container.patch new file mode 100644 index 0000000..e2ce6df --- /dev/null +++ b/runtime/patches/0087-kata-runtime-force-delete-the-sandbox-and-container.patch @@ -0,0 +1,55 @@ +From 1002cdb399b19ec2d82e4254e48f8096be6119f0 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 19 Dec 2020 13:15:12 -0500 +Subject: [PATCH] kata-runtime: force delete the sandbox and container + +reason: force delete the container and sandbox when container +or sandbox state is not running. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-runtime +Change-Id: Iaccf6c2f1de2712e88307259bed305cd366a90b7 +Signed-off-by: jiangpengfei +--- + cli/delete.go | 12 +++--------- + 1 file changed, 3 insertions(+), 9 deletions(-) + +diff --git a/cli/delete.go b/cli/delete.go +index 09552b9..1e6dfa5 100644 +--- a/cli/delete.go ++++ b/cli/delete.go +@@ -117,20 +117,14 @@ func delete(ctx context.Context, containerID string, force bool) error { + return err + } + +- forceStop := false + if oci.StateToOCIState(status.State.State) == oci.StateRunning { + if !force { + return fmt.Errorf("Container still running, should be stopped") + } +- +- forceStop = true + } + +- if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy { +- // Set forceStop and force bool flag to true to force delete everything +- forceStop = true +- force = true +- } ++ // no matter what delete parameter is, just set force to remove the container and sandbox ++ force = true + + switch containerType { + case vc.PodSandbox: +@@ -138,7 +132,7 @@ func delete(ctx context.Context, containerID string, force bool) error { + return err + } + case vc.PodContainer: +- if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil { ++ if err := deleteContainer(ctx, sandboxID, containerID, force); err != nil { + // If err info containers "no such file or directory, because pod_sandbox type + // container is deleted before pod_container type container, just return nil + // and let containerd delete container operations continue +-- +1.8.3.1 + diff --git a/runtime/patches/0088-kata-runtime-check-sandbox-healthy-state-before-call.patch b/runtime/patches/0088-kata-runtime-check-sandbox-healthy-state-before-call.patch new file mode 100644 index 0000000..6138a44 --- /dev/null +++ b/runtime/patches/0088-kata-runtime-check-sandbox-healthy-state-before-call.patch @@ -0,0 +1,219 @@ +From fda8655987010f1c569b71f3cb269a2ba5b999f0 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sun, 20 Dec 2020 18:45:30 -0500 +Subject: [PATCH] kata-runtime: check sandbox healthy state before call + kata-network + +reason: add more strict check operation before calling kata-network +subcommand, because when qemu/kata-proxy in D/T abnormal state,which +may lead to inconsistent result. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-runtime +Signed-off-by: jiangpengfei +--- + cli/network.go | 48 ++++++++++++++++++++++++++++++++++------ + cli/oci.go | 33 +++++++++++++++++++++++++++ + virtcontainers/api.go | 25 +++++++++++++++++++++ + virtcontainers/implementation.go | 4 ++++ + virtcontainers/interfaces.go | 1 + + 5 files changed, 104 insertions(+), 7 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index 7dce052..824c85d 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -234,8 +234,28 @@ var listRoutesCommand = cli.Command{ + } + + func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, op vcTypes.NetworkOp) (err error) { ++ var ( ++ f *os.File ++ output = defaultOutputFile ++ ) ++ ++ sandboxHealthy, err := checkSandboxHealthy(ctx, containerID) ++ if err != nil { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(output).Encode(nil) ++ return err ++ } ++ ++ if !sandboxHealthy { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(output).Encode(nil) ++ return fmt.Errorf("sandbox is not healthy, please check the sandbox status") ++ } ++ + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(output).Encode(nil) + return err + } + +@@ -250,14 +270,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + + // container MUST be running + if status.State.State != types.StateRunning { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(output).Encode(nil) + return fmt.Errorf("container %s is not running", containerID) + } + +- var ( +- f *os.File +- output = defaultOutputFile +- ) +- + if input == "-" { + f = os.Stdin + } else { +@@ -329,8 +346,25 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + } + + func networkListCommand(ctx context.Context, containerID string, opType networkType) (err error) { ++ var file = defaultOutputFile ++ ++ sandboxHealthy, err := checkSandboxHealthy(ctx, containerID) ++ if err != nil { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(file).Encode(nil) ++ return err ++ } ++ ++ if !sandboxHealthy { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(file).Encode(nil) ++ return fmt.Errorf("sandbox is not healthy, please check the sandbox status") ++ } ++ + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(file).Encode(nil) + return err + } + +@@ -345,11 +379,11 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT + + // container MUST be running + if status.State.State != types.StateRunning { ++ // return the null to stdout to indicate networkModifyCommand execute fail ++ json.NewEncoder(file).Encode(nil) + return fmt.Errorf("container %s is not running", containerID) + } + +- var file = defaultOutputFile +- + switch opType { + case interfaceType: + var interfaces []*vcTypes.Interface +diff --git a/cli/oci.go b/cli/oci.go +index bf962d0..1795720 100644 +--- a/cli/oci.go ++++ b/cli/oci.go +@@ -68,6 +68,39 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat + return ctrStatus, sandboxID, nil + } + ++func checkSandboxHealthy(ctx context.Context, containerID string) (bool, error) { ++ // container ID MUST be provided. ++ if containerID == "" { ++ return false, fmt.Errorf("Missing container ID") ++ } ++ ++ if len(containerID) < maxIDLength { ++ fullContainerID, err := getContainerIDbyPrefix(containerID) ++ if err != nil { ++ return false, err ++ } ++ containerID = fullContainerID ++ } ++ ++ sandboxID, err := katautils.FetchContainerIDMapping(containerID) ++ if err != nil { ++ return false, err ++ } ++ if sandboxID == "" { ++ // Not finding a container should not trigger an error as ++ // getContainerInfo is used for checking the existence and ++ // the absence of a container ID. ++ return false, nil ++ } ++ ++ healthy, err := vci.CheckSandboxHealth(ctx, sandboxID) ++ if err != nil { ++ return false, err ++ } ++ ++ return healthy, nil ++} ++ + func getExistingContainerInfo(ctx context.Context, containerID string) (vc.ContainerStatus, string, error) { + cStatus, sandboxID, err := getContainerInfo(ctx, containerID) + if err != nil { +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 0a6ba59..fd4db92 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -1114,6 +1114,31 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force + return nil + } + ++// CheckSandboxHealth is used to check sandbox healthy state to avoid qemu/kata-proxy.\ ++// process is D/T state which make grpc request is blocked. ++func CheckSandboxHealth(ctx context.Context, sandboxID string) (bool, error) { ++ span, ctx := trace(ctx, "CheckSandboxHealth") ++ defer span.Finish() ++ ++ if sandboxID == "" { ++ return false, vcTypes.ErrNeedSandboxID ++ } ++ ++ unlock, err := rwLockSandbox(sandboxID) ++ if err != nil { ++ return false, err ++ } ++ defer unlock() ++ ++ s, err := fetchSandbox(ctx, sandboxID) ++ if err != nil { ++ return false, err ++ } ++ defer s.releaseStatelessSandbox() ++ ++ return s.health(), nil ++} ++ + // procesUnhealthySandbox only change sandbox state to unhealthy + // when caller is kata-runtime kill or kata-runtime delete + func processUnhealthySandbox(sandbox *Sandbox, container *Container) error { +diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go +index fedc51f..30f6807 100644 +--- a/virtcontainers/implementation.go ++++ b/virtcontainers/implementation.go +@@ -208,3 +208,7 @@ func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxC + func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { + return GetSandboxCgroupPath(ctx, sandboxID) + } ++ ++func (impl *VCImpl) CheckSandboxHealth(ctx context.Context, sandboxID string) (bool, error) { ++ return CheckSandboxHealth(ctx, sandboxID) ++} +\ No newline at end of file +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index 4d166e0..3acd435 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -63,6 +63,7 @@ type VC interface { + UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) + + CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error ++ CheckSandboxHealth(ctx context.Context, sandboxID string) (bool, error) + } + + // VCSandbox is the Sandbox interface +-- +1.8.3.1 + diff --git a/runtime/patches/0089-kata-add-support-for-update-iface.patch b/runtime/patches/0089-kata-add-support-for-update-iface.patch new file mode 100644 index 0000000..a4832e5 --- /dev/null +++ b/runtime/patches/0089-kata-add-support-for-update-iface.patch @@ -0,0 +1,31 @@ +From 5fbf47c81a089e90d30927df231bec04f6e83348 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Tue, 2 Feb 2021 15:39:55 +0800 +Subject: [PATCH] kata: add support for update-iface + +reason: add support for update-iface + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: I245ad02dafb96eed4b1525a2e6a49b8621a960c0 +Signed-off-by: yangfeiyu +--- + cli/network.go | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/cli/network.go b/cli/network.go +index 878a01c..dbd5389 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -42,6 +42,7 @@ var kataNetworkCLICommand = cli.Command{ + addIfaceCommand, + delIfaceCommand, + listIfacesCommand, ++ updateIfaceCommand, + updateRoutesCommand, + listRoutesCommand, + addRoutesCommand, +-- +2.23.0 + diff --git a/runtime/patches/0090-kata-set-sandbox-or-container-status-to-unhealthy.patch b/runtime/patches/0090-kata-set-sandbox-or-container-status-to-unhealthy.patch new file mode 100644 index 0000000..59d9d8a --- /dev/null +++ b/runtime/patches/0090-kata-set-sandbox-or-container-status-to-unhealthy.patch @@ -0,0 +1,115 @@ +From 7ec457ebbbeeeaa7f130b45a2533c8459cba2913 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Fri, 5 Feb 2021 16:41:36 +0800 +Subject: [PATCH] kata: set sandbox or container status to unhealthy + +reason: set sandbox or container status to unhealthy when +the cmdline is kill or delete, the unhealthy flag is used +to totally clean the resources of stopped container + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Signed-off-by: yangfeiyu +--- + virtcontainers/api.go | 69 +++++++++++++++++++++++++++++++++++-------- + 1 file changed, 57 insertions(+), 12 deletions(-) + +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index dea0f5b..0223e0c 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -24,6 +24,7 @@ import ( + "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" ++ "github.com/prometheus/procfs" + "github.com/sirupsen/logrus" + ) + +@@ -639,19 +640,22 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err + + // If sandbox is unhealthy, process it correctly + if !sandbox.health() { +- // process podSandbox container type case +- if isPodSandbox { +- if err := processUnhealthySandbox(sandbox, container); err != nil { +- return ContainerStatus{}, err ++ printHypervisorStatus(sandbox) ++ if isCurrentCmdKillOrDelete() { ++ // process podSandbox container type case ++ if isPodSandbox { ++ if err := processUnhealthySandbox(sandbox, container); err != nil { ++ return ContainerStatus{}, err ++ } ++ } else { ++ // If container type is pod_container, which means container operations can not be ++ // processed successfully, we should return the error as soon as possible ++ if err := container.setContainerState(types.StateUnhealthy); err != nil { ++ return ContainerStatus{}, err ++ } ++ ++ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed") + } +- } else { +- // If container type is pod_container, which means container operations can not be +- // processed successfully, we should return the error as soon as possible +- if err := container.setContainerState(types.StateUnhealthy); err != nil { +- return ContainerStatus{}, err +- } +- +- return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed") + } + } + } +@@ -670,6 +674,47 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err + // No matching containers in the sandbox + return ContainerStatus{}, nil + } ++func printHypervisorStatus(s *Sandbox) { ++ pids := s.hypervisor.getPids() ++ ++ for _, pid := range pids { ++ if pid <= 0 { ++ virtLog.Logger.Errorf("Sandbox %v with invalid hypervisor PID: %+v", s.id, pids) ++ continue ++ } ++ ++ proc, err := procfs.NewProc(pid) ++ if err != nil { ++ virtLog.Logger.Warnf("New proc of pid %v failed", pid) ++ } ++ ++ stat, err := proc.NewStat() ++ virtLog.Logger.Debugf("The status of pid %v is %#v, and err is %v", pid, stat, err) ++ } ++} ++ ++func isCurrentCmdKillOrDelete() bool { ++ pid := os.Getpid() ++ proc, err := procfs.NewProc(pid) ++ if err != nil { ++ virtLog.Logger.Warnf("New proc of currrent process %v failed", pid) ++ return false ++ } ++ ++ lines, err := proc.CmdLine() ++ if err != nil { ++ virtLog.Logger.Errorf("Get cmd line of currrent process %v failed", pid) ++ } ++ ++ for _, v := range lines { ++ if v == "kill" || v == "delete" { ++ virtLog.Logger.Debugf("The cmdline is kill or delete %v", lines) ++ return true ++ } ++ } ++ ++ return false ++} + + // KillContainer is the virtcontainers entry point to send a signal + // to a container running inside a sandbox. If all is true, all processes in +-- +2.23.0 + diff --git a/runtime/patches/0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch b/runtime/patches/0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch new file mode 100644 index 0000000..98df72a --- /dev/null +++ b/runtime/patches/0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch @@ -0,0 +1,51 @@ +From 3f422248b2521b5c9f449b4d3d61193426bd2246 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 15 Mar 2021 04:07:43 -0400 +Subject: [PATCH] kata-runtime: add sandbox file lock while call + GetSandboxCgroupPath + +reason: vci.GetSandboxCgroupPath func will load the sandbox persist.json file, +which is shared with all the containers in the same sandbox. So need to add a +sandbox file lock to make sure load sandbox persist.json file is mutexable. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: I07eeb213256bd27f6d5718a65cf7e7ccbec2ecb7 +Signed-off-by: jiangpengfei +--- + virtcontainers/cgroups.go | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index 21708eb..6b1d193 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -22,6 +22,7 @@ import ( + "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" ++ vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + ) + + type cgroupPather interface { +@@ -155,6 +156,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error { + + // GetSandboxCgroupPath return the cgroup path of specified sandbox + func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { ++ if sandboxID == "" { ++ return "", vcTypes.ErrNeedSandboxID ++ } ++ ++ unlock, err := rwLockSandbox(sandboxID) ++ if err != nil { ++ return "", err ++ } ++ defer unlock() ++ + config, err := loadSandboxConfig(sandboxID) + if err != nil { + return "", err +-- +1.8.3.1 + diff --git a/runtime/patches/0092-qemu-add-arm64-to-support-list-of-dimm.patch b/runtime/patches/0092-qemu-add-arm64-to-support-list-of-dimm.patch new file mode 100644 index 0000000..a40d86b --- /dev/null +++ b/runtime/patches/0092-qemu-add-arm64-to-support-list-of-dimm.patch @@ -0,0 +1,32 @@ +From e36bd3400309ae13e4fd20db055b332c671ff530 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Fri, 16 Apr 2021 06:57:53 -0400 +Subject: [PATCH] qemu: add arm64 to support list of dimm + +reason: dimm is supported on arm64, so add is to check list. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: I26774ce6a39a9048f2c63dcb79855b17b36da999 +Signed-off-by: jiangpengfei +--- + vendor/github.com/intel/govmm/qemu/qemu.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go +index 174d377..1e80a62 100644 +--- a/vendor/github.com/intel/govmm/qemu/qemu.go ++++ b/vendor/github.com/intel/govmm/qemu/qemu.go +@@ -127,7 +127,7 @@ const ( + + func isDimmSupported(config *Config) bool { + switch runtime.GOARCH { +- case "amd64", "386": ++ case "amd64", "386", "arm64": + return true + default: + return false +-- +1.8.3.1 + diff --git a/runtime/patches/0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch b/runtime/patches/0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch new file mode 100644 index 0000000..79c1bb0 --- /dev/null +++ b/runtime/patches/0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch @@ -0,0 +1,40 @@ +From 699a5993aebc93e41a71658b32061b918e2a7a39 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Fri, 16 Apr 2021 17:10:48 +0800 +Subject: [PATCH] kata-runtime: add timeout for grpcWaitProcessRequest + +reason: add timeout for grpcWaitProcessRequest, it is only used +in stopping container process + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Signed-off-by: yangfeiyu +--- + virtcontainers/kata_agent.go | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index f7742fe..94b4ef5 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -60,6 +60,7 @@ const ( + ) + + var ( ++ waitProcessTimeOut = 10 * time.Second + checkRequestTimeout = 10 * time.Second + defaultRequestTimeout = 10 * time.Second + createContainerTimeout = 120 * time.Second +@@ -2183,7 +2184,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c + ctx = context.Background() + switch reqName { + case grpcWaitProcessRequest: +- // Wait has no timeout ++ ctx, cancel = context.WithTimeout(ctx, waitProcessTimeOut) + case grpcCheckRequest: + ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout) + case grpcCreateContainerRequest: +-- +2.23.0 + diff --git a/runtime/patches/0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch b/runtime/patches/0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch new file mode 100644 index 0000000..4964976 --- /dev/null +++ b/runtime/patches/0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch @@ -0,0 +1,115 @@ +From 940ddf068c95fd9662f8be79a38316f585e35edf Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 27 May 2021 21:43:11 -0400 +Subject: [PATCH] kata-runtime: fix update-iface clean NIC cause route left + problem + +reason: when kata-network update-iface is called to clean NIC ip info, +the guest os will clean the related routes with NIC, however the routes +info in the persisit.json file is still left. So the followed add-route +command add the same route will cause the problem. + +Conflict: NA +Reference:https://gitee.com/src-openeuler/kata-containers + +Change-Id: I723bee277e8ec537f6de1127de3ec812d5686fe4 +Signed-off-by: jiangpengfei +--- + virtcontainers/network.go | 70 +++++------------------------------------------ + 1 file changed, 7 insertions(+), 63 deletions(-) + +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index 7bbf85b..18cc04c 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -1698,45 +1698,16 @@ func isSameRoute(existing, r *netlink.Route, fuzzy bool) bool { + } + + func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Route, err error) { +- add, err := generateAddRoute(route) ++ _, err = generateAddRoute(route) + if err != nil { + return nil, err + } + +- // add the route for "lo" loopback device +- if route.Device == localHostDeviceName { +- added = &vcTypes.Route{ +- Dest: route.Dest, +- Gateway: route.Gateway, +- Device: route.Device, +- } +- return added, nil +- } +- +- // add the route for exist network enpoints +- for _, ep := range ns.Endpoints { +- if ep.Name() != route.Device { +- continue +- } +- +- netInfo := ep.Properties() +- for _, exist := range ep.Properties().Routes { +- if isSameRoute(&exist, add, false) { +- return nil, fmt.Errorf("route rule %v already exits", add) +- } +- } +- // flush the netInfo.Routes with new added route +- netInfo.Routes = append(netInfo.Routes, *add) +- ep.SetProperties(netInfo) +- added = &vcTypes.Route{ +- Dest: route.Dest, +- Gateway: route.Gateway, +- Device: route.Device, +- } +- break +- } +- +- return added, nil ++ return &vcTypes.Route{ ++ Dest: route.Dest, ++ Gateway: route.Gateway, ++ Device: route.Device, ++ }, nil + } + + func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) { +@@ -1812,34 +1783,7 @@ func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcType + return nil, err + } + +- // remove the lo device related routes +- if route.Device == localHostDeviceName { +- removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false)) +- +- return removed, nil +- } +- +- for _, ep := range ns.Endpoints { +- // if device is empty, means search every device +- if route.Device != "" && ep.Name() != route.Device { +- continue +- } +- +- netInfo := ep.Properties() +- for i, exist := range ep.Properties().Routes { +- if isSameRoute(&exist, del, true) { +- // need remove +- netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...) +- ep.SetProperties(netInfo) +- dev := route.Device +- if route.Device == "" { +- dev = netInfo.Iface.Name +- } +- removed = append(removed, parseToGrpcRoute(dev, del, false)) +- } +- } +- } +- ++ removed = append(removed, parseToGrpcRoute(route.Device, del, false)) + return removed, nil + } + +-- +1.8.3.1 + diff --git a/runtime/patches/0095-kata-runtime-fix-qemu-process-resource-resi.patch b/runtime/patches/0095-kata-runtime-fix-qemu-process-resource-resi.patch new file mode 100644 index 0000000..7ad002c --- /dev/null +++ b/runtime/patches/0095-kata-runtime-fix-qemu-process-resource-resi.patch @@ -0,0 +1,95 @@ +From eae6c9cf1d875b28cde530ac337578bc5e301762 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 14 Jul 2021 18:16:22 -0400 +Subject: [PATCH] kata-runtime: fix qemu process resource residuals +reason: fix qemu process resource residuals +Change-Id: Ic16c949fde62a6f662e00d0608c2285bfaf737d0 +Signed-off-by: jiangpengfei +--- + cli/delete.go | 4 ++++ + pkg/katautils/oci.go | 25 +++++++++++++++++++++++++ + virtcontainers/container.go | 14 +++++++------- + 3 files changed, 36 insertions(+), 7 deletions(-) +diff --git a/cli/delete.go b/cli/delete.go +index 1e6dfa5..cd7eb77 100644 +--- a/cli/delete.go ++++ b/cli/delete.go +@@ -131,6 +131,10 @@ func delete(ctx context.Context, containerID string, force bool) error { + if err := deleteSandbox(ctx, sandboxID, force); err != nil { + return err + } ++ ++ if err := katautils.DelSandboxIDMapping(ctx, sandboxID); err != nil { ++ kataLog.Errorf("delete all sandbox related container mapping failed: %v", err) ++ } + case vc.PodContainer: + if err := deleteContainer(ctx, sandboxID, containerID, force); err != nil { + // If err info containers "no such file or directory, because pod_sandbox type +diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go +index 1334af3..053c284 100644 +--- a/pkg/katautils/oci.go ++++ b/pkg/katautils/oci.go +@@ -113,3 +113,28 @@ func DelContainerIDMapping(ctx context.Context, containerID string) error { + + return os.RemoveAll(path) + } ++ ++func DelSandboxIDMapping(ctx context.Context, sandboxID string) error { ++ if sandboxID == "" { ++ return fmt.Errorf("Missing Sandbox ID") ++ } ++ ++ files, err := ioutil.ReadDir(ctrsMapTreePath) ++ if err != nil { ++ return err ++ } ++ for _, f := range files { ++ parentID, err := FetchContainerIDMapping(f.Name()) ++ if err != nil { ++ return err ++ } ++ ++ if parentID == sandboxID { ++ err = DelContainerIDMapping(ctx, f.Name()) ++ if err != nil { ++ return err ++ } ++ } ++ } ++ return nil ++} +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 724b58c..fa7d0ea 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1126,13 +1126,6 @@ func (c *Container) stop(force bool) error { + return nil + } + +- // If container state is unhealthy, just force kill the container +- if c.state.State == types.StateUnhealthy { +- c.forceKillContainer() +- // after force kill container, then change container state to stopped +- return c.setContainerState(types.StateStopped) +- } +- + if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil { + return err + } +@@ -1155,6 +1148,13 @@ func (c *Container) stop(force bool) error { + + }() + ++ // If container state is unhealthy, just force kill the container ++ if c.state.State == types.StateUnhealthy { ++ c.forceKillContainer() ++ // after force kill container, then change container state to stopped ++ return c.setContainerState(types.StateStopped) ++ } ++ + // Here we expect that stop() has been called because the container + // process returned or because it received a signal. In case of a + // signal, we want to give it some time to end the container process. +-- +1.8.3.1 + diff --git a/runtime/patches/0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch b/runtime/patches/0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch new file mode 100644 index 0000000..6154f1a --- /dev/null +++ b/runtime/patches/0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch @@ -0,0 +1,46 @@ +From dfd1e6aa9be31b0df7866c3a3dbb0ab210a02052 Mon Sep 17 00:00:00 2001 +From: jiangpengfei 00444339 +Date: Fri, 22 Apr 2022 11:52:53 +0800 +Subject: [PATCH] kata-containers: Move from query-cpus to query-cpus-fast + +reason: We recently updated to using qemu-6.1 (from qemu 5.2). Unfortunately one +breaking change in qemu 6.0 wasn't caught by the CI. + +The query-cpus QMP command has been removed, replaced by query-cpus-fast +(which has been available since qemu 2.12). govmm already had support for +query-cpus-fast, we just weren't using it, so the change is quite easy. + +fixes #2643 + +Signed-off-by: David Gibson + +Signed-off-by: jiangpengfei 00444339 +--- + virtcontainers/qemu.go | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index c60ec02..5c55c09 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -2210,7 +2210,7 @@ func (q *qemu) getThreadIDs() (vcpuThreadIDs, error) { + return tid, err + } + +- cpuInfos, err := q.qmpMonitorCh.qmp.ExecQueryCpus(q.qmpMonitorCh.ctx) ++ cpuInfos, err := q.qmpMonitorCh.qmp.ExecQueryCpusFast(q.qmpMonitorCh.ctx) + if err != nil { + q.Logger().WithError(err).Error("failed to query cpu infos") + return tid, err +@@ -2219,7 +2219,7 @@ func (q *qemu) getThreadIDs() (vcpuThreadIDs, error) { + tid.vcpus = make(map[int]int, len(cpuInfos)) + for _, i := range cpuInfos { + if i.ThreadID > 0 { +- tid.vcpus[i.CPU] = i.ThreadID ++ tid.vcpus[i.CPUIndex] = i.ThreadID + } + } + return tid, nil +-- +2.33.0 + diff --git a/runtime/series.conf b/runtime/series.conf index b8c53a7..02a09e3 100644 --- a/runtime/series.conf +++ b/runtime/series.conf @@ -74,3 +74,22 @@ 0074-add-timeout-for-grpcWaitProcessRequest.patch 0075-move-waitProcess-timeout-to-stop-process.patch 0076-kata-runtime-fix-the-problem-that-sandbox-stratovirt.patch +0078-kata-containers-check-file-size-before-creating-cont.patch +0079-kata-runtime-fix-qemu-SCSIBus-info-not-saved-into-pe.patch +0080-kata-runtime-fix-the-block-device-not-removed-in-dev.patch +0081-kata-runtime-cut-too-long-message-in-grpc-log.patch +0082-kata-runtime-change-sandbox-state-to-unhealthy-when-.patch +0083-kata-runtime-add-removeMountBlockDevices-for-contain.patch +0084-kata-runtime-fix-validInterface-func-cause-crash-pro.patch +0085-kata-runtime-fix-kata-netmon-does-not-exit-when-cont.patch +0086-kata-runtime-add-checkCPUSet-before-create-container.patch +0087-kata-runtime-force-delete-the-sandbox-and-container.patch +0088-kata-runtime-check-sandbox-healthy-state-before-call.patch +0089-kata-add-support-for-update-iface.patch +0090-kata-set-sandbox-or-container-status-to-unhealthy.patch +0091-kata-runtime-add-sandbox-file-lock-while-call-GetSan.patch +0092-qemu-add-arm64-to-support-list-of-dimm.patch +0093-kata-runtime-add-timeout-for-grpcWaitProcessRequest.patch +0094-kata-runtime-fix-update-iface-clean-NIC-cause-route-.patch +0095-kata-runtime-fix-qemu-process-resource-resi.patch +0096-kata-containers-Move-from-query-cpus-to-query-cpus-f.patch -- Gitee