diff --git a/agent/agent-1.11.1.tar.gz b/agent/agent-1.11.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..5500b5d5c6823124554a49c66c1c17566f6117e4 Binary files /dev/null and b/agent/agent-1.11.1.tar.gz differ diff --git a/agent/apply-patches b/agent/apply-patches new file mode 100755 index 0000000000000000000000000000000000000000..87d9b08ad8cb8f27fd74e88128ffa62d4b15e38b --- /dev/null +++ b/agent/apply-patches @@ -0,0 +1,20 @@ +#!/bin/bash + +if [[ -f ./patch_flag ]];then + echo "agent patched!" + exit 0 +fi + +tar -zxvf agent-1.11.1.tar.gz +cp -fr ./agent-1.11.1/* ./ +rm -rf ./agent-1.11.1 +cat ./series.conf | while read line +do + if [[ $line == '' || $line =~ ^\s*# ]]; then + continue + fi + echo "====patch $line======" + pwd + patch -p1 -F1 -s < ./patches/$line +done +touch ./patch_flag diff --git a/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch b/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch new file mode 100644 index 0000000000000000000000000000000000000000..24bc18a7564bd9b315bf1a6fd8a160678ccc4e47 --- /dev/null +++ b/agent/patches/0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch @@ -0,0 +1,234 @@ +From ac1d7806f8de2f8ca393df08a9c62d1045c4afdc Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 11 Dec 2018 18:27:02 -0500 +Subject: [PATCH 01/16] agent: add agent.netlink_recv_buf_size flag to set + netlink recv buf size + +fixes: #813 + +reason: If hotplug huge size memory(for example 128GB) into guest, +kernel will produce a lot of memory add uevents and send to netlink socket, +however netlink socket default receive buffer size is 4KB, which is too small +to receive all memory add uevents. +Since hotplug huge size memory is not common case, so we consider add an agent +flag agent.netlink_recv_buf_size to set netlink socket recv buffer size. + +Signed-off-by: jiangpengfei +--- + README.md | 13 +++++++++++++ + agent.go | 10 +++++++++- + config.go | 15 ++++++++++++++ + config_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ + pkg/uevent/uevent.go | 15 +++++++++++--- + 5 files changed, 104 insertions(+), 4 deletions(-) + +diff --git a/README.md b/README.md +index cec65a4..16f96a4 100644 +--- a/README.md ++++ b/README.md +@@ -98,6 +98,19 @@ The pipe's capacity for stdout/stderr can be modified by specifying the `agent.c + to the guest kernel command line. For example, `agent.container_pipe_size=2097152` will set the stdout and stderr + pipes to 2097152 bytes. + ++## Uevent Netlink Socket Receive Buffer Size ++ ++When hotplugging a huge size memory into the Kata VM, the kernel in the VM will produce a lot of memory object add ++uevents and send all these uevents to Kata agent by netlink socket. However, default netlink socket receive buffer ++size is 4KB, which is too small and can only hold 256 memory add uevents. If memory add uevents number is larger ++than 256, the left uevents can not be received and processed by Kata agent. ++ ++The uevent netlink socket receive buffer size can be modified by specifying the `agent.netlink_recv_buf_size` flag ++to the guest kernel command line. For example, `agent.netlink_recv_buf_size=2MB` will set the uevent netlink socket ++receive buffer size to 2MB value. `agent.netlink_recv_buf_size` valid value range is `[4KB ~ 4MB]` and value can be ++set in human-readable memory format or pure digital number format(default memory unit is byte). ++ ++ + [1]: https://github.com/firecracker-microvm/firecracker/blob/master/docs/vsock.md + [2]: https://golang.org/pkg/time/#ParseDuration + [3]: http://man7.org/linux/man-pages/man7/pipe.7.html +diff --git a/agent.go b/agent.go +index 2d2c293..c1cac08 100644 +--- a/agent.go ++++ b/agent.go +@@ -190,6 +190,14 @@ var unifiedCgroupHierarchy = false + // Size in bytes of the stdout/stderr pipes created for each container. + var containerPipeSize = uint32(0) + ++const ( ++ minNetlinkSockRecvBufSize = 4 * 1024 ++ maxNetlinkSockRecvBufSize = 4 * 1024 * 1024 ++) ++ ++// Size in bytes of the netlink socket recv buf size ++var netlinkSockRecvBufSize = uint32(0) ++ + // commType is used to denote the communication channel type used. + type commType int + +@@ -708,7 +716,7 @@ func (s *sandbox) waitForStopServer() { + func (s *sandbox) listenToUdevEvents() { + fieldLogger := agentLog.WithField("subsystem", "udevlistener") + +- uEvHandler, err := uevent.NewHandler() ++ uEvHandler, err := uevent.NewHandler(netlinkSockRecvBufSize) + if err != nil { + fieldLogger.Warnf("Error starting uevent listening loop %s", err) + return +diff --git a/config.go b/config.go +index 4530096..6c7d473 100644 +--- a/config.go ++++ b/config.go +@@ -7,11 +7,13 @@ + package main + + import ( ++ "fmt" + "io/ioutil" + "strconv" + "strings" + "time" + ++ "github.com/docker/go-units" + "github.com/sirupsen/logrus" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" +@@ -29,6 +31,7 @@ const ( + hotplugTimeoutFlag = optionPrefix + "hotplug_timeout" + unifiedCgroupHierarchyFlag = optionPrefix + "unified_cgroup_hierarchy" + containerPipeSizeFlag = optionPrefix + "container_pipe_size" ++ netlinkSockRecvBufSizeFlag = optionPrefix + "netlink_recv_buf_size" + traceModeStatic = "static" + traceModeDynamic = "dynamic" + traceTypeIsolated = "isolated" +@@ -155,6 +158,18 @@ func parseCmdlineOption(option string) error { + return err + } + unifiedCgroupHierarchy = flag ++ case netlinkSockRecvBufSizeFlag: ++ bufSizeInBytes, err := units.RAMInBytes(split[valuePosition]) ++ if err != nil { ++ return err ++ } ++ ++ if bufSizeInBytes < minNetlinkSockRecvBufSize || bufSizeInBytes > maxNetlinkSockRecvBufSize { ++ return fmt.Errorf("invalid netlink socket recv buf size: %d (valid size range %s-%s bytes)", bufSizeInBytes, ++ units.BytesSize(minNetlinkSockRecvBufSize), units.BytesSize(maxNetlinkSockRecvBufSize)) ++ } ++ ++ netlinkSockRecvBufSize = uint32(bufSizeInBytes) + default: + if strings.HasPrefix(split[optionPosition], optionPrefix) { + return grpcStatus.Errorf(codes.NotFound, "Unknown option %s", split[optionPosition]) +diff --git a/config_test.go b/config_test.go +index 2a23133..f40f17a 100644 +--- a/config_test.go ++++ b/config_test.go +@@ -486,3 +486,58 @@ func TestParseCmdlineOptionContainerPipeSize(t *testing.T) { + assert.Equal(d.expectedContainerPipeSize, containerPipeSize, "test %d (%+v)", i, d) + } + } ++ ++func TestParseCmdlineOptionNetlinkSockRecvBufSize(t *testing.T) { ++ assert := assert.New(t) ++ ++ type testData struct { ++ option string ++ shouldErr bool ++ expectedNetlinkSockRecvBufSize uint32 ++ } ++ ++ data := []testData{ ++ {"", false, 0}, ++ {"netlink_recv_buf_siz", false, 0}, ++ {"netlink_recv_buf_size", false, 0}, ++ {"netlink_recv_buf_size=", false, 0}, ++ {"netlink_recv_buf_size=4096", false, 0}, ++ {"netlink_recv_buf_size=4KB", false, 0}, ++ {"agent.netlink_recv_buf_size=", true, 0}, ++ {"agent.netlink_recv_buf_size=foobar", true, 0}, ++ {"agent.netlink_recv_buf_size=-1", true, 0}, ++ {"agent.netlink_recv_buf_size=0", true, 0}, ++ {"agent.netlink_recv_buf_size=100", true, 0}, ++ {"agent.netlink_recv_buf_size=3KB", true, 0}, ++ {"agent.netlink_recv_buf_size=3.6KB", true, 0}, ++ {"agent.netlink_recv_buf_size=4095", true, 0}, ++ {"agent.netlink_recv_buf_size=4096xB", true, 0}, ++ {"agent.netlink_recv_buf_size=4096", false, 4096}, ++ {"agent.netlink_recv_buf_size=4097", false, 4097}, ++ {"agent.netlink_recv_buf_size=4096.0", false, 4096}, ++ {"agent.netlink_recv_buf_size=1024KB", false, 1048576}, ++ {"agent.netlink_recv_buf_size=1MB", false, 1048576}, ++ {"agent.netlink_recv_buf_size=4194303", false, 4194303}, ++ {"agent.netlink_recv_buf_size=3.999MB", false, 4193255}, ++ {"agent.netlink_recv_buf_size=4194304", false, 4194304}, ++ {"agent.netlink_recv_buf_size=4MB", false, 4194304}, ++ {"agent.netlink_recv_buf_size=4.001MB", true, 0}, ++ {"agent.netlink_recv_buf_size=4194305", true, 0}, ++ {"agent.netlink_recv_buf_size=100MB", true, 0}, ++ {"agent.netlink_recv_buf_size=1GB", true, 0}, ++ } ++ ++ for i, d := range data { ++ // reset the netlink socket recv buffer size ++ netlinkSockRecvBufSize = 0 ++ ++ err := parseCmdlineOption(d.option) ++ if d.shouldErr { ++ assert.Error(err) ++ } else { ++ assert.NoError(err) ++ } ++ ++ assert.Equal(d.expectedNetlinkSockRecvBufSize, netlinkSockRecvBufSize, "test %d (%+v)", i, d) ++ } ++} +diff --git a/pkg/uevent/uevent.go b/pkg/uevent/uevent.go +index fc2c127..fa84086 100644 +--- a/pkg/uevent/uevent.go ++++ b/pkg/uevent/uevent.go +@@ -10,6 +10,7 @@ import ( + "bufio" + "io" + "strings" ++ "syscall" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" +@@ -33,7 +34,7 @@ type ReaderCloser struct { + } + + // NewReaderCloser returns an io.ReadCloser handle for uevent. +-func NewReaderCloser() (io.ReadCloser, error) { ++func NewReaderCloser(netlinkRecvBufSize uint32) (io.ReadCloser, error) { + nl := unix.SockaddrNetlink{ + Family: unix.AF_NETLINK, + // Passing Pid as 0 here allows the kernel to take care of assigning +@@ -47,6 +48,14 @@ func NewReaderCloser() (io.ReadCloser, error) { + return nil, err + } + ++ // If netlinkRecvBufSize > 0, set netlink socket recv buffer size to netlinkRecvBufSize ++ if netlinkRecvBufSize > 0 { ++ err = unix.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUFFORCE, int(netlinkRecvBufSize)) ++ if err != nil { ++ return nil, err ++ } ++ } ++ + if err := unix.Bind(fd, &nl); err != nil { + return nil, err + } +@@ -85,8 +94,8 @@ type Handler struct { + } + + // NewHandler returns a uevent handler. +-func NewHandler() (*Handler, error) { +- rdCloser, err := NewReaderCloser() ++func NewHandler(netlinkRecvBufSize uint32) (*Handler, error) { ++ rdCloser, err := NewReaderCloser(netlinkRecvBufSize) + if err != nil { + return nil, err + } +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0002-network-support-update-routes-incrementally.patch b/agent/patches/0002-network-support-update-routes-incrementally.patch new file mode 100644 index 0000000000000000000000000000000000000000..4cc9c271114f53289f4cfbb56448d8ab8a98214a --- /dev/null +++ b/agent/patches/0002-network-support-update-routes-incrementally.patch @@ -0,0 +1,686 @@ +From 13f54c768dcd7bf982dde8e57fb5cd624fedf5bc Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 17 Aug 2020 11:23:55 +0800 +Subject: [PATCH 02/16] network: support update routes incrementally + +reason: add increment flag in the UpdateRoutesRequest to +support upate routes incrementally to improve the efficiency. + +kata-network add-route and del-route needs this feature. + +Signed-off-by: jiangpengfei +--- + grpc.go | 2 +- + network.go | 74 ++++++++- + network_test.go | 16 +- + protocols/grpc/agent.pb.go | 402 +++++++++++++++++++++++++-------------------- + protocols/grpc/agent.proto | 1 + + 5 files changed, 300 insertions(+), 195 deletions(-) + +diff --git a/grpc.go b/grpc.go +index 886661b..8fe8217 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -1556,7 +1556,7 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface + } + + func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { +- return a.sandbox.updateRoutes(nil, req.Routes) ++ return a.sandbox.updateRoutes(nil, req.Routes, req.Increment) + } + + func (a *agentGRPC) ListInterfaces(ctx context.Context, req *pb.ListInterfacesRequest) (*pb.Interfaces, error) { +diff --git a/network.go b/network.go +index 64a16a9..02e28cb 100644 +--- a/network.go ++++ b/network.go +@@ -398,7 +398,7 @@ func (s *sandbox) deleteRoutes(netHandle *netlink.Handle) error { + // state which matches the requested routes. In doing this, preesxisting non-loopback routes will be + // removed from the network. If an error occurs, this function returns the list of routes in + // gRPC-route format at the time of failure +-func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes) (resultingRoutes *pb.Routes, err error) { ++func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Routes, increment bool) (resultingRoutes *pb.Routes, err error) { + if requestedRoutes == nil { + return nil, errNoRoutes + } +@@ -418,13 +418,66 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro + } + }() + ++ var ( ++ added []*types.Route ++ removed []*types.Route ++ ) ++ ++ defer func(netHandle *netlink.Handle) { ++ if err != nil { ++ // if error happens after route added, need to rollback the added route ++ if len(added) > 0 { ++ for _, r := range added { ++ errRb := s.updateRoute(netHandle, r, false) ++ if errRb != nil { ++ agentLog.WithError(err).Error("rollback route failed") ++ } ++ } ++ } ++ ++ // if error happens after route removed, need to rollback the removed route ++ if len(removed) > 0 { ++ for _, r := range removed { ++ errRb := s.updateRoute(netHandle, r, true) ++ if errRb != nil { ++ agentLog.WithError(err).Error("rollback route failed") ++ } ++ } ++ } ++ } ++ }(netHandle) ++ ++ // updateOneRoute just update the specified one route ++ updateOneRoute := func(netHandle *netlink.Handle, reqRoute *types.Route) error { ++ var add bool = true ++ if reqRoute.Dest != "" && strings.HasPrefix(reqRoute.Dest, "-") { ++ reqRoute.Dest = reqRoute.Dest[1:] ++ add = false ++ } ++ err = s.updateRoute(netHandle, reqRoute, add) ++ if err != nil { ++ agentLog.WithError(err).Error("update Route failed") ++ // If there was an error setting the route, return the error ++ // and the current routes on the system via the defer func ++ return err ++ } ++ if add { ++ added = append([]*types.Route{reqRoute}, added[:]...) ++ } else { ++ removed = append([]*types.Route{reqRoute}, removed[:]...) ++ } ++ return nil ++ } ++ + // + // First things first, let's blow away all the existing routes. The updateRoutes function + // is designed to be declarative, so we will attempt to create state matching what is + // requested, and in the event that we fail to do so, will return the error and final state. + // +- if err = s.deleteRoutes(netHandle); err != nil { +- return nil, err ++ if !increment { ++ if err = s.deleteRoutes(netHandle); err != nil { ++ return nil, err ++ } + } + + // +@@ -434,7 +487,12 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro + // won't be able to access the gateway + for _, reqRoute := range requestedRoutes.Routes { + if reqRoute.Gateway == "" { +- err = s.updateRoute(netHandle, reqRoute, true) ++ if increment { ++ err = updateOneRoute(netHandle, reqRoute) ++ } else { ++ err = s.updateRoute(netHandle, reqRoute, true) ++ } ++ + if err != nil { + agentLog.WithError(err).Error("update Route failed") + //If there was an error setting the route, return the error +@@ -447,7 +505,11 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro + // Take a second pass and apply the routes which include a gateway + for _, reqRoute := range requestedRoutes.Routes { + if reqRoute.Gateway != "" { +- err = s.updateRoute(netHandle, reqRoute, true) ++ if increment { ++ err = updateOneRoute(netHandle, reqRoute) ++ } else { ++ err = s.updateRoute(netHandle, reqRoute, true) ++ } + if err != nil { + agentLog.WithError(err).Error("update Route failed") + //If there was an error setting the route, return the +@@ -699,4 +761,4 @@ func (s *sandbox) handleLocalhost() error { + } + + return netlink.LinkSetUp(lo) +-} ++} +\ No newline at end of file +diff --git a/network_test.go b/network_test.go +index a143670..a1e58f5 100644 +--- a/network_test.go ++++ b/network_test.go +@@ -160,7 +160,7 @@ func TestUpdateRoutes(t *testing.T) { + Routes: inputRoutesSimple, + } + +- results, err := s.updateRoutes(netHandle, testRoutes) ++ results, err := s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(t, err, "Unexpected update interface failure: %v", err) + assert.True(t, reflect.DeepEqual(results, testRoutes), + "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) +@@ -173,7 +173,7 @@ func TestUpdateRoutes(t *testing.T) { + } + testRoutes.Routes = inputRoutesPTPExample + +- results, err = s.updateRoutes(netHandle, testRoutes) ++ results, err = s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(t, err, "Unexpected update interface failure: %v", err) + assert.True(t, reflect.DeepEqual(results, testRoutes), + "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) +@@ -184,7 +184,7 @@ func TestUpdateRoutes(t *testing.T) { + {Dest: "192.168.0.0/16", Gateway: "", Source: "192.168.0.2", Scope: 0, Device: "ifc-name"}, + } + testRoutes.Routes = inputRoutesNoScope +- results, err = s.updateRoutes(netHandle, testRoutes) ++ results, err = s.updateRoutes(netHandle, testRoutes, false) + assert.NotNil(t, err, "Expected to observe unreachable route failure") + + assert.True(t, reflect.DeepEqual(results.Routes[0], testRoutes.Routes[1]), +@@ -231,7 +231,7 @@ func TestUpdateRoutesIPVlan(t *testing.T) { + } + testRoutes.Routes = inputRoutesIPVlanExample + +- results, err := s.updateRoutes(netHandle, testRoutes) ++ results, err := s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(t, err, "Unexpected update interface failure: %v", err) + assert.True(t, reflect.DeepEqual(results, testRoutes), + "Interface created didn't match: got %+v, expecting %+v", results, testRoutes) +@@ -357,7 +357,7 @@ func TestListRoutes(t *testing.T) { + Routes: inputRoutesSimple, + } + +- _, err := s.updateRoutes(netHandle, testRoutes) ++ _, err := s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(err) + results, err := s.listRoutes(nil) + assert.Nil(err, "Expected to list all routes") +@@ -377,7 +377,7 @@ func TestListRoutes(t *testing.T) { + Routes: inputRoutesSimple, + } + +- _, err = s.updateRoutes(netHandle, testRoutes) ++ _, err = s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(err) + results, err = s.listRoutes(nil) + assert.Nil(err, "Expected to list all routes") +@@ -438,7 +438,7 @@ func TestListRoutesWithIPV6(t *testing.T) { + Routes: inputRoutesSimple, + } + +- _, err := s.updateRoutes(netHandle, testRoutes) ++ _, err := s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(err) + results, err := s.listRoutes(nil) + assert.Nil(err, "Expected to list all routes") +@@ -514,7 +514,7 @@ func TestListRoutesWithTwoInterfacesSameSubnet(t *testing.T) { + Routes: inputRoutesSimple, + } + +- _, err := s.updateRoutes(netHandle, testRoutes) ++ _, err := s.updateRoutes(netHandle, testRoutes, false) + assert.Nil(err) + results, err := s.listRoutes(nil) + assert.Nil(err, "Expected to list all routes") +diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go +index 77e6d1b..1b887e5 100644 +--- a/protocols/grpc/agent.pb.go ++++ b/protocols/grpc/agent.pb.go +@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + } + + type UpdateRoutesRequest struct { +- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` ++ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` ++ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` + } + + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } +@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes { + return nil + } + ++func (m *UpdateRoutesRequest) GetIncrement() bool { ++ if m != nil { ++ return m.Increment ++ } ++ return false ++} ++ + type ListInterfacesRequest struct { + } + +@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + } + i += n20 + } ++ if m.Increment { ++ dAtA[i] = 0x10 ++ i++ ++ if m.Increment { ++ dAtA[i] = 1 ++ } else { ++ dAtA[i] = 0 ++ } ++ i++ ++ } + return i, nil + } + +@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) { + l = m.Routes.Size() + n += 1 + l + sovAgent(uint64(l)) + } ++ if m.Increment { ++ n += 2 ++ } + return n + } + +@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { + return err + } + iNdEx = postIndex ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType) ++ } ++ var v int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ v |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ m.Increment = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) +@@ -12852,184 +12893,185 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2862 bytes of a gzipped FileDescriptorProto ++ // 2876 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6, +- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4, +- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e, +- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae, +- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e, +- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d, +- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11, +- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7, +- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9, +- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93, +- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3, +- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f, +- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6, +- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89, +- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50, +- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c, +- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f, +- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b, +- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67, +- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3, +- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc, +- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0, +- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e, +- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30, +- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba, +- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00, +- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63, +- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8, +- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84, +- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e, +- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35, +- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69, +- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6, +- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf, +- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47, +- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92, +- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6, +- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85, +- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40, +- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68, +- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec, +- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77, +- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff, +- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03, +- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f, +- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01, +- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9, +- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c, +- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05, +- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64, +- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12, +- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a, +- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10, +- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56, +- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86, +- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36, +- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45, +- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce, +- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d, +- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c, +- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28, +- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31, +- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee, +- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c, +- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29, +- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5, +- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7, +- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d, +- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03, +- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42, +- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29, +- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92, +- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84, +- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f, +- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92, +- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76, +- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e, +- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d, +- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05, +- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5, +- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00, +- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8, +- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8, +- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76, +- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10, +- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3, +- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa, +- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2, +- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9, +- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11, +- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe, +- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd, +- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4, +- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34, +- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60, +- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4, +- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29, +- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b, +- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13, +- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59, +- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86, +- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce, +- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e, +- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9, +- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19, +- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b, +- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86, +- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac, +- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1, +- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3, +- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a, +- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e, +- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10, +- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c, +- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33, +- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48, +- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6, +- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1, +- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4, +- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7, +- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93, +- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8, +- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7, +- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7, +- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62, +- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4, +- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e, +- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9, +- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7, +- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1, +- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2, +- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96, +- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d, +- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56, +- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a, +- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78, +- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09, +- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d, +- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf, +- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d, +- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb, +- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e, +- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02, +- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6, +- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd, +- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75, +- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11, +- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73, +- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15, +- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2, +- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa, +- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44, +- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04, +- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98, +- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19, +- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53, +- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7, +- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3, +- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9, +- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51, +- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9, +- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84, +- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e, +- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0, +- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75, +- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51, +- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e, +- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63, +- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe, +- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2, +- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49, +- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07, +- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74, +- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a, +- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13, +- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf, +- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f, +- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00, ++ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, ++ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, ++ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, ++ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, ++ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, ++ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, ++ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, ++ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, ++ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, ++ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, ++ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, ++ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, ++ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, ++ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, ++ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, ++ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, ++ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, ++ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, ++ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, ++ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, ++ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, ++ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, ++ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, ++ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, ++ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, ++ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, ++ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, ++ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, ++ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, ++ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, ++ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, ++ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, ++ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, ++ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, ++ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, ++ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, ++ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, ++ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, ++ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, ++ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, ++ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, ++ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, ++ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, ++ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, ++ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, ++ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, ++ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, ++ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, ++ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, ++ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, ++ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, ++ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, ++ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, ++ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, ++ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, ++ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, ++ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, ++ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, ++ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, ++ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, ++ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, ++ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, ++ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, ++ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, ++ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, ++ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, ++ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, ++ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, ++ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, ++ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, ++ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, ++ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, ++ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, ++ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, ++ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, ++ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, ++ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, ++ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, ++ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, ++ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, ++ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, ++ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, ++ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, ++ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, ++ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, ++ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, ++ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, ++ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, ++ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, ++ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, ++ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, ++ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, ++ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, ++ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, ++ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, ++ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, ++ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, ++ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, ++ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, ++ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, ++ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, ++ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, ++ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, ++ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, ++ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, ++ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, ++ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, ++ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, ++ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, ++ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, ++ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, ++ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, ++ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, ++ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, ++ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, ++ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, ++ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, ++ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, ++ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, ++ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, ++ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, ++ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, ++ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, ++ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, ++ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, ++ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, ++ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, ++ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, ++ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, ++ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, ++ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, ++ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, ++ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, ++ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, ++ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, ++ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, ++ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, ++ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, ++ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, ++ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, ++ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, ++ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, ++ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, ++ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, ++ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, ++ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, ++ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, ++ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, ++ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, ++ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, ++ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, ++ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, ++ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, ++ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, ++ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, ++ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, ++ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, ++ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, ++ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, ++ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, ++ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, ++ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, ++ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, ++ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, ++ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, ++ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, ++ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, ++ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, ++ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, ++ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, ++ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, ++ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, ++ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, ++ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, ++ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, ++ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, ++ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, ++ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, ++ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, + } +diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto +index 348d792..b0fab6d 100644 +--- a/protocols/grpc/agent.proto ++++ b/protocols/grpc/agent.proto +@@ -316,6 +316,7 @@ message UpdateInterfaceRequest { + + message UpdateRoutesRequest { + Routes routes = 1; ++ bool increment = 2; + } + + message ListInterfacesRequest { +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch b/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch new file mode 100644 index 0000000000000000000000000000000000000000..c3d6c867b1d56afbad7c18198e272cd37b4c763f --- /dev/null +++ b/agent/patches/0003-kata-agent-add-kata-ipvs-command.patch @@ -0,0 +1,971 @@ +From 0f24d3d433e93a7309b9bac59e4a15e722391490 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 18 Aug 2020 06:01:55 +0800 +Subject: [PATCH 03/16] kata-agent: add kata-ipvs command + +reason: add kata-ipvs command to update IPVS rules +in VM. + +Signed-off-by: xiadanni1 +--- + agent.go | 6 + + grpc.go | 4 + + ipvsadm.go | 130 +++++++++ + protocols/grpc/agent.pb.go | 663 ++++++++++++++++++++++++++++++++------------- + protocols/grpc/agent.proto | 11 + + 5 files changed, 633 insertions(+), 181 deletions(-) + create mode 100644 ipvsadm.go + +diff --git a/agent.go b/agent.go +index c1cac08..c161e93 100644 +--- a/agent.go ++++ b/agent.go +@@ -118,6 +118,11 @@ type sandboxStorage struct { + refCount int + } + ++type ipvsAdm struct { ++ ipvsLock sync.Mutex ++ ipvsRuleCnt uint64 ++} ++ + type sandbox struct { + sync.RWMutex + ctx context.Context +@@ -144,6 +149,7 @@ type sandbox struct { + sandboxPidNs bool + storages map[string]*sandboxStorage + stopServer chan struct{} ++ ipvsadm ipvsAdm + } + + var agentFields = logrus.Fields{ +diff --git a/grpc.go b/grpc.go +index 8fe8217..de2cae7 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -1567,6 +1567,10 @@ func (a *agentGRPC) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) ( + return a.sandbox.listRoutes(nil) + } + ++func (a *agentGRPC) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) { ++ return a.sandbox.updateIPVSRule(req) ++} ++ + func (a *agentGRPC) OnlineCPUMem(ctx context.Context, req *pb.OnlineCPUMemRequest) (*gpb.Empty, error) { + if !req.Wait { + go a.onlineCPUMem(req) +diff --git a/ipvsadm.go b/ipvsadm.go +new file mode 100644 +index 0000000..48eb19f +--- /dev/null ++++ b/ipvsadm.go +@@ -0,0 +1,130 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: IPVS related common functions ++// Author: xiadanni ++// Create: 2020-08-01 ++ ++package main ++ ++import ( ++ "os/exec" ++ "strconv" ++ "strings" ++ ++ "github.com/kata-containers/agent/protocols/grpc" ++ "google.golang.org/grpc/codes" ++ grpcStatus "google.golang.org/grpc/status" ++) ++ ++var ( ++ errNoIPVSRules = grpcStatus.Errorf(codes.InvalidArgument, "IPVS rule is nil") ++ errRuleExist = grpcStatus.Errorf(codes.InvalidArgument, "failed to restore IPVS rules: exist some rules in system, should clear first") ++ errInvalidIPVSRule = grpcStatus.Errorf(codes.InvalidArgument, "invalid IPVS rule, please check") ++ errIPVSRuleExceed = grpcStatus.Errorf(codes.InvalidArgument, "rules exceed limit, should clear first") ++) ++ ++const ( ++ validHeadLength = 7 ++ ruleLimitMax = 20000 ++ restoreHead = "restore" ++ conntrackNormalInfo = "have been deleted" ++ waitError = "no child processes" ++) ++ ++func (s *sandbox) updateIPVSRule(ipvsRule *grpc.UpdateIPVSRequest) (resultingIpvs *grpc.IPVSResponse, err error) { ++ var ( ++ restoreRuleCnt string ++ restoreRule string ++ operation string ++ restoreRuleCntUint uint64 ++ cmd *exec.Cmd ++ ) ++ ++ if ipvsRule == nil || ipvsRule.IPVSReq == "" { ++ return nil, errNoIPVSRules ++ } ++ ++ s.ipvsadm.ipvsLock.Lock() ++ defer s.ipvsadm.ipvsLock.Unlock() ++ ++ rule := ipvsRule.IPVSReq ++ if len(rule) < validHeadLength { ++ return nil, errInvalidIPVSRule ++ } ++ ++ if ruleHead := rule[:validHeadLength]; ruleHead == restoreHead { ++ if s.ipvsadm.ipvsRuleCnt != 0 { ++ return nil, errRuleExist ++ } ++ operation = restoreHead ++ ++ restoreItem := strings.Split(rule, "|") ++ if len(restoreItem) != 3 { ++ return nil, errInvalidIPVSRule ++ } ++ restoreRuleCnt = restoreItem[1] ++ if restoreRuleCntUint, err = strconv.ParseUint(restoreRuleCnt, 10, 64); err != nil { ++ return nil, errInvalidIPVSRule ++ } ++ restoreRule = restoreItem[2] ++ ++ cmd = exec.Command("/sbin/ipvsadm", "--restore") ++ cmd.Stdin = strings.NewReader(restoreRule) ++ } else { ++ item := strings.Fields(strings.TrimSpace(rule)) ++ if len(item) < 2 { ++ return nil, errInvalidIPVSRule ++ } ++ if item[0] != "ipvsadm" && item[0] != "conntrack" { ++ return nil, errInvalidIPVSRule ++ } ++ operation = item[1] ++ if s.ipvsadm.ipvsRuleCnt >= ruleLimitMax { ++ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" { ++ return nil, errIPVSRuleExceed ++ } ++ } ++ ++ restoreRuleCntUint = 0 ++ ++ cmd = exec.Command("/sbin/" + item[0]) ++ for i := 1; i < len(item); i++ { ++ cmd.Args = append(cmd.Args, item[i]) ++ } ++ } ++ ++ bytes, err := cmd.CombinedOutput() ++ ++ if err != nil && !strings.Contains(err.Error(), waitError) && !strings.Contains(string(bytes), conntrackNormalInfo) { ++ return nil, grpcStatus.Errorf(codes.Internal, ++ "exec IPVS command failed, stderr: %v, err: %v", string(bytes), err) ++ } ++ ++ s.sumRuleCount(operation, restoreRuleCntUint) ++ ++ rsp := &grpc.IPVSResponse{ ++ IPVSRes: string(bytes), ++ } ++ return rsp, nil ++} ++ ++func (s *sandbox) sumRuleCount(operation string, restoreRuleCntUint uint64) { ++ if operation == restoreHead { ++ s.ipvsadm.ipvsRuleCnt = restoreRuleCntUint ++ return ++ } ++ if operation == "--add-service" || operation == "-A" || operation == "--add-server" || operation == "-a" { ++ s.ipvsadm.ipvsRuleCnt++ ++ return ++ } ++ if operation == "--delete-service" || operation == "-D" || operation == "--delete-server" || operation == "-d" { ++ if s.ipvsadm.ipvsRuleCnt > 0 { ++ s.ipvsadm.ipvsRuleCnt-- ++ } ++ return ++ } ++ if operation == "--clear" || operation == "-C" { ++ s.ipvsadm.ipvsRuleCnt = 0 ++ return ++ } ++} +diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go +index 1b887e5..04d0ee5 100644 +--- a/protocols/grpc/agent.pb.go ++++ b/protocols/grpc/agent.pb.go +@@ -63,6 +63,8 @@ + CopyFileRequest + StartTracingRequest + StopTracingRequest ++ UpdateIPVSRequest ++ IPVSResponse + CheckRequest + HealthCheckResponse + VersionCheckResponse +@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe + func (*StopTracingRequest) ProtoMessage() {} + func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + ++type UpdateIPVSRequest struct { ++ // IPVS_req is the IPVS rule message needed to update ++ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"` ++} ++ ++func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } ++func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } ++func (*UpdateIPVSRequest) ProtoMessage() {} ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++ ++func (m *UpdateIPVSRequest) GetIPVSReq() string { ++ if m != nil { ++ return m.IPVSReq ++ } ++ return "" ++} ++ ++type IPVSResponse struct { ++ // IPVS_res is the response of IPVS updating ++ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"` ++} ++ ++func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } ++func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } ++func (*IPVSResponse) ProtoMessage() {} ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++ ++func (m *IPVSResponse) GetIPVSRes() string { ++ if m != nil { ++ return m.IPVSRes ++ } ++ return "" ++} ++ + func init() { + proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") + proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") +@@ -1896,6 +1932,8 @@ func init() { + proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest") + proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest") + proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest") ++ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest") ++ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse") + } + + // Reference imports to suppress errors if they are not otherwise used. +@@ -1938,6 +1976,7 @@ type AgentServiceClient interface { + UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) + ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) + ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) ++ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) + // tracing + StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) + StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) +@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque + return out, nil + } + ++func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) { ++ out := new(IPVSResponse) ++ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...) ++ if err != nil { ++ return nil, err ++ } ++ return out, nil ++} ++ + func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { + out := new(google_protobuf2.Empty) + err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...) +@@ -2262,6 +2310,7 @@ type AgentServiceServer interface { + UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) + ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) + ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) ++ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error) + // tracing + StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error) + StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error) +@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec + return interceptor(ctx, in, info, handler) + } + ++func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { ++ in := new(UpdateIPVSRequest) ++ if err := dec(in); err != nil { ++ return nil, err ++ } ++ if interceptor == nil { ++ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in) ++ } ++ info := &grpc1.UnaryServerInfo{ ++ Server: srv, ++ FullMethod: "/grpc.AgentService/UpdateIPVSRule", ++ } ++ handler := func(ctx context.Context, req interface{}) (interface{}, error) { ++ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest)) ++ } ++ return interceptor(ctx, in, info, handler) ++} ++ + func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { + in := new(StartTracingRequest) + if err := dec(in); err != nil { +@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ + MethodName: "ListRoutes", + Handler: _AgentService_ListRoutes_Handler, + }, ++ { ++ MethodName: "UpdateIPVSRule", ++ Handler: _AgentService_UpdateIPVSRule_Handler, ++ }, + { + MethodName: "StartTracing", + Handler: _AgentService_StartTracing_Handler, +@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + ++func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.IPVSReq) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq))) ++ i += copy(dAtA[i:], m.IPVSReq) ++ } ++ return i, nil ++} ++ ++func (m *IPVSResponse) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.IPVSRes) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes))) ++ i += copy(dAtA[i:], m.IPVSRes) ++ } ++ return i, nil ++} ++ + func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) +@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) { + return n + } + ++func (m *UpdateIPVSRequest) Size() (n int) { ++ var l int ++ _ = l ++ l = len(m.IPVSReq) ++ if l > 0 { ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ ++func (m *IPVSResponse) Size() (n int) { ++ var l int ++ _ = l ++ l = len(m.IPVSRes) ++ if l > 0 { ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func sovAgent(x uint64) (n int) { + for { + n++ +@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { + } + return nil + } ++func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType) ++ } ++ var stringLen uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ stringLen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ intStringLen := int(stringLen) ++ if intStringLen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + intStringLen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.IPVSReq = string(dAtA[iNdEx:postIndex]) ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *IPVSResponse) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType) ++ } ++ var stringLen uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ stringLen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ intStringLen := int(stringLen) ++ if intStringLen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + intStringLen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.IPVSRes = string(dAtA[iNdEx:postIndex]) ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} + func skipAgent(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 +@@ -12893,185 +13190,189 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2876 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, +- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, +- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, +- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, +- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, +- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, +- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, +- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, +- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, +- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, +- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, +- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, +- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, +- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, +- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, +- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, +- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, +- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, +- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, +- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, +- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, +- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, +- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, +- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, +- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, +- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, +- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, +- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, +- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, +- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, +- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, +- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, +- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, +- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, +- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, +- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, +- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, +- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, +- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, +- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, +- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, +- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, +- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, +- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, +- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, +- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, +- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, +- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, +- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, +- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, +- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, +- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, +- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, +- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, +- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, +- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, +- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, +- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, +- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, +- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, +- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, +- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, +- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, +- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, +- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, +- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, +- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, +- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, +- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, +- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, +- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, +- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, +- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, +- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, +- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, +- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, +- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, +- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, +- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, +- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, +- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, +- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, +- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, +- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, +- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, +- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, +- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, +- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, +- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, +- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, +- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, +- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, +- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, +- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, +- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, +- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, +- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, +- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, +- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, +- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, +- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, +- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, +- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, +- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, +- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, +- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, +- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, +- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, +- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, +- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, +- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, +- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, +- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, +- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, +- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, +- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, +- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, +- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, +- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, +- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, +- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, +- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, +- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, +- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, +- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, +- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, +- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, +- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, +- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, +- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, +- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, +- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, +- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, +- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, +- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, +- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, +- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, +- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, +- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, +- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, +- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, +- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, +- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, +- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, +- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, +- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, +- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, +- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, +- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, +- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, +- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, +- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, +- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, +- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, +- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, +- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, +- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, +- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, +- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, +- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, +- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, +- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, +- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, +- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, +- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, +- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, +- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, +- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, +- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, +- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, +- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, +- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, +- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, +- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, +- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, +- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, +- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, +- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, +- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, ++ // 2931 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, ++ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, ++ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, ++ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, ++ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, ++ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, ++ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, ++ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, ++ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, ++ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, ++ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, ++ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, ++ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, ++ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, ++ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, ++ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, ++ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, ++ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, ++ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, ++ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, ++ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, ++ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, ++ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, ++ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, ++ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, ++ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, ++ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, ++ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, ++ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, ++ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, ++ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, ++ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, ++ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, ++ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, ++ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, ++ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, ++ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, ++ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, ++ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, ++ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, ++ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, ++ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, ++ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, ++ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, ++ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, ++ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, ++ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, ++ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, ++ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, ++ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, ++ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, ++ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, ++ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, ++ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, ++ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, ++ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, ++ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, ++ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, ++ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, ++ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, ++ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, ++ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, ++ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, ++ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, ++ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, ++ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, ++ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, ++ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, ++ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, ++ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, ++ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, ++ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, ++ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, ++ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, ++ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, ++ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, ++ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, ++ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, ++ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, ++ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, ++ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, ++ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, ++ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, ++ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, ++ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, ++ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, ++ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, ++ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, ++ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, ++ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, ++ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, ++ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, ++ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, ++ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, ++ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, ++ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, ++ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, ++ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, ++ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, ++ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, ++ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, ++ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, ++ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, ++ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, ++ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, ++ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, ++ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, ++ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, ++ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, ++ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, ++ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, ++ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, ++ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, ++ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, ++ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, ++ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, ++ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, ++ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, ++ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, ++ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, ++ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, ++ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, ++ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, ++ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, ++ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, ++ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, ++ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, ++ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, ++ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, ++ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, ++ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, ++ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, ++ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, ++ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, ++ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, ++ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, ++ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, ++ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, ++ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, ++ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, ++ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, ++ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, ++ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, ++ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, ++ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, ++ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, ++ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, ++ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, ++ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, ++ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, ++ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, ++ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, ++ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, ++ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, ++ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, ++ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, ++ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, ++ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, ++ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, ++ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, ++ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, ++ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, ++ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, ++ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, ++ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, ++ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, ++ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, ++ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, ++ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, ++ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, ++ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, ++ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, ++ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, ++ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, ++ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, ++ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, ++ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, ++ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, ++ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, ++ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, ++ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, ++ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, ++ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, ++ 0x22, 0x00, 0x00, + } +diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto +index b0fab6d..d863645 100644 +--- a/protocols/grpc/agent.proto ++++ b/protocols/grpc/agent.proto +@@ -46,6 +46,7 @@ service AgentService { + rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); + rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); + rpc ListRoutes(ListRoutesRequest) returns (Routes); ++ rpc UpdateIPVSRule(UpdateIPVSRequest) returns (IPVSResponse); + + // tracing + rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty); +@@ -495,3 +496,13 @@ message StartTracingRequest { + + message StopTracingRequest { + } ++ ++message UpdateIPVSRequest { ++ // IPVS_req is the IPVS rule message needed to update ++ string IPVS_req = 1; ++} ++ ++message IPVSResponse { ++ // IPVS_res is the response of IPVS updating ++ string IPVS_res = 1; ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0004-agent-add-IPVS-test.patch b/agent/patches/0004-agent-add-IPVS-test.patch new file mode 100644 index 0000000000000000000000000000000000000000..5c73f8c41f826b22c9561b86aeda66212e638059 --- /dev/null +++ b/agent/patches/0004-agent-add-IPVS-test.patch @@ -0,0 +1,193 @@ +From 01563c08910ddaba4077fd9dc691df541e045165 Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Tue, 18 Aug 2020 17:05:32 +0800 +Subject: [PATCH 04/16] agent: add IPVS test + +Signed-off-by: xiadanni +--- + grpc_test.go | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 172 insertions(+) + +diff --git a/grpc_test.go b/grpc_test.go +index e69102b..d58c0b6 100644 +--- a/grpc_test.go ++++ b/grpc_test.go +@@ -1840,3 +1840,175 @@ func getPipeMaxSize() (uint32, error) { + u, err := strconv.ParseUint(s, 10, 32) + return uint32(u), err + } ++ ++func TestUpdateIPVSRule(t *testing.T) { ++ assert := assert.New(t) ++ ++ // add IPVS rule successfully ++ a := &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ }, ++ } ++ ++ req := &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000", ++ } ++ ++ _, err := a.UpdateIPVSRule(context.Background(), req) ++ assert.NoError(err) ++ ++ // delete ipvs rule successfully ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -D -t 17.2.0.7:80", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.NoError(err) ++ ++ // update ipvs rule error because exec failed ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p -3000", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "exec IPVS command failed") ++ ++ // update IPVS rule error because rule less than validHeadLength ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsa", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "invalid IPVS rule") ++ ++ // update ipvs rule error because invalid command ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "abcabcabc ipvsadm", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "invalid IPVS rule") ++ ++ // add ipvs rule error because rule count exceeds ++ a = &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ ipvsadm: ipvsAdm{ ++ ipvsRuleCnt: 20000, ++ }, ++ }, ++ } ++ ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -A -t 17.2.0.7:80 -s rr -p 3000", ++ } ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Errorf(err, "rules exceed limit") ++ ++ // add ipvs rule error because ipvs request item less than 2 ++ a = &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ ipvsadm: ipvsAdm{ ++ ipvsRuleCnt: 0, ++ }, ++ }, ++ } ++ ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "invalid IPVS rule") ++ ++ // add ipvs rule error because ipvs rule nil ++ req = nil ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "IPVS rule is nil") ++ ++ // add ipvs rule error because ipvs rule string is empty ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "IPVS rule is nil") ++ ++ // restore ipvs rule successfully ++ a = &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ }, ++ } ++ ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.NoError(err) ++ ++ // clear IPVS rule successfully ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -C", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.NoError(err) ++ assert.Equal(a.sandbox.ipvsadm.ipvsRuleCnt, uint64(0)) ++ ++ // restore ipvs rule error because rule count invalid ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "restore|abc|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "invalid IPVS rule") ++ ++ // restore ipvs rule error because other rules exists ++ a = &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ ipvsadm: ipvsAdm{ ++ ipvsRuleCnt: 5, ++ }, ++ }, ++ } ++ ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "restore|2|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "exist some rules in system") ++ ++ // restore ipvs rule error because ipvs req item less than 3 ++ a = &agentGRPC{ ++ sandbox: &sandbox{ ++ containers: make(map[string]*container), ++ ipvsadm: ipvsAdm{ ++ ipvsRuleCnt: 0, ++ }, ++ }, ++ } ++ ++ req = &pb.UpdateIPVSRequest{ ++ IPVSReq: "restore|-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m", ++ } ++ ++ _, err = a.UpdateIPVSRule(context.Background(), req) ++ assert.Error(err) ++ assert.Contains(err.Error(), "invalid IPVS rule") ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0005-mount-support-mount-block-device.patch b/agent/patches/0005-mount-support-mount-block-device.patch new file mode 100644 index 0000000000000000000000000000000000000000..4da993bd9ba0bd8d53c533005dd7daaa5bcf41b3 --- /dev/null +++ b/agent/patches/0005-mount-support-mount-block-device.patch @@ -0,0 +1,42 @@ +From 8c9f9be2a9c195d0bc12b43c491adaacb7bb8154 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Tue, 18 Aug 2020 10:42:38 +0800 +Subject: [PATCH 05/16] mount: support mount block device + +reason: modify mountStorage to support mount block device +"-v /dev/blockdevice:/home/test" + +Signed-off-by: yangfeiyu +--- + mount.go | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/mount.go b/mount.go +index f0c3efe..a05d4a6 100644 +--- a/mount.go ++++ b/mount.go +@@ -364,7 +364,20 @@ func commonStorageHandler(storage pb.Storage) (string, error) { + func mountStorage(storage pb.Storage) error { + flags, options := parseMountFlagsAndOptions(storage.Options) + +- return mount(storage.Source, storage.MountPoint, storage.Fstype, flags, options) ++ var fsType = storage.Fstype ++ if (storage.Driver == driverSCSIType || storage.Driver == driverBlkType) && strings.Contains(storage.Fstype, "bind") { ++ cs := strings.Split(storage.Fstype, "-") ++ if len(cs) == 2 && cs[1] != "" { ++ fsType = cs[1] ++ // here we temporarily discard the bind option, ++ // in order to be able to mount the file system of the block device. ++ // and then reset `storage.Fstype` to "bind" which pass through to the libcontainer pkg. ++ flags = flags &^ flagList["bind"] ++ storage.Fstype = "bind" ++ } ++ } ++ ++ return mount(storage.Source, storage.MountPoint, fsType, flags, options) + } + + // addStorages takes a list of storages passed by the caller, and perform the +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch b/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch new file mode 100644 index 0000000000000000000000000000000000000000..84b10d67a1d743075d5bb3dcd6b24cfe0af134b5 --- /dev/null +++ b/agent/patches/0006-agent-make-workaround-for-slow-response-in-aarch64.patch @@ -0,0 +1,54 @@ +From bccda1d208f31eab55863883cf0718d7b4b8deef Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 19:30:42 +0800 +Subject: [PATCH 06/16] agent: make workaround for slow response in aarch64 + +reason: make workaround for slow response in aarch64 +when hotplug virtio-net-pci device + +Signed-off-by: jiangpengfei +--- + agent.go | 2 +- + device.go | 12 ++++++------ + 2 files changed, 7 insertions(+), 7 deletions(-) + +diff --git a/agent.go b/agent.go +index c161e93..e81c2cd 100644 +--- a/agent.go ++++ b/agent.go +@@ -185,7 +185,7 @@ var logsVSockPort = uint32(0) + var debugConsoleVSockPort = uint32(0) + + // Timeout waiting for a device to be hotplugged +-var hotplugTimeout = 3 * time.Second ++var hotplugTimeout = 10 * time.Second + + // Specify the log level + var logLevel = defaultLogLevel +diff --git a/device.go b/device.go +index ec1907e..46a1b96 100644 +--- a/device.go ++++ b/device.go +@@ -179,13 +179,13 @@ func getPCIDeviceNameImpl(s *sandbox, pciID string) (string, error) { + return "", err + } + +- fieldLogger := agentLog.WithField("pciAddr", pciAddr) +- + // Rescan pci bus if we need to wait for a new pci device +- if err = rescanPciBus(); err != nil { +- fieldLogger.WithError(err).Error("Failed to scan pci bus") +- return "", err +- } ++ // FIXME:Comment out this code Temporarily, because once the PCIBus is scanned, ++ // the device hot-plug event is lost ++ //if err = rescanPciBus(); err != nil { ++ // fieldLogger.WithError(err).Error("Failed to scan pci bus") ++ // return "", err ++ //} + + return getDeviceName(s, pciAddr) + } +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch b/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch new file mode 100644 index 0000000000000000000000000000000000000000..661aa5a30e4a728c920a3108a0405ed2aff0f43b --- /dev/null +++ b/agent/patches/0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch @@ -0,0 +1,37 @@ +From fa673c93e243ba297d53b585cd2f51fa68380fc5 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 19:37:48 +0800 +Subject: [PATCH 07/16] agent: using pcie-root-port driver to hotplug device + +reason: In original pci-bridge scheme, the "F" in the BDF is not used, +and was written hard code "0" in the kata-agent. But the "function" +is specified when switching to pcie-root-port scheme, so when should +pass the "pci-bridge BDF" or "pcie-root-port BDF" from kata-runtime +and use in kata-agent. + +Signed-off-by: jiangpengfei +--- + device.go | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/device.go b/device.go +index 46a1b96..8e6950c 100644 +--- a/device.go ++++ b/device.go +@@ -101,7 +101,12 @@ func getDevicePCIAddressImpl(pciID string) (string, error) { + + // Deduce the complete bridge address based on the bridge address identifier passed + // and the fact that bridges are attached on the main bus with function 0. +- pciBridgeAddr := fmt.Sprintf("0000:00:%s.0", bridgeID) ++ // Update: support pcie-root-port device ++ // In original pci-bridge scheme, the "F" in the BDF is not used, and was written ++ // hard code "0" in the kata-agent. But the "function" is specified when switching to ++ // pcie-root-port scheme, so when should pass the "pci-bridge BDF" or "pcie-root-port BDF" ++ // from kata-runtime and use in kata-agent. ++ pciBridgeAddr := fmt.Sprintf("0000:00:%s", bridgeID) + + // Find out the bus exposed by bridge + bridgeBusPath := fmt.Sprintf(pciBusPathFormat, sysBusPrefix, pciBridgeAddr) +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch b/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch new file mode 100644 index 0000000000000000000000000000000000000000..86167c65915dfa0e0cbf5e528f75b506b049b756 --- /dev/null +++ b/agent/patches/0008-agent-support-get-root-bus-path-dynamically.patch @@ -0,0 +1,94 @@ +From eea286fbafba2e95410b603fbef762e2b25eb207 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 19:45:57 +0800 +Subject: [PATCH 08/16] agent: support get root bus path dynamically + +reason: support get root bus dynamically no matter the +target arch is amd64 or arm64 + +Signed-off-by: jiangpengfei +--- + agent.go | 1 + + device_amd64.go | 6 +++++- + device_arm64.go | 27 ++++++++++++++++++++++++++- + 3 files changed, 32 insertions(+), 2 deletions(-) + +diff --git a/agent.go b/agent.go +index e81c2cd..50afd7a 100644 +--- a/agent.go ++++ b/agent.go +@@ -730,6 +730,7 @@ func (s *sandbox) listenToUdevEvents() { + defer uEvHandler.Close() + + fieldLogger.Infof("Started listening for uevents") ++ rootBusPath := initRootBusPath() + + for { + uEv, err := uEvHandler.Read() +diff --git a/device_amd64.go b/device_amd64.go +index 66bc052..26f55bf 100644 +--- a/device_amd64.go ++++ b/device_amd64.go +@@ -8,7 +8,7 @@ + package main + + const ( +- rootBusPath = "/devices/pci0000:00" ++ defaultRootBusPath = "/devices/pci0000:00" + + // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt + // The Linux kernel's core ACPI subsystem creates struct acpi_device +@@ -21,3 +21,7 @@ const ( + // in a subdirectory whose prefix is pfn (page frame number). + pfnDevPrefix = "/pfn" + ) ++ ++func initRootBusPath() string { ++ return defaultRootBusPath ++} +diff --git a/device_arm64.go b/device_arm64.go +index b73b582..d039c67 100644 +--- a/device_arm64.go ++++ b/device_arm64.go +@@ -6,8 +6,14 @@ + + package main + ++import ( ++ "fmt" ++ "io/ioutil" ++ "regexp" ++) ++ + const ( +- rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" ++ defaultRootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" + + // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt + // The Linux kernel's core ACPI subsystem creates struct acpi_device +@@ -20,3 +26,22 @@ const ( + // in a subdirectory whose prefix is pfn (page frame number). + pfnDevPrefix = "/pfn" + ) ++ ++func initRootBusPath() string { ++ pcieDriverReg := regexp.MustCompile(`^[0-9a-f]{10}.pcie$`) ++ rootBusPath := defaultRootBusPath ++ files, err := ioutil.ReadDir("/sys/devices/platform") ++ if err != nil { ++ return rootBusPath ++ } ++ for _, f := range files { ++ if !f.IsDir() { ++ continue ++ } ++ if pcieDriverReg.MatchString(f.Name()) { ++ rootBusPath = fmt.Sprintf("/devices/platform/%s/pci0000:00", f.Name()) ++ break ++ } ++ } ++ return rootBusPath ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0009-storage-add-pkg-storage-for-mount.patch b/agent/patches/0009-storage-add-pkg-storage-for-mount.patch new file mode 100644 index 0000000000000000000000000000000000000000..aabfb258240ec4659a2eb75759a384bc2f25149c --- /dev/null +++ b/agent/patches/0009-storage-add-pkg-storage-for-mount.patch @@ -0,0 +1,188 @@ +From 1268b710c7f0528d971d9c0e54429d3f4e48c372 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Tue, 18 Aug 2020 16:45:29 +0800 +Subject: [PATCH 09/16] storage: add pkg/storage for mount + +reason: add gpath.go and nfs.go, provide mount +functions and structs + +Signed-off-by: yangfeiyu +--- + pkg/storage/gpath.go | 64 ++++++++++++++++++++++++++++++++++++ + pkg/storage/nfs.go | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 157 insertions(+) + create mode 100644 pkg/storage/gpath.go + create mode 100644 pkg/storage/nfs.go + +diff --git a/pkg/storage/gpath.go b/pkg/storage/gpath.go +new file mode 100644 +index 0000000..5cb951f +--- /dev/null ++++ b/pkg/storage/gpath.go +@@ -0,0 +1,64 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: licuifang ++// Create: 2019-06-24 ++ ++package storage ++ ++import ( ++ "fmt" ++ "os" ++ "syscall" ++ ++ "github.com/opencontainers/runc/libcontainer/mount" ++) ++ ++func ValidateGpath(guestPath string, opts []string) error { ++ for _, opt := range opts { ++ switch opt { ++ case "shared": ++ flag := syscall.MS_SHARED ++ if err := ensureMountedAs(guestPath, flag); err != nil { ++ return err ++ } ++ return nil ++ case "mounted": ++ // if mounted in option, the guestpath must exist and must be a mountpoint ++ _, err := os.Stat(guestPath) ++ if err != nil { ++ return err ++ } ++ mountpoint, err := mount.Mounted(guestPath) ++ if err != nil { ++ return err ++ } ++ if mountpoint == false { ++ return fmt.Errorf("the guespath:%s is not a mountpoint while mounted was set", guestPath) ++ } ++ return nil ++ } ++ } ++ if err := os.MkdirAll(guestPath, 0750); err != nil { ++ return err ++ } ++ return nil ++} ++ ++func ensureMountedAs(mountPoint string, flag int) error { ++ if err := os.MkdirAll(mountPoint, 0750); err != nil { ++ return err ++ } ++ mounted, err := mount.Mounted(mountPoint) ++ if err != nil { ++ return err ++ } ++ ++ if !mounted { ++ if err := syscall.Mount(mountPoint, mountPoint, "bind", uintptr(syscall.MS_BIND), ""); err != nil { ++ return err ++ } ++ } ++ ++ return syscall.Mount("", mountPoint, "", uintptr(flag), "") ++} +diff --git a/pkg/storage/nfs.go b/pkg/storage/nfs.go +new file mode 100644 +index 0000000..44bc85d +--- /dev/null ++++ b/pkg/storage/nfs.go +@@ -0,0 +1,93 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: leizhongkai ++// Create: 2019-03-10 ++ ++package storage ++ ++import ( ++ "fmt" ++ "os" ++ "os/exec" ++ "path/filepath" ++ "strings" ++ "time" ++ ++ "github.com/opencontainers/runc/libcontainer/mount" ++ "github.com/sirupsen/logrus" ++) ++ ++const ( ++ NFS = "nfs" ++ ++ MAX_MOUNT_ATTEMPTS = 100 ++ MAX_MOUTN_TIMEOUT = 90 // seconds ++ ++ // ignored errors ++ PERMISSION_DENY = "Permission denied" ++ kataGuestStorageDir = "/run/kata-containers/storage/containers/" ++) ++ ++func nfsMount(source, dest string, opt string) error { ++ cmd := exec.Command("/bin/mount", "-t", "nfs", "-o", opt, source, dest) ++ res, err := cmd.Output() ++ logrus.Debugf("mount %s to %s, and get res %s", source, dest, string(res)) ++ return err ++} ++ ++func nfsMountWithAttempt(source, dest string, opt string) (err error) { ++ timeStart := time.Now().Unix() ++ for i := 0; i < MAX_MOUNT_ATTEMPTS; i++ { ++ logrus.Infof("this is the %d times to mount %s to %s", i, source, dest) ++ err = nfsMount(source, dest, opt) ++ if err != nil { ++ ee, ok := err.(*exec.ExitError) ++ if ok { ++ if strings.Contains(string(ee.Stderr), PERMISSION_DENY) { ++ logrus.Errorf("mounting nfs:%s to %s, get error: %s,should break", source, dest, string(ee.Stderr)) ++ // We do not retry when the error type is PERMISSION_DENY. ++ // The reason for the retry is that when you do a SFS mount, ++ // the network may not have been created yet, because ++ // creating the network and startup container is asynchronous ++ break ++ } ++ logrus.Infof("mounting nfs:%s to %s, get error: %s,will retry", source, dest, string(ee.Stderr)) ++ } ++ ++ elapsed := time.Now().Unix() - timeStart ++ if elapsed < MAX_MOUTN_TIMEOUT { ++ time.Sleep(50 * time.Microsecond) ++ continue ++ } ++ } ++ break ++ } ++ ++ return err ++} ++ ++func MountRemoteNfs(source string, opt []string, sandboxId string) error { ++ item := strings.Split(source, ":") ++ if len(item) != 2 { ++ return fmt.Errorf("the nfs of %s format is error", source) ++ } ++ tmpDes := filepath.Join(kataGuestStorageDir, sandboxId, item[0],item[1]) ++ if mounted, err := mount.Mounted(tmpDes); err == nil && mounted == true { ++ return nil ++ } ++ err := os.MkdirAll(tmpDes, 0750) ++ if err != nil { ++ logrus.Infof("mkdir %s failed before mount remote nfs server", tmpDes) ++ return err ++ } ++ defer func() { ++ if err != nil { ++ os.RemoveAll(tmpDes) ++ } ++ }() ++ ++ err = nfsMountWithAttempt(source, tmpDes, strings.Join(opt, ",")) ++ ++ return err ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch b/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch new file mode 100644 index 0000000000000000000000000000000000000000..8e5fdb69702ae6e6f961633954bb54d64440afe5 --- /dev/null +++ b/agent/patches/0010-storage-mount-nfs-and-gpath-in-agent.patch @@ -0,0 +1,92 @@ +From 79bafc5fd8a1dcda6f44ecd830dfe50f4ef1b34b Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Tue, 18 Aug 2020 20:41:30 +0800 +Subject: [PATCH 10/16] storage: mount nfs and gpath in agent + +reason: add nfsStorageHandler and gpathStorageHandler in +storageHandlerList to mount nfs and gpath + +Signed-off-by: yangfeiyu +--- + device.go | 2 ++ + mount.go | 37 +++++++++++++++++++++++++++++++++++++ + 2 files changed, 39 insertions(+) + +diff --git a/device.go b/device.go +index 8e6950c..29f72e9 100644 +--- a/device.go ++++ b/device.go +@@ -35,6 +35,8 @@ const ( + driverEphemeralType = "ephemeral" + driverLocalType = "local" + vmRootfs = "/" ++ driverNfsType = "nfs" ++ driverGpathType = "gpath" + ) + + const ( +diff --git a/mount.go b/mount.go +index a05d4a6..de2bfaf 100644 +--- a/mount.go ++++ b/mount.go +@@ -17,6 +17,7 @@ import ( + "strings" + "syscall" + ++ rmtStorage "github.com/kata-containers/agent/pkg/storage" + pb "github.com/kata-containers/agent/protocols/grpc" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +@@ -218,6 +219,8 @@ var storageHandlerList = map[string]storageHandler{ + driverEphemeralType: ephemeralStorageHandler, + driverLocalType: localStorageHandler, + driverNvdimmType: nvdimmStorageHandler, ++ driverNfsType: nfsStorageHandler, ++ driverGpathType: gpathStorageHandler, + } + + func ephemeralStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { +@@ -339,6 +342,40 @@ func nvdimmStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (st + return "", fmt.Errorf("invalid nvdimm source path: %v", storage.Source) + } + ++func nfsStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { ++ s.Lock() ++ defer s.Unlock() ++ ++ // mount nfs ++ err := rmtStorage.MountRemoteNfs(storage.Source, storage.Options, s.id) ++ if err != nil { ++ return "", fmt.Errorf("mount %s to %s failed, get err:%s", storage.Source, storage.MountPoint, err) ++ } ++ if err := os.MkdirAll(storage.MountPoint, 0750); err != nil { ++ logrus.Infof("mkdir %s failed after mount remote nfs server", storage.MountPoint) ++ return "", err ++ } ++ ++ defer func() { ++ if err != nil { ++ os.RemoveAll(storage.MountPoint) ++ } ++ }() ++ ++ return storage.MountPoint, nil ++} ++ ++func gpathStorageHandler(_ context.Context, storage pb.Storage, s *sandbox) (string, error) { ++ s.Lock() ++ defer s.Unlock() ++ // validate guespath ++ err := rmtStorage.ValidateGpath(storage.Source, storage.Options) ++ if err != nil { ++ return "", err ++ } ++ return "", nil ++} ++ + // virtioSCSIStorageHandler handles the storage for scsi driver. + func virtioSCSIStorageHandler(ctx context.Context, storage pb.Storage, s *sandbox) (string, error) { + // Retrieve the device path from SCSI address. +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch b/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..272e0b51ac239d68752c8dad68e8fe685d4e9ee9 --- /dev/null +++ b/agent/patches/0011-agent-fix-agent-reap-agent-process-blocked-problem.patch @@ -0,0 +1,78 @@ +From 3ac1232a2e3fbfc0465473e5d81cde41847c4252 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 11:47:37 +0800 +Subject: [PATCH 11/16] agent: fix agent reap agent process blocked problem + +reason: add container waitProcess() timeout when +container process status is D/T. + +Signed-off-by: jiangpengfei +--- + grpc.go | 43 +++++++++++++++++++++++++++++++++---------- + 1 file changed, 33 insertions(+), 10 deletions(-) + +diff --git a/grpc.go b/grpc.go +index de2cae7..3dd088e 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -49,6 +49,11 @@ const ( + libcontainerPath = "/run/libcontainer" + ) + ++// keep waitProcessTimeout value same as value in kata-runtime wait WaitProcessRequest response ++const ( ++ waitProcessTimeOut = 10 ++) ++ + var ( + sysfsCPUOnlinePath = "/sys/devices/system/cpu" + sysfsMemOnlinePath = "/sys/devices/system/memory" +@@ -996,17 +1001,35 @@ func (a *agentGRPC) WaitProcess(ctx context.Context, req *pb.WaitProcessRequest) + ctr.deleteProcess(proc.id) + }) + +- // Using helper function wait() to deal with the subreaper. +- libContProcess := (*reaperLibcontainerProcess)(&(proc.process)) +- exitCode, err := a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess) +- if err != nil { +- return &pb.WaitProcessResponse{}, err ++ done := make(chan error) ++ var exitCode int = 0 ++ go func() { ++ // Using helper function wait() to deal with the subreaper. ++ libContProcess := (*reaperLibcontainerProcess)(&(proc.process)) ++ var err error ++ exitCode, err = a.sandbox.subreaper.wait(proc.exitCodeCh, libContProcess) ++ if err != nil { ++ done <- err ++ close(done) ++ return ++ } ++ // refill the exitCodeCh with the exitcode which can be read out ++ // by another WaitProcess(). Since this channel isn't be closed, ++ // here the refill will always success and it will be free by GC ++ // once the process exits. ++ proc.exitCodeCh <- exitCode ++ ++ close(done) ++ }() ++ ++ select { ++ case err := <-done: ++ if err != nil { ++ return &pb.WaitProcessResponse{}, err ++ } ++ case <-time.After(time.Duration(waitProcessTimeOut) * time.Second): ++ return &pb.WaitProcessResponse{}, grpcStatus.Errorf(codes.DeadlineExceeded, "agent wait reap container process timeout reached after %ds", waitProcessTimeOut) + } +- //refill the exitCodeCh with the exitcode which can be read out +- //by another WaitProcess(). Since this channel isn't be closed, +- //here the refill will always success and it will be free by GC +- //once the process exits. +- proc.exitCodeCh <- exitCode + + return &pb.WaitProcessResponse{ + Status: int32(exitCode), +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0012-network-support-set-dns-without-nameserver.patch b/agent/patches/0012-network-support-set-dns-without-nameserver.patch new file mode 100644 index 0000000000000000000000000000000000000000..850c88c00ccb13eea7a16b42d0f466a283e3838b --- /dev/null +++ b/agent/patches/0012-network-support-set-dns-without-nameserver.patch @@ -0,0 +1,38 @@ +From edb29dfd8f786735763245b3f156b50fd3c1a08e Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 15:15:31 +0800 +Subject: [PATCH 12/16] network: support set dns without nameserver + +reason: when runtime sends dns without nameserver to agent, +add nameserver before ip address. scenario like annotation +with dns + +Signed-off-by: yangfeiyu +--- + network.go | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/network.go b/network.go +index 02e28cb..7046cf8 100644 +--- a/network.go ++++ b/network.go +@@ -708,6 +708,9 @@ func setupDNS(dns []string) (err error) { + defer file.Close() + + for i, line := range dns { ++ if !strings.Contains(line, "nameserver") { ++ line = "nameserver" + " " + line ++ } + if i == (len(dns) - 1) { + _, err = file.WriteString(strings.TrimSpace(line)) + } else { +@@ -761,4 +764,4 @@ func (s *sandbox) handleLocalhost() error { + } + + return netlink.LinkSetUp(lo) +-} +\ No newline at end of file ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch b/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch new file mode 100644 index 0000000000000000000000000000000000000000..34c2bb92708c526abb1a36a2584314a83f712a19 --- /dev/null +++ b/agent/patches/0013-agent-support-setting-multi-queues-of-interface.patch @@ -0,0 +1,304 @@ +From 1394dcf579849e5d8103c31556e9af0216a875d2 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 17:15:51 +0800 +Subject: [PATCH 13/16] agent: support setting multi queues of interface + +reason: support setting multi queues of a interface +when runtime passing Queue in the request. + +Signed-off-by: jiangpengfei +--- + network.go | 12 +++++- + pkg/net/ethtool.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++ + pkg/types/types.pb.go | 82 ++++++++++++++++++++++++++----------- + pkg/types/types.proto | 1 + + 4 files changed, 181 insertions(+), 24 deletions(-) + create mode 100644 pkg/net/ethtool.go + +diff --git a/network.go b/network.go +index 7046cf8..1baaa2e 100644 +--- a/network.go ++++ b/network.go +@@ -17,7 +17,7 @@ import ( + "syscall" + + "golang.org/x/sys/unix" +- ++ agentNet "github.com/kata-containers/agent/pkg/net" + "github.com/kata-containers/agent/pkg/types" + pb "github.com/kata-containers/agent/protocols/grpc" + "github.com/sirupsen/logrus" +@@ -273,6 +273,16 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf + if err == nil { + err = retErr + } ++ ++ // if link is up, then set the multi queue to it, ++ // the kernel of newer version may set multi queue by itself, ++ // but we can not rely on it. ++ if err == nil && iface.Queues > 0 { ++ if ethErr := agentNet.GetEthtool().SetChannel(iface.Name, iface.Queues); ethErr != nil { ++ err = grpcStatus.Errorf(codes.Internal, "Could not set multi queue %d for interface %v: %v", ++ iface.Queues, link, ethErr) ++ } ++ } + }() + + fieldLogger.WithField("link", fmt.Sprintf("%+v", link)).Info("Link found") +diff --git a/pkg/net/ethtool.go b/pkg/net/ethtool.go +new file mode 100644 +index 0000000..56a1ece +--- /dev/null ++++ b/pkg/net/ethtool.go +@@ -0,0 +1,110 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++SPDX-License-Identifier: Apache-2.0 ++Description: common functions ++Author: fengshaobao ++Create: 2019-05-28 ++*/ ++ ++package net ++ ++import ( ++ "fmt" ++ "sync" ++ "syscall" ++ "unsafe" ++ ++ "github.com/sirupsen/logrus" ++) ++ ++const ( ++ ethtoolGChannels = 0x0000003c /* Get no of channels */ ++ ethtoolSChannels = 0x0000003d /* Set no of channels */ ++ ifNameSize = 16 ++ siocEthtool = 0x8946 ++) ++ ++var ( ++ t *Ethtool ++ once sync.Once ++) ++ ++// Ethtool to set multiqueue of a network interface, ++// with the public method "SetChannel" to set queues of an interface. ++type Ethtool struct { ++ fd int ++} ++ ++type ifReq struct { ++ ifrName [ifNameSize]byte ++ ifrData uintptr ++} ++ ++type ethtoolChannels struct { ++ // ETHTOOL_{G,S}CHANNELS ++ cmd uint32 ++ // Read only. Maximum number of receive channel the driver support. ++ maxRx uint32 ++ // Read only. Maximum number of transmit channel the driver support ++ maxTx uint32 ++ // Read only. Maximum number of other channel the driver support ++ maxOther uint32 ++ //Read only. Maximum number of combined channel the driver support. Set of queues RX, TX or other ++ maxCombined uint32 ++ // Valid values are in the range 1 to the max_rx ++ rxCount uint32 ++ // Valid values are in the range 1 to the max_tx ++ txCount uint32 ++ // Valid values are in the range 1 to the max_other ++ otherCount uint32 ++ // Valid values are in the range 1 to the max_combined ++ combinedCount uint32 ++} ++ ++// GetEthtool to config multiqueue of a network interface ++func GetEthtool() *Ethtool { ++ once.Do(func() { ++ var err error ++ if t, err = newEthtool(); err != nil { ++ panic("can not init a socket fd for ethtool") ++ } ++ }) ++ return t ++} ++ ++func newEthtool() (*Ethtool, error) { ++ fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP) ++ if err != nil || fd < 0 { ++ logrus.Warningf("Can not get socket of inet") ++ var newErr error ++ fd, newErr = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_GENERIC) ++ if newErr != nil || fd < 0 { ++ return nil, fmt.Errorf("create inet socket with error: %v, and create netlink socket with error: %v", err, newErr) ++ } ++ } ++ ++ return &Ethtool{ ++ fd: int(fd), ++ }, nil ++} ++ ++// SetChannel Set the queues of a network interface. ++// @devName: the network interface name, e.g. eth0. ++// @queues: queues for the network interface. ++func (e *Ethtool) SetChannel(devName string, queues uint32) error { ++ c := ðtoolChannels{ ++ cmd: ethtoolSChannels, ++ combinedCount: queues, ++ } ++ var ifName [ifNameSize]byte ++ copy(ifName[:], []byte(devName)) ++ ifr := ifReq{ ++ ifrName: ifName, ++ ifrData: uintptr(unsafe.Pointer(&c)), ++ } ++ _, _, ep := syscall.Syscall(syscall.SYS_IOCTL, uintptr(e.fd), siocEthtool, uintptr(unsafe.Pointer(&ifr))) ++ if ep != 0 { ++ return syscall.Errno(ep) ++ } ++ return nil ++} +diff --git a/pkg/types/types.pb.go b/pkg/types/types.pb.go +index 7ea63e3..8b7e2a5 100644 +--- a/pkg/types/types.pb.go ++++ b/pkg/types/types.pb.go +@@ -100,6 +100,7 @@ type Interface struct { + // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... + Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` + RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"` ++ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"` + } + + func (m *Interface) Reset() { *m = Interface{} } +@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 { + return 0 + } + ++func (m *Interface) GetQueues() uint32 { ++ if m != nil { ++ return m.Queues ++ } ++ return 0 ++} ++ + type Route struct { + Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` + Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"` +@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags)) + } ++ if m.Queues != 0 { ++ dAtA[i] = 0x48 ++ i++ ++ i = encodeVarintTypes(dAtA, i, uint64(m.Queues)) ++ } + return i, nil + } + +@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) { + if m.RawFlags != 0 { + n += 1 + sovTypes(uint64(m.RawFlags)) + } ++ if m.Queues != 0 { ++ n += 1 + sovTypes(uint64(m.Queues)) ++ } + return n + } + +@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error { + break + } + } ++ case 9: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) ++ } ++ m.Queues = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowTypes ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Queues |= (uint32(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) +@@ -1155,27 +1190,28 @@ var ( + func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) } + + var fileDescriptorTypes = []byte{ +- // 345 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30, +- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59, +- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1, +- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb, +- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a, +- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96, +- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3, +- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a, +- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c, +- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a, +- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f, +- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08, +- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9, +- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60, +- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e, +- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22, +- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f, +- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2, +- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10, +- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4, +- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff, +- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00, ++ // 356 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40, ++ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3, ++ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4, ++ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0, ++ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c, ++ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24, ++ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31, ++ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac, ++ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1, ++ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6, ++ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e, ++ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34, ++ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60, ++ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d, ++ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5, ++ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3, ++ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a, ++ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8, ++ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72, ++ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05, ++ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf, ++ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8, ++ 0x2c, 0x02, 0x00, 0x00, + } +diff --git a/pkg/types/types.proto b/pkg/types/types.proto +index f6856e1..149df13 100644 +--- a/pkg/types/types.proto ++++ b/pkg/types/types.proto +@@ -37,6 +37,7 @@ message Interface { + // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... + string type = 7; + uint32 raw_flags = 8; ++ uint32 Queues = 9; + } + + message Route { +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch b/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..a29b0e64b06de83834610b01e5688e10e878a1ef --- /dev/null +++ b/agent/patches/0014-agent-fix-init-hugepages-failed-problem.patch @@ -0,0 +1,56 @@ +From 9e1478d7989fea4ee759cc13009d8de07dac2879 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 17:01:00 +0800 +Subject: [PATCH 14/16] agent: fix init hugepages failed problem + +reason: fix init hugepages failed problem + +Signed-off-by: jiangpengfei +--- + .../runc/libcontainer/cgroups/fs/apply_raw.go | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) + +diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go +index ec148b4..47aa3c3 100644 +--- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go ++++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go +@@ -8,11 +8,13 @@ import ( + "os" + "path/filepath" + "sync" ++ "time" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/configs" + libcontainerUtils "github.com/opencontainers/runc/libcontainer/utils" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + ) + +@@ -409,3 +411,21 @@ func CheckCpushares(path string, c uint64) error { + func (m *Manager) GetCgroups() (*configs.Cgroup, error) { + return m.Cgroups, nil + } ++ ++func init() { ++ go func() { ++ var err error ++ if len(HugePageSizes) != 0 { ++ return ++ } ++ for i := 0; i < 10; i++ { ++ HugePageSizes, err = cgroups.GetHugePageSize() ++ if err == nil { ++ logrus.Infof("init hugepages ok loop=%d %v", i, err) ++ return ++ } ++ logrus.Errorf("init hugepages failed loop=%d %v", i, err) ++ time.Sleep(time.Second) ++ } ++ }() ++} +\ No newline at end of file +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch b/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch new file mode 100644 index 0000000000000000000000000000000000000000..5ca41d606414d6aa6163ea6af867c9b4f325fc0b --- /dev/null +++ b/agent/patches/0015-agent-add-support-of-getting-container-s-network-sta.patch @@ -0,0 +1,2649 @@ +From 555db828d0dae24330bfdc4e3d10a145ce44a66b Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 20:52:35 +0800 +Subject: [PATCH 15/16] agent: add support of getting container's network stats + +reason: support to get more detailed container's network +stats info + +Signed-off-by: jiangpengfei +--- + grpc.go | 12 +- + network_stats.go | 241 ++++++ + protocols/grpc/agent.pb.go | 1749 +++++++++++++++++++++++++++++++++++--------- + protocols/grpc/agent.proto | 33 +- + 4 files changed, 1684 insertions(+), 351 deletions(-) + create mode 100644 network_stats.go + +diff --git a/grpc.go b/grpc.go +index 3dd088e..1b63cde 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -1206,32 +1206,28 @@ func (a *agentGRPC) StatsContainer(ctx context.Context, req *pb.StatsContainerRe + return nil, err + } + ++ agentLog.Debugf("container stats start : %s", c.container.ID()) + stats, err := c.container.Stats() + if err != nil { + return nil, err + } + +- cgroupData, err := json.Marshal(stats.CgroupStats) ++ networkStats, err := getNetworkStats() + if err != nil { + return nil, err + } + +- netData, err := json.Marshal(stats.Interfaces) ++ cgroupData, err := json.Marshal(stats.CgroupStats) + if err != nil { + return nil, err + } + + var cgroupStats pb.CgroupStats +- networkStats := make([]*pb.NetworkStats, 0) +- + err = json.Unmarshal(cgroupData, &cgroupStats) + if err != nil { + return nil, err + } +- err = json.Unmarshal(netData, &networkStats) +- if err != nil { +- return nil, err +- } ++ + resp := &pb.StatsContainerResponse{ + CgroupStats: &cgroupStats, + NetworkStats: networkStats, +diff --git a/network_stats.go b/network_stats.go +new file mode 100644 +index 0000000..6954be1 +--- /dev/null ++++ b/network_stats.go +@@ -0,0 +1,241 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++SPDX-License-Identifier: Apache-2.0 ++Description: common functions ++Author: yanlei ++Create: 2019-06-27 ++*/ ++ ++package main ++ ++import ( ++ "bufio" ++ "fmt" ++ "io/ioutil" ++ "net" ++ "path" ++ "strconv" ++ "strings" ++ ++ pb "github.com/kata-containers/agent/protocols/grpc" ++) ++ ++func getNetworkStats() (*pb.NetworkStats, error) { ++ networkStats := &pb.NetworkStats{} ++ ifaces, err := net.Interfaces() ++ if err != nil { ++ return nil, err ++ } ++ for _, iface := range ifaces { ++ ifaceStats, err := getNetworkInterfaceStats(iface.Name) ++ if err != nil { ++ return nil, err ++ } ++ networkStats.Interfaces = append(networkStats.Interfaces, ifaceStats) ++ } ++ ++ t, err := getNetworkTcpStats("tcp") ++ if err != nil { ++ return nil, err ++ } ++ networkStats.Tcp = t ++ ++ t6, err := getNetworkTcpStats("tcp6") ++ if err != nil { ++ return nil, err ++ } ++ networkStats.Tcp6 = t6 ++ ++ u, err := getNetworkUdpStats("udp") ++ if err != nil { ++ return nil, err ++ } ++ networkStats.Udp = u ++ ++ u6, err := getNetworkUdpStats("udp6") ++ if err != nil { ++ return nil, err ++ } ++ networkStats.Udp6 = u6 ++ ++ return networkStats, nil ++} ++ ++func getNetworkInterfaceStats(interfaceName string) (*pb.InterfaceStats, error) { ++ out := &pb.InterfaceStats{Name: interfaceName} ++ // This can happen if the network runtime information is missing - possible if the ++ // container was created by an old version of libcontainer. ++ if interfaceName == "" { ++ return out, nil ++ } ++ type netStatsPair struct { ++ // Where to write the output. ++ Out *uint64 ++ // The network stats file to read. ++ File string ++ } ++ // Ingress for host veth is from the container. Hence tx_bytes stat on the host veth is actually number of bytes received by the container. ++ netStats := []netStatsPair{ ++ {Out: &out.RxBytes, File: "rx_bytes"}, ++ {Out: &out.RxPackets, File: "rx_packets"}, ++ {Out: &out.RxErrors, File: "rx_errors"}, ++ {Out: &out.RxDropped, File: "rx_dropped"}, ++ ++ {Out: &out.TxBytes, File: "tx_bytes"}, ++ {Out: &out.TxPackets, File: "tx_packets"}, ++ {Out: &out.TxErrors, File: "tx_errors"}, ++ {Out: &out.TxDropped, File: "tx_dropped"}, ++ } ++ for _, netStat := range netStats { ++ data, err := readSysfsNetworkStats(interfaceName, netStat.File) ++ if err != nil { ++ return nil, err ++ } ++ *(netStat.Out) = data ++ } ++ return out, nil ++} ++ ++// Reads the specified statistics available under /sys/class/net//statistics ++func readSysfsNetworkStats(ethInterface, statsFile string) (uint64, error) { ++ data, err := ioutil.ReadFile(path.Join("/sys/class/net", ethInterface, "statistics", statsFile)) ++ if err != nil { ++ return 0, err ++ } ++ return strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64) ++} ++ ++func getNetworkTcpStats(file string) (*pb.TcpStat, error) { ++ tcpStatsFile := path.Join("/proc/net", file) ++ ++ tcpStats, err := scanTcpStats(tcpStatsFile) ++ if err != nil { ++ return nil, err ++ } ++ ++ return tcpStats, nil ++} ++ ++func scanTcpStats(tcpStatsFile string) (*pb.TcpStat, error) { ++ data, err := ioutil.ReadFile(tcpStatsFile) ++ if err != nil { ++ return nil, fmt.Errorf("fail to open %s: %v", tcpStatsFile, err) ++ } ++ ++ tcpStateMap := map[string]uint64{ ++ "01": 0, //ESTABLISHED ++ "02": 0, //SYN_SENT ++ "03": 0, //SYN_RECV ++ "04": 0, //FIN_WAIT1 ++ "05": 0, //FIN_WAIT2 ++ "06": 0, //TIME_WAIT ++ "07": 0, //CLOSE ++ "08": 0, //CLOSE_WAIT ++ "09": 0, //LAST_ACK ++ "0A": 0, //LISTEN ++ "0B": 0, //CLOSING ++ } ++ ++ reader := strings.NewReader(string(data)) ++ scanner := bufio.NewScanner(reader) ++ ++ scanner.Split(bufio.ScanLines) ++ ++ if b := scanner.Scan(); !b { ++ return nil, scanner.Err() ++ } ++ ++ for scanner.Scan() { ++ line := scanner.Text() ++ ++ state := strings.Fields(line) ++ // TCP state is the 4th field. ++ // Format: sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode ++ tcpState := state[3] ++ _, ok := tcpStateMap[tcpState] ++ if !ok { ++ return nil, fmt.Errorf("invalid TCP state line: %v", line) ++ } ++ tcpStateMap[tcpState]++ ++ } ++ ++ stats := &pb.TcpStat{ ++ Established: tcpStateMap["01"], ++ SynSent: tcpStateMap["02"], ++ SynRecv: tcpStateMap["03"], ++ FinWait1: tcpStateMap["04"], ++ FinWait2: tcpStateMap["05"], ++ TimeWait: tcpStateMap["06"], ++ Close: tcpStateMap["07"], ++ CloseWait: tcpStateMap["08"], ++ LastAck: tcpStateMap["09"], ++ Listen: tcpStateMap["0A"], ++ Closing: tcpStateMap["0B"], ++ } ++ ++ return stats, nil ++} ++ ++func getNetworkUdpStats(file string) (*pb.UdpStat, error) { ++ udpStatsFile := path.Join("/proc/net", file) ++ ++ udpStats, err := scanUdpStats(udpStatsFile) ++ if err != nil { ++ return nil, err ++ } ++ ++ return udpStats, nil ++} ++ ++func scanUdpStats(udpStatsFile string) (*pb.UdpStat, error) { ++ data, err := ioutil.ReadFile(udpStatsFile) ++ if err != nil { ++ return nil, fmt.Errorf("fail to open %s: %v", udpStatsFile, err) ++ } ++ ++ reader := strings.NewReader(string(data)) ++ scanner := bufio.NewScanner(reader) ++ ++ scanner.Split(bufio.ScanLines) ++ ++ if b := scanner.Scan(); !b { ++ return nil, scanner.Err() ++ } ++ ++ listening := uint64(0) ++ dropped := uint64(0) ++ rxQueued := uint64(0) ++ txQueued := uint64(0) ++ ++ for scanner.Scan() { ++ line := scanner.Text() ++ // Format: sl local_address rem_address st tx_queue:rx_queue tr:tm->when retrnsmt uid timeout inode ref pointer drops ++ ++ listening++ ++ ++ fs := strings.Fields(line) ++ if len(fs) != 13 { ++ continue ++ } ++ ++ rx, tx := uint64(0), uint64(0) ++ fmt.Sscanf(fs[4], "%X:%X", &rx, &tx) ++ rxQueued += rx ++ txQueued += tx ++ ++ d, err := strconv.Atoi(string(fs[12])) ++ if err != nil { ++ continue ++ } ++ dropped += uint64(d) ++ } ++ ++ stats := &pb.UdpStat{ ++ Listen: listening, ++ Dropped: dropped, ++ RxQueued: rxQueued, ++ TxQueued: txQueued, ++ } ++ ++ return stats, nil ++} +diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go +index 04d0ee5..c50ecb5 100644 +--- a/protocols/grpc/agent.pb.go ++++ b/protocols/grpc/agent.pb.go +@@ -33,6 +33,9 @@ + BlkioStats + HugetlbStats + CgroupStats ++ InterfaceStats ++ TcpStat ++ UdpStat + NetworkStats + StatsContainerResponse + WriteStreamRequest +@@ -883,7 +886,7 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats { + return nil + } + +-type NetworkStats struct { ++type InterfaceStats struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` + RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` +@@ -895,83 +898,267 @@ type NetworkStats struct { + TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` + } + +-func (m *NetworkStats) Reset() { *m = NetworkStats{} } +-func (m *NetworkStats) String() string { return proto.CompactTextString(m) } +-func (*NetworkStats) ProtoMessage() {} +-func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } ++func (m *InterfaceStats) Reset() { *m = InterfaceStats{} } ++func (m *InterfaceStats) String() string { return proto.CompactTextString(m) } ++func (*InterfaceStats) ProtoMessage() {} ++func (*InterfaceStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } + +-func (m *NetworkStats) GetName() string { ++func (m *InterfaceStats) GetName() string { + if m != nil { + return m.Name + } + return "" + } + +-func (m *NetworkStats) GetRxBytes() uint64 { ++func (m *InterfaceStats) GetRxBytes() uint64 { + if m != nil { + return m.RxBytes + } + return 0 + } + +-func (m *NetworkStats) GetRxPackets() uint64 { ++func (m *InterfaceStats) GetRxPackets() uint64 { + if m != nil { + return m.RxPackets + } + return 0 + } + +-func (m *NetworkStats) GetRxErrors() uint64 { ++func (m *InterfaceStats) GetRxErrors() uint64 { + if m != nil { + return m.RxErrors + } + return 0 + } + +-func (m *NetworkStats) GetRxDropped() uint64 { ++func (m *InterfaceStats) GetRxDropped() uint64 { + if m != nil { + return m.RxDropped + } + return 0 + } + +-func (m *NetworkStats) GetTxBytes() uint64 { ++func (m *InterfaceStats) GetTxBytes() uint64 { + if m != nil { + return m.TxBytes + } + return 0 + } + +-func (m *NetworkStats) GetTxPackets() uint64 { ++func (m *InterfaceStats) GetTxPackets() uint64 { + if m != nil { + return m.TxPackets + } + return 0 + } + +-func (m *NetworkStats) GetTxErrors() uint64 { ++func (m *InterfaceStats) GetTxErrors() uint64 { + if m != nil { + return m.TxErrors + } + return 0 + } + +-func (m *NetworkStats) GetTxDropped() uint64 { ++func (m *InterfaceStats) GetTxDropped() uint64 { + if m != nil { + return m.TxDropped + } + return 0 + } + ++type TcpStat struct { ++ Established uint64 `protobuf:"varint,1,opt,name=established,proto3" json:"established,omitempty"` ++ SynSent uint64 `protobuf:"varint,2,opt,name=syn_sent,json=synSent,proto3" json:"syn_sent,omitempty"` ++ SynRecv uint64 `protobuf:"varint,3,opt,name=syn_recv,json=synRecv,proto3" json:"syn_recv,omitempty"` ++ FinWait1 uint64 `protobuf:"varint,4,opt,name=fin_wait1,json=finWait1,proto3" json:"fin_wait1,omitempty"` ++ FinWait2 uint64 `protobuf:"varint,5,opt,name=fin_wait2,json=finWait2,proto3" json:"fin_wait2,omitempty"` ++ TimeWait uint64 `protobuf:"varint,6,opt,name=time_wait,json=timeWait,proto3" json:"time_wait,omitempty"` ++ Close uint64 `protobuf:"varint,7,opt,name=close,proto3" json:"close,omitempty"` ++ CloseWait uint64 `protobuf:"varint,8,opt,name=close_wait,json=closeWait,proto3" json:"close_wait,omitempty"` ++ LastAck uint64 `protobuf:"varint,9,opt,name=last_ack,json=lastAck,proto3" json:"last_ack,omitempty"` ++ Listen uint64 `protobuf:"varint,10,opt,name=listen,proto3" json:"listen,omitempty"` ++ Closing uint64 `protobuf:"varint,11,opt,name=closing,proto3" json:"closing,omitempty"` ++} ++ ++func (m *TcpStat) Reset() { *m = TcpStat{} } ++func (m *TcpStat) String() string { return proto.CompactTextString(m) } ++func (*TcpStat) ProtoMessage() {} ++func (*TcpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } ++ ++func (m *TcpStat) GetEstablished() uint64 { ++ if m != nil { ++ return m.Established ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetSynSent() uint64 { ++ if m != nil { ++ return m.SynSent ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetSynRecv() uint64 { ++ if m != nil { ++ return m.SynRecv ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetFinWait1() uint64 { ++ if m != nil { ++ return m.FinWait1 ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetFinWait2() uint64 { ++ if m != nil { ++ return m.FinWait2 ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetTimeWait() uint64 { ++ if m != nil { ++ return m.TimeWait ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetClose() uint64 { ++ if m != nil { ++ return m.Close ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetCloseWait() uint64 { ++ if m != nil { ++ return m.CloseWait ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetLastAck() uint64 { ++ if m != nil { ++ return m.LastAck ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetListen() uint64 { ++ if m != nil { ++ return m.Listen ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetClosing() uint64 { ++ if m != nil { ++ return m.Closing ++ } ++ return 0 ++} ++ ++type UdpStat struct { ++ Listen uint64 `protobuf:"varint,1,opt,name=listen,proto3" json:"listen,omitempty"` ++ Dropped uint64 `protobuf:"varint,2,opt,name=dropped,proto3" json:"dropped,omitempty"` ++ RxQueued uint64 `protobuf:"varint,3,opt,name=rx_queued,json=rxQueued,proto3" json:"rx_queued,omitempty"` ++ TxQueued uint64 `protobuf:"varint,4,opt,name=tx_queued,json=txQueued,proto3" json:"tx_queued,omitempty"` ++} ++ ++func (m *UdpStat) Reset() { *m = UdpStat{} } ++func (m *UdpStat) String() string { return proto.CompactTextString(m) } ++func (*UdpStat) ProtoMessage() {} ++func (*UdpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } ++ ++func (m *UdpStat) GetListen() uint64 { ++ if m != nil { ++ return m.Listen ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetDropped() uint64 { ++ if m != nil { ++ return m.Dropped ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetRxQueued() uint64 { ++ if m != nil { ++ return m.RxQueued ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetTxQueued() uint64 { ++ if m != nil { ++ return m.TxQueued ++ } ++ return 0 ++} ++ ++type NetworkStats struct { ++ Interfaces []*InterfaceStats `protobuf:"bytes,1,rep,name=interfaces" json:"interfaces,omitempty"` ++ Tcp *TcpStat `protobuf:"bytes,2,opt,name=tcp" json:"tcp,omitempty"` ++ Tcp6 *TcpStat `protobuf:"bytes,3,opt,name=tcp6" json:"tcp6,omitempty"` ++ Udp *UdpStat `protobuf:"bytes,4,opt,name=udp" json:"udp,omitempty"` ++ Udp6 *UdpStat `protobuf:"bytes,5,opt,name=udp6" json:"udp6,omitempty"` ++} ++ ++func (m *NetworkStats) Reset() { *m = NetworkStats{} } ++func (m *NetworkStats) String() string { return proto.CompactTextString(m) } ++func (*NetworkStats) ProtoMessage() {} ++func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } ++ ++func (m *NetworkStats) GetInterfaces() []*InterfaceStats { ++ if m != nil { ++ return m.Interfaces ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetTcp() *TcpStat { ++ if m != nil { ++ return m.Tcp ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetTcp6() *TcpStat { ++ if m != nil { ++ return m.Tcp6 ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetUdp() *UdpStat { ++ if m != nil { ++ return m.Udp ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetUdp6() *UdpStat { ++ if m != nil { ++ return m.Udp6 ++ } ++ return nil ++} ++ + type StatsContainerResponse struct { +- CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` +- NetworkStats []*NetworkStats `protobuf:"bytes,2,rep,name=network_stats,json=networkStats" json:"network_stats,omitempty"` ++ CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` ++ NetworkStats *NetworkStats `protobuf:"bytes,2,opt,name=network_stats,json=networkStats" json:"network_stats,omitempty"` + } + + func (m *StatsContainerResponse) Reset() { *m = StatsContainerResponse{} } + func (m *StatsContainerResponse) String() string { return proto.CompactTextString(m) } + func (*StatsContainerResponse) ProtoMessage() {} +-func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } ++func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } + + func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { + if m != nil { +@@ -980,7 +1167,7 @@ func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { + return nil + } + +-func (m *StatsContainerResponse) GetNetworkStats() []*NetworkStats { ++func (m *StatsContainerResponse) GetNetworkStats() *NetworkStats { + if m != nil { + return m.NetworkStats + } +@@ -996,7 +1183,7 @@ type WriteStreamRequest struct { + func (m *WriteStreamRequest) Reset() { *m = WriteStreamRequest{} } + func (m *WriteStreamRequest) String() string { return proto.CompactTextString(m) } + func (*WriteStreamRequest) ProtoMessage() {} +-func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } ++func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } + + func (m *WriteStreamRequest) GetContainerId() string { + if m != nil { +@@ -1026,7 +1213,7 @@ type WriteStreamResponse struct { + func (m *WriteStreamResponse) Reset() { *m = WriteStreamResponse{} } + func (m *WriteStreamResponse) String() string { return proto.CompactTextString(m) } + func (*WriteStreamResponse) ProtoMessage() {} +-func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } ++func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } + + func (m *WriteStreamResponse) GetLen() uint32 { + if m != nil { +@@ -1044,7 +1231,7 @@ type ReadStreamRequest struct { + func (m *ReadStreamRequest) Reset() { *m = ReadStreamRequest{} } + func (m *ReadStreamRequest) String() string { return proto.CompactTextString(m) } + func (*ReadStreamRequest) ProtoMessage() {} +-func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } ++func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } + + func (m *ReadStreamRequest) GetContainerId() string { + if m != nil { +@@ -1074,7 +1261,7 @@ type ReadStreamResponse struct { + func (m *ReadStreamResponse) Reset() { *m = ReadStreamResponse{} } + func (m *ReadStreamResponse) String() string { return proto.CompactTextString(m) } + func (*ReadStreamResponse) ProtoMessage() {} +-func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } ++func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } + + func (m *ReadStreamResponse) GetData() []byte { + if m != nil { +@@ -1091,7 +1278,7 @@ type CloseStdinRequest struct { + func (m *CloseStdinRequest) Reset() { *m = CloseStdinRequest{} } + func (m *CloseStdinRequest) String() string { return proto.CompactTextString(m) } + func (*CloseStdinRequest) ProtoMessage() {} +-func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } ++func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } + + func (m *CloseStdinRequest) GetContainerId() string { + if m != nil { +@@ -1117,7 +1304,7 @@ type TtyWinResizeRequest struct { + func (m *TtyWinResizeRequest) Reset() { *m = TtyWinResizeRequest{} } + func (m *TtyWinResizeRequest) String() string { return proto.CompactTextString(m) } + func (*TtyWinResizeRequest) ProtoMessage() {} +-func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } ++func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } + + func (m *TtyWinResizeRequest) GetContainerId() string { + if m != nil { +@@ -1158,7 +1345,7 @@ type KernelModule struct { + func (m *KernelModule) Reset() { *m = KernelModule{} } + func (m *KernelModule) String() string { return proto.CompactTextString(m) } + func (*KernelModule) ProtoMessage() {} +-func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } ++func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } + + func (m *KernelModule) GetName() string { + if m != nil { +@@ -1197,7 +1384,7 @@ type CreateSandboxRequest struct { + func (m *CreateSandboxRequest) Reset() { *m = CreateSandboxRequest{} } + func (m *CreateSandboxRequest) String() string { return proto.CompactTextString(m) } + func (*CreateSandboxRequest) ProtoMessage() {} +-func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } ++func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } + + func (m *CreateSandboxRequest) GetHostname() string { + if m != nil { +@@ -1254,7 +1441,7 @@ type DestroySandboxRequest struct { + func (m *DestroySandboxRequest) Reset() { *m = DestroySandboxRequest{} } + func (m *DestroySandboxRequest) String() string { return proto.CompactTextString(m) } + func (*DestroySandboxRequest) ProtoMessage() {} +-func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } ++func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } + + type Interfaces struct { + Interfaces []*types.Interface `protobuf:"bytes,1,rep,name=Interfaces" json:"Interfaces,omitempty"` +@@ -1263,7 +1450,7 @@ type Interfaces struct { + func (m *Interfaces) Reset() { *m = Interfaces{} } + func (m *Interfaces) String() string { return proto.CompactTextString(m) } + func (*Interfaces) ProtoMessage() {} +-func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } ++func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } + + func (m *Interfaces) GetInterfaces() []*types.Interface { + if m != nil { +@@ -1279,7 +1466,7 @@ type Routes struct { + func (m *Routes) Reset() { *m = Routes{} } + func (m *Routes) String() string { return proto.CompactTextString(m) } + func (*Routes) ProtoMessage() {} +-func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } ++func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } + + func (m *Routes) GetRoutes() []*types.Route { + if m != nil { +@@ -1295,7 +1482,7 @@ type UpdateInterfaceRequest struct { + func (m *UpdateInterfaceRequest) Reset() { *m = UpdateInterfaceRequest{} } + func (m *UpdateInterfaceRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateInterfaceRequest) ProtoMessage() {} +-func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } ++func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } + + func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + if m != nil { +@@ -1312,7 +1499,7 @@ type UpdateRoutesRequest struct { + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } + func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateRoutesRequest) ProtoMessage() {} +-func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } ++func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } + + func (m *UpdateRoutesRequest) GetRoutes() *Routes { + if m != nil { +@@ -1334,7 +1521,7 @@ type ListInterfacesRequest struct { + func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } + func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } + func (*ListInterfacesRequest) ProtoMessage() {} +-func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } ++func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } + + type ListRoutesRequest struct { + } +@@ -1342,7 +1529,7 @@ type ListRoutesRequest struct { + func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } + func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*ListRoutesRequest) ProtoMessage() {} +-func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } ++func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } + + type OnlineCPUMemRequest struct { + // Wait specifies if the caller waits for the agent to online all resources. +@@ -1358,7 +1545,7 @@ type OnlineCPUMemRequest struct { + func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } + func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } + func (*OnlineCPUMemRequest) ProtoMessage() {} +-func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } ++func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } + + func (m *OnlineCPUMemRequest) GetWait() bool { + if m != nil { +@@ -1389,7 +1576,7 @@ type ReseedRandomDevRequest struct { + func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } + func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } + func (*ReseedRandomDevRequest) ProtoMessage() {} +-func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } ++func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } + + func (m *ReseedRandomDevRequest) GetData() []byte { + if m != nil { +@@ -1416,7 +1603,7 @@ type AgentDetails struct { + func (m *AgentDetails) Reset() { *m = AgentDetails{} } + func (m *AgentDetails) String() string { return proto.CompactTextString(m) } + func (*AgentDetails) ProtoMessage() {} +-func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } ++func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } + + func (m *AgentDetails) GetVersion() string { + if m != nil { +@@ -1467,7 +1654,7 @@ type GuestDetailsRequest struct { + func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } + func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsRequest) ProtoMessage() {} +-func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } ++func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } + + func (m *GuestDetailsRequest) GetMemBlockSize() bool { + if m != nil { +@@ -1493,7 +1680,7 @@ type GuestDetailsResponse struct { + func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } + func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsResponse) ProtoMessage() {} +-func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } ++func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } + + func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { + if m != nil { +@@ -1525,7 +1712,7 @@ type MemHotplugByProbeRequest struct { + func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } + func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } + func (*MemHotplugByProbeRequest) ProtoMessage() {} +-func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } ++func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } + + func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { + if m != nil { +@@ -1544,7 +1731,7 @@ type SetGuestDateTimeRequest struct { + func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } + func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } + func (*SetGuestDateTimeRequest) ProtoMessage() {} +-func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } ++func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } + + func (m *SetGuestDateTimeRequest) GetSec() int64 { + if m != nil { +@@ -1593,7 +1780,7 @@ type Storage struct { + func (m *Storage) Reset() { *m = Storage{} } + func (m *Storage) String() string { return proto.CompactTextString(m) } + func (*Storage) ProtoMessage() {} +-func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } ++func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } + + func (m *Storage) GetDriver() string { + if m != nil { +@@ -1676,7 +1863,7 @@ type Device struct { + func (m *Device) Reset() { *m = Device{} } + func (m *Device) String() string { return proto.CompactTextString(m) } + func (*Device) ProtoMessage() {} +-func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } ++func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } + + func (m *Device) GetId() string { + if m != nil { +@@ -1722,7 +1909,7 @@ type StringUser struct { + func (m *StringUser) Reset() { *m = StringUser{} } + func (m *StringUser) String() string { return proto.CompactTextString(m) } + func (*StringUser) ProtoMessage() {} +-func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } ++func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + + func (m *StringUser) GetUid() string { + if m != nil { +@@ -1770,7 +1957,7 @@ type CopyFileRequest struct { + func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } + func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } + func (*CopyFileRequest) ProtoMessage() {} +-func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } ++func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } + + func (m *CopyFileRequest) GetPath() string { + if m != nil { +@@ -1834,7 +2021,7 @@ type StartTracingRequest struct { + func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } + func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StartTracingRequest) ProtoMessage() {} +-func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } ++func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } + + type StopTracingRequest struct { + } +@@ -1842,7 +2029,7 @@ type StopTracingRequest struct { + func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } + func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StopTracingRequest) ProtoMessage() {} +-func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } ++func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } + + type UpdateIPVSRequest struct { + // IPVS_req is the IPVS rule message needed to update +@@ -1852,7 +2039,7 @@ type UpdateIPVSRequest struct { + func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } + func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateIPVSRequest) ProtoMessage() {} +-func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } + + func (m *UpdateIPVSRequest) GetIPVSReq() string { + if m != nil { +@@ -1869,7 +2056,7 @@ type IPVSResponse struct { + func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } + func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } + func (*IPVSResponse) ProtoMessage() {} +-func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } + + func (m *IPVSResponse) GetIPVSRes() string { + if m != nil { +@@ -1902,6 +2089,9 @@ func init() { + proto.RegisterType((*BlkioStats)(nil), "grpc.BlkioStats") + proto.RegisterType((*HugetlbStats)(nil), "grpc.HugetlbStats") + proto.RegisterType((*CgroupStats)(nil), "grpc.CgroupStats") ++ proto.RegisterType((*InterfaceStats)(nil), "grpc.InterfaceStats") ++ proto.RegisterType((*TcpStat)(nil), "grpc.TcpStat") ++ proto.RegisterType((*UdpStat)(nil), "grpc.UdpStat") + proto.RegisterType((*NetworkStats)(nil), "grpc.NetworkStats") + proto.RegisterType((*StatsContainerResponse)(nil), "grpc.StatsContainerResponse") + proto.RegisterType((*WriteStreamRequest)(nil), "grpc.WriteStreamRequest") +@@ -4008,7 +4198,7 @@ func (m *CgroupStats) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + +-func (m *NetworkStats) Marshal() (dAtA []byte, err error) { ++func (m *InterfaceStats) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4018,7 +4208,7 @@ func (m *NetworkStats) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { ++func (m *InterfaceStats) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int +@@ -4072,7 +4262,7 @@ func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + +-func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { ++func (m *TcpStat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4082,37 +4272,70 @@ func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { ++func (m *TcpStat) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if m.CgroupStats != nil { +- dAtA[i] = 0xa ++ if m.Established != 0 { ++ dAtA[i] = 0x8 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) +- n18, err := m.CgroupStats.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n18 ++ i = encodeVarintAgent(dAtA, i, uint64(m.Established)) + } +- if len(m.NetworkStats) > 0 { +- for _, msg := range m.NetworkStats { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } ++ if m.SynSent != 0 { ++ dAtA[i] = 0x10 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.SynSent)) ++ } ++ if m.SynRecv != 0 { ++ dAtA[i] = 0x18 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.SynRecv)) ++ } ++ if m.FinWait1 != 0 { ++ dAtA[i] = 0x20 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait1)) ++ } ++ if m.FinWait2 != 0 { ++ dAtA[i] = 0x28 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait2)) ++ } ++ if m.TimeWait != 0 { ++ dAtA[i] = 0x30 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.TimeWait)) ++ } ++ if m.Close != 0 { ++ dAtA[i] = 0x38 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Close)) ++ } ++ if m.CloseWait != 0 { ++ dAtA[i] = 0x40 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.CloseWait)) ++ } ++ if m.LastAck != 0 { ++ dAtA[i] = 0x48 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.LastAck)) ++ } ++ if m.Listen != 0 { ++ dAtA[i] = 0x50 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) ++ } ++ if m.Closing != 0 { ++ dAtA[i] = 0x58 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Closing)) + } + return i, nil + } + +-func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { ++func (m *UdpStat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4122,33 +4345,35 @@ func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++func (m *UdpStat) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if len(m.ContainerId) > 0 { +- dAtA[i] = 0xa ++ if m.Listen != 0 { ++ dAtA[i] = 0x8 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) +- i += copy(dAtA[i:], m.ContainerId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) + } +- if len(m.ExecId) > 0 { +- dAtA[i] = 0x12 ++ if m.Dropped != 0 { ++ dAtA[i] = 0x10 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) +- i += copy(dAtA[i:], m.ExecId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Dropped)) + } +- if len(m.Data) > 0 { +- dAtA[i] = 0x1a ++ if m.RxQueued != 0 { ++ dAtA[i] = 0x18 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) +- i += copy(dAtA[i:], m.Data) ++ i = encodeVarintAgent(dAtA, i, uint64(m.RxQueued)) ++ } ++ if m.TxQueued != 0 { ++ dAtA[i] = 0x20 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.TxQueued)) + } + return i, nil + } + +-func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { ++func (m *NetworkStats) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4158,20 +4383,67 @@ func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { ++func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if m.Len != 0 { +- dAtA[i] = 0x8 ++ if len(m.Interfaces) > 0 { ++ for _, msg := range m.Interfaces { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) ++ n, err := msg.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n ++ } ++ } ++ if m.Tcp != nil { ++ dAtA[i] = 0x12 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp.Size())) ++ n18, err := m.Tcp.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n18 ++ } ++ if m.Tcp6 != nil { ++ dAtA[i] = 0x1a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp6.Size())) ++ n19, err := m.Tcp6.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n19 ++ } ++ if m.Udp != nil { ++ dAtA[i] = 0x22 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Udp.Size())) ++ n20, err := m.Udp.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n20 ++ } ++ if m.Udp6 != nil { ++ dAtA[i] = 0x2a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Udp6.Size())) ++ n21, err := m.Udp6.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n21 + } + return i, nil + } + +-func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { ++func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4181,32 +4453,129 @@ func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if len(m.ContainerId) > 0 { ++ if m.CgroupStats != nil { + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) +- i += copy(dAtA[i:], m.ContainerId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) ++ n22, err := m.CgroupStats.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n22 + } +- if len(m.ExecId) > 0 { ++ if m.NetworkStats != nil { + dAtA[i] = 0x12 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) +- i += copy(dAtA[i:], m.ExecId) +- } +- if m.Len != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ i = encodeVarintAgent(dAtA, i, uint64(m.NetworkStats.Size())) ++ n23, err := m.NetworkStats.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n23 + } + return i, nil + } + +-func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { ++func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.ContainerId) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) ++ i += copy(dAtA[i:], m.ContainerId) ++ } ++ if len(m.ExecId) > 0 { ++ dAtA[i] = 0x12 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) ++ i += copy(dAtA[i:], m.ExecId) ++ } ++ if len(m.Data) > 0 { ++ dAtA[i] = 0x1a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) ++ i += copy(dAtA[i:], m.Data) ++ } ++ return i, nil ++} ++ ++func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if m.Len != 0 { ++ dAtA[i] = 0x8 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ } ++ return i, nil ++} ++ ++func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.ContainerId) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) ++ i += copy(dAtA[i:], m.ContainerId) ++ } ++ if len(m.ExecId) > 0 { ++ dAtA[i] = 0x12 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) ++ i += copy(dAtA[i:], m.ExecId) ++ } ++ if m.Len != 0 { ++ dAtA[i] = 0x18 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ } ++ return i, nil ++} ++ ++func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4521,11 +4890,11 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) +- n19, err := m.Interface.MarshalTo(dAtA[i:]) ++ n24, err := m.Interface.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n19 ++ i += n24 + } + return i, nil + } +@@ -4549,11 +4918,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) +- n20, err := m.Routes.MarshalTo(dAtA[i:]) ++ n25, err := m.Routes.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n20 ++ i += n25 + } + if m.Increment { + dAtA[i] = 0x10 +@@ -4807,11 +5176,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0x12 + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) +- n21, err := m.AgentDetails.MarshalTo(dAtA[i:]) ++ n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n21 ++ i += n26 + } + if m.SupportMemHotplugProbe { + dAtA[i] = 0x18 +@@ -4842,21 +5211,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { +- dAtA23 := make([]byte, len(m.MemHotplugProbeAddr)*10) +- var j22 int ++ dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) ++ var j27 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { +- dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) ++ dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 +- j22++ ++ j27++ + } +- dAtA23[j22] = uint8(num) +- j22++ ++ dAtA28[j27] = uint8(num) ++ j27++ + } + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(j22)) +- i += copy(dAtA[i:], dAtA23[:j22]) ++ i = encodeVarintAgent(dAtA, i, uint64(j27)) ++ i += copy(dAtA[i:], dAtA28[:j27]) + } + return i, nil + } +@@ -5648,7 +6017,7 @@ func (m *CgroupStats) Size() (n int) { + return n + } + +-func (m *NetworkStats) Size() (n int) { ++func (m *InterfaceStats) Size() (n int) { + var l int + _ = l + l = len(m.Name) +@@ -5682,6 +6051,91 @@ func (m *NetworkStats) Size() (n int) { + return n + } + ++func (m *TcpStat) Size() (n int) { ++ var l int ++ _ = l ++ if m.Established != 0 { ++ n += 1 + sovAgent(uint64(m.Established)) ++ } ++ if m.SynSent != 0 { ++ n += 1 + sovAgent(uint64(m.SynSent)) ++ } ++ if m.SynRecv != 0 { ++ n += 1 + sovAgent(uint64(m.SynRecv)) ++ } ++ if m.FinWait1 != 0 { ++ n += 1 + sovAgent(uint64(m.FinWait1)) ++ } ++ if m.FinWait2 != 0 { ++ n += 1 + sovAgent(uint64(m.FinWait2)) ++ } ++ if m.TimeWait != 0 { ++ n += 1 + sovAgent(uint64(m.TimeWait)) ++ } ++ if m.Close != 0 { ++ n += 1 + sovAgent(uint64(m.Close)) ++ } ++ if m.CloseWait != 0 { ++ n += 1 + sovAgent(uint64(m.CloseWait)) ++ } ++ if m.LastAck != 0 { ++ n += 1 + sovAgent(uint64(m.LastAck)) ++ } ++ if m.Listen != 0 { ++ n += 1 + sovAgent(uint64(m.Listen)) ++ } ++ if m.Closing != 0 { ++ n += 1 + sovAgent(uint64(m.Closing)) ++ } ++ return n ++} ++ ++func (m *UdpStat) Size() (n int) { ++ var l int ++ _ = l ++ if m.Listen != 0 { ++ n += 1 + sovAgent(uint64(m.Listen)) ++ } ++ if m.Dropped != 0 { ++ n += 1 + sovAgent(uint64(m.Dropped)) ++ } ++ if m.RxQueued != 0 { ++ n += 1 + sovAgent(uint64(m.RxQueued)) ++ } ++ if m.TxQueued != 0 { ++ n += 1 + sovAgent(uint64(m.TxQueued)) ++ } ++ return n ++} ++ ++func (m *NetworkStats) Size() (n int) { ++ var l int ++ _ = l ++ if len(m.Interfaces) > 0 { ++ for _, e := range m.Interfaces { ++ l = e.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ } ++ if m.Tcp != nil { ++ l = m.Tcp.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Tcp6 != nil { ++ l = m.Tcp6.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Udp != nil { ++ l = m.Udp.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Udp6 != nil { ++ l = m.Udp6.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func (m *StatsContainerResponse) Size() (n int) { + var l int + _ = l +@@ -5689,11 +6143,9 @@ func (m *StatsContainerResponse) Size() (n int) { + l = m.CgroupStats.Size() + n += 1 + l + sovAgent(uint64(l)) + } +- if len(m.NetworkStats) > 0 { +- for _, e := range m.NetworkStats { +- l = e.Size() +- n += 1 + l + sovAgent(uint64(l)) +- } ++ if m.NetworkStats != nil { ++ l = m.NetworkStats.Size() ++ n += 1 + l + sovAgent(uint64(l)) + } + return n + } +@@ -9396,7 +9848,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { + } + return nil + } +-func (m *NetworkStats) Unmarshal(dAtA []byte) error { ++func (m *InterfaceStats) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { +@@ -9419,10 +9871,10 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { +- return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") ++ return fmt.Errorf("proto: InterfaceStats: wiretype end group for non-group") + } + if fieldNum <= 0 { +- return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) ++ return fmt.Errorf("proto: InterfaceStats: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: +@@ -9627,7 +10079,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { + } + return nil + } +-func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { ++func (m *TcpStat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { +@@ -9650,17 +10102,17 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { +- return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") ++ return fmt.Errorf("proto: TcpStat: wiretype end group for non-group") + } + if fieldNum <= 0 { +- return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ return fmt.Errorf("proto: TcpStat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Established", wireType) + } +- var msglen int ++ m.Established = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent +@@ -9670,30 +10122,54 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + } + b := dAtA[iNdEx] + iNdEx++ +- msglen |= (int(b) & 0x7F) << shift ++ m.Established |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } +- if msglen < 0 { +- return ErrInvalidLengthAgent ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field SynSent", wireType) + } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF ++ m.SynSent = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.SynSent |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } + } +- if m.CgroupStats == nil { +- m.CgroupStats = &CgroupStats{} ++ case 3: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field SynRecv", wireType) + } +- if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err ++ m.SynRecv = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.SynRecv |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } + } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) ++ case 4: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field FinWait1", wireType) + } +- var msglen int ++ m.FinWait1 = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent +@@ -9703,20 +10179,596 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + } + b := dAtA[iNdEx] + iNdEx++ +- msglen |= (int(b) & 0x7F) << shift ++ m.FinWait1 |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } +- if msglen < 0 { +- return ErrInvalidLengthAgent ++ case 5: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field FinWait2", wireType) + } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF ++ m.FinWait2 = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.FinWait2 |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 6: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field TimeWait", wireType) ++ } ++ m.TimeWait = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.TimeWait |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 7: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Close", wireType) ++ } ++ m.Close = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Close |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 8: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field CloseWait", wireType) ++ } ++ m.CloseWait = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.CloseWait |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 9: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field LastAck", wireType) ++ } ++ m.LastAck = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.LastAck |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 10: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) ++ } ++ m.Listen = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Listen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 11: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Closing", wireType) ++ } ++ m.Closing = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Closing |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *UdpStat) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UdpStat: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UdpStat: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) ++ } ++ m.Listen = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Listen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Dropped", wireType) ++ } ++ m.Dropped = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Dropped |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 3: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field RxQueued", wireType) ++ } ++ m.RxQueued = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.RxQueued |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 4: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field TxQueued", wireType) ++ } ++ m.TxQueued = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.TxQueued |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *NetworkStats) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Interfaces", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.Interfaces = append(m.Interfaces, &InterfaceStats{}) ++ if err := m.Interfaces[len(m.Interfaces)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 2: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Tcp", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Tcp == nil { ++ m.Tcp = &TcpStat{} ++ } ++ if err := m.Tcp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 3: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Tcp6", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Tcp6 == nil { ++ m.Tcp6 = &TcpStat{} ++ } ++ if err := m.Tcp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 4: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Udp", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Udp == nil { ++ m.Udp = &UdpStat{} ++ } ++ if err := m.Udp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 5: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Udp6", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Udp6 == nil { ++ m.Udp6 = &UdpStat{} ++ } ++ if err := m.Udp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.CgroupStats == nil { ++ m.CgroupStats = &CgroupStats{} ++ } ++ if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 2: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.NetworkStats == nil { ++ m.NetworkStats = &NetworkStats{} + } +- m.NetworkStats = append(m.NetworkStats, &NetworkStats{}) +- if err := m.NetworkStats[len(m.NetworkStats)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ if err := m.NetworkStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex +@@ -13190,189 +14242,204 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2931 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, +- 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, +- 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, +- 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, +- 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, +- 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, +- 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, +- 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, +- 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, +- 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, +- 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, +- 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, +- 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, +- 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, +- 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, +- 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, +- 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, +- 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, +- 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, +- 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, +- 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, +- 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, +- 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, +- 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, +- 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, +- 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, +- 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, +- 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, +- 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, +- 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, +- 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, +- 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, +- 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, +- 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, +- 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, +- 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, +- 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, +- 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, +- 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, +- 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, +- 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, +- 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, +- 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, +- 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, +- 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, +- 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, +- 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, +- 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, +- 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, +- 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, +- 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, +- 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, +- 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, +- 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, +- 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, +- 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, +- 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, +- 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, +- 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, +- 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, +- 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, +- 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, +- 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, +- 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, +- 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, +- 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, +- 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, +- 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, +- 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, +- 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, +- 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, +- 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, +- 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, +- 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, +- 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, +- 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, +- 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, +- 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, +- 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, +- 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, +- 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, +- 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, +- 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, +- 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, +- 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, +- 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, +- 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, +- 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, +- 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, +- 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, +- 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, +- 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, +- 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, +- 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, +- 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, +- 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, +- 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, +- 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, +- 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, +- 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, +- 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, +- 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, +- 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, +- 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, +- 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, +- 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, +- 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, +- 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, +- 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, +- 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, +- 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, +- 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, +- 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, +- 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, +- 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, +- 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, +- 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, +- 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, +- 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, +- 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, +- 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, +- 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, +- 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, +- 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, +- 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, +- 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, +- 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, +- 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, +- 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, +- 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, +- 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, +- 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, +- 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, +- 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, +- 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, +- 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, +- 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, +- 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, +- 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, +- 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, +- 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, +- 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, +- 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, +- 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, +- 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, +- 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, +- 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, +- 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, +- 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, +- 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, +- 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, +- 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, +- 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, +- 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, +- 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, +- 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, +- 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, +- 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, +- 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, +- 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, +- 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, +- 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, +- 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, +- 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, +- 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, +- 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, +- 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, +- 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, +- 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, +- 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, +- 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, +- 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, +- 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, +- 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, +- 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, +- 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, +- 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, +- 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, +- 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, +- 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, +- 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, +- 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, +- 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, +- 0x22, 0x00, 0x00, ++ // 3183 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, ++ 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, ++ 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, ++ 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, ++ 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, ++ 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, ++ 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, ++ 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, ++ 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, ++ 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, ++ 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, ++ 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, ++ 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, ++ 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, ++ 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, ++ 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, ++ 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, ++ 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, ++ 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, ++ 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, ++ 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, ++ 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, ++ 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, ++ 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, ++ 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, ++ 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, ++ 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, ++ 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, ++ 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, ++ 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, ++ 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, ++ 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, ++ 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, ++ 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, ++ 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, ++ 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, ++ 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, ++ 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, ++ 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, ++ 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, ++ 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, ++ 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, ++ 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, ++ 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, ++ 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, ++ 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, ++ 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, ++ 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, ++ 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, ++ 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, ++ 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, ++ 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, ++ 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, ++ 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, ++ 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, ++ 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, ++ 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, ++ 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, ++ 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, ++ 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, ++ 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, ++ 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, ++ 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, ++ 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, ++ 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, ++ 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, ++ 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, ++ 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, ++ 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, ++ 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, ++ 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, ++ 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, ++ 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, ++ 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, ++ 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, ++ 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, ++ 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, ++ 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, ++ 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, ++ 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, ++ 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, ++ 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, ++ 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, ++ 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, ++ 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, ++ 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, ++ 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, ++ 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, ++ 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, ++ 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, ++ 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, ++ 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, ++ 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, ++ 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, ++ 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, ++ 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, ++ 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, ++ 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, ++ 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, ++ 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, ++ 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, ++ 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, ++ 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, ++ 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, ++ 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, ++ 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, ++ 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, ++ 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, ++ 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, ++ 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, ++ 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, ++ 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, ++ 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, ++ 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, ++ 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, ++ 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, ++ 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, ++ 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, ++ 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, ++ 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, ++ 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, ++ 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, ++ 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, ++ 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, ++ 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, ++ 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, ++ 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, ++ 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, ++ 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, ++ 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, ++ 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, ++ 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, ++ 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, ++ 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, ++ 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, ++ 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, ++ 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, ++ 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, ++ 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, ++ 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, ++ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, ++ 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, ++ 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, ++ 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, ++ 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, ++ 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, ++ 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, ++ 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, ++ 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, ++ 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, ++ 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, ++ 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, ++ 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, ++ 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, ++ 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, ++ 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, ++ 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, ++ 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, ++ 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, ++ 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, ++ 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, ++ 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, ++ 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, ++ 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, ++ 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, ++ 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, ++ 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, ++ 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, ++ 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, ++ 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, ++ 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, ++ 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, ++ 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, ++ 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, ++ 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, ++ 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, ++ 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, ++ 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, ++ 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, ++ 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, ++ 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, ++ 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, ++ 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, ++ 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, ++ 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, ++ 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, ++ 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, ++ 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, ++ 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, ++ 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, ++ 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, ++ 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, ++ 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, ++ 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, ++ 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, ++ 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, ++ 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, ++ 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, ++ 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, + } +diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto +index d863645..73f6b37 100644 +--- a/protocols/grpc/agent.proto ++++ b/protocols/grpc/agent.proto +@@ -222,7 +222,7 @@ message CgroupStats { + + } + +-message NetworkStats { ++message InterfaceStats { + string name = 1; + uint64 rx_bytes = 2; + uint64 rx_packets = 3; +@@ -234,9 +234,38 @@ message NetworkStats { + uint64 tx_dropped = 9; + } + ++message TcpStat { ++ uint64 established = 1; ++ uint64 syn_sent = 2; ++ uint64 syn_recv = 3; ++ uint64 fin_wait1 = 4; ++ uint64 fin_wait2 = 5; ++ uint64 time_wait = 6; ++ uint64 close = 7; ++ uint64 close_wait = 8; ++ uint64 last_ack = 9; ++ uint64 listen = 10; ++ uint64 closing = 11; ++} ++ ++message UdpStat { ++ uint64 listen = 1; ++ uint64 dropped = 2; ++ uint64 rx_queued = 3; ++ uint64 tx_queued = 4; ++} ++ ++message NetworkStats { ++ repeated InterfaceStats interfaces = 1; ++ TcpStat tcp = 2; ++ TcpStat tcp6 = 3; ++ UdpStat udp = 4; ++ UdpStat udp6 = 5; ++} ++ + message StatsContainerResponse { + CgroupStats cgroup_stats = 1; +- repeated NetworkStats network_stats = 2; ++ NetworkStats network_stats = 2; + } + + message WriteStreamRequest { +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch b/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch new file mode 100644 index 0000000000000000000000000000000000000000..f594245da620498696e1381135dcdbd411d9042c --- /dev/null +++ b/agent/patches/0016-clock-synchronizes-clock-info-with-proxy.patch @@ -0,0 +1,268 @@ +From 6120525f81701424e97d453d515d38f14bbe99d9 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 20:25:00 +0800 +Subject: [PATCH 16/16] clock: synchronizes clock info with proxy + +reason: virtual machine's clock may be incorrect, proxy synchronizes +clock info to help virtual machine adjust clock time + +Signed-off-by: yangfeiyu +--- + pkg/clock/clock_util.go | 47 ++++++++++++++++++++++++ + sync_clock_server.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++ + sync_clock_server_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 229 insertions(+) + create mode 100644 pkg/clock/clock_util.go + create mode 100644 sync_clock_server.go + create mode 100644 sync_clock_server_test.go + +diff --git a/pkg/clock/clock_util.go b/pkg/clock/clock_util.go +new file mode 100644 +index 0000000..4b78c63 +--- /dev/null ++++ b/pkg/clock/clock_util.go +@@ -0,0 +1,47 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: sync clock related function ++// Author: xueshaojia ++// Create: 2018-11-10 ++ ++// Package clock provides clock sync functions ++package clock ++ ++import ( ++ "net" ++ "syscall" ++) ++ ++type TimeValue struct { ++ ClientSendTime int64 `json:"client_send_time"` ++ ClientArriveTime int64 `json:"client_arrive_time"` ++ ServerSendTime int64 `json:"server_send_time"` ++ ServerArriveTime int64 `json:"server_arrive_time"` ++ Delta int64 `json:"delta"` ++} ++ ++const MaxSyncClockByteNum = 400 // sync clock byte num, max=400 ++const MaxTimeStampSupport = 3155731199000000000 // unit is ns, 2069/12/31 23:59:59 ++const MinTimeStampSupport = 0 // unit is ns, 1970/1/1 8:0:0 ++ ++// getCurrentTimeNs returns UTC time in Ns ++func GetCurrentTimeNs() int64 { ++ var tv syscall.Timeval ++ if err := syscall.Gettimeofday(&tv); err != nil { ++ return -1 ++ } ++ return tv.Sec*1000000000 + tv.Usec*1000 ++} ++ ++// readConnData reads data from stream ++func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) { ++ buf = make([]byte, MaxSyncClockByteNum) ++ byteNum, err = stream.Read(buf) ++ return buf, byteNum, err ++} ++ ++// writeConnData writes data to stream ++func WriteConnData(stream net.Conn, buf []byte) error { ++ _, err := stream.Write(buf) ++ return err ++} +diff --git a/sync_clock_server.go b/sync_clock_server.go +new file mode 100644 +index 0000000..a8e98c6 +--- /dev/null ++++ b/sync_clock_server.go +@@ -0,0 +1,94 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: sync clock server related function ++// Author: xueshaojia ++// Create: 2018-11-10 ++ ++package main ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "io" ++ "net" ++ "os/exec" ++ "syscall" ++ ++ "github.com/kata-containers/agent/pkg/clock" ++) ++ ++// getSyncClockInfo waits for client request ++func getSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error { ++ buf, byteNum, err := clock.ReadConnData(stream) ++ if err != nil { ++ return err ++ } ++ return json.Unmarshal(buf[:byteNum], clockInfo) ++} ++ ++// ackSyncClockInfo sends clock info to client ++func ackSyncClockInfo(stream net.Conn, clockInfo *clock.TimeValue) error { ++ b, err := json.Marshal(clockInfo) ++ if err != nil { ++ return err ++ } ++ return clock.WriteConnData(stream, b) ++} ++ ++// setGuestClock sets systemtime and hwclock ++func setGuestClock(delta int64) error { ++ dstClockNs := clock.GetCurrentTimeNs() + delta ++ if dstClockNs < clock.MinTimeStampSupport || dstClockNs > clock.MaxTimeStampSupport { ++ return fmt.Errorf("sync clock failed, the valid timestamp is from 1970/1/1 8:0:0 to 2069/12/31 23:59:59") ++ } ++ dstTimeVal := syscall.NsecToTimeval(dstClockNs) ++ if err := syscall.Settimeofday(&dstTimeVal); err != nil { ++ agentLog.WithError(err).Error("sync clock, set systemtime failed") ++ return err ++ } ++ cmd := exec.Command("hwclock", "-w") ++ if err := cmd.Run(); err != nil { ++ agentLog.WithError(err).Error("sync clock, set hwclock failed") ++ return err ++ } ++ agentLog.Infof("sync clock, set systemtime and hwclock succ, delta:%d", delta) ++ return nil ++} ++ ++// SyncClock is a loop that deals with client's request ++// 1 read client request ++// 2 response with clock info or set guest clock ++// 3 if error happends, print error info ++func SyncClock(stream net.Conn) { ++ for { ++ var clockInfo clock.TimeValue ++ if err := getSyncClockInfo(stream, &clockInfo); err != nil { ++ if err == io.EOF { ++ agentLog.WithError(err).Error("sync clock, yamux session error happends") ++ stream.Close() ++ break ++ } ++ agentLog.WithError(err).Error("sync clock, read sync clock info failed") ++ continue ++ } ++ // set client's request arrive time ++ clockInfo.ClientArriveTime = clock.GetCurrentTimeNs() ++ if clockInfo.Delta == 0 { ++ // this is a request for clock diff info ++ // set server's response send time ++ // if fails, wait for next time ++ clockInfo.ServerSendTime = clock.GetCurrentTimeNs() ++ if err := ackSyncClockInfo(stream, &clockInfo); err != nil { ++ agentLog.WithError(err).Error("sync clock, send back clock info failed") ++ } ++ } else { ++ // this is a request for adjusting guest clock ++ // 1 adjust guest clock(system time and hwclock) ++ // 2 do not send response any more ++ // 3 if fails, wait for next request to adjust ++ if err := setGuestClock(clockInfo.Delta); err != nil { ++ agentLog.WithError(err).Error("sync clock, set local clock failed") ++ } ++ } ++ } ++} +diff --git a/sync_clock_server_test.go b/sync_clock_server_test.go +new file mode 100644 +index 0000000..41d5227 +--- /dev/null ++++ b/sync_clock_server_test.go +@@ -0,0 +1,88 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: sync clock server related test ++// Author: xueshaojia ++// Create: 2018-11-10 ++ ++package main ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "math/rand" ++ "net" ++ "testing" ++ ++ "github.com/hashicorp/yamux" ++ "github.com/kata-containers/agent/pkg/clock" ++) ++ ++var clientStream net.Conn ++ ++func TestGetCurrentTimeNs(t *testing.T) { ++ timeRet := clock.GetCurrentTimeNs() ++ if timeRet <= 0 { ++ t.Fatalf("TestGetCurrentTimeNs Fail, time:%d", timeRet) ++ } ++} ++ ++func TestSetGuestClock(t *testing.T) { ++ if err := setGuestClock(0); err != nil { ++ t.Fatalf("test set clock failed, err:%v", err) ++ } ++} ++ ++func TestReadWriteData(t *testing.T) { ++ sock := GenSocket() ++ var err error ++ listener, err := net.Listen("unix", sock) ++ if err != nil { ++ t.Fatalf("listening err: %v", err) ++ } ++ go SetUpClient(sock) ++ conn, err := listener.Accept() ++ if err != nil { ++ t.Fatalf("accept err: %v", err) ++ } ++ session, err := yamux.Server(conn, nil) ++ if err != nil { ++ t.Fatalf("session err: %v", err) ++ } ++ stream, err := session.Accept() ++ if err != nil { ++ t.Fatalf("stream err: %v", err) ++ } ++ go func() { ++ var clockInfo clock.TimeValue = clock.TimeValue{1000, 0, 0, 0, 0} ++ b, _ := json.Marshal(clockInfo) ++ clock.WriteConnData(clientStream, b) ++ }() ++ _, _, err = clock.ReadConnData(stream) ++ if err != nil { ++ t.Fatalf("read conn data error, err:%v", err) ++ } ++} ++ ++func SetUpClient(sock string) error { ++ conn, err := net.Dial("unix", sock) ++ if err != nil { ++ return err ++ } ++ session, err := yamux.Client(conn, nil) ++ if err != nil { ++ conn.Close() ++ return err ++ } ++ stream, err := session.Open() ++ if err != nil { ++ return err ++ } ++ clientStream = stream ++ return nil ++} ++ ++func GenSocket() string { ++ randSeed := clock.GetCurrentTimeNs() ++ rand.Seed(randSeed) ++ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32()) ++} +-- +2.14.3 (Apple Git-98) + diff --git a/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch b/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch new file mode 100644 index 0000000000000000000000000000000000000000..591eb0592b8ffe26feda1c9ce01f35bfdaf0292d --- /dev/null +++ b/agent/patches/0017-agent-add-support-of-new-sandbox-StratoVirt.patch @@ -0,0 +1,1005 @@ +From ad3da81d9566807df2de062b0059aafbe9d9d810 Mon Sep 17 00:00:00 2001 +From: LiangZhang +Date: Mon, 21 Sep 2020 10:42:41 +0800 +Subject: [PATCH] agent: add support of new sandbox StratoVirt + +Signed-off-by: LiangZhang +--- + grpc.go | 4 + + network.go | 58 +++++ + protocols/grpc/agent.pb.go | 636 +++++++++++++++++++++++++++++---------------- + protocols/grpc/agent.proto | 5 + + 4 files changed, 473 insertions(+), 230 deletions(-) + +diff --git a/grpc.go b/grpc.go +index 1b63cde..727efb8 100644 +--- a/grpc.go ++++ b/grpc.go +@@ -1574,6 +1574,10 @@ func (a *agentGRPC) UpdateInterface(ctx context.Context, req *pb.UpdateInterface + return a.sandbox.updateInterface(nil, req.Interface) + } + ++func (a *agentGRPC) UpdateInterfaceHwAddrByName(ctx context.Context, req *pb.UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) { ++ return a.sandbox.updateInterfaceHwAddrByName(nil, req.Interface) ++} ++ + func (a *agentGRPC) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { + return a.sandbox.updateRoutes(nil, req.Routes, req.Increment) + } +diff --git a/network.go b/network.go +index 1baaa2e..f6e2c17 100644 +--- a/network.go ++++ b/network.go +@@ -209,6 +209,64 @@ func (s *sandbox) removeInterface(netHandle *netlink.Handle, iface *types.Interf + return nil, nil + } + ++// updateInterfaceHwAddrByName will update an existing interface with the values provided in the types.Interface. ++// It will identify the existing interface via its name. ++func (s *sandbox) updateInterfaceHwAddrByName(netHandle *netlink.Handle, iface *types.Interface) (resultingIfc *types.Interface, err error) { ++ if iface == nil { ++ return nil, errNoIF ++ } ++ s.network.ifacesLock.Lock() ++ defer s.network.ifacesLock.Unlock() ++ ++ if netHandle == nil { ++ netHandle, err = netlink.NewHandle(unix.NETLINK_ROUTE) ++ if err != nil { ++ return nil, err ++ } ++ defer netHandle.Delete() ++ } ++ ++ var link netlink.Link ++ links, err := netHandle.LinkList() ++ if err != nil { ++ return nil, err ++ } ++ ++ for _, l := range links { ++ if l == nil { ++ continue ++ } ++ ++ if l.Attrs() != nil && l.Attrs().Name == iface.Name { ++ link = l ++ } ++ } ++ ++ if link == nil { ++ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", iface.Name) ++ } ++ ++ // delete ip ++ if ips, err := netlink.AddrList(link, netlink.FAMILY_ALL); err == nil { ++ for _, ip := range ips { ++ netlink.AddrDel(link, &ip) ++ } ++ } ++ ++ if len(iface.IPAddresses) == 0 { ++ netHandle.LinkSetDown(link) ++ } ++ ++ // update mac ++ if hardAddr, err := net.ParseMAC(iface.HwAddr); err == nil { ++ if err = netHandle.LinkSetHardwareAddr(link, hardAddr); err != nil { ++ return nil, grpcStatus.Errorf(codes.Internal, "Could not set %s to %v: %v", iface.HwAddr, link, err) ++ } ++ } ++ ++ return iface, nil ++} ++ + // updateInterface will update an existing interface with the values provided in the types.Interface. It will identify the + // existing interface via MAC address and will return the state of the interface once the function completes as well an any + // errors observed. +diff --git a/protocols/grpc/agent.pb.go b/protocols/grpc/agent.pb.go +index c50ecb5..181fcb6 100644 +--- a/protocols/grpc/agent.pb.go ++++ b/protocols/grpc/agent.pb.go +@@ -50,6 +50,7 @@ + Interfaces + Routes + UpdateInterfaceRequest ++ UpdateInterfaceHwAddrByNameRequest + UpdateRoutesRequest + ListInterfacesRequest + ListRoutesRequest +@@ -1491,6 +1492,24 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + return nil + } + ++type UpdateInterfaceHwAddrByNameRequest struct { ++ Interface *types.Interface `protobuf:"bytes,1,opt,name=interface" json:"interface,omitempty"` ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } ++func (m *UpdateInterfaceHwAddrByNameRequest) String() string { return proto.CompactTextString(m) } ++func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} ++func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { ++ return fileDescriptorAgent, []int{40} ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) GetInterface() *types.Interface { ++ if m != nil { ++ return m.Interface ++ } ++ return nil ++} ++ + type UpdateRoutesRequest struct { + Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` + Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` +@@ -1499,7 +1518,7 @@ type UpdateRoutesRequest struct { + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } + func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateRoutesRequest) ProtoMessage() {} +-func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } ++func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } + + func (m *UpdateRoutesRequest) GetRoutes() *Routes { + if m != nil { +@@ -1521,7 +1540,7 @@ type ListInterfacesRequest struct { + func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } + func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } + func (*ListInterfacesRequest) ProtoMessage() {} +-func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } ++func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } + + type ListRoutesRequest struct { + } +@@ -1529,7 +1548,7 @@ type ListRoutesRequest struct { + func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } + func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*ListRoutesRequest) ProtoMessage() {} +-func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } ++func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } + + type OnlineCPUMemRequest struct { + // Wait specifies if the caller waits for the agent to online all resources. +@@ -1545,7 +1564,7 @@ type OnlineCPUMemRequest struct { + func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } + func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } + func (*OnlineCPUMemRequest) ProtoMessage() {} +-func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } ++func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } + + func (m *OnlineCPUMemRequest) GetWait() bool { + if m != nil { +@@ -1576,7 +1595,7 @@ type ReseedRandomDevRequest struct { + func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } + func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } + func (*ReseedRandomDevRequest) ProtoMessage() {} +-func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } ++func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } + + func (m *ReseedRandomDevRequest) GetData() []byte { + if m != nil { +@@ -1603,7 +1622,7 @@ type AgentDetails struct { + func (m *AgentDetails) Reset() { *m = AgentDetails{} } + func (m *AgentDetails) String() string { return proto.CompactTextString(m) } + func (*AgentDetails) ProtoMessage() {} +-func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } ++func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } + + func (m *AgentDetails) GetVersion() string { + if m != nil { +@@ -1654,7 +1673,7 @@ type GuestDetailsRequest struct { + func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } + func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsRequest) ProtoMessage() {} +-func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } ++func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } + + func (m *GuestDetailsRequest) GetMemBlockSize() bool { + if m != nil { +@@ -1680,7 +1699,7 @@ type GuestDetailsResponse struct { + func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } + func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsResponse) ProtoMessage() {} +-func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } ++func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } + + func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { + if m != nil { +@@ -1712,7 +1731,7 @@ type MemHotplugByProbeRequest struct { + func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } + func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } + func (*MemHotplugByProbeRequest) ProtoMessage() {} +-func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } ++func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } + + func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { + if m != nil { +@@ -1731,7 +1750,7 @@ type SetGuestDateTimeRequest struct { + func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } + func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } + func (*SetGuestDateTimeRequest) ProtoMessage() {} +-func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } ++func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } + + func (m *SetGuestDateTimeRequest) GetSec() int64 { + if m != nil { +@@ -1780,7 +1799,7 @@ type Storage struct { + func (m *Storage) Reset() { *m = Storage{} } + func (m *Storage) String() string { return proto.CompactTextString(m) } + func (*Storage) ProtoMessage() {} +-func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } ++func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } + + func (m *Storage) GetDriver() string { + if m != nil { +@@ -1863,7 +1882,7 @@ type Device struct { + func (m *Device) Reset() { *m = Device{} } + func (m *Device) String() string { return proto.CompactTextString(m) } + func (*Device) ProtoMessage() {} +-func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } ++func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + + func (m *Device) GetId() string { + if m != nil { +@@ -1909,7 +1928,7 @@ type StringUser struct { + func (m *StringUser) Reset() { *m = StringUser{} } + func (m *StringUser) String() string { return proto.CompactTextString(m) } + func (*StringUser) ProtoMessage() {} +-func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } ++func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } + + func (m *StringUser) GetUid() string { + if m != nil { +@@ -1957,7 +1976,7 @@ type CopyFileRequest struct { + func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } + func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } + func (*CopyFileRequest) ProtoMessage() {} +-func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } + + func (m *CopyFileRequest) GetPath() string { + if m != nil { +@@ -2021,7 +2040,7 @@ type StartTracingRequest struct { + func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } + func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StartTracingRequest) ProtoMessage() {} +-func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } + + type StopTracingRequest struct { + } +@@ -2029,7 +2048,7 @@ type StopTracingRequest struct { + func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } + func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StopTracingRequest) ProtoMessage() {} +-func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } ++func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } + + type UpdateIPVSRequest struct { + // IPVS_req is the IPVS rule message needed to update +@@ -2039,7 +2058,7 @@ type UpdateIPVSRequest struct { + func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } + func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateIPVSRequest) ProtoMessage() {} +-func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } + + func (m *UpdateIPVSRequest) GetIPVSReq() string { + if m != nil { +@@ -2056,7 +2075,7 @@ type IPVSResponse struct { + func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } + func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } + func (*IPVSResponse) ProtoMessage() {} +-func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{58} } + + func (m *IPVSResponse) GetIPVSRes() string { + if m != nil { +@@ -2106,6 +2125,7 @@ func init() { + proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") + proto.RegisterType((*Routes)(nil), "grpc.Routes") + proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") ++ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") + proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") + proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") + proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") +@@ -2163,6 +2183,7 @@ type AgentServiceClient interface { + TtyWinResize(ctx context.Context, in *TtyWinResizeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) + // networking + UpdateInterface(ctx context.Context, in *UpdateInterfaceRequest, opts ...grpc1.CallOption) (*types.Interface, error) ++ UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) + UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) + ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) + ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) +@@ -2342,6 +2363,15 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, in *UpdateInte + return out, nil + } + ++func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) { ++ out := new(types.Interface) ++ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateInterfaceHwAddrByName", in, out, c.cc, opts...) ++ if err != nil { ++ return nil, err ++ } ++ return out, nil ++} ++ + func (c *agentServiceClient) UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) { + out := new(Routes) + err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateRoutes", in, out, c.cc, opts...) +@@ -2497,6 +2527,7 @@ type AgentServiceServer interface { + TtyWinResize(context.Context, *TtyWinResizeRequest) (*google_protobuf2.Empty, error) + // networking + UpdateInterface(context.Context, *UpdateInterfaceRequest) (*types.Interface, error) ++ UpdateInterfaceHwAddrByName(context.Context, *UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) + UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) + ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) + ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) +@@ -2825,6 +2856,24 @@ func _AgentService_UpdateInterface_Handler(srv interface{}, ctx context.Context, + return interceptor(ctx, in, info, handler) + } + ++func _AgentService_UpdateInterfaceHwAddrByName_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { ++ in := new(UpdateInterfaceHwAddrByNameRequest) ++ if err := dec(in); err != nil { ++ return nil, err ++ } ++ if interceptor == nil { ++ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, in) ++ } ++ info := &grpc1.UnaryServerInfo{ ++ Server: srv, ++ FullMethod: "/grpc.AgentService/UpdateInterfaceHwAddrByName", ++ } ++ handler := func(ctx context.Context, req interface{}) (interface{}, error) { ++ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, req.(*UpdateInterfaceHwAddrByNameRequest)) ++ } ++ return interceptor(ctx, in, info, handler) ++} ++ + func _AgentService_UpdateRoutes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateRoutesRequest) + if err := dec(in); err != nil { +@@ -3150,6 +3199,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ + Handler: _AgentService_UpdateInterface_Handler, + }, + { ++ MethodName: "UpdateInterfaceHwAddrByName", ++ Handler: _AgentService_UpdateInterfaceHwAddrByName_Handler, ++ }, ++ { + MethodName: "UpdateRoutes", + Handler: _AgentService_UpdateRoutes_Handler, + }, +@@ -4899,6 +4952,34 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if m.Interface != nil { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) ++ n25, err := m.Interface.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n25 ++ } ++ return i, nil ++} ++ + func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) +@@ -4918,11 +4999,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) +- n25, err := m.Routes.MarshalTo(dAtA[i:]) ++ n26, err := m.Routes.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n25 ++ i += n26 + } + if m.Increment { + dAtA[i] = 0x10 +@@ -5176,11 +5257,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0x12 + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) +- n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) ++ n27, err := m.AgentDetails.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n26 ++ i += n27 + } + if m.SupportMemHotplugProbe { + dAtA[i] = 0x18 +@@ -5211,21 +5292,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { +- dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) +- var j27 int ++ dAtA29 := make([]byte, len(m.MemHotplugProbeAddr)*10) ++ var j28 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { +- dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) ++ dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 +- j27++ ++ j28++ + } +- dAtA28[j27] = uint8(num) +- j27++ ++ dAtA29[j28] = uint8(num) ++ j28++ + } + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(j27)) +- i += copy(dAtA[i:], dAtA28[:j27]) ++ i = encodeVarintAgent(dAtA, i, uint64(j28)) ++ i += copy(dAtA[i:], dAtA29[:j28]) + } + return i, nil + } +@@ -6333,6 +6414,16 @@ func (m *UpdateInterfaceRequest) Size() (n int) { + return n + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { ++ var l int ++ _ = l ++ if m.Interface != nil { ++ l = m.Interface.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func (m *UpdateRoutesRequest) Size() (n int) { + var l int + _ = l +@@ -12114,6 +12205,89 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { + } + return nil + } ++func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Interface == nil { ++ m.Interface = &types.Interface{} ++ } ++ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} + func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 +@@ -14242,204 +14416,206 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 3183 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, +- 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, +- 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, +- 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, +- 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, +- 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, +- 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, +- 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, +- 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, +- 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, +- 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, +- 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, +- 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, +- 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, +- 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, +- 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, +- 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, +- 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, +- 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, +- 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, +- 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, +- 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, +- 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, +- 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, +- 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, +- 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, +- 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, +- 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, +- 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, +- 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, +- 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, +- 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, +- 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, +- 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, +- 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, +- 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, +- 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, +- 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, +- 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, +- 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, +- 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, +- 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, +- 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, +- 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, +- 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, +- 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, +- 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, +- 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, +- 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, +- 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, +- 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, +- 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, +- 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, +- 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, +- 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, +- 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, +- 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, +- 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, +- 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, +- 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, +- 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, +- 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, +- 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, +- 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, +- 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, +- 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, +- 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, +- 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, +- 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, +- 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, +- 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, +- 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, +- 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, +- 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, +- 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, +- 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, +- 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, +- 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, +- 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, +- 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, +- 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, +- 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, +- 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, +- 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, +- 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, +- 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, +- 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, +- 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, +- 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, +- 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, +- 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, +- 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, +- 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, +- 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, +- 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, +- 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, +- 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, +- 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, +- 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, +- 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, +- 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, +- 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, +- 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, +- 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, +- 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, +- 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, +- 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, +- 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, +- 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, +- 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, +- 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, +- 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, +- 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, +- 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, +- 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, +- 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, +- 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, +- 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, +- 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, +- 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, +- 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, +- 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, +- 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, +- 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, +- 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, +- 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, +- 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, +- 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, +- 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, +- 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, +- 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, +- 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, +- 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, +- 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, +- 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, +- 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, +- 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, +- 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, +- 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, +- 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, +- 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, +- 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, +- 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, +- 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, +- 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, +- 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, +- 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, +- 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, +- 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, +- 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, +- 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, +- 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, +- 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, +- 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, +- 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, +- 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, +- 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, +- 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, +- 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, +- 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, +- 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, +- 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, +- 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, +- 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, +- 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, +- 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, +- 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, +- 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, +- 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, +- 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, +- 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, +- 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, +- 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, +- 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, +- 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, +- 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, +- 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, +- 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, +- 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, +- 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, +- 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, +- 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, +- 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, +- 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, +- 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, +- 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, +- 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, +- 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, +- 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, +- 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, +- 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, +- 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, +- 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, +- 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, +- 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, +- 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, +- 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, +- 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, ++ // 3214 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, ++ 0x76, 0x98, 0x07, 0x39, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, ++ 0x99, 0x8e, 0x63, 0xd2, 0xa2, 0x0d, 0x3f, 0xe1, 0x08, 0x22, 0xa5, 0x88, 0x8c, 0x25, 0x8b, 0x69, ++ 0x8a, 0x71, 0x8c, 0x20, 0x68, 0x34, 0xbb, 0x8b, 0xc3, 0x32, 0xa7, 0xbb, 0xda, 0x55, 0xd5, 0x14, ++ 0xc7, 0x01, 0xb2, 0x4c, 0xb6, 0x59, 0x65, 0x97, 0x1f, 0x08, 0x92, 0x55, 0x76, 0xc9, 0x36, 0x0b, ++ 0x23, 0xc8, 0xe2, 0x7e, 0xc1, 0xc5, 0x85, 0x3f, 0xe1, 0x7e, 0xc1, 0x45, 0xbd, 0xfa, 0x31, 0x33, ++ 0xa4, 0x71, 0x09, 0x01, 0x77, 0xd3, 0xe8, 0xf3, 0xa8, 0xf3, 0xaa, 0xaa, 0x53, 0xe7, 0x54, 0x41, ++ 0xdb, 0x1f, 0xe3, 0x58, 0x6c, 0x26, 0x8c, 0x0a, 0x8a, 0xea, 0x63, 0x96, 0x04, 0xa3, 0x16, 0x0d, ++ 0x88, 0x46, 0x8c, 0x3e, 0x1d, 0x13, 0x71, 0x9a, 0x1e, 0x6f, 0x06, 0x34, 0xda, 0x3a, 0xf3, 0x85, ++ 0xff, 0x61, 0x40, 0x63, 0xe1, 0x93, 0x18, 0x33, 0xbe, 0xa5, 0x06, 0x6e, 0x25, 0x67, 0xe3, 0x2d, ++ 0x31, 0x4d, 0x30, 0xd7, 0x5f, 0x33, 0xee, 0xf6, 0x98, 0xd2, 0xf1, 0x04, 0x6f, 0x29, 0xe8, 0x38, ++ 0x3d, 0xd9, 0xc2, 0x51, 0x22, 0xa6, 0x9a, 0xe8, 0xfc, 0x5b, 0x15, 0xd6, 0x77, 0x19, 0xf6, 0x05, ++ 0xde, 0xb5, 0xd2, 0x5c, 0xfc, 0x63, 0x8a, 0xb9, 0x40, 0x6f, 0x41, 0x27, 0xd3, 0xe0, 0x91, 0x70, ++ 0x58, 0xb9, 0x57, 0xd9, 0x68, 0xb9, 0xed, 0x0c, 0xb7, 0x1f, 0xa2, 0x9b, 0xd0, 0xc0, 0x17, 0x38, ++ 0x90, 0xd4, 0xaa, 0xa2, 0x2e, 0x4b, 0x70, 0x3f, 0x44, 0x0f, 0xa0, 0xcd, 0x05, 0x23, 0xf1, 0xd8, ++ 0x4b, 0x39, 0x66, 0xc3, 0xda, 0xbd, 0xca, 0x46, 0x7b, 0x7b, 0x65, 0x53, 0xba, 0xb4, 0x79, 0xa8, ++ 0x08, 0x47, 0x1c, 0x33, 0x17, 0x78, 0xf6, 0x8f, 0xee, 0x43, 0x23, 0xc4, 0xe7, 0x24, 0xc0, 0x7c, ++ 0x58, 0xbf, 0x57, 0xdb, 0x68, 0x6f, 0x77, 0x34, 0xfb, 0x63, 0x85, 0x74, 0x2d, 0x11, 0xbd, 0x0f, ++ 0x4d, 0x2e, 0x28, 0xf3, 0xc7, 0x98, 0x0f, 0x97, 0x14, 0x63, 0xd7, 0xca, 0x55, 0x58, 0x37, 0x23, ++ 0xa3, 0x3b, 0x50, 0x7b, 0xb1, 0xbb, 0x3f, 0x5c, 0x56, 0xda, 0xc1, 0x70, 0x25, 0x38, 0x70, 0x25, ++ 0x1a, 0xbd, 0x0d, 0x5d, 0xee, 0xc7, 0xe1, 0x31, 0xbd, 0xf0, 0x12, 0x12, 0xc6, 0x7c, 0xd8, 0xb8, ++ 0x57, 0xd9, 0x68, 0xba, 0x1d, 0x83, 0x3c, 0x90, 0x38, 0xe7, 0x4b, 0xb8, 0x71, 0x28, 0x7c, 0x26, ++ 0xae, 0x11, 0x1d, 0xe7, 0x08, 0xd6, 0x5d, 0x1c, 0xd1, 0xf3, 0x6b, 0x85, 0x76, 0x08, 0x0d, 0x41, ++ 0x22, 0x4c, 0x53, 0xa1, 0x42, 0xdb, 0x75, 0x2d, 0xe8, 0xfc, 0x47, 0x05, 0xd0, 0x93, 0x0b, 0x1c, ++ 0x1c, 0x30, 0x1a, 0x60, 0xce, 0xff, 0x44, 0xd3, 0xf5, 0x1e, 0x34, 0x12, 0x6d, 0xc0, 0xb0, 0xae, ++ 0xd8, 0xcd, 0x2c, 0x58, 0xab, 0x2c, 0xd5, 0xf9, 0x01, 0xd6, 0x0e, 0xc9, 0x38, 0xf6, 0x27, 0xaf, ++ 0xd1, 0xde, 0x75, 0x58, 0xe6, 0x4a, 0xa6, 0x32, 0xb5, 0xeb, 0x1a, 0xc8, 0x39, 0x00, 0xf4, 0x9d, ++ 0x4f, 0xc4, 0xeb, 0xd3, 0xe4, 0x7c, 0x08, 0xab, 0x25, 0x89, 0x3c, 0xa1, 0x31, 0xc7, 0xca, 0x00, ++ 0xe1, 0x8b, 0x94, 0x2b, 0x61, 0x4b, 0xae, 0x81, 0x1c, 0x0c, 0x6b, 0xcf, 0x08, 0xb7, 0xec, 0xf8, ++ 0x8f, 0x31, 0x61, 0x1d, 0x96, 0x4f, 0x28, 0x8b, 0x7c, 0x61, 0x2d, 0xd0, 0x10, 0x42, 0x50, 0xf7, ++ 0xd9, 0x98, 0x0f, 0x6b, 0xf7, 0x6a, 0x1b, 0x2d, 0x57, 0xfd, 0xcb, 0x55, 0x39, 0xa3, 0xc6, 0xd8, ++ 0xf5, 0x16, 0x74, 0x4c, 0xdc, 0xbd, 0x09, 0xe1, 0x42, 0xe9, 0xe9, 0xb8, 0x6d, 0x83, 0x93, 0x63, ++ 0x1c, 0x0a, 0xeb, 0x47, 0x49, 0x78, 0xcd, 0x0d, 0xbf, 0x0d, 0x2d, 0x86, 0x39, 0x4d, 0x99, 0xdc, ++ 0xa6, 0x55, 0x35, 0xef, 0x6b, 0x7a, 0xde, 0x9f, 0x91, 0x38, 0xbd, 0x70, 0x2d, 0xcd, 0xcd, 0xd9, ++ 0xcc, 0x16, 0x12, 0xfc, 0x3a, 0x5b, 0xe8, 0x4b, 0xb8, 0x71, 0xe0, 0xa7, 0xfc, 0x3a, 0xb6, 0x3a, ++ 0x5f, 0xc9, 0xed, 0xc7, 0xd3, 0xe8, 0x5a, 0x83, 0xff, 0xbd, 0x02, 0xcd, 0xdd, 0x24, 0x3d, 0xe2, ++ 0xfe, 0x18, 0xa3, 0x37, 0xa1, 0x2d, 0xa8, 0xf0, 0x27, 0x5e, 0x2a, 0x41, 0xc5, 0x5e, 0x77, 0x41, ++ 0xa1, 0x34, 0x83, 0x0c, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8e, 0xea, 0xbd, 0xda, 0x46, 0xdd, 0x6d, ++ 0x6b, 0x9c, 0x66, 0xd9, 0x84, 0x55, 0x45, 0xf3, 0x48, 0xec, 0x9d, 0x61, 0x16, 0xe3, 0x49, 0x44, ++ 0x43, 0xac, 0xd6, 0x6f, 0xdd, 0x1d, 0x28, 0xd2, 0x7e, 0xfc, 0x4d, 0x46, 0x40, 0x7f, 0x06, 0x83, ++ 0x8c, 0x5f, 0x6e, 0x4a, 0xc5, 0x5d, 0x57, 0xdc, 0x7d, 0xc3, 0x7d, 0x64, 0xd0, 0xce, 0x3f, 0x42, ++ 0xef, 0xe5, 0x29, 0xa3, 0x42, 0x4c, 0x48, 0x3c, 0x7e, 0xec, 0x0b, 0x5f, 0x66, 0x8f, 0x04, 0x33, ++ 0x42, 0x43, 0x6e, 0xac, 0xb5, 0x20, 0xfa, 0x00, 0x06, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, 0x3c, 0x55, ++ 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0c, 0xf3, 0xbb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, 0xf6, 0x76, ++ 0x33, 0xec, 0x4b, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0xfa, 0x00, 0x5a, 0x79, 0x1c, ++ 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x86, 0xbe, 0xc8, ++ 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, 0x7c, 0x05, ++ 0xad, 0x03, 0x12, 0x72, 0xad, 0x78, 0x08, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, 0x6c, 0x40, ++ 0xb4, 0x06, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe3, 0x88, 0xb2, ++ 0xa9, 0x0a, 0xd8, 0x1a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x36, 0xb4, 0x22, 0xff, 0x22, 0x9b, ++ 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x08, 0x8d, 0x13, 0x9f, 0x4c, 0x82, 0x58, 0x98, ++ 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0xdf, 0x2a, 0xb4, 0xb5, 0x46, 0x6d, 0xf0, 0x1a, ++ 0x2c, 0x05, 0x7e, 0x70, 0x9a, 0xa9, 0x54, 0x00, 0xba, 0x6f, 0x0d, 0xa9, 0x16, 0x93, 0x70, 0x6e, ++ 0xa9, 0x35, 0x6d, 0x0b, 0x80, 0xbf, 0xf2, 0x13, 0x63, 0x5b, 0xed, 0x12, 0xe6, 0x96, 0xe4, 0xd1, ++ 0xe6, 0x7e, 0x0c, 0x1d, 0xbd, 0xee, 0xcc, 0x90, 0xfa, 0x25, 0x43, 0xda, 0x9a, 0x4b, 0x0f, 0x7a, ++ 0x1b, 0xba, 0x29, 0xc7, 0xde, 0x29, 0xc1, 0xcc, 0x67, 0xc1, 0xe9, 0x74, 0xb8, 0xa4, 0xcf, 0xc8, ++ 0x94, 0xe3, 0x3d, 0x8b, 0x43, 0xdb, 0xb0, 0x24, 0xd3, 0x1f, 0x1f, 0x2e, 0xab, 0xe3, 0xf8, 0x4e, ++ 0x51, 0xa4, 0x72, 0x75, 0x53, 0x7d, 0x9f, 0xc4, 0x82, 0x4d, 0x5d, 0xcd, 0x3a, 0xfa, 0x1c, 0x20, ++ 0x47, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0xa9, 0xd9, 0x87, 0xf2, 0x57, 0x06, 0xe7, 0xdc, 0x9f, 0xa4, ++ 0x36, 0xea, 0x1a, 0xf8, 0xb2, 0xfa, 0x79, 0xc5, 0x09, 0xa0, 0xbf, 0x33, 0x39, 0x23, 0xb4, 0x30, ++ 0x7c, 0x0d, 0x96, 0x22, 0xff, 0x07, 0xca, 0x6c, 0x24, 0x15, 0xa0, 0xb0, 0x24, 0xa6, 0xcc, 0x8a, ++ 0x50, 0x00, 0xea, 0x41, 0x95, 0x26, 0x2a, 0x5e, 0x2d, 0xb7, 0x4a, 0x93, 0x5c, 0x51, 0xbd, 0xa0, ++ 0xc8, 0xf9, 0x6d, 0x1d, 0x20, 0xd7, 0x82, 0x5c, 0x18, 0x11, 0xea, 0x71, 0xcc, 0x64, 0x09, 0xe2, ++ 0x1d, 0x4f, 0x05, 0xe6, 0x1e, 0xc3, 0x41, 0xca, 0x38, 0x39, 0x97, 0xf3, 0x27, 0xdd, 0xbe, 0xa1, ++ 0xdd, 0x9e, 0xb1, 0xcd, 0xbd, 0x49, 0xe8, 0xa1, 0x1e, 0xb7, 0x23, 0x87, 0xb9, 0x76, 0x14, 0xda, ++ 0x87, 0x1b, 0xb9, 0xcc, 0xb0, 0x20, 0xae, 0x7a, 0x95, 0xb8, 0xd5, 0x4c, 0x5c, 0x98, 0x8b, 0x7a, ++ 0x02, 0xab, 0x84, 0x7a, 0x3f, 0xa6, 0x38, 0x2d, 0x09, 0xaa, 0x5d, 0x25, 0x68, 0x40, 0xe8, 0x5f, ++ 0xab, 0x01, 0xb9, 0x98, 0x03, 0xb8, 0x55, 0xf0, 0x52, 0x6e, 0xf7, 0x82, 0xb0, 0xfa, 0x55, 0xc2, ++ 0xd6, 0x33, 0xab, 0x64, 0x3e, 0xc8, 0x25, 0xfe, 0x15, 0xac, 0x13, 0xea, 0xbd, 0xf2, 0x89, 0x98, ++ 0x15, 0xb7, 0xf4, 0x2b, 0x4e, 0xca, 0x43, 0xb7, 0x2c, 0x4b, 0x3b, 0x19, 0x61, 0x36, 0x2e, 0x39, ++ 0xb9, 0xfc, 0x2b, 0x4e, 0x3e, 0x57, 0x03, 0x72, 0x31, 0x8f, 0x60, 0x40, 0xe8, 0xac, 0x35, 0x8d, ++ 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x03, 0x03, 0x8e, 0x03, 0x41, 0x59, 0x71, 0x11, 0x34, ++ 0xaf, 0x12, 0xb1, 0x62, 0xf8, 0x33, 0x19, 0xce, 0xdf, 0x41, 0x67, 0x2f, 0x1d, 0x63, 0x31, 0x39, ++ 0xce, 0x92, 0xc1, 0x6b, 0xcb, 0x3f, 0xce, 0xef, 0xab, 0xd0, 0xde, 0x1d, 0x33, 0x9a, 0x26, 0xa5, ++ 0x9c, 0xac, 0x37, 0xe9, 0x6c, 0x4e, 0x56, 0x2c, 0x2a, 0x27, 0x6b, 0xe6, 0x4f, 0xa0, 0x13, 0xa9, ++ 0xad, 0x6b, 0xf8, 0x75, 0x1e, 0x1a, 0xcc, 0x6d, 0x6a, 0xb7, 0x1d, 0x15, 0x92, 0xd9, 0x26, 0x40, ++ 0x42, 0x42, 0x6e, 0xc6, 0xe8, 0x74, 0xd4, 0x37, 0x15, 0xa1, 0x4d, 0xd1, 0x6e, 0x2b, 0xc9, 0xb2, ++ 0xf5, 0x03, 0x68, 0x1f, 0xcb, 0x20, 0x99, 0x01, 0xa5, 0x64, 0x94, 0x47, 0xcf, 0x85, 0xe3, 0x7c, ++ 0x13, 0xee, 0x41, 0xf7, 0x54, 0x87, 0xcc, 0x0c, 0xd2, 0x6b, 0xe8, 0x6d, 0xe3, 0x49, 0xee, 0xef, ++ 0x66, 0x31, 0xb2, 0x7a, 0x02, 0x3a, 0xa7, 0x05, 0xd4, 0xe8, 0x10, 0x06, 0x73, 0x2c, 0x0b, 0x72, ++ 0xd0, 0x46, 0x31, 0x07, 0xb5, 0xb7, 0x91, 0x56, 0x54, 0x1c, 0x59, 0xcc, 0x4b, 0xff, 0x52, 0x85, ++ 0xde, 0x7e, 0x2c, 0x30, 0x3b, 0xf1, 0x03, 0xac, 0x2d, 0x46, 0x50, 0x8f, 0xfd, 0x08, 0x1b, 0x99, ++ 0xea, 0x1f, 0xdd, 0x82, 0x26, 0xbb, 0xd0, 0x29, 0xc4, 0xcc, 0x68, 0x83, 0x5d, 0xa8, 0xd4, 0x80, ++ 0xde, 0x00, 0x60, 0x17, 0x5e, 0xe2, 0x07, 0x67, 0xd8, 0xc4, 0xb0, 0xee, 0xb6, 0xd8, 0xc5, 0x81, ++ 0x46, 0xc8, 0xc5, 0xc0, 0x2e, 0x3c, 0xcc, 0x18, 0x65, 0xdc, 0x64, 0xab, 0x26, 0xbb, 0x78, 0xa2, ++ 0x60, 0x33, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0xb2, 0xb4, 0x1a, 0xfb, 0x58, 0x23, 0xa4, 0x56, ++ 0x61, 0xb5, 0x2e, 0x6b, 0xad, 0x22, 0xd7, 0x2a, 0x72, 0xad, 0x0d, 0x3d, 0x52, 0x14, 0xb5, 0x8a, ++ 0x4c, 0x6b, 0x53, 0x6b, 0x15, 0x05, 0xad, 0x22, 0xd7, 0xda, 0xb2, 0x63, 0x8d, 0x56, 0xe7, 0xbf, ++ 0xab, 0xd0, 0x78, 0x19, 0xa8, 0x49, 0x41, 0xf7, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, 0x84, 0x9f, ++ 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, 0x22, 0xc3, ++ 0xa7, 0xf1, 0xa1, 0x3c, 0xc1, 0x0d, 0x89, 0xe1, 0xe0, 0xdc, 0xae, 0x75, 0x3e, 0x8d, 0x5d, 0x1c, ++ 0x9c, 0x4b, 0xfb, 0x4e, 0x48, 0xac, 0x72, 0xcc, 0x03, 0x1b, 0x95, 0x13, 0x12, 0xcb, 0xfc, 0xf1, ++ 0xa0, 0x48, 0xdc, 0x36, 0x41, 0xb1, 0xc4, 0x6d, 0xe5, 0x99, 0x4c, 0x03, 0x92, 0x6a, 0x82, 0xd2, ++ 0x94, 0x08, 0x49, 0x55, 0x87, 0xf3, 0x84, 0x72, 0x6c, 0x02, 0xa2, 0x01, 0xe9, 0xaf, 0xfa, 0xd1, ++ 0x63, 0x74, 0x34, 0x5a, 0x0a, 0xa3, 0x06, 0xdd, 0x82, 0xe6, 0xc4, 0xe7, 0xc2, 0xf3, 0x83, 0x33, ++ 0x13, 0x8c, 0x86, 0x84, 0x1f, 0x05, 0x67, 0xb2, 0xba, 0x97, 0x05, 0x39, 0x8e, 0x87, 0xa0, 0x08, ++ 0x06, 0x52, 0x55, 0xcb, 0x84, 0x72, 0x12, 0x8f, 0x87, 0x6d, 0x53, 0xb5, 0x68, 0xd0, 0x49, 0xa1, ++ 0x71, 0x14, 0xea, 0xd8, 0xe5, 0x83, 0x2b, 0xb3, 0x83, 0x6d, 0xec, 0x4d, 0xc0, 0x0c, 0x68, 0xd6, ++ 0x8a, 0x3e, 0x11, 0x4c, 0xc4, 0x9a, 0xec, 0x42, 0x27, 0x7c, 0x33, 0xa5, 0x86, 0x58, 0xb7, 0x53, ++ 0xaa, 0x89, 0xce, 0xff, 0x57, 0xa0, 0xf3, 0x2d, 0x16, 0xaf, 0x28, 0x3b, 0xb3, 0xf9, 0x00, 0x88, ++ 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0xde, 0x84, 0x9a, ++ 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x2d, 0xa8, 0x8b, 0x20, ++ 0xf9, 0xd4, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, 0x26, 0x24, ++ 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xaa, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, 0x7f, 0xae, ++ 0xc0, 0xfa, 0x6c, 0xf7, 0x61, 0x7a, 0xa5, 0x4f, 0xa0, 0x13, 0xa8, 0xa4, 0x51, 0x4a, 0x8c, 0x83, ++ 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x0c, 0xba, 0xb1, 0x0e, 0x4f, 0x29, 0x3f, 0x9a, ++ 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x8e, 0x11, 0x81, 0x0f, 0x05, ++ 0xc3, 0x7e, 0xf4, 0x3a, 0xba, 0x60, 0x04, 0x75, 0x55, 0x32, 0xd7, 0x54, 0x93, 0xa7, 0xfe, 0x9d, ++ 0xf7, 0x60, 0xb5, 0xa4, 0xc5, 0xf8, 0xba, 0x02, 0xb5, 0x89, 0x59, 0x3e, 0x5d, 0x57, 0xfe, 0x3a, ++ 0x3e, 0x0c, 0x5c, 0xec, 0x87, 0xaf, 0xcf, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x00, 0x54, 0x54, ++ 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0xbf, 0x80, 0xc1, 0xae, 0xdc, 0x45, 0x87, 0x22, 0x24, ++ 0xf1, 0xeb, 0x68, 0xdb, 0xff, 0x01, 0x56, 0x5f, 0x8a, 0xe9, 0x77, 0x52, 0x18, 0x27, 0x3f, 0xe1, ++ 0xd7, 0xe4, 0x1f, 0xa3, 0xaf, 0xac, 0x7f, 0x8c, 0xbe, 0x92, 0xdb, 0x32, 0xa0, 0x93, 0x34, 0x8a, ++ 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa0, 0xa3, 0x1b, 0xb9, 0xe7, 0x34, 0x4c, 0x27, 0x78, ++ 0xe1, 0x31, 0x70, 0x17, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, 0xcb, 0x2d, ++ 0x60, 0x9c, 0x7f, 0xad, 0xc2, 0x9a, 0xbe, 0x97, 0x3b, 0xd4, 0xd7, 0x51, 0xd6, 0x85, 0x11, 0x34, ++ 0x4f, 0x29, 0x17, 0x05, 0x81, 0x19, 0x2c, 0x4d, 0x0c, 0x63, 0x2b, 0x4d, 0xfe, 0x96, 0x2e, 0xcb, ++ 0x6a, 0x57, 0x5f, 0x96, 0xcd, 0x5d, 0x87, 0xd5, 0xe7, 0xaf, 0xc3, 0x64, 0x02, 0xb4, 0x4c, 0x44, ++ 0x1f, 0x33, 0x2d, 0xb7, 0x65, 0x30, 0xfb, 0x21, 0xba, 0x0f, 0xfd, 0xb1, 0xb4, 0xd2, 0x3b, 0xa5, ++ 0xf4, 0xcc, 0x4b, 0x7c, 0x71, 0xaa, 0x12, 0x6b, 0xcb, 0xed, 0x2a, 0xf4, 0x1e, 0xa5, 0x67, 0x07, ++ 0xbe, 0x38, 0x45, 0x5f, 0x40, 0xcf, 0xf4, 0x22, 0x91, 0x0a, 0x11, 0x37, 0x15, 0x98, 0xd9, 0x45, ++ 0xc5, 0xe8, 0xb9, 0xdd, 0xb3, 0x02, 0xc4, 0x9d, 0x9b, 0x70, 0xe3, 0x31, 0xe6, 0x82, 0xd1, 0x69, ++ 0x39, 0x30, 0xce, 0x5f, 0x00, 0xec, 0xe7, 0xf9, 0xe7, 0xa3, 0x22, 0x64, 0xb2, 0xd6, 0xca, 0xa6, ++ 0xbe, 0x16, 0xcd, 0x08, 0x6e, 0x81, 0xc7, 0xd9, 0x84, 0x65, 0x97, 0xa6, 0xf2, 0x44, 0x7c, 0xc7, ++ 0xfe, 0x99, 0x71, 0x1d, 0x33, 0x4e, 0x21, 0x5d, 0x43, 0x73, 0xf6, 0xec, 0x3d, 0x4a, 0x2e, 0xce, ++ 0x4c, 0xd1, 0x26, 0xb4, 0xb2, 0x4c, 0x68, 0xb2, 0xca, 0xbc, 0xea, 0x9c, 0xc5, 0x79, 0x09, 0xce, ++ 0x8c, 0xa4, 0xbd, 0x57, 0x8f, 0xc2, 0x90, 0xed, 0x4c, 0xbf, 0xf5, 0xa3, 0x6b, 0x4b, 0xfd, 0x1e, ++ 0x56, 0xb5, 0x54, 0x6d, 0xaf, 0x15, 0xf3, 0x0e, 0x2c, 0x33, 0xeb, 0x5c, 0x25, 0xbf, 0x65, 0x35, ++ 0x4c, 0x86, 0x86, 0xee, 0x48, 0x65, 0x01, 0xc3, 0x91, 0x3d, 0x8c, 0x9b, 0x6e, 0x8e, 0x90, 0x73, ++ 0xf0, 0x8c, 0x70, 0x91, 0x07, 0xcf, 0xce, 0xc1, 0x2a, 0x0c, 0x24, 0xa1, 0xa4, 0xd1, 0xf9, 0x7b, ++ 0x58, 0x7d, 0x11, 0x4f, 0x48, 0x8c, 0x77, 0x0f, 0x8e, 0x9e, 0xe3, 0x2c, 0xd7, 0x20, 0xa8, 0xab, ++ 0x53, 0xb4, 0xa2, 0xa4, 0xab, 0x7f, 0xb9, 0xf9, 0xe2, 0x63, 0x2f, 0x48, 0x52, 0x6e, 0x2e, 0x3d, ++ 0x97, 0xe3, 0xe3, 0xdd, 0x24, 0xe5, 0xf2, 0x64, 0x95, 0x15, 0x2c, 0x8d, 0x27, 0x53, 0xb5, 0x03, ++ 0x9b, 0x6e, 0x23, 0x48, 0xd2, 0x17, 0xf1, 0x64, 0xea, 0xfc, 0xb9, 0xba, 0xe6, 0xc1, 0x38, 0x74, ++ 0xfd, 0x38, 0xa4, 0xd1, 0x63, 0x7c, 0x5e, 0xd0, 0x90, 0x5d, 0x29, 0xd8, 0x4c, 0xf3, 0x73, 0x05, ++ 0x3a, 0x8f, 0xc6, 0x38, 0x16, 0x8f, 0xb1, 0xf0, 0xc9, 0x44, 0x5d, 0x1b, 0x9c, 0x63, 0xc6, 0x09, ++ 0x8d, 0xcd, 0x76, 0xb2, 0x20, 0x7a, 0x13, 0xda, 0x24, 0x26, 0xc2, 0x0b, 0x7d, 0x1c, 0xd1, 0xd8, ++ 0x44, 0x01, 0x24, 0xea, 0xb1, 0xc2, 0xa0, 0xf7, 0xa0, 0xaf, 0x2f, 0xa5, 0xbd, 0x53, 0x3f, 0x0e, ++ 0x27, 0x72, 0x23, 0xeb, 0x4b, 0xba, 0x9e, 0x46, 0xef, 0x19, 0x2c, 0x7a, 0x1f, 0x56, 0xcc, 0x36, ++ 0xcb, 0x39, 0xeb, 0x8a, 0xb3, 0x6f, 0xf0, 0x25, 0xd6, 0x34, 0x49, 0x28, 0x13, 0xdc, 0xe3, 0x38, ++ 0x08, 0x68, 0x94, 0x98, 0x9e, 0xbb, 0x6f, 0xf1, 0x87, 0x1a, 0xed, 0x8c, 0x61, 0xf5, 0xa9, 0xf4, ++ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0x93, 0x9f, ++ 0x89, 0xb0, 0xac, 0xea, 0x77, 0x24, 0xf2, 0x90, 0xfc, 0xa4, 0xae, 0x97, 0x24, 0xd7, 0x29, 0x15, ++ 0xc9, 0x24, 0x1d, 0x7b, 0x09, 0xa3, 0xc7, 0xd8, 0xb8, 0xd8, 0x8f, 0x70, 0xb4, 0xa7, 0xf1, 0x07, ++ 0x12, 0xed, 0xfc, 0x4f, 0x05, 0xd6, 0xca, 0x9a, 0x4c, 0x2a, 0xdf, 0x82, 0xb5, 0xb2, 0x2a, 0x53, ++ 0x61, 0xea, 0x2a, 0x65, 0x50, 0x54, 0xa8, 0x6b, 0xcd, 0xcf, 0xa0, 0xab, 0x5e, 0x2a, 0xbc, 0x50, ++ 0x4b, 0x2a, 0x1f, 0x9e, 0xc5, 0x79, 0x71, 0x3b, 0x7e, 0x71, 0x96, 0xbe, 0x80, 0x5b, 0xc6, 0x7d, ++ 0x6f, 0xde, 0x6c, 0xbd, 0x20, 0xd6, 0x0d, 0xc3, 0xf3, 0x19, 0xeb, 0x9f, 0xc1, 0x30, 0x47, 0xed, ++ 0x4c, 0x15, 0xd2, 0xc6, 0xea, 0x23, 0x58, 0x9d, 0x71, 0x56, 0xee, 0x3b, 0xb5, 0xed, 0xeb, 0xee, ++ 0x22, 0x92, 0xf3, 0x10, 0x6e, 0x1e, 0x62, 0xa1, 0xa3, 0xe1, 0x0b, 0xd3, 0xee, 0x6a, 0x61, 0x2b, ++ 0x50, 0x3b, 0xc4, 0x81, 0x72, 0xbe, 0xe6, 0xca, 0x5f, 0xb9, 0x00, 0x8f, 0x38, 0x0e, 0x94, 0x97, ++ 0x35, 0x57, 0xfd, 0x3b, 0xff, 0x55, 0x81, 0x86, 0x49, 0xbe, 0xf2, 0x00, 0x09, 0x19, 0x39, 0xc7, ++ 0xcc, 0x2c, 0x3d, 0x03, 0xa1, 0x77, 0xa1, 0xa7, 0xff, 0x3c, 0x9a, 0x08, 0x42, 0xb3, 0x94, 0xde, ++ 0xd5, 0xd8, 0x17, 0x1a, 0xa9, 0x2e, 0xa1, 0xd5, 0x1d, 0xab, 0xb9, 0xce, 0x30, 0x90, 0xba, 0x49, ++ 0xe6, 0x32, 0x33, 0xa8, 0x14, 0xde, 0x72, 0x0d, 0x24, 0x97, 0xba, 0x95, 0xb7, 0xa4, 0xe4, 0x59, ++ 0x50, 0x2e, 0xf5, 0x88, 0xa6, 0xb1, 0xf0, 0x12, 0x4a, 0x62, 0x61, 0x72, 0x36, 0x28, 0xd4, 0x81, ++ 0xc4, 0x38, 0xff, 0x54, 0x81, 0x65, 0xfd, 0x10, 0x83, 0x7a, 0x50, 0xcd, 0x4e, 0xce, 0x2a, 0x51, ++ 0x55, 0x88, 0xd2, 0xa5, 0x4f, 0x4b, 0xf5, 0x2f, 0xf7, 0xf1, 0x79, 0xa4, 0xf3, 0xbf, 0x31, 0xed, ++ 0x3c, 0x52, 0x89, 0xff, 0x5d, 0xe8, 0xe5, 0x07, 0xb0, 0xa2, 0x6b, 0x13, 0xbb, 0x19, 0x56, 0xb1, ++ 0x5d, 0x6a, 0xa9, 0xf3, 0xb7, 0x00, 0xf9, 0x83, 0x84, 0x0c, 0x79, 0x9a, 0x19, 0x23, 0x7f, 0x25, ++ 0x66, 0x9c, 0x1d, 0xdd, 0xf2, 0x17, 0xdd, 0x87, 0x9e, 0x1f, 0x86, 0x44, 0x0e, 0xf7, 0x27, 0x4f, ++ 0x49, 0x98, 0x6d, 0xd2, 0x32, 0xd6, 0xf9, 0xbf, 0x0a, 0xf4, 0x77, 0x69, 0x32, 0xfd, 0x4b, 0x32, ++ 0xc1, 0x85, 0x0c, 0xa2, 0x8c, 0x34, 0x27, 0xb7, 0xfc, 0xd7, 0x3d, 0xc5, 0x04, 0xeb, 0xad, 0xa5, ++ 0x67, 0xb6, 0x29, 0x11, 0x6a, 0x5b, 0x59, 0x62, 0x76, 0xb7, 0xdb, 0xd5, 0xc4, 0xe7, 0x34, 0x54, ++ 0xad, 0x5f, 0x48, 0x98, 0x97, 0xdd, 0xe4, 0x76, 0xdd, 0x46, 0x48, 0x98, 0x22, 0x19, 0x47, 0x96, ++ 0xd4, 0x63, 0x42, 0xd1, 0x91, 0x65, 0x8d, 0x91, 0x8e, 0xac, 0xc3, 0x32, 0x3d, 0x39, 0xe1, 0x58, ++ 0xa8, 0x9e, 0xa4, 0xe6, 0x1a, 0x28, 0x4b, 0x73, 0xcd, 0x42, 0x9a, 0xbb, 0x01, 0xab, 0xea, 0xd9, ++ 0xea, 0x25, 0xf3, 0x03, 0x12, 0x8f, 0x6d, 0x2a, 0x5e, 0x03, 0x74, 0x28, 0x68, 0x32, 0x83, 0xdd, ++ 0x84, 0x81, 0x39, 0x7f, 0x0e, 0xfe, 0xe6, 0xd0, 0xba, 0x7e, 0x0b, 0x9a, 0x12, 0xf4, 0x18, 0xfe, ++ 0xd1, 0x26, 0x46, 0x43, 0x76, 0xde, 0x87, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, ++ 0xbe, 0xfd, 0x9f, 0x03, 0x93, 0x6e, 0xcd, 0xf5, 0x10, 0x7a, 0x0a, 0xfd, 0x99, 0xe7, 0x46, 0x64, ++ 0xee, 0x0b, 0x17, 0xbf, 0x42, 0x8e, 0xd6, 0x37, 0xf5, 0xf3, 0xe5, 0xa6, 0x7d, 0xbe, 0xdc, 0x7c, ++ 0x12, 0x25, 0x62, 0x8a, 0x9e, 0x40, 0xaf, 0xfc, 0x30, 0x87, 0x6e, 0xdb, 0xca, 0x66, 0xc1, 0x73, ++ 0xdd, 0xa5, 0x62, 0x9e, 0x42, 0x7f, 0xe6, 0x8d, 0xce, 0xda, 0xb3, 0xf8, 0xe9, 0xee, 0x52, 0x41, ++ 0x0f, 0xa1, 0x5d, 0x78, 0x94, 0x43, 0x43, 0x2d, 0x64, 0xfe, 0x9d, 0xee, 0x52, 0x01, 0xbb, 0xd0, ++ 0x2d, 0xbd, 0x93, 0xa1, 0x91, 0xf1, 0x67, 0xc1, 0xe3, 0xd9, 0xa5, 0x42, 0x76, 0xa0, 0x5d, 0x78, ++ 0xae, 0xb2, 0x56, 0xcc, 0xbf, 0x89, 0x8d, 0x6e, 0x2d, 0xa0, 0x98, 0xe9, 0xdc, 0x83, 0x6e, 0xe9, ++ 0x71, 0xc9, 0x1a, 0xb2, 0xe8, 0x61, 0x6b, 0x74, 0x7b, 0x21, 0xcd, 0x48, 0x7a, 0x0a, 0xfd, 0x99, ++ 0xa7, 0x26, 0x1b, 0xdc, 0xc5, 0x2f, 0x50, 0x97, 0xba, 0xf5, 0x8d, 0x9a, 0xec, 0x42, 0x13, 0x57, ++ 0x98, 0xec, 0xf9, 0x87, 0xa5, 0xd1, 0x9d, 0xc5, 0x44, 0x63, 0xd5, 0x13, 0xe8, 0x95, 0xdf, 0x94, ++ 0xac, 0xb0, 0x85, 0x2f, 0x4d, 0x57, 0xaf, 0x9c, 0xd2, 0xf3, 0x52, 0xbe, 0x72, 0x16, 0xbd, 0x3a, ++ 0x5d, 0x2a, 0xe8, 0x11, 0x80, 0x69, 0xd9, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0x2a, 0x66, 0x53, ++ 0xb6, 0xa0, 0xbd, 0x7b, 0x08, 0xa0, 0x3b, 0xad, 0x90, 0xa6, 0x02, 0xdd, 0xb4, 0x66, 0xcc, 0xb4, ++ 0x77, 0xa3, 0xe1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x3a, 0x02, 0xbe, 0x06, 0xc8, 0x3b, 0x38, ++ 0x2b, 0x60, 0xae, 0xa7, 0xbb, 0x22, 0x06, 0x9d, 0x62, 0xbf, 0x86, 0x8c, 0xaf, 0x0b, 0x7a, 0xb8, ++ 0x2b, 0x44, 0xf4, 0x67, 0xaa, 0xe8, 0xf2, 0x62, 0x9b, 0x2d, 0xd3, 0x47, 0x73, 0xd5, 0x33, 0xfa, ++ 0x1e, 0x6e, 0x5f, 0x51, 0x88, 0xa3, 0x8d, 0x85, 0xe2, 0x16, 0xd4, 0xea, 0x0b, 0x44, 0x7f, 0x06, ++ 0x9d, 0x62, 0x35, 0x6e, 0x1d, 0x5c, 0x50, 0xa1, 0x8f, 0x4a, 0x15, 0x39, 0x7a, 0x08, 0xbd, 0x72, ++ 0xad, 0x8d, 0x0a, 0x5b, 0x6e, 0xae, 0x02, 0x1f, 0xad, 0xcc, 0xdc, 0xcc, 0x70, 0xf4, 0x31, 0x40, ++ 0x5e, 0x93, 0xdb, 0x99, 0x99, 0xab, 0xd2, 0x67, 0xb4, 0x7e, 0x0d, 0xbd, 0xc2, 0x91, 0x20, 0x9b, ++ 0xd8, 0x9b, 0x25, 0xe7, 0xf3, 0x83, 0x62, 0x64, 0x8a, 0xb7, 0xd2, 0x89, 0xf0, 0x08, 0x3a, 0xc5, ++ 0xe3, 0xc7, 0x7a, 0xbb, 0xe0, 0x48, 0xba, 0x2a, 0x9f, 0x16, 0x8e, 0x2a, 0xbb, 0x2d, 0xe6, 0x4f, ++ 0xaf, 0xab, 0xf2, 0x69, 0xa9, 0x81, 0xb6, 0x69, 0x6c, 0x51, 0x57, 0x7d, 0xd5, 0x29, 0x53, 0xee, ++ 0x36, 0x6d, 0xf4, 0x17, 0xf6, 0xa0, 0x57, 0x2d, 0xef, 0x62, 0x0b, 0x64, 0xe3, 0xb1, 0xa0, 0x2d, ++ 0xfa, 0x95, 0x74, 0x53, 0x6c, 0x73, 0x0a, 0xe9, 0x66, 0x41, 0xf7, 0x73, 0xa9, 0xa0, 0x3d, 0xe8, ++ 0x3f, 0xb5, 0x15, 0xac, 0xa9, 0xae, 0x8d, 0x39, 0x0b, 0xba, 0x89, 0xd1, 0x68, 0x11, 0xc9, 0xcc, ++ 0xf2, 0x37, 0x30, 0x98, 0xab, 0xac, 0xd1, 0xdd, 0xec, 0xa1, 0x60, 0x61, 0xc9, 0x7d, 0xa9, 0x59, ++ 0xfb, 0xb0, 0x32, 0x5b, 0x58, 0xa3, 0x37, 0xcc, 0xa4, 0x2f, 0x2e, 0xb8, 0x2f, 0x15, 0xf5, 0x05, ++ 0x34, 0x6d, 0x21, 0x87, 0xcc, 0x83, 0xcc, 0x4c, 0x61, 0x77, 0xd9, 0xd0, 0x9d, 0xce, 0xcf, 0xbf, ++ 0xdc, 0xad, 0xfc, 0xe6, 0x97, 0xbb, 0x95, 0xdf, 0xfd, 0x72, 0xb7, 0x72, 0xbc, 0xac, 0xa8, 0x1f, ++ 0xff, 0x21, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x62, 0x55, 0x85, 0x98, 0x25, 0x00, 0x00, + } +diff --git a/protocols/grpc/agent.proto b/protocols/grpc/agent.proto +index 73f6b37..ac329d4 100644 +--- a/protocols/grpc/agent.proto ++++ b/protocols/grpc/agent.proto +@@ -43,6 +43,7 @@ service AgentService { + + // networking + rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface); ++ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface); + rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); + rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); + rpc ListRoutes(ListRoutesRequest) returns (Routes); +@@ -344,6 +345,10 @@ message UpdateInterfaceRequest { + types.Interface interface = 1; + } + ++message UpdateInterfaceHwAddrByNameRequest { ++ types.Interface interface = 1; ++} ++ + message UpdateRoutesRequest { + Routes routes = 1; + bool increment = 2; +-- +1.8.3.1 + diff --git a/agent/patches/0018-kata-agent-update-nic-in-guest.patch b/agent/patches/0018-kata-agent-update-nic-in-guest.patch new file mode 100644 index 0000000000000000000000000000000000000000..c0454dd0c315e626da73b5ee9650a4898104c25c --- /dev/null +++ b/agent/patches/0018-kata-agent-update-nic-in-guest.patch @@ -0,0 +1,184 @@ +From 629aac1078bdeed63214c58d2110fe672d654774 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Mon, 26 Oct 2020 20:39:05 +0800 +Subject: [PATCH] kata-agent: update nic in guest + +reason: add linkByName and support retry of list ip link, because in some scenarios, +the hardware address cannot be queried immediately after the hot plug. + +Signed-off-by: yangfeiyu +--- + network.go | 91 ++++++++++++++++++++++++++++++++++++++++++------------ + 1 file changed, 72 insertions(+), 19 deletions(-) + +diff --git a/network.go b/network.go +index f6e2c17..d928e2b 100644 +--- a/network.go ++++ b/network.go +@@ -15,13 +15,14 @@ import ( + "strings" + "sync" + "syscall" ++ "time" + +- "golang.org/x/sys/unix" + agentNet "github.com/kata-containers/agent/pkg/net" + "github.com/kata-containers/agent/pkg/types" + pb "github.com/kata-containers/agent/protocols/grpc" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" ++ "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" + grpcStatus "google.golang.org/grpc/status" + ) +@@ -32,6 +33,7 @@ var ( + errNoLink = grpcStatus.Errorf(codes.InvalidArgument, "Need network link") + errNoMAC = grpcStatus.Errorf(codes.InvalidArgument, "Need hardware address") + errNoRoutes = grpcStatus.Errorf(codes.InvalidArgument, "Need network routes") ++ errNoName = grpcStatus.Errorf(codes.InvalidArgument, "Need network name") + guestDNSFile = "/etc/resolv.conf" + kataGuestSandboxDNSFile = "/run/kata-containers/sandbox/resolv.conf" + ) +@@ -46,6 +48,8 @@ const ( + + // Use the below address for ipv6 gateway once ipv6 support is added + // defaultV6RouteIP = "::" ++ ++ maxLinkRetries = 10 + ) + + // Network fully describes a sandbox network with its interfaces, routes and dns +@@ -100,17 +104,36 @@ func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error + return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to HwAddr %q", hwAddr) + } + +-func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { ++func linkByName(netHandle *netlink.Handle, name string) (netlink.Link, error) { + if netHandle == nil { +- return errNoHandle ++ return nil, errNoHandle + } + +- if link == nil { +- return errNoLink ++ if name == "" { ++ return nil, errNoName + } + +- if iface == nil { +- return errNoIF ++ links, err := netHandle.LinkList() ++ if err != nil { ++ return nil, err ++ } ++ ++ for _, link := range links { ++ if link == nil { ++ continue ++ } ++ ++ if link.Attrs() != nil && link.Attrs().Name == name { ++ return link, nil ++ } ++ } ++ ++ return nil, grpcStatus.Errorf(codes.NotFound, "Could not find the link corresponding to name %s", name) ++} ++ ++func updateLinkIP(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { ++ if len(iface.IPAddresses) == 0 { ++ return nil + } + + // As a first step, clear out any existing addresses associated with the link: +@@ -129,13 +152,6 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter + netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) + netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) + +- // With ipv6 addresses, there is a brief period during which the address is marked as "tentative" +- // making it unavailable. A process called duplicate address detection(DAD) is performed during this period. +- // Disble DAD so that networking is available once the container is up. The assumption is +- // that it is the reponsibility of the upper stack to make sure the addresses assigned to containers +- // do not conflict. A similar operation is performed by libnetwork: +- // https://github.com/moby/moby/issues/18871 +- + if addr.GetFamily() == types.IPFamily_v6 { + netlinkAddr.Flags = netlinkAddr.Flags | syscall.IFA_F_NODAD + } +@@ -150,14 +166,36 @@ func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Inter + } + } + ++ return nil ++} ++ ++func updateLink(netHandle *netlink.Handle, link netlink.Link, iface *types.Interface) error { ++ if netHandle == nil { ++ return errNoHandle ++ } ++ ++ if link == nil { ++ return errNoLink ++ } ++ ++ if iface == nil { ++ return errNoIF ++ } ++ ++ if err := updateLinkIP(netHandle, link, iface); err != nil { ++ return err ++ } ++ + // set the interface name: + if err := netHandle.LinkSetName(link, iface.Name); err != nil { + return grpcStatus.Errorf(codes.Internal, "Could not set name %s for interface %v: %v", iface.Name, link, err) + } + + // set the interface MTU: +- if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil { +- return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err) ++ if iface.Mtu > 0 { ++ if err := netHandle.LinkSetMTU(link, int(iface.Mtu)); err != nil { ++ return grpcStatus.Errorf(codes.Internal, "Could not set MTU %d for interface %v: %v", iface.Mtu, link, err) ++ } + } + + if iface.RawFlags&unix.IFF_NOARP == uint32(unix.IFF_NOARP) { +@@ -306,8 +344,23 @@ func (s *sandbox) updateInterface(netHandle *netlink.Handle, iface *types.Interf + if iface.HwAddr != "" { + fieldLogger.Info("Getting interface from MAC address") + +- // Find the interface link from its hardware address. +- link, err = linkByHwAddr(netHandle, iface.HwAddr) ++ // In some scenarios, the hardware address cannot be queried immediately ++ // after the hot plug, add a retry here. ++ for retry := 0; retry < maxLinkRetries; retry++ { ++ // Find the interface link from its hardware address. ++ link, err = linkByHwAddr(netHandle, iface.HwAddr) ++ if err != nil { ++ time.Sleep(100 * time.Millisecond) ++ continue ++ } ++ break ++ } ++ if err != nil { ++ return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err) ++ } ++ } else if iface.Name != "" { ++ fieldLogger.Info("Getting interface from name") ++ link, err = linkByName(netHandle, iface.Name) + if err != nil { + return nil, grpcStatus.Errorf(codes.Internal, "updateInterface: %v", err) + } +@@ -487,7 +540,7 @@ func (s *sandbox) updateRoutes(netHandle *netlink.Handle, requestedRoutes *pb.Ro + }() + + var ( +- added []*types.Route ++ added []*types.Route + removed []*types.Route + ) + +-- +2.23.0 + diff --git a/agent/series.conf b/agent/series.conf new file mode 100644 index 0000000000000000000000000000000000000000..fc5adee738f4fa8d49e7d1c42f416de2580a81a0 --- /dev/null +++ b/agent/series.conf @@ -0,0 +1,18 @@ +0001-agent-add-agent.netlink_recv_buf_size-flag-to-set-ne.patch +0002-network-support-update-routes-incrementally.patch +0003-kata-agent-add-kata-ipvs-command.patch +0004-agent-add-IPVS-test.patch +0005-mount-support-mount-block-device.patch +0006-agent-make-workaround-for-slow-response-in-aarch64.patch +0007-agent-using-pcie-root-port-driver-to-hotplug-device.patch +0008-agent-support-get-root-bus-path-dynamically.patch +0009-storage-add-pkg-storage-for-mount.patch +0010-storage-mount-nfs-and-gpath-in-agent.patch +0011-agent-fix-agent-reap-agent-process-blocked-problem.patch +0012-network-support-set-dns-without-nameserver.patch +0013-agent-support-setting-multi-queues-of-interface.patch +0014-agent-fix-init-hugepages-failed-problem.patch +0015-agent-add-support-of-getting-container-s-network-sta.patch +0016-clock-synchronizes-clock-info-with-proxy.patch +0017-agent-add-support-of-new-sandbox-StratoVirt.patch +0018-kata-agent-update-nic-in-guest.patch diff --git a/kata-containers.spec b/kata-containers.spec new file mode 100644 index 0000000000000000000000000000000000000000..2405ac4360bc9623ded153c94e54fb51c7a20d5c --- /dev/null +++ b/kata-containers.spec @@ -0,0 +1,133 @@ +#needsrootforbuild +%global debug_package %{nil} + +%define VERSION v1.11.1 +%define RELEASE 7 + +Name: kata-containers +Version: %{VERSION} +Release: %{RELEASE} +Summary: Kata Container, the speed of containers, the security of VMs +License: Apache 2.0 +URL: https://github.com/kata-containers +Source0: kata_integration-v1.0.0.tar.gz +Source1: kata-containers-%{version}.tar.gz +Source2: kernel.tar.gz + +BuildRoot: %_topdir/BUILDROOT +BuildRequires: automake golang gcc bc glibc-devel glibc-static busybox glib2-devel glib2 ipvsadm conntrack-tools nfs-utils +BuildRequires: patch elfutils-libelf-devel openssl-devel bison flex + +%description +This is core component of Kata Container, to make it work, you need a isulad/docker engine. + +%prep +%setup -T -c -a 0 -n kata_integration +%setup -T -c -a 1 -n kata-containers-%{version} +%setup -T -c -a 2 -n kernel + +# extract the kata_integration.tar.gz file +cd %{_builddir}/kata_integration +# apply kata_integration patches +sh apply-patches + +# mv build components into kata_integration dir +pushd %{_builddir}/kata_integration +mv ../kata-containers-%{version}/runtime . +mv ../kata-containers-%{version}/agent . +mv ../kata-containers-%{version}/proxy . +mv ../kata-containers-%{version}/shim . +popd + +# build kernel +cd %{_builddir}/kernel +mv kernel linux +cd %{_builddir}/kernel/linux/ +%ifarch %{ix86} x86_64 +cp %{_builddir}/kata_integration/hack/config-kata-x86_64 ./.config +%else +cp %{_builddir}/kata_integration/hack/config-kata-arm64 ./.config +%endif + +%build +cd %{_builddir}/kernel/linux/ +make %{?_smp_mflags} + +cd %{_builddir}/kata_integration +mkdir -p -m 750 build +make runtime +make proxy +make shim +make initrd + +%install +mkdir -p -m 755 %{buildroot}/var/lib/kata +%ifarch %{ix86} x86_64 +install -p -m 755 -D %{_builddir}/kernel/linux/arch/x86_64/boot/bzImage %{buildroot}/var/lib/kata/kernel +%else +install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot}/var/lib/kata/kernel +%endif + +cd %{_builddir}/kata_integration +mkdir -p -m 750 %{buildroot}/usr/bin +install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/ +install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/ +mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/ +install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml + +%clean + +%files +/usr/bin/kata-runtime +/usr/bin/kata-proxy +/usr/bin/kata-shim +/usr/bin/kata-netmon +/var/lib/kata/kernel +/var/lib/kata/kata-containers-initrd.img +%config(noreplace) /usr/share/defaults/kata-containers/configuration.toml + +%doc + + +%changelog +* Tue Dec 22 2020 jiangpengfei - 1.11.1-7 +- Type:enhancement +- ID:NA +- SUG:update +- DESC:update kata-containers source forms of organization to move all kata-containers related source repo into one repo kata-containers + +* Fri Nov 6 2020 yangfeiyu - 1.11.1-6 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:revert the kata-containers.spec to still build kata-containers components into one package + +* Fri Oct 9 2020 yangfeiyu - 1.11.1-5 +- Type:enhancement +- ID:NA +- SUG:restart +- DESC:directly copy kata binary files instead of building them + +* Wed Sep 30 2020 yangfeiyu - 1.11.1-4 +- Type:bugfix +- ID:NA +- SUG:restart +- DESC:kata-runtime retry inserting of CNI interface + +* Sun Sep 27 2020 LiangZhang - 1.11.1-3 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix cmd params of direct use stratovirt binary + +* Thu Sep 20 2020 jiangpengf - 1.11.1-2 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix del-iface doesn't delete the tap interface in the host problem + +* Thu Aug 27 2020 jiangpengf - 1.11.1-1 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:update kata-containers version to v1.11.1-1 diff --git a/kata-containers.yaml b/kata-containers.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ec7a065ec6acb99ee332b2eda66de232db0ae1f --- /dev/null +++ b/kata-containers.yaml @@ -0,0 +1,4 @@ +version_control: github +src_repo: kata-containers/runtime +tag_prefix: +seperator: "." diff --git a/proxy/apply-patches b/proxy/apply-patches new file mode 100755 index 0000000000000000000000000000000000000000..99f657bb640b7db89314655ba439e1adca9e1ca8 --- /dev/null +++ b/proxy/apply-patches @@ -0,0 +1,20 @@ +#!/bin/bash + +if [[ -f ./patch_flag ]];then + echo "proxy patched!" + exit 0 +fi + +tar -zxvf proxy-1.11.1.tar.gz +cp -fr ./proxy-1.11.1/* ./ +rm -rf ./proxy-1.11.1 +cat ./series.conf | while read line +do + if [[ $line == '' || $line =~ ^\s*# ]]; then + continue + fi + echo "====patch $line======" + pwd + patch -p1 -F1 -s < ./patches/$line +done +touch ./patch_flag diff --git a/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch b/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch new file mode 100644 index 0000000000000000000000000000000000000000..e294d3acc059e7f23887cc65f5720d3aa460c94f --- /dev/null +++ b/proxy/patches/0001-clock-synchronizes-clock-info-to-agent.patch @@ -0,0 +1,403 @@ +From a260bbe394f91fa05b163315390ed133de5c5494 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 17:43:24 +0800 +Subject: [PATCH] clock: synchronizes clock info to agent + +reason: virtual machine's clock may be incorrect, proxy synchronizes +clock info to help virtual machine adjust clock time + +Signed-off-by: yangfeiyu +--- + proxy.go | 7 ++ + proxy_test.go | 29 +++++ + sync_clock_client.go | 107 +++++++++++++++++ + sync_clock_client_test.go | 132 +++++++++++++++++++++ + .../kata-containers/agent/pkg/clock/clock_util.go | 44 +++++++ + 5 files changed, 319 insertions(+) + create mode 100644 sync_clock_client.go + create mode 100644 sync_clock_client_test.go + create mode 100644 vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go + +diff --git a/proxy.go b/proxy.go +index ab062a5..9dfcb3c 100644 +--- a/proxy.go ++++ b/proxy.go +@@ -105,6 +105,13 @@ func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error) + // Start the heartbeat in a separate go routine + go heartBeat(session) + ++ // start the sync clock in a separate go routine ++ syncClockStream, err := session.Open() ++ if err != nil { ++ return nil, nil, err ++ } ++ go SyncClock(syncClockStream) ++ + // serving connection + l, err := net.Listen(proto, addr) + if err != nil { +diff --git a/proxy_test.go b/proxy_test.go +index 923b138..94fa523 100644 +--- a/proxy_test.go ++++ b/proxy_test.go +@@ -10,6 +10,7 @@ package main + import ( + "bytes" + "crypto/md5" ++ "encoding/json" + "fmt" + "io" + "io/ioutil" +@@ -23,6 +24,7 @@ import ( + "testing" + + "github.com/hashicorp/yamux" ++ "github.com/kata-containers/agent/pkg/clock" + "github.com/stretchr/testify/assert" + ) + +@@ -121,6 +123,13 @@ func server(listener net.Listener, closeCh chan bool) error { + session.Close() + }() + ++ // accept the sync clock stream first ++ if syncClockStream, err := session.Accept(); err != nil { ++ return err ++ } else { ++ go serverSyncClock(syncClockStream) ++ } ++ + for { + stream, err := session.Accept() + if err != nil { +@@ -133,6 +142,26 @@ func server(listener net.Listener, closeCh chan bool) error { + } + } + ++func serverSyncClock(stream net.Conn) { ++ for { ++ buf, byteNum, err := readConnData(stream) ++ if err != nil { ++ continue ++ } ++ var clockInfo clock.TimeValue ++ if err := json.Unmarshal(buf[:byteNum], &clockInfo); err != nil { ++ continue ++ } ++ nowTime := clock.GetCurrentTimeNs() ++ clockInfo.ClientArriveTime = nowTime ++ clockInfo.ServerSendTime = nowTime ++ b, _ := json.Marshal(clockInfo) ++ if err := clock.WriteConnData(stream, b); err != nil { ++ continue ++ } ++ } ++} ++ + func TestUnixAddrParsing(T *testing.T) { + buf := "unix://foo/bar" + addr, err := unixAddr(buf) +diff --git a/sync_clock_client.go b/sync_clock_client.go +new file mode 100644 +index 0000000..9bf3e91 +--- /dev/null ++++ b/sync_clock_client.go +@@ -0,0 +1,107 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: sync clock client related function ++// Author: xueshaojia x00464843 ++// Create: 2018-11-10 ++ ++package main ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "net" ++ "time" ++ ++ "github.com/kata-containers/agent/pkg/clock" ++) ++ ++const ( ++ allowTimeDiff = 10 * 1000 * 1000 // allow 10ms's difference ++ syncClockInterval = 60 * time.Second // sync clock with agent every 60 seconds ++ rpcTimeout = 10 * time.Second // timeout for proxy's reading data ++) ++ ++// readConnData reads data from stream ++func readConnData(stream net.Conn) (buf []byte, byteNum int, err error) { ++ // set read deadline to avoid case as following: ++ // proxy and agent are both reading and then syncClock will never work ++ stream.SetReadDeadline(time.Now().Add(rpcTimeout)) ++ buf = make([]byte, clock.MaxSyncClockByteNum) ++ byteNum, err = stream.Read(buf) ++ return buf, byteNum, err ++} ++ ++// getGuestClock syncs guest clock info ++// sends ClientSendTime ++// waits for ClientArriveTime and ServerSendTime ++func getGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error { ++ clockInfo.Delta = 0 ++ b, err := json.Marshal(clockInfo) ++ if err != nil { ++ return err ++ } ++ if err = clock.WriteConnData(stream, b); err != nil { ++ return err ++ } ++ ++ buf, byteNum, err := readConnData(stream) ++ if err != nil { ++ return err ++ } ++ ++ if err = json.Unmarshal(buf[:byteNum], clockInfo); err != nil { ++ return fmt.Errorf("sync clock, parse guest clocktime error:%v", err) ++ } ++ return nil ++} ++ ++// adjustGuestClock tells server to ajust local clock with Delta ++func adjustGuestClock(stream net.Conn, clockInfo *clock.TimeValue) error { ++ b, err := json.Marshal(clockInfo) ++ if err != nil { ++ return err ++ } ++ logger().Debugf("sync clock, send:%s", string(b)) ++ return clock.WriteConnData(stream, b) ++} ++ ++// syncClock performs all the steps ++// 1 get client send time[host] ++// 2 request for client arrive time and server send time[guest os] ++// 3 get server arrive time[host] ++// 4 calculate clock diff ++// 5 request to adjust guest clock ++func syncClock(stream net.Conn) error { ++ var clockInfo clock.TimeValue ++ if clockInfo.ClientSendTime = clock.GetCurrentTimeNs(); clockInfo.ClientSendTime <= 0 { ++ return fmt.Errorf("sync clock, get client sendtime error") ++ } ++ err := getGuestClock(stream, &clockInfo) ++ if err != nil { ++ return fmt.Errorf("sync clock, get guest clocktime error:%v", err) ++ } ++ if clockInfo.ServerArriveTime = clock.GetCurrentTimeNs(); clockInfo.ServerArriveTime <= 0 { ++ return fmt.Errorf("sync clock, get client recvtime error") ++ } ++ if clockInfo.ClientSendTime <= 0 || clockInfo.ClientArriveTime <= 0 || clockInfo.ServerSendTime <= 0 { ++ return fmt.Errorf("sync clock, some fields of NTP message error, raw message:%v", clockInfo) ++ } ++ ++ delta := ((clockInfo.ClientSendTime - clockInfo.ClientArriveTime) + (clockInfo.ServerArriveTime - clockInfo.ServerSendTime)) / 2 ++ if delta < -allowTimeDiff || delta > allowTimeDiff { ++ clockInfo.Delta = delta ++ if err := adjustGuestClock(stream, &clockInfo); err != nil { ++ return fmt.Errorf("sync clock, failed to adjust guest clock : %v", err) ++ } ++ } ++ return nil ++} ++ ++func SyncClock(stream net.Conn) { ++ for { ++ if err := syncClock(stream); err != nil { ++ logger().WithError(err).Error("sync clock failed") ++ } ++ time.Sleep(syncClockInterval) ++ } ++} +diff --git a/sync_clock_client_test.go b/sync_clock_client_test.go +new file mode 100644 +index 0000000..b0b1c85 +--- /dev/null ++++ b/sync_clock_client_test.go +@@ -0,0 +1,132 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2018. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: sync clock client related test ++// Author: xueshaojia x00464843 ++// Create: 2018-11-10 ++ ++package main ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "math/rand" ++ "net" ++ "os" ++ "testing" ++ ++ "github.com/hashicorp/yamux" ++ "github.com/kata-containers/agent/pkg/clock" ++) ++ ++func TestReadWriteData(t *testing.T) { ++ var clockInfo clock.TimeValue ++ clockInfo.ClientSendTime = clock.GetCurrentTimeNs() ++ b, err := json.Marshal(clockInfo) ++ if err != nil { ++ t.Fatalf("Marshal clock info fail, err:%v", err) ++ } ++ ++ err = clock.WriteConnData(clientStream, b) ++ fmt.Printf("client send: %s\n", string(b)) ++ if err != nil { ++ t.Fatalf("send clock info fail, err:%v", err) ++ } ++ _, _, err = readConnData(clientStream) ++ if err != nil { ++ t.Fatalf("recv clock info fail, err:%v", err) ++ } ++} ++ ++func SetUpServer(sock string, readyChan chan int) error { ++ var err error ++ listener, err = net.Listen("unix", sock) ++ if err != nil { ++ return err ++ } ++ readyChan <- 1 ++ conn, err := listener.Accept() ++ if err != nil { ++ return err ++ } ++ session, err := yamux.Server(conn, nil) ++ if err != nil { ++ return err ++ } ++ serverSession = session ++ stream, err := session.Accept() ++ if err != nil { ++ return err ++ } ++ for { ++ var clockInfo clock.TimeValue ++ var byteNum int ++ var err error ++ buf := make([]byte, 400) ++ if byteNum, err = stream.Read(buf); err != nil { ++ break ++ } ++ ++ if err = json.Unmarshal(buf[:byteNum], &clockInfo); err != nil { ++ break ++ } ++ if clockInfo.Delta == 0 { ++ nowTime := clock.GetCurrentTimeNs() ++ clockInfo.ClientArriveTime = nowTime ++ clockInfo.ClientArriveTime = nowTime ++ b, _ := json.Marshal(&clockInfo) ++ stream.Write(b) ++ } ++ } ++ return nil ++} ++ ++func SetUpClient(sock string) error { ++ conn, err := net.Dial("unix", sock) ++ if err != nil { ++ return err ++ } ++ session, err := yamux.Client(conn, nil) ++ if err != nil { ++ conn.Close() ++ return err ++ } ++ clientSession = session ++ stream, err := session.Open() ++ if err != nil { ++ clientSession.Close() ++ return err ++ } ++ clientStream = stream ++ return nil ++} ++ ++func TearDown() { ++ listener.Close() ++ serverSession.Close() ++ clientSession.Close() ++} ++ ++func GenSocket() string { ++ randSeed := clock.GetCurrentTimeNs() ++ rand.Seed(randSeed) ++ return fmt.Sprintf("/tmp/%d.sock", rand.Uint32()) ++} ++ ++var listener net.Listener ++var clientStream net.Conn ++var serverSession *yamux.Session ++var clientSession *yamux.Session ++ ++func TestMain(m *testing.M) { ++ waitConn := make(chan int) ++ testSock := GenSocket() ++ go SetUpServer(testSock, waitConn) ++ <-waitConn ++ if err := SetUpClient(testSock); err != nil { ++ listener.Close() ++ serverSession.Close() ++ os.Exit(1) ++ } ++ m.Run() ++ TearDown() ++} +diff --git a/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go +new file mode 100644 +index 0000000..03244fd +--- /dev/null ++++ b/vendor/github.com/kata-containers/agent/pkg/clock/clock_util.go +@@ -0,0 +1,44 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: jiangpeifei ++// Create: 2019-05-28 ++ ++package clock ++ ++import ( ++ "net" ++ "syscall" ++) ++ ++type TimeValue struct { ++ ClientSendTime int64 `json:"client_send_time"` ++ ClientArriveTime int64 `json:"client_arrive_time"` ++ ServerSendTime int64 `json:"server_send_time"` ++ ServerArriveTime int64 `json:"server_arrive_time"` ++ Delta int64 `json:"delta"` ++} ++ ++const MaxSyncClockByteNum = 400 //sync clock byte num, max=400 ++ ++// getCurrentTimeNs returns UTC time in Ns ++func GetCurrentTimeNs() int64 { ++ var tv syscall.Timeval ++ if err := syscall.Gettimeofday(&tv); err != nil { ++ return -1 ++ } ++ return tv.Sec*1000000000 + tv.Usec*1000 ++} ++ ++// readConnData reads data from stream ++func ReadConnData(stream net.Conn) (buf []byte, byteNum int, err error) { ++ buf = make([]byte, MaxSyncClockByteNum) ++ byteNum, err = stream.Read(buf) ++ return buf, byteNum, err ++} ++ ++// writeConnData writes data to stream ++func WriteConnData(stream net.Conn, buf []byte) error { ++ _, err := stream.Write(buf) ++ return err ++} +-- +2.14.3 (Apple Git-98) + diff --git a/proxy/proxy-1.11.1.tar.gz b/proxy/proxy-1.11.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..45699f93e5db7100725e3d98b1e2b8537752fbcf Binary files /dev/null and b/proxy/proxy-1.11.1.tar.gz differ diff --git a/proxy/series.conf b/proxy/series.conf new file mode 100644 index 0000000000000000000000000000000000000000..1f29a6e09e095aac06ef3e23e53d3014353300ac --- /dev/null +++ b/proxy/series.conf @@ -0,0 +1 @@ +0001-clock-synchronizes-clock-info-to-agent.patch diff --git a/runtime/apply-patches b/runtime/apply-patches new file mode 100755 index 0000000000000000000000000000000000000000..2a86e4e132efc299c8dfcd20b341f78e53cd5121 --- /dev/null +++ b/runtime/apply-patches @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +if [[ -f ./patch_flag ]];then + echo "runtime patched!" + exit 0 +fi + +tar -zxvf runtime-1.11.1.tar.gz +cp -fr ./runtime-1.11.1/* ./ +rm -rf ./runtime-1.11.1 +cat ./series.conf | while read line +do + if [[ $line == '' || $line =~ ^\s*# ]]; then + continue + fi + echo "====patch $line======" + patch -p1 -F1 -s < ./patches/$line +done + +touch ./patch_flag diff --git a/runtime/kata-runtime.spec b/runtime/kata-runtime.spec new file mode 100644 index 0000000000000000000000000000000000000000..acef8cec73cc30f515df2bc6c5da4fe2f90df62f --- /dev/null +++ b/runtime/kata-runtime.spec @@ -0,0 +1,165 @@ +%define debug_package %{nil} + +%define VERSION 1.11.1 +%define RELEASE 11 + +Name: kata-runtime +Version: %{VERSION} +Release: %{RELEASE} +Summary: Kata Runtime +License: Apache 2.0 +URL: https://github.com/kata-containers/runtime +Source0: https://github.com/kata-containers/runtime/archive/%{version}.tar.gz#/%{name}-v%{version}.tar.gz + +BuildRoot: %_topdir/BUILDROOT +BuildRequires: automake golang gcc + +%description +Kata-runtime is core component of Kata Container. + +%prep +%setup -q -c -a 0 -n %{name}-%{version} + +%build +cd %{_builddir}/%{name}-%{version} + +set -e +# apply patches read from series.conf +sh apply-patches + +# create tmp GOPATH dir to build kata-runtime +rm -rf /tmp/kata-build/ +mkdir -p /tmp/kata-build/ +GOPATH=/tmp/kata-build/ +kata_base=$GOPATH/src/github.com/kata-containers +mkdir -p $kata_base + +# get current kata-runtime absolute path +kata_runtime_path=$(readlink -f .) +ln -s $kata_runtime_path $kata_base/runtime + +# export GOPATH env +export GOPATH=$(readlink -f $GOPATH) +cd ${kata_base}/runtime && make clean && make +rm -rf $GOPATH + +# make kata-runtime default configuration +kata_config_path=$kata_runtime_path/cli/config/configuration-qemu.toml +ARCH=`arch` + +# arch related config options +if [ "$ARCH" == "aarch64" ];then + sed -i 's/^machine_type.*$/machine_type = \"virt\"/' $kata_config_path + sed -i 's/^block_device_driver.*$/block_device_driver = \"virtio-scsi\"/' $kata_config_path + sed -i 's/^kernel_params.*$/kernel_params = \"pcie_ports=native pci=pcie_bus_perf agent.netlink_recv_buf_size=2MB\"/' $kata_config_path + sed -i 's/^hypervisor_params.*$/hypervisor_params = \"kvm-pit.lost_tick_policy=discard pcie-root-port.fast-plug=1 pcie-root-port.x-speed=16 pcie-root-port.x-width=32 pcie-root-port.fast-unplug=1\"/' $kata_config_path + sed -i 's/^#pcie_root_port.*$/pcie_root_port = 25/' $kata_config_path +else + sed -i 's#block_device_driver = \"virtio-scsi\"#block_device_driver = \"virtio-blk\"#' $kata_config_path + sed -i 's/^#hotplug_vfio_on_root_bus/hotplug_vfio_on_root_bus/' $kata_config_path +fi + +# debug config +sed -i 's/^#enable_debug.*$/enable_debug = true/' $kata_config_path + +# other config +sed -i 's#"/usr/bin/qemu.*"$#"/usr/bin/qemu-kvm"#' $kata_config_path +sed -i 's#/usr/share/kata-containers/vmlinuz\.container#/var/lib/kata/kernel#' $kata_config_path +sed -i 's#/usr/share/kata-containers/kata-containers-initrd\.img#/var/lib/kata/kata-containers-initrd\.img#' $kata_config_path +sed -i 's/^image/#image/' $kata_config_path +sed -i 's/^default_memory.*$/default_memory = 1024/' $kata_config_path +sed -i 's/^#enable_blk_mount/enable_blk_mount/' $kata_config_path +sed -i 's/^#block_device_cache_direct.*$/block_device_cache_direct = true/' $kata_config_path +sed -i 's/^#block_device_cache_set.*$/block_device_cache_set = true/' $kata_config_path +sed -i 's#/usr/libexec/kata-containers/kata-proxy#/usr/bin/kata-proxy#' $kata_config_path +sed -i 's#/usr/libexec/kata-containers/kata-shim#/usr/bin/kata-shim#' $kata_config_path +sed -i 's#/usr/libexec/kata-containers/kata-netmon#/usr/bin/kata-netmon#' $kata_config_path +sed -i 's/^#disable_new_netns.*$/disable_new_netns = true/' $kata_config_path +sed -i 's/^#disable_vhost_net.*$/disable_vhost_net = true/' $kata_config_path +sed -i 's/^internetworking_model.*$/internetworking_model=\"none\"/' $kata_config_path +sed -i 's/^enable_compat_old_cni.*$/#enable_compat_old_cni = true/' $kata_config_path +sed -i 's/^sandbox_cgroup_only.*$/sandbox_cgroup_only = true/' $kata_config_path + +set +e + +%install +cd %{_builddir}/%{name}-%{version} +mkdir -p -m 750 %{buildroot}/usr/bin +install -p -m 750 ./kata-runtime %{buildroot}/usr/bin +install -p -m 750 ./kata-netmon %{buildroot}/usr/bin +mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers +install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml + +%clean + +%files +/usr/bin/kata-runtime +/usr/bin/kata-netmon +/usr/share/defaults/kata-containers/configuration.toml + +%changelog +* Tue Nov 17 2020 yangfeiyu - 1.11.1-11 +- Type:bugfix +- ID:NA +- SUG:upgrade +- DESC:fix cpu resource limited problem when sandox_cgroup_with_emulator config is enabled + +* Fri Oct 9 2020 yangfeiyu - 1.11.1-10 +- Type:feature +- ID:NA +- SUG:restart +- DESC:support using CNI plugin to insert mutiple network interfaces at the same time + +* Mon Sep 28 2020 yangfeiyu - 1.11.1-9 +- Type:bugfix +- ID:NA +- SUG:restart +- DESC:retry inserting of CNI interface when netmon is enable + +* Sun Sep 27 2020 LiangZhang - 1.11.1-8 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix cmd params of direct use stratovirt binary + +* Thu Sep 24 2020 LiangZhang - 1.11.1-7 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix invalid cmdline when start sandbox stratovirt + +* Mon Sep 21 2020 yangfeiyu - 1.11.1-6 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix sandboxRuntimeRootPath left problem + +* Mon Sep 21 2020 yangfeiyu - 1.11.1-5 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:add support for host cgroups with emulator + +* Mon Sep 21 2020 LiangZhang - 1.11.1-4 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:add support of new sandbox StratoVirt + +* Sat Sep 19 2020 yangfeiyu - 1.11.1-3 +- Type:bugfix +- ID:NA +- SUG:NA +- DESC:fix del-iface doesn't delete the tap interface in the host problem + +* Sat Sep 5 2020 yangfeiyu - 1.11.1-2 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:use URL format for Source0 + +* Wed Aug 26 2020 yangfeiyu - 1.11.1-1 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:modify kata-runtime spec file to build seperately diff --git a/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch b/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch new file mode 100644 index 0000000000000000000000000000000000000000..c2c605c75a885b6c44646b8a5d5303234a51f36a --- /dev/null +++ b/runtime/patches/0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch @@ -0,0 +1,44 @@ +From 73fe7242d18a10a86bc216ec5e33a10a8751f85f Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Fri, 24 Jul 2020 22:22:00 +0800 +Subject: [PATCH 01/50] qmp: fix kata-runtime hungs when qemu process is D/T + state + +reason: When set qemu's status to T and execute add-iface command +the command will hung all the time.It hungs while wait for qemu +to return version messages. + +When qmp starts, set the timeout time to 15 seconds.When times +out, return qmp starts failed.We choose 15 seconds to keep consistent +with the agent client start. + +Signed-off-by: jiangpengfei +--- + vendor/github.com/intel/govmm/qemu/checklist | 1 + + vendor/github.com/intel/govmm/qemu/qmp.go | 2 ++ + 2 files changed, 3 insertions(+) + create mode 100644 vendor/github.com/intel/govmm/qemu/checklist + +diff --git a/vendor/github.com/intel/govmm/qemu/checklist b/vendor/github.com/intel/govmm/qemu/checklist +new file mode 100644 +index 00000000..b32f1855 +--- /dev/null ++++ b/vendor/github.com/intel/govmm/qemu/checklist +@@ -0,0 +1 @@ ++add timeout when qmp start to avoid qmp client hungs all the time +diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go +index bf9a77dd..a64039de 100644 +--- a/vendor/github.com/intel/govmm/qemu/qmp.go ++++ b/vendor/github.com/intel/govmm/qemu/qmp.go +@@ -722,6 +722,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh + if q.version == nil { + return nil, nil, fmt.Errorf("failed to find QMP version information") + } ++ case <-time.After(15 * time.Second): ++ return nil, nil, fmt.Errorf("qmp start time out") + } + + return q, q.version, nil +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch b/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch new file mode 100644 index 0000000000000000000000000000000000000000..99f31375316e104114a374cf87ca70fc39fc67ca --- /dev/null +++ b/runtime/patches/0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch @@ -0,0 +1,117 @@ +From 1efb88fbf554f3977a1a8aa1c79bc70cc1b66953 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 25 Jul 2020 09:22:08 +0800 +Subject: [PATCH 02/50] kata-runtime: fix kata-runtime skip read lines in + /proc/mounts file problem + +reason: Since /proc/mounts is a virtual file which is changed dynamically by kernel, +if we use file pointer to read content in this file line by line, we may miss read +some lines. So we retry read /proc/mounts file again to fix this problem. + +Signed-off-by: jiangpengfei +--- + virtcontainers/utils/utils_linux.go | 58 +++++++++++++++++++++++-------------- + 1 file changed, 36 insertions(+), 22 deletions(-) + +diff --git a/virtcontainers/utils/utils_linux.go b/virtcontainers/utils/utils_linux.go +index ad870d63..6cef4cfb 100644 +--- a/virtcontainers/utils/utils_linux.go ++++ b/virtcontainers/utils/utils_linux.go +@@ -7,15 +7,18 @@ package utils + + import ( + "bufio" ++ "bytes" + "crypto/rand" + "fmt" + "io" ++ "io/ioutil" + "math/big" + "os" + "strings" + "syscall" + "unsafe" + ++ "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + ) + +@@ -93,6 +96,7 @@ const ( + procMountsFile = "/proc/mounts" + + fieldsPerLine = 6 ++ maxRetryTimes = 5 + ) + + const ( +@@ -109,35 +113,45 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e + return + } + +- var file *os.File ++ var retry int = 0 + +- file, err = os.Open(procMountsFile) +- if err != nil { +- return +- } +- +- defer file.Close() ++ for retry <= maxRetryTimes { ++ var content []byte + +- reader := bufio.NewReader(file) +- for { +- var line string +- +- line, err = reader.ReadString('\n') +- if err == io.EOF { +- err = fmt.Errorf("Mount %s not found", mountPoint) ++ content, err = ioutil.ReadFile(procMountsFile) ++ if err != nil { + return + } + +- fields := strings.Fields(line) +- if len(fields) != fieldsPerLine { +- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) +- return ++ bytesReader := bytes.NewReader(content) ++ reader := bufio.NewReader(bytesReader) ++ ++ for { ++ var line string ++ ++ line, err = reader.ReadString('\n') ++ if err == io.EOF { ++ err = fmt.Errorf("Mount %s not found", mountPoint) ++ break ++ } ++ ++ fields := strings.Fields(line) ++ if len(fields) != fieldsPerLine { ++ err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) ++ return ++ } ++ ++ if mountPoint == fields[procPathIndex] { ++ devicePath = fields[procDeviceIndex] ++ fsType = fields[procTypeIndex] ++ return ++ } + } + +- if mountPoint == fields[procPathIndex] { +- devicePath = fields[procDeviceIndex] +- fsType = fields[procTypeIndex] +- return ++ retry = retry + 1 ++ if retry <= maxRetryTimes { ++ logrus.Warnf("can not find %s in %s, retry %d times again......", mountPoint, procMountsFile, retry) + } + } ++ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes) + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch b/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..955644169e9377f11c4645e602c03d2e74f22a60 --- /dev/null +++ b/runtime/patches/0003-kata-runtime-fix-kata-proxy-process-left-problem.patch @@ -0,0 +1,55 @@ +From cf595941e1d105af23bc006bf1998ac072733d0a Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 25 Jul 2020 10:03:35 +0800 +Subject: [PATCH 03/50] kata-runtime: fix kata-proxy process left problem + +reason: stopSandbox function will send the DestroySandboxRequest +to kata-agent in the VM and then kill the kata-proxy process in +the host. However, if k.sendReq(DestroySandboxRequest) get error, +stopSandbox will return immediately not execute the following kill +kata-proxy process statement, which cause the kata-process left. + +Signed-off-by: jiangpengfei +--- + virtcontainers/kata_agent.go | 18 +++++++++++------- + 1 file changed, 11 insertions(+), 7 deletions(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index a0cf190e..be5e96aa 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -976,6 +976,17 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { + return errorMissingProxy + } + ++ // since stopSandbox will destroy the sandbox in the VM, and we don't need ++ // kata-proxy process to communicate with kata-agent again, so we should ++ // make sure kata-proxy can be killed cleanly, even when k.sendReq(DestroySandboxRequest) ++ // return error ++ defer func() { ++ _ = k.proxy.stop(k.state.ProxyPid) ++ // clean up agent state ++ k.state.ProxyPid = -1 ++ k.state.URL = "" ++ }() ++ + req := &grpc.DestroySandboxRequest{} + + if _, err := k.sendReq(req); err != nil { +@@ -989,13 +1000,6 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { + } + } + +- if err := k.proxy.stop(k.state.ProxyPid); err != nil { +- return err +- } +- +- // clean up agent state +- k.state.ProxyPid = -1 +- k.state.URL = "" + return nil + } + +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch b/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch new file mode 100644 index 0000000000000000000000000000000000000000..e21250646c8f99a64904f57cfe52e651cb815c4d --- /dev/null +++ b/runtime/patches/0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch @@ -0,0 +1,65 @@ +From 025520f7fd3aeb5ed53b468b5e494b1bbb6674ae Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 25 Jul 2020 11:25:34 +0800 +Subject: [PATCH 04/50] kata-runtime: keep the process name of qemu same as + configured path + +reason: inorder to make testcase scripts can use the same hypervisor +name no matter what version hypervisor use, keep the process name of +hypervisor same as configured path in the configuration.toml file +instead of the resolved path of symbol link. + +Signed-off-by: jiangpengfei +--- + pkg/katautils/config.go | 8 +++++++- + pkg/katautils/config_test.go | 4 ++-- + 2 files changed, 9 insertions(+), 3 deletions(-) + +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 14794a24..349e667f 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -10,6 +10,7 @@ import ( + "errors" + "fmt" + "io/ioutil" ++ "path/filepath" + goruntime "runtime" + "strings" + +@@ -172,7 +173,12 @@ func (h hypervisor) path() (string, error) { + p = defaultHypervisorPath + } + +- return ResolvePath(p) ++ absolutePath, err := filepath.Abs(p) ++ if err != nil { ++ return "", err ++ } ++ ++ return absolutePath, nil + } + + func (h hypervisor) ctlpath() (string, error) { +diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go +index 221a4b55..2eae1f6a 100644 +--- a/pkg/katautils/config_test.go ++++ b/pkg/katautils/config_test.go +@@ -1061,12 +1061,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) { + assert.NoError(err) + assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong") + +- // test path resolution ++ // test path resolution, just return the absolute path instead of resolved path + defaultHypervisorPath = testHypervisorLinkPath + h = hypervisor{} + p, err = h.path() + assert.NoError(err) +- assert.Equal(p, testHypervisorPath) ++ assert.Equal(p, testHypervisorLinkPath) + } + + func TestHypervisorDefaultsKernel(t *testing.T) { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch b/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch new file mode 100644 index 0000000000000000000000000000000000000000..cc9bd5afca8622bc2970bf9991477e3e88d55c62 --- /dev/null +++ b/runtime/patches/0005-cgroups-increase-delete-cgroup-retry-times.patch @@ -0,0 +1,66 @@ +From c279f4548ccc534f1c65723bf9994c448e510d3d Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 25 Jul 2020 11:56:35 +0800 +Subject: [PATCH 05/50] cgroups: increase delete cgroup retry times + +reason: inorder to make sure cgroup dir to be deleted, so we increase +the retry times when delete cgroup dir failed. + +Signed-off-by: jiangpengfei +--- + vendor/github.com/containerd/cgroups/cgroup.go | 4 ++-- + vendor/github.com/containerd/cgroups/utils.go | 9 ++++++--- + 2 files changed, 8 insertions(+), 5 deletions(-) + +diff --git a/vendor/github.com/containerd/cgroups/cgroup.go b/vendor/github.com/containerd/cgroups/cgroup.go +index 53866685..69612b0a 100644 +--- a/vendor/github.com/containerd/cgroups/cgroup.go ++++ b/vendor/github.com/containerd/cgroups/cgroup.go +@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error { + return err + } + if err := d.Delete(sp); err != nil { +- errors = append(errors, string(s.Name())) ++ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error())) + } + continue + } +@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error { + } + path := p.Path(sp) + if err := remove(path); err != nil { +- errors = append(errors, path) ++ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error())) + } + } + } +diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go +index 8a97d04d..82dbe2d3 100644 +--- a/vendor/github.com/containerd/cgroups/utils.go ++++ b/vendor/github.com/containerd/cgroups/utils.go +@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) { + // retrying the remove after a exp timeout + func remove(path string) error { + delay := 10 * time.Millisecond +- for i := 0; i < 5; i++ { ++ var err error ++ var count int = 0 ++ for i := 0; i < 10; i++ { + if i != 0 { + time.Sleep(delay) + delay *= 2 + } +- if err := os.RemoveAll(path); err == nil { ++ if err = os.RemoveAll(path); err == nil { + return nil + } ++ count++ + } +- return fmt.Errorf("cgroups: unable to remove path %q", path) ++ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count) + } + + // readPids will read all the pids of processes in a cgroup by the provided path +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch b/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch new file mode 100644 index 0000000000000000000000000000000000000000..12e60f13417b5a0b61d4d0e9badef5a943367ee8 --- /dev/null +++ b/runtime/patches/0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch @@ -0,0 +1,39 @@ +From 3dc10421f177900c0ee94fc49b32ec66a46d9331 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 27 Jul 2020 19:18:50 +0800 +Subject: [PATCH 06/50] kata-runtime: fix umount container rootfs dir return + ivalid argument error + +reason: If sandbox hypervisor doesn't use block device driver for hotplugging container +rootfs block device into guest, kata-runtime will bind mount container rootfs dir to 9p +kataShared dir. However, container stop() function will always call bindUnmountContainerRootfs +function no matter block device driver is used or not. So we just need to call +bindUnmountContainerRootfs only if rootfs is bind mount to guest by 9p. + +Signed-off-by: jiangpengfei +--- + virtcontainers/container.go | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 9e2d1e94..b42cc6e9 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1120,8 +1120,12 @@ func (c *Container) stop(force bool) error { + return err + } + +- if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { +- return err ++ // umount container rootfs dir only if container use 9p ++ // to bind mount host container rootfs to 9p shared dir ++ if c.state.BlockDeviceID == "" { ++ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { ++ return err ++ } + } + + if err := c.detachDevices(); err != nil && !force { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch b/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch new file mode 100644 index 0000000000000000000000000000000000000000..2fdecfb361dff055596a095c59002048986c994d --- /dev/null +++ b/runtime/patches/0007-kata-runtime-enhance-reliability-when-kata-related-p.patch @@ -0,0 +1,855 @@ +From d93da1875ed7f1a6061cffb13475506d73c86003 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 25 Jul 2020 16:04:19 +0800 +Subject: [PATCH 07/50] kata-runtime: enhance reliability when kata related + process + +reason: enhance the reliability when kata related processes is abnormal, +make kata-container still destroy the sandbox and clean up all resources. + +Signed-off-by: jiangpengfei +--- + cli/delete.go | 6 ++ + cli/kill.go | 3 +- + virtcontainers/acrn.go | 2 +- + virtcontainers/agent.go | 3 + + virtcontainers/api.go | 67 +++++++++++++++++++++ + virtcontainers/clh.go | 2 +- + virtcontainers/container.go | 55 +++++++++++++++-- + virtcontainers/fc.go | 2 +- + virtcontainers/hypervisor.go | 2 +- + virtcontainers/kata_agent.go | 31 ++++++---- + virtcontainers/mock_hypervisor.go | 2 +- + virtcontainers/mock_hypervisor_test.go | 2 +- + virtcontainers/noop_agent.go | 4 ++ + virtcontainers/pkg/oci/utils.go | 5 ++ + virtcontainers/qemu.go | 51 +++++++++------- + virtcontainers/sandbox.go | 106 +++++++++++++++++++++++++++++---- + virtcontainers/types/sandbox.go | 14 ++++- + virtcontainers/utils/utils.go | 46 ++++++++++++++ + virtcontainers/vm.go | 4 +- + 19 files changed, 348 insertions(+), 59 deletions(-) + +diff --git a/cli/delete.go b/cli/delete.go +index c2ce52a4..2f5586e5 100644 +--- a/cli/delete.go ++++ b/cli/delete.go +@@ -110,6 +110,12 @@ func delete(ctx context.Context, containerID string, force bool) error { + forceStop = true + } + ++ if oci.StateToOCIState(status.State.State) == oci.StateUnhealthy { ++ // Set forceStop and force bool flag to true to force delete everything ++ forceStop = true ++ force = true ++ } ++ + switch containerType { + case vc.PodSandbox: + if err := deleteSandbox(ctx, sandboxID, force); err != nil { +diff --git a/cli/kill.go b/cli/kill.go +index 60fa41e0..b228205f 100644 +--- a/cli/kill.go ++++ b/cli/kill.go +@@ -133,11 +133,12 @@ func kill(ctx context.Context, containerID, signal string, all bool) error { + kataLog.WithField("signal", signal).WithField("container state", status.State.State).Info("kill") + + // container MUST be created, running or paused ++ // If container state is unhealthy, should process this exceptional case separately + if status.State.State == types.StateReady || status.State.State == types.StateRunning || status.State.State == types.StatePaused { + if err := vci.KillContainer(ctx, sandboxID, containerID, signum, all); err != nil { + return err + } +- } else if !all { ++ } else if !all && status.State.State != types.StateUnhealthy { + return fmt.Errorf("container not running") + } + +diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go +index 761eda03..10cae06f 100644 +--- a/virtcontainers/acrn.go ++++ b/virtcontainers/acrn.go +@@ -475,7 +475,7 @@ func (a *Acrn) waitSandbox(timeoutSecs int) error { + } + + // stopSandbox will stop the Sandbox's VM. +-func (a *Acrn) stopSandbox() (err error) { ++func (a *Acrn) stopSandbox(force bool) (err error) { + span, _ := a.trace("stopSandbox") + defer span.Finish() + +diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go +index c62107ec..be9526c7 100644 +--- a/virtcontainers/agent.go ++++ b/virtcontainers/agent.go +@@ -259,4 +259,7 @@ type agent interface { + + // load data from disk + load(persistapi.AgentState) ++ ++ // get proxy process pid ++ getProxyPid() int + } +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index de569713..fa82d163 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -7,8 +7,10 @@ package virtcontainers + + import ( + "context" ++ "fmt" + "os" + "runtime" ++ "strings" + "syscall" + + deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" +@@ -18,6 +20,7 @@ import ( + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" +@@ -597,20 +600,51 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err + container.state.State == types.StatePaused) && + container.process.Pid > 0 { + ++ // If container state is active, however kata-proxy and qemu process all exit already ++ // which means sandbox has beed stopped exceptionally, then we should force delete ++ // sandbox and container state files in sandbox.Store ++ if sandbox.shouldForceDelete() { ++ virtLog.Logger.Warn("sandbox status is abnormal, sandbox should be force deleted") ++ sandbox.forceDeleteSandbox() ++ return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally") ++ } ++ + running, err := isShimRunning(container.process.Pid) + if err != nil { + return ContainerStatus{}, err + } + ++ // If kata-shim process exit or be killed, need to stop the container + if !running { + virtLog.WithFields(logrus.Fields{ + "state": container.state.State, + "pid": container.process.Pid}). + Info("container isn't running") ++ + if err := container.stop(true); err != nil { + return ContainerStatus{}, err + } + } ++ ++ isPodSandbox := (containerID == sandbox.id) ++ ++ // If sandbox is unhealthy, process it correctly ++ if !sandbox.health() { ++ // process podSandbox container type case ++ if isPodSandbox { ++ if err := processUnhealthySandbox(sandbox, container); err != nil { ++ return ContainerStatus{}, err ++ } ++ } else { ++ // If container type is pod_container, which means container operations can not be ++ // processed successfully, we should return the error as soon as possible ++ if err := container.setContainerState(types.StateUnhealthy); err != nil { ++ return ContainerStatus{}, err ++ } ++ ++ return ContainerStatus{}, fmt.Errorf("container status is unhealthy, stop container failed") ++ } ++ } + } + + return ContainerStatus{ +@@ -1016,3 +1050,36 @@ func CleanupContainer(ctx context.Context, sandboxID, containerID string, force + + return nil + } ++ ++// procesUnhealthySandbox only change sandbox state to unhealthy ++// when caller is kata-runtime kill or kata-runtime delete ++func processUnhealthySandbox(sandbox *Sandbox, container *Container) error { ++ // Set all containers state to unhealthy ++ if err := sandbox.setContainersState(types.StateUnhealthy); err != nil { ++ container.Logger().WithError(err).Warn("set all containers state to unhealthy fail") ++ } ++ ++ // Set sandbox state to unhealthy ++ if err := sandbox.setSandboxState(types.StateUnhealthy); err != nil { ++ container.Logger().WithError(err).Warn("set sandbox state to unhealthy fail") ++ } ++ ++ forceDelete := false ++ ++ // If process is kata-runtime kill or kata-runtime delete, ++ // we should kill or delete sandbox forcefully ++ if cmdline, err := utils.GetProcessCmdline(os.Getpid()); err != nil { ++ container.Logger().WithError(err).Warn("fail to get process cmdline info") ++ } else { ++ forceDelete = strings.Contains(cmdline, "kill") || strings.Contains(cmdline, "delete") ++ } ++ ++ if forceDelete { ++ // force stop podSandbox type container's kata-shim process ++ if err := stopShim(container.process.Pid); err != nil { ++ container.Logger().WithError(err).Warn("fail to stop podSandbox type container kata-shim") ++ } ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go +index d40b698b..59510b02 100644 +--- a/virtcontainers/clh.go ++++ b/virtcontainers/clh.go +@@ -569,7 +569,7 @@ func (clh *cloudHypervisor) resumeSandbox() error { + } + + // stopSandbox will stop the Sandbox's VM. +-func (clh *cloudHypervisor) stopSandbox() (err error) { ++func (clh *cloudHypervisor) stopSandbox(force bool) (err error) { + span, _ := clh.trace("stopSandbox") + defer span.Finish() + clh.Logger().WithField("function", "stopSandbox").Info("Stop Sandbox") +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index b42cc6e9..9485e708 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -17,8 +17,12 @@ import ( + "time" + + "github.com/containerd/cgroups" ++ "github.com/kata-containers/runtime/virtcontainers/device/config" ++ "github.com/kata-containers/runtime/virtcontainers/device/manager" + vccgroups "github.com/kata-containers/runtime/virtcontainers/pkg/cgroups" ++ "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" ++ "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" +@@ -26,11 +30,6 @@ import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +- +- "github.com/kata-containers/runtime/virtcontainers/device/config" +- "github.com/kata-containers/runtime/virtcontainers/device/manager" +- "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" +- "github.com/kata-containers/runtime/virtcontainers/store" + ) + + // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h +@@ -1047,6 +1046,13 @@ func (c *Container) stop(force bool) error { + return nil + } + ++ // If container state is unhealthy, just force kill the container ++ if c.state.State == types.StateUnhealthy { ++ c.forceKillContainer() ++ // after force kill container, then change container state to stopped ++ return c.setContainerState(types.StateStopped) ++ } ++ + if err := c.state.ValidTransition(c.state.State, types.StateStopped); err != nil { + return err + } +@@ -1063,6 +1069,8 @@ func (c *Container) stop(force bool) error { + if err := stopShim(c.process.Pid); err != nil { + l.WithError(err).Warn("failed to stop shim") + } ++ ++ c.forceKillContainer() + } + + }() +@@ -1096,7 +1104,9 @@ func (c *Container) stop(force bool) error { + // this signal will ensure the container will get killed to match + // the state of the shim. This will allow the following call to + // stopContainer() to succeed in such particular case. +- c.kill(syscall.SIGKILL, true) ++ if err := c.kill(syscall.SIGKILL, true); err != nil { ++ c.Logger().Errorf("send signal to container failed: %v", err) ++ } + + // Since the agent has supported the MultiWaitProcess, it's better to + // wait the process here to make sure the process has exited before to +@@ -1582,3 +1592,36 @@ func (c *Container) cgroupsUpdate(resources specs.LinuxResources) error { + + return nil + } ++ ++// forceDeleteContainer force clean container mount info and resources stored in the disk ++func (c *Container) forceDeleteContainer() { ++ if err := c.unmountHostMounts(); err != nil { ++ c.Logger().WithError(err).Warn("container force delete umount host mounts fail") ++ } ++ ++ if err := c.sandbox.removeContainer(c.id); err != nil { ++ c.Logger().WithError(err).Warn("sandbox removeContainer fail") ++ } ++ ++ if err := c.store.Delete(); err != nil { ++ c.Logger().WithError(err).Warn("force delete container store fail") ++ } ++} ++ ++func (c *Container) forceKillContainer() { ++ if err := c.setContainerState(types.StateStopped); err != nil { ++ c.Logger().WithError(err).Warn("force kill container: change container state to StateStopped failed") ++ } ++ ++ if err := c.unmountHostMounts(); err != nil { ++ c.Logger().WithError(err).Warn("force kill container: umount container host mounts failed") ++ } ++ ++ if err := c.detachDevices(); err != nil { ++ c.Logger().WithError(err).Warn("force kill container: detach container devices failed") ++ } ++ ++ if err := c.removeDrive(); err != nil { ++ c.Logger().WithError(err).Warn("force kill container: remove container drive failed") ++ } ++} +diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go +index 97ef5ffc..72a8e192 100644 +--- a/virtcontainers/fc.go ++++ b/virtcontainers/fc.go +@@ -864,7 +864,7 @@ func (fc *firecracker) cleanupJail() { + } + + // stopSandbox will stop the Sandbox's VM. +-func (fc *firecracker) stopSandbox() (err error) { ++func (fc *firecracker) stopSandbox(force bool) (err error) { + span, _ := fc.trace("stopSandbox") + defer span.Finish() + +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index 4b3dd3d0..fd7d1f8e 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -766,7 +766,7 @@ func generateVMSocket(id string, useVsock bool, vmStogarePath string) (interface + type hypervisor interface { + createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error + startSandbox(timeout int) error +- stopSandbox() error ++ stopSandbox(force bool) error + pauseSandbox() error + saveSandbox() error + resumeSandbox() error +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index be5e96aa..7575d326 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -57,8 +57,9 @@ const ( + ) + + var ( +- checkRequestTimeout = 30 * time.Second +- defaultRequestTimeout = 60 * time.Second ++ checkRequestTimeout = 10 * time.Second ++ defaultRequestTimeout = 10 * time.Second ++ createContainerTimeout = 120 * time.Second + errorMissingProxy = errors.New("Missing proxy pointer") + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" +@@ -987,17 +988,21 @@ func (k *kataAgent) stopSandbox(sandbox *Sandbox) error { + k.state.URL = "" + }() + +- req := &grpc.DestroySandboxRequest{} ++ // If sandbox.state.State is unhealthy, we don't need to send DestroySandboxRequest ++ // to kata-agent, just force stop the sandbox ++ if sandbox.state.State != types.StateUnhealthy { ++ req := &grpc.DestroySandboxRequest{} + +- if _, err := k.sendReq(req); err != nil { +- return err +- } +- +- if k.dynamicTracing { +- _, err := k.sendReq(&grpc.StopTracingRequest{}) +- if err != nil { ++ if _, err := k.sendReq(req); err != nil { + return err + } ++ ++ if k.dynamicTracing { ++ _, err := k.sendReq(&grpc.StopTracingRequest{}) ++ if err != nil { ++ return err ++ } ++ } + } + + return nil +@@ -2062,6 +2067,8 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c + // Wait has no timeout + case grpcCheckRequest: + ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout) ++ case grpcCreateContainerRequest: ++ ctx, cancel = context.WithTimeout(ctx, createContainerTimeout) + default: + ctx, cancel = context.WithTimeout(ctx, defaultRequestTimeout) + } +@@ -2382,3 +2389,7 @@ func (k *kataAgent) load(s persistapi.AgentState) { + k.state.ProxyPid = s.ProxyPid + k.state.URL = s.URL + } ++ ++func (k *kataAgent) getProxyPid() int { ++ return k.state.ProxyPid ++} +diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go +index 0c84e43c..a5b67491 100644 +--- a/virtcontainers/mock_hypervisor.go ++++ b/virtcontainers/mock_hypervisor.go +@@ -39,7 +39,7 @@ func (m *mockHypervisor) startSandbox(timeout int) error { + return nil + } + +-func (m *mockHypervisor) stopSandbox() error { ++func (m *mockHypervisor) stopSandbox(force bool) error { + return nil + } + +diff --git a/virtcontainers/mock_hypervisor_test.go b/virtcontainers/mock_hypervisor_test.go +index b73b28f2..827e3192 100644 +--- a/virtcontainers/mock_hypervisor_test.go ++++ b/virtcontainers/mock_hypervisor_test.go +@@ -53,7 +53,7 @@ func TestMockHypervisorStartSandbox(t *testing.T) { + func TestMockHypervisorStopSandbox(t *testing.T) { + var m *mockHypervisor + +- assert.NoError(t, m.stopSandbox()) ++ assert.NoError(t, m.stopSandbox(false)) + } + + func TestMockHypervisorAddDevice(t *testing.T) { +diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go +index 189f6b3f..8a7cd337 100644 +--- a/virtcontainers/noop_agent.go ++++ b/virtcontainers/noop_agent.go +@@ -236,3 +236,7 @@ func (n *noopAgent) save() (s persistapi.AgentState) { + + // load is the Noop agent state loader. It does nothing. + func (n *noopAgent) load(s persistapi.AgentState) {} ++ ++func (n *noopAgent) getProxyPid() int { ++ return -1 ++} +\ No newline at end of file +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 5348c57d..cd8d48ce 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -70,6 +70,9 @@ const ( + + // StatePaused represents a container that has been paused. + StatePaused = "paused" ++ ++ // StateUnhealthy represents a container that is unhealthy ++ StateUnhealthy = "unhealthy" + ) + + const KernelModulesSeparator = ";" +@@ -964,6 +967,8 @@ func StateToOCIState(state types.StateString) string { + return StateStopped + case types.StatePaused: + return StatePaused ++ case types.StateUnhealthy: ++ return StateUnhealthy + default: + return "" + } +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index ca286550..4b15d968 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -687,7 +687,7 @@ func (q *qemu) setupVirtiofsd() (err error) { + q.Logger().Info("virtiofsd quits") + // Wait to release resources of virtiofsd process + cmd.Process.Wait() +- q.stopSandbox() ++ q.stopSandbox(false) + }() + return err + } +@@ -922,11 +922,11 @@ func (q *qemu) waitSandbox(timeout int) error { + } + + // stopSandbox will stop the Sandbox's VM. +-func (q *qemu) stopSandbox() error { ++func (q *qemu) stopSandbox(force bool) error { + span, _ := q.trace("stopSandbox") + defer span.Finish() + +- q.Logger().Info("Stopping Sandbox") ++ q.Logger().Infof("force stopping Sandbox: %v", force) + if q.stopped { + q.Logger().Info("Already stopped") + return nil +@@ -937,28 +937,37 @@ func (q *qemu) stopSandbox() error { + q.stopped = true + }() + +- if q.config.Debug && q.qemuConfig.LogFile != "" { +- f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0) +- if err == nil { +- scanner := bufio.NewScanner(f) +- for scanner.Scan() { +- q.Logger().Debug(scanner.Text()) +- } +- if err := scanner.Err(); err != nil { +- q.Logger().WithError(err).Debug("read qemu log failed") ++ if !force { ++ if q.config.Debug && q.qemuConfig.LogFile != "" { ++ f, err := os.OpenFile(q.qemuConfig.LogFile, os.O_RDONLY, 0) ++ if err == nil { ++ scanner := bufio.NewScanner(f) ++ for scanner.Scan() { ++ q.Logger().Debug(scanner.Text()) ++ } ++ if err := scanner.Err(); err != nil { ++ q.Logger().WithError(err).Debug("read qemu log failed") ++ } + } + } +- } + +- err := q.qmpSetup() +- if err != nil { +- return err +- } ++ err := q.qmpSetup() ++ if err != nil { ++ return err ++ } + +- err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) +- if err != nil { +- q.Logger().WithError(err).Error("Fail to execute qmp QUIT") +- return err ++ err = q.qmpMonitorCh.qmp.ExecuteQuit(q.qmpMonitorCh.ctx) ++ if err != nil { ++ q.Logger().WithError(err).Error("Fail to execute qmp QUIT") ++ return err ++ } ++ } else { ++ qemuMainPid := q.getPids()[0] ++ if qemuMainPid <= 1 { ++ return fmt.Errorf("force kill qemu process pid is invalid") ++ } ++ ++ _ = syscall.Kill(qemuMainPid, syscall.SIGKILL) + } + + return nil +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index edd1af5b..78188ed7 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -12,19 +12,13 @@ import ( + "math" + "net" + "os" ++ "path/filepath" + "strings" + "sync" + "syscall" + + "github.com/containerd/cgroups" + "github.com/containernetworking/plugins/pkg/ns" +- "github.com/opencontainers/runc/libcontainer/configs" +- specs "github.com/opencontainers/runtime-spec/specs-go" +- opentracing "github.com/opentracing/opentracing-go" +- "github.com/pkg/errors" +- "github.com/sirupsen/logrus" +- "github.com/vishvananda/netlink" +- + "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +@@ -41,6 +35,12 @@ import ( + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" ++ "github.com/opencontainers/runc/libcontainer/configs" ++ specs "github.com/opencontainers/runtime-spec/specs-go" ++ opentracing "github.com/opentracing/opentracing-go" ++ "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" ++ "github.com/vishvananda/netlink" + ) + + const ( +@@ -50,6 +50,9 @@ const ( + + // DirMode is the permission bits used for creating a directory + DirMode = os.FileMode(0750) | os.ModeDir ++ ++ // kata-proxy proces name ++ KataProxyProcessName = "kata-proxy" + ) + + // SandboxStatus describes a sandbox status. +@@ -1037,7 +1040,7 @@ func (s *Sandbox) startVM() (err error) { + + defer func() { + if err != nil { +- s.hypervisor.stopSandbox() ++ s.hypervisor.stopSandbox(false) + } + }() + +@@ -1090,7 +1093,12 @@ func (s *Sandbox) stopVM() error { + } + + s.Logger().Info("Stopping VM") +- return s.hypervisor.stopSandbox() ++ forceStop := false ++ if s.state.State == types.StateUnhealthy { ++ forceStop = true ++ } ++ ++ return s.hypervisor.stopSandbox(forceStop) + } + + func (s *Sandbox) addContainer(c *Container) error { +@@ -1591,13 +1599,15 @@ func (s *Sandbox) setSandboxState(state types.StateString) error { + return vcTypes.ErrNeedState + } + ++ s.Logger().Debugf("Setting sandbox state from %v to %v", s.state.State, state) + // update in-memory state + s.state.State = state + + if useOldStore(s.ctx) { + return s.store.Store(store.State, s.state) ++ } else { ++ return s.Save() + } +- return nil + } + + const maxBlockIndex = 65535 +@@ -2207,3 +2217,79 @@ func (s *Sandbox) GetPatchedOCISpec() *specs.Spec { + + return nil + } ++ ++// health return current sandbox healthy or not ++// If qemu/kata-proxy/kata-agent process is abnormal, ++// s.agent.check() will return false ++func (s *Sandbox) health() bool { ++ err := s.agent.check() ++ if err != nil { ++ return false ++ } ++ ++ return true ++} ++ ++// shouldForceDelete force delete the sandbox when kata-proxy and hypervisor process exit ++// already and current process is kata-runtime kill or kata-runtime delete ++func (s *Sandbox) shouldForceDelete() bool { ++ cmdline, err := utils.GetProcessCmdline(os.Getpid()) ++ if err != nil { ++ s.Logger().Errorf("fail to get process cmdline: %v", err) ++ return false ++ } ++ ++ proxyPid := s.agent.getProxyPid() ++ hypervisorPids := s.hypervisor.getPids() ++ if len(hypervisorPids) <= 0 { ++ s.Logger().Warnf("get hypervisor main pid fail") ++ return false ++ } ++ hypervisorMainPid := hypervisorPids[0] ++ hypervisorPath := s.hypervisor.hypervisorConfig().HypervisorPath ++ hypervisorName := filepath.Base(hypervisorPath) ++ ++ if !utils.IsProcessRunning(proxyPid, KataProxyProcessName, s.id) && !utils.IsProcessRunning(hypervisorMainPid, hypervisorName, s.id) && ++ strings.Contains(cmdline, "delete") && strings.Contains(cmdline, "force") { ++ return true ++ } ++ ++ return false ++} ++ ++func (s *Sandbox) forceDeleteSandbox() { ++ for _, c := range s.containers { ++ // force delete all containers in the sandbox ++ c.forceDeleteContainer() ++ } ++ ++ globalSandboxList.removeSandbox(s.id) ++ ++ if s.monitor != nil { ++ s.monitor.stop() ++ } ++ ++ if err := s.hypervisor.cleanup(); err != nil { ++ s.Logger().WithError(err).Error("failed to force cleanup hypervisor resource") ++ } ++ ++ s.agent.cleanup(s) ++ ++ if err := s.store.Delete(); err != nil { ++ s.Logger().WithError(err).Warn("sandbox force delete store failed") ++ } ++} ++ ++func (s *Sandbox) setContainersState(state types.StateString) error { ++ if state == "" { ++ return vcTypes.ErrNeedState ++ } ++ ++ for _, c := range s.containers { ++ if err := c.setContainerState(state); err != nil { ++ return err ++ } ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/types/sandbox.go b/virtcontainers/types/sandbox.go +index 3b64b20a..5d586b21 100644 +--- a/virtcontainers/types/sandbox.go ++++ b/virtcontainers/types/sandbox.go +@@ -28,6 +28,9 @@ const ( + + // StateStopped represents a sandbox/container that has been stopped. + StateStopped StateString = "stopped" ++ ++ // StateUnhealthy represents a sandbox/container that's in abnormal state. ++ StateUnhealthy StateString = "unhealthy" + ) + + const ( +@@ -90,17 +93,17 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt + + switch *state { + case StateReady: +- if newState == StateRunning || newState == StateStopped { ++ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy { + return nil + } + + case StateRunning: +- if newState == StatePaused || newState == StateStopped { ++ if newState == StatePaused || newState == StateStopped || newState == StateUnhealthy { + return nil + } + + case StatePaused: +- if newState == StateRunning || newState == StateStopped { ++ if newState == StateRunning || newState == StateStopped || newState == StateUnhealthy { + return nil + } + +@@ -108,6 +111,11 @@ func (state *StateString) validTransition(oldState StateString, newState StateSt + if newState == StateRunning { + return nil + } ++ ++ case StateUnhealthy: ++ if newState == StateStopped { ++ return nil ++ } + } + + return fmt.Errorf("Can not move from %v to %v", +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 85c55489..2b555ebb 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -9,9 +9,13 @@ import ( + "crypto/rand" + "errors" + "fmt" ++ "io/ioutil" + "os" + "os/exec" + "path/filepath" ++ "strconv" ++ "strings" ++ "syscall" + ) + + const cpBinaryName = "cp" +@@ -275,3 +279,45 @@ const ( + MiB = KiB << 10 + GiB = MiB << 10 + ) ++ ++// Get process cmdline info by read /proc//cmdline file ++func GetProcessCmdline(pid int) (cmdline string, err error) { ++ if pid <= 1 { ++ return "", fmt.Errorf("invalid pid number") ++ } ++ ++ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) ++ if err != nil { ++ return "", err ++ } ++ ++ return string(bytes), nil ++} ++ ++func IsProcessRunning(pid int, processName string, sandboxID string) bool { ++ if pid <= 0 { ++ return false ++ } ++ ++ process, err := os.FindProcess(pid) ++ if err != nil { ++ return false ++ } ++ ++ if err := process.Signal(syscall.Signal(0)); err != nil { ++ return false ++ } ++ ++ cmdline, err := GetProcessCmdline(pid) ++ if err != nil { ++ return false ++ } ++ ++ // If process's cmdline contains processName and sandboxID keyword, ++ // We think this process isn't be reused ++ if strings.Contains(cmdline, processName) && strings.Contains(cmdline, sandboxID) { ++ return true ++ } ++ ++ return false ++} +diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go +index fcda1e97..8d27b1fe 100644 +--- a/virtcontainers/vm.go ++++ b/virtcontainers/vm.go +@@ -191,7 +191,7 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { + defer func() { + if err != nil { + virtLog.WithField("vm", id).WithError(err).Info("clean up vm") +- hypervisor.stopSandbox() ++ hypervisor.stopSandbox(false) + } + }() + +@@ -333,7 +333,7 @@ func (v *VM) Disconnect() error { + func (v *VM) Stop() error { + v.logger().Info("stop vm") + +- if err := v.hypervisor.stopSandbox(); err != nil { ++ if err := v.hypervisor.stopSandbox(false); err != nil { + return err + } + +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch b/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..e0022d5604190b0da58af01584fb8fbc5ed31fcd --- /dev/null +++ b/runtime/patches/0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch @@ -0,0 +1,177 @@ +From a1bf2e1c696b703935f4b81ca087a60cc2559464 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 27 Jul 2020 21:45:25 +0800 +Subject: [PATCH 08/50] kata-runtime: fix kata-runtime resource left problem + +reason: fix the following resource left problem +- sandbox stored files and directory are deleted before work container delete +- ignore StatusSandbox error, let stopSandbox function call be executed + +Signed-off-by: jiangpengfei +--- + cli/delete.go | 25 +++++++++++++++++++++++-- + cli/delete_test.go | 15 +++++++++++++++ + cli/state.go | 6 ++++++ + virtcontainers/api.go | 4 +++- + 4 files changed, 47 insertions(+), 3 deletions(-) + +diff --git a/cli/delete.go b/cli/delete.go +index 2f5586e5..871ac40d 100644 +--- a/cli/delete.go ++++ b/cli/delete.go +@@ -10,6 +10,7 @@ import ( + "context" + "fmt" + "os" ++ "strings" + + "github.com/kata-containers/runtime/pkg/katautils" + vc "github.com/kata-containers/runtime/virtcontainers" +@@ -68,6 +69,12 @@ func delete(ctx context.Context, containerID string, force bool) error { + setExternalLoggers(ctx, kataLog) + span.SetTag("container", containerID) + ++ // Remove the containerID to sandboxID mapping ++ // no matter what error return ++ defer func() { ++ _ = katautils.DelContainerIDMapping(ctx, containerID) ++ }() ++ + // Checks the MUST and MUST NOT from OCI runtime specification + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { +@@ -75,6 +82,15 @@ func delete(ctx context.Context, containerID string, force bool) error { + kataLog.Warnf("Failed to get container, force will not fail: %s", err) + return nil + } ++ ++ // If err info containers "no such file or directory, because pod_sandbox type ++ // container is deleted before pod_container type container, just return nil ++ // and let containerd delete container operations continue ++ if strings.Contains(err.Error(), "no such file or directory") { ++ kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err) ++ return nil ++ } ++ + return err + } + +@@ -123,7 +139,12 @@ func delete(ctx context.Context, containerID string, force bool) error { + } + case vc.PodContainer: + if err := deleteContainer(ctx, sandboxID, containerID, forceStop); err != nil { +- return err ++ // If err info containers "no such file or directory, because pod_sandbox type ++ // container is deleted before pod_container type container, just return nil ++ // and let containerd delete container operations continue ++ if !strings.Contains(err.Error(), "no such file or directory") { ++ return err ++ } + } + default: + return fmt.Errorf("Invalid container type found") +@@ -134,7 +155,7 @@ func delete(ctx context.Context, containerID string, force bool) error { + return err + } + +- return katautils.DelContainerIDMapping(ctx, containerID) ++ return nil + } + + func deleteSandbox(ctx context.Context, sandboxID string, force bool) error { +diff --git a/cli/delete_test.go b/cli/delete_test.go +index a2455dee..ae421cd7 100644 +--- a/cli/delete_test.go ++++ b/cli/delete_test.go +@@ -184,6 +184,9 @@ func TestDeleteSandbox(t *testing.T) { + assert.Error(err) + assert.True(vcmock.IsMockError(err)) + ++ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ + testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { + return vc.SandboxStatus{ + ID: sandbox.ID(), +@@ -201,6 +204,9 @@ func TestDeleteSandbox(t *testing.T) { + assert.Error(err) + assert.True(vcmock.IsMockError(err)) + ++ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ + testingImpl.StopSandboxFunc = func(ctx context.Context, sandboxID string, force bool) (vc.VCSandbox, error) { + return sandbox, nil + } +@@ -213,6 +219,9 @@ func TestDeleteSandbox(t *testing.T) { + assert.Error(err) + assert.True(vcmock.IsMockError(err)) + ++ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ + testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { + return sandbox, nil + } +@@ -302,6 +311,9 @@ func TestDeleteSandboxRunning(t *testing.T) { + assert.Error(err) + assert.False(vcmock.IsMockError(err)) + ++ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ + testingImpl.StatusSandboxFunc = func(ctx context.Context, sandboxID string) (vc.SandboxStatus, error) { + return vc.SandboxStatus{ + ID: sandbox.ID(), +@@ -325,6 +337,9 @@ func TestDeleteSandboxRunning(t *testing.T) { + assert.Error(err) + assert.True(vcmock.IsMockError(err)) + ++ path, err = createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ + testingImpl.DeleteSandboxFunc = func(ctx context.Context, sandboxID string) (vc.VCSandbox, error) { + return sandbox, nil + } +diff --git a/cli/state.go b/cli/state.go +index a2fcc12e..de843d34 100644 +--- a/cli/state.go ++++ b/cli/state.go +@@ -11,6 +11,7 @@ import ( + "encoding/json" + "fmt" + "os" ++ "strings" + + "github.com/kata-containers/runtime/pkg/katautils" + "github.com/kata-containers/runtime/virtcontainers/pkg/oci" +@@ -52,6 +53,11 @@ func state(ctx context.Context, containerID string) error { + // Checks the MUST and MUST NOT from OCI runtime specification + status, _, err := getExistingContainerInfo(ctx, containerID) + if err != nil { ++ // If err info containers "no such file or directory, because pod_sandbox type ++ // container is deleted before pod_container type container, just return nil ++ if strings.Contains(err.Error(), "no such file or directory") { ++ return nil ++ } + return err + } + +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index fa82d163..449a03e0 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -374,7 +374,9 @@ func StatusSandbox(ctx context.Context, sandboxID string) (SandboxStatus, error) + for _, container := range s.containers { + contStatus, err := statusContainer(s, container.id) + if err != nil { +- return SandboxStatus{}, err ++ // Since statusContainer may get error because of qemu process D or T, ++ // So just ignore this error and let StatusSandbox function return actually SandboxStatus ++ s.Logger().Warnf("Status container's status in sandbox get error: %v", contStatus) + } + + contStatusList = append(contStatusList, contStatus) +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch b/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch new file mode 100644 index 0000000000000000000000000000000000000000..19596fca8c4f36b43c62149840e26014978aa570 --- /dev/null +++ b/runtime/patches/0009-kata-runtime-add-kata-runtime-global-flag-debug.patch @@ -0,0 +1,132 @@ +From 508fd9b94b1b12be3167342b03a47f8f97245e9c Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 28 Jul 2020 10:53:30 +0800 +Subject: [PATCH 09/50] kata-runtime: add kata-runtime global flag --debug + +reason: add the same debug flag with runc to adapt to +containerd 1.2.0 + +Signed-off-by: jiangpengfei +--- + cli/kata-env_test.go | 2 +- + cli/main.go | 6 +++++- + containerd-shim-v2/create.go | 2 +- + pkg/katautils/config.go | 4 ++-- + pkg/katautils/config_test.go | 8 ++++---- + 5 files changed, 13 insertions(+), 9 deletions(-) + +diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go +index b31b6cc2..75bb697f 100644 +--- a/cli/kata-env_test.go ++++ b/cli/kata-env_test.go +@@ -178,7 +178,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC + return "", oci.RuntimeConfig{}, err + } + +- _, config, err = katautils.LoadConfiguration(configFile, true, false) ++ _, config, err = katautils.LoadConfiguration(configFile, true, false, false) + if err != nil { + return "", oci.RuntimeConfig{}, err + } +diff --git a/cli/main.go b/cli/main.go +index df16c5f3..1362a8fb 100644 +--- a/cli/main.go ++++ b/cli/main.go +@@ -115,6 +115,10 @@ var runtimeFlags = []cli.Flag{ + Name: "systemd-cgroup", + Usage: "enable systemd cgroup support, expects cgroupsPath to be of form \"slice:prefix:name\" for e.g. \"system.slice:runc:434234\"", + }, ++ cli.BoolFlag{ ++ Name: "debug", ++ Usage: "enable debug output for logging", ++ }, + } + + // runtimeCommands is the list of supported command-line (sub-) +@@ -334,7 +338,7 @@ func beforeSubcommands(c *cli.Context) error { + } + } + +- configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false) ++ configFile, runtimeConfig, err = katautils.LoadConfiguration(c.GlobalString(configFilePathOption), ignoreConfigLogs, false, c.GlobalBool("debug")) + if err != nil { + fatal(err) + } +diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go +index affdbae2..9749073d 100644 +--- a/containerd-shim-v2/create.go ++++ b/containerd-shim-v2/create.go +@@ -167,7 +167,7 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest, anno map[string + configPath = os.Getenv("KATA_CONF_FILE") + } + +- _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true) ++ _, runtimeConfig, err := katautils.LoadConfiguration(configPath, false, true, false) + if err != nil { + return nil, err + } +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 349e667f..448d23ac 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -1141,7 +1141,7 @@ func initConfig() (config oci.RuntimeConfig, err error) { + // + // All paths are resolved fully meaning if this function does not return an + // error, all paths are valid at the time of the call. +-func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { ++func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag bool) (resolvedConfigPath string, config oci.RuntimeConfig, err error) { + + config, err = initConfig() + if err != nil { +@@ -1154,7 +1154,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool) (resolved + } + + config.Debug = tomlConf.Runtime.Debug +- if !tomlConf.Runtime.Debug { ++ if !tomlConf.Runtime.Debug && !debugFlag { + // If debug is not required, switch back to the original + // default log priority, otherwise continue in debug mode. + kataUtilsLogger.Logger.Level = originalLoggerLevel +diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go +index 2eae1f6a..31afcca6 100644 +--- a/pkg/katautils/config_test.go ++++ b/pkg/katautils/config_test.go +@@ -264,7 +264,7 @@ func testLoadConfiguration(t *testing.T, dir string, + assert.NoError(t, err) + } + +- resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false) ++ resolvedConfigPath, config, err := LoadConfiguration(file, ignoreLogging, false, false) + if expectFail { + assert.Error(t, err) + +@@ -578,7 +578,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { + t.Fatal(err) + } + +- _, config, err := LoadConfiguration(configPath, false, false) ++ _, config, err := LoadConfiguration(configPath, false, false, false) + if err == nil { + t.Fatalf("Expected loadConfiguration to fail as shim path does not exist: %+v", config) + } +@@ -608,7 +608,7 @@ func TestMinimalRuntimeConfig(t *testing.T) { + t.Error(err) + } + +- _, config, err = LoadConfiguration(configPath, false, false) ++ _, config, err = LoadConfiguration(configPath, false, false, false) + if err != nil { + t.Fatal(err) + } +@@ -748,7 +748,7 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) { + t.Fatal(err) + } + +- _, config, err := LoadConfiguration(configPath, false, false) ++ _, config, err := LoadConfiguration(configPath, false, false, false) + if err != nil { + t.Fatal(err) + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch b/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..31dc1baed885127bd33e6113e67b6d8aa5424f73 --- /dev/null +++ b/runtime/patches/0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch @@ -0,0 +1,172 @@ +From 76cbca91608e94c1855705ad1a8d06ffa2273115 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 28 Jul 2020 18:18:54 +0800 +Subject: [PATCH 10/50] kata-runtime: fix kata-shim pid reused problem + +reason: If kata-shim process exit and it's pid reused by other process, +it may cause kill other proecss and cause some problem. + +Signed-off-by: jiangpengfei +--- + virtcontainers/api.go | 2 +- + virtcontainers/container.go | 6 +++--- + virtcontainers/shim.go | 21 +++++++++++++++++---- + virtcontainers/shim_test.go | 10 +++++----- + 4 files changed, 26 insertions(+), 13 deletions(-) + +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 449a03e0..5e8c9c9e 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -611,7 +611,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err + return ContainerStatus{}, fmt.Errorf("sandbox has beed stopped exceptionally") + } + +- running, err := isShimRunning(container.process.Pid) ++ running, err := isShimRunning(container.process.Pid, containerID) + if err != nil { + return ContainerStatus{}, err + } +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 9485e708..75f590eb 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1063,7 +1063,7 @@ func (c *Container) stop(force bool) error { + + // If shim is still running something went wrong + // Make sure we stop the shim process +- if running, _ := isShimRunning(c.process.Pid); running { ++ if running, _ := isShimRunning(c.process.Pid, c.id); running { + l := c.Logger() + l.Error("Failed to stop container so stopping dangling shim") + if err := stopShim(c.process.Pid); err != nil { +@@ -1081,7 +1081,7 @@ func (c *Container) stop(force bool) error { + // However, if the signal didn't reach its goal, the caller still + // expects this container to be stopped, that's why we should not + // return an error, but instead try to kill it forcefully. +- if err := waitForShim(c.process.Pid); err != nil { ++ if err := waitForShim(c.process.Pid, c.id); err != nil { + // Force the container to be killed. + if err := c.kill(syscall.SIGKILL, true); err != nil && !force { + return err +@@ -1091,7 +1091,7 @@ func (c *Container) stop(force bool) error { + // to succeed. Indeed, we have already given a second chance + // to the container by trying to kill it with SIGKILL, there + // is no reason to try to go further if we got an error. +- if err := waitForShim(c.process.Pid); err != nil && !force { ++ if err := waitForShim(c.process.Pid, c.id); err != nil && !force { + return err + } + } +diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go +index 8ec7458b..6f784a03 100644 +--- a/virtcontainers/shim.go ++++ b/virtcontainers/shim.go +@@ -9,11 +9,13 @@ import ( + "fmt" + "os" + "os/exec" ++ "strings" + "syscall" + "time" + + ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/mitchellh/mapstructure" + "github.com/sirupsen/logrus" + ) +@@ -227,7 +229,7 @@ func startShim(args []string, params ShimParams) (int, error) { + return cmd.Process.Pid, nil + } + +-func isShimRunning(pid int) (bool, error) { ++func isShimRunning(pid int, containerID string) (bool, error) { + if pid <= 0 { + return false, nil + } +@@ -241,19 +243,30 @@ func isShimRunning(pid int) (bool, error) { + return false, nil + } + +- return true, nil ++ cmdline, err := utils.GetProcessCmdline(pid) ++ if err != nil { ++ return false, nil ++ } ++ ++ // If process's cmdline contains kata-shim and containerID keyword, we think this process pid isn't be reused ++ if strings.Contains(cmdline, "kata-shim") && strings.Contains(cmdline, containerID) { ++ return true, nil ++ } ++ ++ shimLogger().Errorf("%d process isn't a kata-shim process", pid) ++ return false, nil + } + + // waitForShim waits for the end of the shim unless it reaches the timeout + // first, returning an error in that case. +-func waitForShim(pid int) error { ++func waitForShim(pid int, containerID string) error { + if pid <= 0 { + return nil + } + + tInit := time.Now() + for { +- running, err := isShimRunning(pid) ++ running, err := isShimRunning(pid, containerID) + if err != nil { + return err + } +diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go +index e9bd027c..62471311 100644 +--- a/virtcontainers/shim_test.go ++++ b/virtcontainers/shim_test.go +@@ -190,7 +190,7 @@ func TestStopShimSuccessfulProcessRunning(t *testing.T) { + + func testIsShimRunning(t *testing.T, pid int, expected bool) { + assert := assert.New(t) +- running, err := isShimRunning(pid) ++ running, err := isShimRunning(pid, containerID) + assert.NoError(err) + assert.Equal(running, expected) + } +@@ -205,7 +205,7 @@ func TestIsShimRunningTrue(t *testing.T) { + cmd := testRunSleep999AndGetCmd(t) + assert := assert.New(t) + +- testIsShimRunning(t, cmd.Process.Pid, true) ++ testIsShimRunning(t, cmd.Process.Pid, false) + + err := syscall.Kill(cmd.Process.Pid, syscall.SIGKILL) + assert.NoError(err) +@@ -216,7 +216,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) { + assert := assert.New(t) + + for _, val := range wrongValuesList { +- err := waitForShim(val) ++ err := waitForShim(val, containerID) + assert.NoError(err) + } + } +@@ -224,7 +224,7 @@ func TestWaitForShimInvalidPidSuccessful(t *testing.T) { + func TestWaitForShimNotRunningSuccessful(t *testing.T) { + pid := testRunSleep0AndGetPid(t) + assert := assert.New(t) +- assert.NoError(waitForShim(pid)) ++ assert.NoError(waitForShim(pid, containerID)) + } + + func TestWaitForShimRunningForTooLongFailure(t *testing.T) { +@@ -232,6 +232,6 @@ func TestWaitForShimRunningForTooLongFailure(t *testing.T) { + assert := assert.New(t) + + waitForShimTimeout = 0.1 +- assert.Error(waitForShim(cmd.Process.Pid)) ++ assert.NoError(waitForShim(cmd.Process.Pid, containerID)) + assert.NoError(syscall.Kill(cmd.Process.Pid, syscall.SIGKILL)) + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch b/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch new file mode 100644 index 0000000000000000000000000000000000000000..6a021bef6b8b5a5814dfa4d0a5ff23f856adbb75 --- /dev/null +++ b/runtime/patches/0011-kata-runtime-check-the-process-info-before-send-SIGK.patch @@ -0,0 +1,119 @@ +From 0aeff2632eac58eefdc8ae438891303332831ec5 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 28 Jul 2020 20:48:24 +0800 +Subject: [PATCH 11/50] kata-runtime: check the process info before send + SIGKILL + +reason: In order to avoid the pid reuse problem, check the +process info before send SIGKILL signal to process. + +Signed-off-by: jiangpengfei +--- + virtcontainers/kata_proxy.go | 18 ++++++++++++++++++ + virtcontainers/qemu.go | 5 +++++ + virtcontainers/shim.go | 9 +++++++++ + virtcontainers/shim_test.go | 8 ++++---- + 4 files changed, 36 insertions(+), 4 deletions(-) + +diff --git a/virtcontainers/kata_proxy.go b/virtcontainers/kata_proxy.go +index e04b4cff..ed272bad 100644 +--- a/virtcontainers/kata_proxy.go ++++ b/virtcontainers/kata_proxy.go +@@ -6,8 +6,12 @@ + package virtcontainers + + import ( ++ "fmt" + "os/exec" ++ "strings" + "syscall" ++ ++ "github.com/kata-containers/runtime/virtcontainers/utils" + ) + + // This is the Kata Containers implementation of the proxy interface. +@@ -61,6 +65,20 @@ func (p *kataProxy) start(params proxyParams) (int, string, error) { + + // stop is kataProxy stop implementation for proxy interface. + func (p *kataProxy) stop(pid int) error { ++ if pid <= 1 { ++ return nil ++ } ++ ++ // check process info before send SIGKILL signal ++ cmdline, err := utils.GetProcessCmdline(pid) ++ if err != nil { ++ return fmt.Errorf("get kata-proxy %d cmdline error: %v", pid, err) ++ } ++ ++ if !strings.Contains(cmdline, KataProxyProcessName) { ++ return fmt.Errorf("%d is not kata-proxy process, don't kill wrong process", pid) ++ } ++ + // Signal the proxy with SIGTERM. + return syscall.Kill(pid, syscall.SIGTERM) + } +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index 4b15d968..4789101d 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -967,6 +967,11 @@ func (q *qemu) stopSandbox(force bool) error { + return fmt.Errorf("force kill qemu process pid is invalid") + } + ++ cmdline, _ := utils.GetProcessCmdline(qemuMainPid) ++ if !strings.Contains(cmdline, string(QemuHypervisor)) { ++ return fmt.Errorf("force kill %d process is not qemu process, don't kill wrong process", qemuMainPid) ++ } ++ + _ = syscall.Kill(qemuMainPid, syscall.SIGKILL) + } + +diff --git a/virtcontainers/shim.go b/virtcontainers/shim.go +index 6f784a03..b192b258 100644 +--- a/virtcontainers/shim.go ++++ b/virtcontainers/shim.go +@@ -143,6 +143,15 @@ func stopShim(pid int) error { + return nil + } + ++ cmdline, err := utils.GetProcessCmdline(pid) ++ if err != nil { ++ return err ++ } ++ ++ if !strings.Contains(cmdline, "kata-shim") { ++ return fmt.Errorf("%d process is not kata-shim process, don't kill wrong process", pid) ++ } ++ + if err := signalShim(pid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { + return err + } +diff --git a/virtcontainers/shim_test.go b/virtcontainers/shim_test.go +index 62471311..dc15eab0 100644 +--- a/virtcontainers/shim_test.go ++++ b/virtcontainers/shim_test.go +@@ -176,16 +176,16 @@ func testRunSleep999AndGetCmd(t *testing.T) *exec.Cmd { + return cmd + } + +-func TestStopShimSuccessfulProcessNotRunning(t *testing.T) { ++func TestStopShimFailProcessNotRunning(t *testing.T) { + assert := assert.New(t) + pid := testRunSleep0AndGetPid(t) +- assert.NoError(stopShim(pid)) ++ assert.Error(stopShim(pid)) + } + +-func TestStopShimSuccessfulProcessRunning(t *testing.T) { ++func TestStopShimFailProcessRunning(t *testing.T) { + assert := assert.New(t) + cmd := testRunSleep999AndGetCmd(t) +- assert.NoError(stopShim(cmd.Process.Pid)) ++ assert.Error(stopShim(cmd.Process.Pid)) + } + + func testIsShimRunning(t *testing.T, pid int, expected bool) { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch b/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch new file mode 100644 index 0000000000000000000000000000000000000000..5fc9679f862fa2858fe33ef03150aac757077137 --- /dev/null +++ b/runtime/patches/0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch @@ -0,0 +1,37 @@ +From 441d80f55f4dc5efb4c92d91608a3c8db3d087cb Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 28 Jul 2020 21:43:15 +0800 +Subject: [PATCH 12/50] kata-runtime: truncate the log.json file before + kata-runtime subcommand executed + +reason: since we have redirect the kata-runtime log to /var/log/messages, and avoid the +path of log.json file to be large in the tmpfs, so we truncate the log.json file every +time before subcommand is executed. + +Signed-off-by: jiangpengfei +--- + cli/main.go | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/cli/main.go b/cli/main.go +index 1362a8fb..5eb2fb19 100644 +--- a/cli/main.go ++++ b/cli/main.go +@@ -300,6 +300,14 @@ func beforeSubcommands(c *cli.Context) error { + ignoreConfigLogs = true + } else { + if path := c.GlobalString("log"); path != "" { ++ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the ++ // path of log.json file to be large in the tmpfs, so we truncate the log.json file ++ // every time before subcommand is executed. ++ if path != "/dev/null" && katautils.FileExists(path) { ++ if err := os.Truncate(path, 0); err != nil { ++ return err ++ } ++ } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) + if err != nil { + return err +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch b/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch new file mode 100644 index 0000000000000000000000000000000000000000..a377bd893d47bae067998d79bf2a1d4016815fb7 --- /dev/null +++ b/runtime/patches/0013-kata-runtime-get-container-info-by-containerID-prefi.patch @@ -0,0 +1,98 @@ +From fd63d26a5b0542f35d61b0c19c80795f052b4518 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 28 Jul 2020 22:05:44 +0800 +Subject: [PATCH 13/50] kata-runtime: get container info by containerID prefix + +reason: get container info by containerID prefix, so we just +need to input the prefix containerID when call kata-runtime +subcommand + +Signed-off-by: jiangpengfei +--- + cli/oci.go | 35 +++++++++++++++++++++++++++++++++++ + pkg/katautils/oci.go | 5 +++++ + 2 files changed, 40 insertions(+) + +diff --git a/cli/oci.go b/cli/oci.go +index 8ffac2df..bf962d03 100644 +--- a/cli/oci.go ++++ b/cli/oci.go +@@ -9,6 +9,7 @@ import ( + "bufio" + "context" + "fmt" ++ "io/ioutil" + "net" + "os" + "path/filepath" +@@ -25,6 +26,8 @@ const ( + // Filesystem type corresponding to CGROUP_SUPER_MAGIC as listed + // here: http://man7.org/linux/man-pages/man2/statfs.2.html + cgroupFsType = 0x27e0eb ++ ++ maxIDLength = 64 + ) + + var cgroupsDirPath string +@@ -38,6 +41,14 @@ func getContainerInfo(ctx context.Context, containerID string) (vc.ContainerStat + return vc.ContainerStatus{}, "", fmt.Errorf("Missing container ID") + } + ++ if len(containerID) < maxIDLength { ++ fullContainerID, err := getContainerIDbyPrefix(containerID) ++ if err != nil { ++ return vc.ContainerStatus{}, "", err ++ } ++ containerID = fullContainerID ++ } ++ + sandboxID, err := katautils.FetchContainerIDMapping(containerID) + if err != nil { + return vc.ContainerStatus{}, "", err +@@ -211,3 +222,27 @@ func getCgroupsDirPath(mountInfoFile string) (string, error) { + + return cgroupRootPath, nil + } ++ ++func getContainerIDbyPrefix(prefix string) (string, error) { ++ files, err := ioutil.ReadDir(katautils.GetCtrsMapTreePath()) ++ if err != nil { ++ return "", err ++ } ++ ++ containers := []string{} ++ for _, file := range files { ++ if file.IsDir() && strings.HasPrefix(file.Name(), prefix) { ++ containers = append(containers, file.Name()) ++ } ++ } ++ ++ if len(containers) == 0 { ++ return "", fmt.Errorf("no such container ID (%v)", prefix) ++ } ++ ++ if len(containers) > 1 { ++ return "", fmt.Errorf("multiple containers found (%v)", prefix) ++ } ++ ++ return containers[0], nil ++} +diff --git a/pkg/katautils/oci.go b/pkg/katautils/oci.go +index 6de8101e..1334af35 100644 +--- a/pkg/katautils/oci.go ++++ b/pkg/katautils/oci.go +@@ -25,6 +25,11 @@ func SetCtrsMapTreePath(path string) { + ctrsMapTreePath = path + } + ++// GetCtrsMapTreePath return the containerID to SandboxID mapping dir ++func GetCtrsMapTreePath() string { ++ return ctrsMapTreePath ++} ++ + // doUpdatePath returns whether a ctrsMapTreePath needs to be updated with a rootless prefix + func doUpdatePath() bool { + return rootless.IsRootless() && !strings.HasPrefix(ctrsMapTreePath, rootless.GetRootlessDir()) +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch b/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch new file mode 100644 index 0000000000000000000000000000000000000000..3dc1a5fff40cd9bfc92fd57cca20543c93a0be70 --- /dev/null +++ b/runtime/patches/0014-kata-runtime-add-self-defined-annotations-framework.patch @@ -0,0 +1,76 @@ +From 1bd3cb85a1cf0e94b3280412d5fb47cecc4721fd Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 29 Jul 2020 10:42:49 +0800 +Subject: [PATCH 14/50] kata-runtime: add self defined annotations framework + +reason: add self defined annotations framework to pass some +self defined resource for sandbox + +Signed-off-by: jiangpengfei +--- + virtcontainers/pkg/oci/utils.go | 38 ++++++++++++++++++++++++++++++++++++++ + 1 file changed, 38 insertions(+) + +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index cd8d48ce..05181efd 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -33,6 +33,10 @@ type annotationContainerType struct { + containerType vc.ContainerType + } + ++type annotationHandler func(value string) error ++ ++var annotationHandlerList = map[string]annotationHandler{} ++ + var ( + // ErrNoLinux is an error for missing Linux sections in the OCI configuration file. + ErrNoLinux = errors.New("missing Linux section") +@@ -342,6 +346,10 @@ func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { + if err := addAgentConfigOverrides(ocispec, config); err != nil { + return err + } ++ ++ if err := addOtherSandboxAnnotation(ocispec, config); err != nil { ++ return err ++ } + return nil + } + +@@ -1016,3 +1024,33 @@ func GetOCIConfig(status vc.ContainerStatus) (specs.Spec, error) { + + return *status.Spec, nil + } ++ ++// validateOtherSandboxAnnotations validate the value of annotation ++func validateOtherSandboxAnnotations(annotation, value string) error { ++ validateHandler, ok := annotationHandlerList[annotation] ++ if !ok { ++ return fmt.Errorf("unsupport Sandbox annotation type") ++ } ++ ++ return validateHandler(value) ++} ++ ++// addOtherSandboxAnnotation add self defined annotation for sandbox ++func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { ++ otherSandboxAnnotationsKey := []string{} ++ ++ for _, a := range otherSandboxAnnotationsKey { ++ value, ok := ocispec.Annotations[a] ++ if !ok { ++ continue ++ } ++ ++ if err := validateOtherSandboxAnnotations(a, value); err != nil { ++ return err ++ } ++ ++ sbConfig.Annotations[a] = value ++ } ++ ++ return nil ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch b/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch new file mode 100644 index 0000000000000000000000000000000000000000..5624a46aaf04ff0e5af66a1876beefe45503b334 --- /dev/null +++ b/runtime/patches/0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch @@ -0,0 +1,143 @@ +From c0a33c4584e1fbd9b39ff1ca3ed632efe85de65e Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sun, 2 Aug 2020 15:51:10 +0800 +Subject: [PATCH 15/50] kata-runtime: add reuse hypervisor cpu and memory + feature + +reason: If default hypervisor cpu and memory is set too large, +which may waste resource, so we add enable_reuse_cpu_memory +config in the configuration.toml file to choose share the +hypervisor cpu and memory with container or not. + +Signed-off-by: jiangpengfei +--- + cli/config/configuration-qemu.toml.in | 5 +++++ + pkg/katautils/config.go | 2 ++ + virtcontainers/hypervisor.go | 3 +++ + virtcontainers/persist.go | 2 ++ + virtcontainers/persist/api/config.go | 3 +++ + virtcontainers/sandbox.go | 22 ++++++++++++++++++---- + 6 files changed, 33 insertions(+), 4 deletions(-) + +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index 46ce7d9b..82461732 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -95,6 +95,11 @@ default_memory = @DEFMEMSZ@ + # Default false + #enable_virtio_mem = true + ++# Specifies share hypervisor default cpu and memory resource ++# between hypervisor and container process. ++# Default false ++#enable_reuse_cpu_memory = false ++ + # Disable block device from being used for a container's rootfs. + # In case of a storage driver like devicemapper where a container's + # root file system is backed by a block device, the block device is passed +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 448d23ac..d1883c94 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -112,6 +112,7 @@ type hypervisor struct { + MemorySize uint32 `toml:"default_memory"` + MemSlots uint32 `toml:"memory_slots"` + MemOffset uint32 `toml:"memory_offset"` ++ EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"` + DefaultBridges uint32 `toml:"default_bridges"` + Msize9p uint32 `toml:"msize_9p"` + PCIeRootPort uint32 `toml:"pcie_root_port"` +@@ -640,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), + MemOffset: h.defaultMemOffset(), ++ EnableCPUMemoryReuse: h.EnableCPUMemoryReuse, + VirtioMem: h.VirtioMem, + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index fd7d1f8e..5f8d24f9 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -250,6 +250,9 @@ type HypervisorConfig struct { + // MemOffset specifies memory space for nvdimm device + MemOffset uint32 + ++ // Enable hypervisor cpu and memory resource share with container ++ EnableCPUMemoryReuse bool ++ + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + +diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go +index d96a3890..aef4d18d 100644 +--- a/virtcontainers/persist.go ++++ b/virtcontainers/persist.go +@@ -214,6 +214,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + Msize9p: sconfig.HypervisorConfig.Msize9p, + MemSlots: sconfig.HypervisorConfig.MemSlots, + MemOffset: sconfig.HypervisorConfig.MemOffset, ++ EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse, + VirtioMem: sconfig.HypervisorConfig.VirtioMem, + VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize, + KernelPath: sconfig.HypervisorConfig.KernelPath, +@@ -503,6 +504,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { + Msize9p: hconf.Msize9p, + MemSlots: hconf.MemSlots, + MemOffset: hconf.MemOffset, ++ EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse, + VirtioMem: hconf.VirtioMem, + VirtioFSCacheSize: hconf.VirtioFSCacheSize, + KernelPath: hconf.KernelPath, +diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go +index 34a5fd0f..a3c6ec91 100644 +--- a/virtcontainers/persist/api/config.go ++++ b/virtcontainers/persist/api/config.go +@@ -35,6 +35,9 @@ type HypervisorConfig struct { + // MemOffset specifies memory space for nvdimm device + MemOffset uint32 + ++ // Enable hypervisor cpu and memory resource share with container ++ EnableCPUMemoryReuse bool ++ + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 78188ed7..e766d1f7 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1833,12 +1833,26 @@ func (s *Sandbox) updateResources() error { + } + + sandboxVCPUs := s.calculateSandboxCPUs() +- // Add default vcpus for sandbox +- sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs ++ // Share the hypervisor vcpu with container process ++ if s.config.HypervisorConfig.EnableCPUMemoryReuse { ++ if sandboxVCPUs <= s.config.HypervisorConfig.NumVCPUs { ++ sandboxVCPUs = s.config.HypervisorConfig.NumVCPUs ++ } ++ } else { ++ // Add default vcpus for sandbox ++ sandboxVCPUs += s.hypervisor.hypervisorConfig().NumVCPUs ++ } + + sandboxMemoryByte := s.calculateSandboxMemory() +- // Add default / rsvd memory for sandbox. +- sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift ++ // Share the hypervisor memory with container process ++ if s.config.HypervisorConfig.EnableCPUMemoryReuse { ++ if sandboxMemoryByte <= (int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift) { ++ sandboxMemoryByte = int64(s.config.HypervisorConfig.MemorySize) << utils.MibToBytesShift ++ } ++ } else { ++ // Add default / rsvd memory for sandbox. ++ sandboxMemoryByte += int64(s.hypervisor.hypervisorConfig().MemorySize) << utils.MibToBytesShift ++ } + + // Update VCPUs + s.Logger().WithField("cpus-sandbox", sandboxVCPUs).Debugf("Request to hypervisor to update vCPUs") +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch b/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch new file mode 100644 index 0000000000000000000000000000000000000000..3d12a3fa8d663c272f5fad56184868bfa6a33b27 --- /dev/null +++ b/runtime/patches/0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch @@ -0,0 +1,288 @@ +From dc9de8bb181e2cec2f3e0a76d02833fef45b46af Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 6 Aug 2020 09:28:34 -0400 +Subject: [PATCH 16/50] virtcontainers: fix hotplug huge size memory cause + agent hang bug + +fixes: #2872 + +reason: If hotplug huge size memory into kata VM at once time, +guest kernel will allocate some extra memory for memory management, +which may cause kata-agent hang and out of responding. +And hotplug more memory into VM, more extra memory is needed. + +Inorder to solve this problem, we divide hotplug huge memory into +two steps. First, hotplug the max allowed memory into VM and wait +all first step hotplugged memory online. Second Step, hotplug the +left memory into VM. + +Signed-off-by: jiangpengfei +--- + virtcontainers/acrn.go | 4 ++++ + virtcontainers/agent.go | 3 ++- + virtcontainers/clh.go | 4 ++++ + virtcontainers/fc.go | 4 ++++ + virtcontainers/hypervisor.go | 3 +++ + virtcontainers/kata_agent.go | 4 ++-- + virtcontainers/kata_agent_test.go | 2 +- + virtcontainers/mock_hypervisor.go | 4 ++++ + virtcontainers/noop_agent.go | 2 +- + virtcontainers/qemu.go | 4 ++++ + virtcontainers/sandbox.go | 30 ++++++++++++++++++++++++++++-- + virtcontainers/sandbox_test.go | 12 ++++++++++++ + virtcontainers/utils/utils.go | 3 +++ + virtcontainers/vm.go | 2 +- + 14 files changed, 73 insertions(+), 8 deletions(-) + +diff --git a/virtcontainers/acrn.go b/virtcontainers/acrn.go +index 10cae06f..c9a0fe0b 100644 +--- a/virtcontainers/acrn.go ++++ b/virtcontainers/acrn.go +@@ -811,3 +811,7 @@ func (a *Acrn) loadInfo() error { + } + return nil + } ++ ++func (a *Acrn) getMemorySize() uint32 { ++ return a.config.MemorySize ++} +diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go +index be9526c7..b1dea816 100644 +--- a/virtcontainers/agent.go ++++ b/virtcontainers/agent.go +@@ -201,7 +201,8 @@ type agent interface { + // This function should be called after hot adding vCPUs or Memory. + // cpus specifies the number of CPUs that were added and the agent should online + // cpuOnly specifies that we should online cpu or online memory or both +- onlineCPUMem(cpus uint32, cpuOnly bool) error ++ // wait specifies that we should wait all cpu or memory online in the VM synchronously ++ onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error + + // memHotplugByProbe will notify the guest kernel about memory hotplug event through + // probe interface. +diff --git a/virtcontainers/clh.go b/virtcontainers/clh.go +index 59510b02..8afcd4bf 100644 +--- a/virtcontainers/clh.go ++++ b/virtcontainers/clh.go +@@ -1210,3 +1210,7 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) { + return info, openAPIClientError(err) + + } ++ ++func (clh *cloudHypervisor) getMemorySize() uint32 { ++ return clh.config.MemorySize ++} +diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go +index 72a8e192..15726156 100644 +--- a/virtcontainers/fc.go ++++ b/virtcontainers/fc.go +@@ -1212,3 +1212,7 @@ func (fc *firecracker) watchConsole() (*os.File, error) { + + return stdio, nil + } ++ ++func (fc *firecracker) getMemorySize() uint32 { ++ return fc.config.MemorySize ++} +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index 5f8d24f9..9cd685ad 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -778,6 +778,9 @@ type hypervisor interface { + hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) + resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) + resizeVCPUs(vcpus uint32) (uint32, uint32, error) ++ // getMemorySize return the total memory in the guest include default memory size + hot plugged memory ++ // return memory size unit is MB ++ getMemorySize() uint32 + getSandboxConsole(sandboxID string) (string, error) + disconnect() + capabilities() types.Capabilities +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 7575d326..8e073339 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -1806,9 +1806,9 @@ func (k *kataAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS + return err + } + +-func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { ++func (k *kataAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error { + req := &grpc.OnlineCPUMemRequest{ +- Wait: false, ++ Wait: wait, + NbCpus: cpus, + CpuOnly: cpuOnly, + } +diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go +index 62d31c93..2a2ddada 100644 +--- a/virtcontainers/kata_agent_test.go ++++ b/virtcontainers/kata_agent_test.go +@@ -324,7 +324,7 @@ func TestKataAgentSendReq(t *testing.T) { + err = k.resumeContainer(sandbox, Container{}) + assert.Nil(err) + +- err = k.onlineCPUMem(1, true) ++ err = k.onlineCPUMem(1, true, false) + assert.Nil(err) + + _, err = k.statsContainer(sandbox, Container{}) +diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go +index a5b67491..f1c6106d 100644 +--- a/virtcontainers/mock_hypervisor.go ++++ b/virtcontainers/mock_hypervisor.go +@@ -128,3 +128,7 @@ func (m *mockHypervisor) check() error { + func (m *mockHypervisor) generateSocket(id string, useVsock bool) (interface{}, error) { + return types.Socket{HostPath: "/tmp/socket", Name: "socket"}, nil + } ++ ++func (m *mockHypervisor) getMemorySize() uint32 { ++ return 0 ++} +diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go +index 8a7cd337..6e211bca 100644 +--- a/virtcontainers/noop_agent.go ++++ b/virtcontainers/noop_agent.go +@@ -102,7 +102,7 @@ func (n *noopAgent) memHotplugByProbe(addr uint64, sizeMB uint32, memorySectionS + } + + // onlineCPUMem is the Noop agent Container online CPU and Memory implementation. It does nothing. +-func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool) error { ++func (n *noopAgent) onlineCPUMem(cpus uint32, cpuOnly bool, wait bool) error { + return nil + } + +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index 4789101d..7bae3278 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -2273,3 +2273,7 @@ func (q *qemu) check() error { + func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) { + return generateVMSocket(id, useVsock, q.store.RunVMStoragePath()) + } ++ ++func (q *qemu) getMemorySize() uint32 { ++ return q.config.MemorySize + uint32(q.state.HotpluggedMemory) ++} +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index e766d1f7..a318d677 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1864,7 +1864,7 @@ func (s *Sandbox) updateResources() error { + // If the CPUs were increased, ask agent to online them + if oldCPUs < newCPUs { + vcpusAdded := newCPUs - oldCPUs +- if err := s.agent.onlineCPUMem(vcpusAdded, true); err != nil { ++ if err := s.agent.onlineCPUMem(vcpusAdded, true, false); err != nil { + return err + } + } +@@ -1872,6 +1872,20 @@ func (s *Sandbox) updateResources() error { + + // Update Memory + s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") ++ reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift) ++ currentMemMB := s.hypervisor.getMemorySize() ++ ++ // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime, ++ // inorder to avoid hotplug memory oom problem, we need to hotplug large memory ++ // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into ++ // guest and wait all hotplug memory online. Then, hotplug the left unplugged memory ++ // into the guest ++ if currentMemMB < reqMemMB && (reqMemMB-currentMemMB) > utils.MaxHotplugMemMBOnceTime { ++ if err := s.beforeHotplugHugeMem(currentMemMB); err != nil { ++ return err ++ } ++ } ++ + newMemory, updatedMemoryDevice, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) + if err != nil { + return err +@@ -1884,7 +1898,7 @@ func (s *Sandbox) updateResources() error { + return err + } + } +- if err := s.agent.onlineCPUMem(0, false); err != nil { ++ if err := s.agent.onlineCPUMem(0, false, false); err != nil { + return err + } + return nil +@@ -1926,6 +1940,18 @@ func (s *Sandbox) calculateSandboxCPUs() uint32 { + return utils.CalculateVCpusFromMilliCpus(mCPU) + } + ++func (s *Sandbox) beforeHotplugHugeMem(currentMemSizeInMB uint32) error { ++ wantedTotalMemSize := currentMemSizeInMB + utils.MaxHotplugMemMBOnceTime ++ newMemory, _, err := s.hypervisor.resizeMemory(wantedTotalMemSize, s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) ++ if err != nil { ++ return err ++ } ++ ++ s.Logger().Debugf("first part hotplug memory size: %d MB", newMemory) ++ // wait all first part hotplugged memory online in the guest ++ return s.agent.onlineCPUMem(0, false, true) ++} ++ + // GetHypervisorType is used for getting Hypervisor name currently used. + // Sandbox implement DeviceReceiver interface from device/api/interface.go + func (s *Sandbox) GetHypervisorType() string { +diff --git a/virtcontainers/sandbox_test.go b/virtcontainers/sandbox_test.go +index 85c712e8..4b02b3f3 100644 +--- a/virtcontainers/sandbox_test.go ++++ b/virtcontainers/sandbox_test.go +@@ -25,6 +25,7 @@ import ( + "github.com/kata-containers/runtime/virtcontainers/persist/fs" + "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "golang.org/x/sys/unix" +@@ -1522,6 +1523,17 @@ func TestSandboxUpdateResources(t *testing.T) { + } + err = s.updateResources() + assert.NoError(t, err) ++ ++ // add a container with huge memory equal utils.MaxHotplugMemMBOnceTime ++ contConfig3 := newTestContainerConfigNoop("cont-00003") ++ contConfig3.Resources.Memory = &specs.LinuxMemory{ ++ Limit: new(int64), ++ } ++ container3MemLimitInBytes := int64(utils.MaxHotplugMemMBOnceTime << utils.MibToBytesShift) ++ contConfig3.Resources.Memory.Limit = &container3MemLimitInBytes ++ s.config.Containers = append(s.config.Containers, contConfig3) ++ err = s.updateResources() ++ assert.NoError(t, err) + } + + func TestSandboxExperimentalFeature(t *testing.T) { +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 2b555ebb..3ae95aef 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -25,6 +25,9 @@ const fileMode0755 = os.FileMode(0755) + // MibToBytesShift the number to shift needed to convert MiB to Bytes + const MibToBytesShift = 20 + ++// Max Hotplug Memory size at once time, unit is MB ++const MaxHotplugMemMBOnceTime = 32 * 1024 ++ + // MaxSocketPathLen is the effective maximum Unix domain socket length. + // + // See unix(7). +diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go +index 8d27b1fe..2e5fef44 100644 +--- a/virtcontainers/vm.go ++++ b/virtcontainers/vm.go +@@ -370,7 +370,7 @@ func (v *VM) AddMemory(numMB uint32) error { + // OnlineCPUMemory puts the hotplugged CPU and memory online. + func (v *VM) OnlineCPUMemory() error { + v.logger().Infof("online CPU %d and memory", v.cpuDelta) +- err := v.agent.onlineCPUMem(v.cpuDelta, false) ++ err := v.agent.onlineCPUMem(v.cpuDelta, false, false) + if err == nil { + v.cpuDelta = 0 + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch b/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch new file mode 100644 index 0000000000000000000000000000000000000000..62b2847252784023b3d84b33253dec41f2536988 --- /dev/null +++ b/runtime/patches/0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch @@ -0,0 +1,355 @@ +From e6051eb1a8c54b91c46f68eab9a63ad0e73a9221 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Fri, 7 Aug 2020 20:01:15 +0800 +Subject: [PATCH 17/50] kata-runtime: validate sandbox cpu and memory size + +reason: +1. add more strict sandbox cpu and memory size check. +2. use GetPhysicalCPUNumber func to replace goruntime.NumCPU(), +because goruntime.NumCPU() return the CPU number of cpuset cgroup +that current process joined, however kata-runtime process will +change the cpuset cgroup when setupSanboxCgroup is called, which +may cause goruntime.NumCPU() return value changed. + +Signed-off-by: jiangpengfei +--- + pkg/katautils/config.go | 39 ++++++++++++++++++++++++++++++++++--- + pkg/katautils/config_test.go | 11 ++++++++--- + virtcontainers/pkg/oci/utils.go | 11 +++++------ + virtcontainers/qemu_arm64.go | 4 ++-- + virtcontainers/qemu_arm64_test.go | 4 ++-- + virtcontainers/sandbox.go | 7 ++++++- + virtcontainers/utils/utils.go | 41 +++++++++++++++++++++++++++++++++++++++ + 7 files changed, 100 insertions(+), 17 deletions(-) + +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index d1883c94..b9b08469 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -11,7 +11,6 @@ import ( + "fmt" + "io/ioutil" + "path/filepath" +- goruntime "runtime" + "strings" + + "github.com/BurntSushi/toml" +@@ -285,7 +284,7 @@ func (h hypervisor) GetEntropySource() string { + } + + func (h hypervisor) defaultVCPUs() uint32 { +- numCPUs := goruntime.NumCPU() ++ numCPUs := utils.GetPhysicalCPUNumber() + + if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) { + return uint32(numCPUs) +@@ -297,8 +296,22 @@ func (h hypervisor) defaultVCPUs() uint32 { + return uint32(h.NumVCPUs) + } + ++func (h hypervisor) checkVCPUs() error { ++ numCPUs := utils.GetPhysicalCPUNumber() ++ ++ if h.NumVCPUs <= 0 { ++ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0") ++ } ++ ++ if h.NumVCPUs > int32(numCPUs) { ++ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs) ++ } ++ ++ return nil ++} ++ + func (h hypervisor) defaultMaxVCPUs() uint32 { +- numcpus := uint32(goruntime.NumCPU()) ++ numcpus := uint32(utils.GetPhysicalCPUNumber()) + maxvcpus := vc.MaxQemuVCPUs() + reqVCPUs := h.DefaultMaxVCPUs + +@@ -324,6 +337,18 @@ func (h hypervisor) defaultMemSz() uint32 { + return h.MemorySize + } + ++func (h hypervisor) checkMemSz() error { ++ if h.MemorySize < utils.MinMemorySizeInMB { ++ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB) ++ } ++ ++ if h.MemorySize > utils.MaxMemorySizeInMB { ++ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB) ++ } ++ ++ return nil ++} ++ + func (h hypervisor) defaultMemSlots() uint32 { + slots := h.MemSlots + if slots == 0 { +@@ -627,6 +652,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + } + } + ++ if err = h.checkVCPUs(); err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ ++ if err = h.checkMemSz(); err != nil { ++ return vc.HypervisorConfig{}, err ++ } ++ + return vc.HypervisorConfig{ + HypervisorPath: hypervisor, + KernelPath: kernel, +diff --git a/pkg/katautils/config_test.go b/pkg/katautils/config_test.go +index 31afcca6..37aa16d0 100644 +--- a/pkg/katautils/config_test.go ++++ b/pkg/katautils/config_test.go +@@ -14,7 +14,6 @@ import ( + "path" + "path/filepath" + "reflect" +- goruntime "runtime" + "strings" + "syscall" + "testing" +@@ -152,7 +151,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), + HypervisorMachineType: machineType, + NumVCPUs: defaultVCPUCount, +- DefaultMaxVCPUs: uint32(goruntime.NumCPU()), ++ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()), + MemorySize: defaultMemSize, + DisableBlockDeviceUse: disableBlockDevice, + BlockDeviceDriver: defaultBlockDeviceDriver, +@@ -727,6 +726,8 @@ func TestMinimalRuntimeConfigWithVsock(t *testing.T) { + [hypervisor.qemu] + use_vsock = true + image = "` + imagePath + `" ++ default_vcpus = 1 ++ default_memory = 1024 + + [proxy.kata] + path = "` + proxyPath + `" +@@ -786,6 +787,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) { + utils.VHostVSockDevicePath = orgVHostVSockDevicePath + }() + utils.VHostVSockDevicePath = "/dev/abc/xyz" ++ defaultVCPUs := int32(1) ++ defaultMemory := uint32(1024) + + hypervisor := hypervisor{ + Path: hypervisorPath, +@@ -797,6 +800,8 @@ func TestNewQemuHypervisorConfig(t *testing.T) { + HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, + PCIeRootPort: pcieRootPort, + UseVSock: true, ++ NumVCPUs: defaultVCPUs, ++ MemorySize: defaultMemory, + } + + files := []string{hypervisorPath, kernelPath, imagePath} +@@ -996,7 +1001,7 @@ func TestNewShimConfig(t *testing.T) { + func TestHypervisorDefaults(t *testing.T) { + assert := assert.New(t) + +- numCPUs := goruntime.NumCPU() ++ numCPUs := utils.GetPhysicalCPUNumber() + + h := hypervisor{} + +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 05181efd..0a6f08c7 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -10,22 +10,21 @@ import ( + "errors" + "fmt" + "path/filepath" +- goruntime "runtime" + "strconv" + "strings" + "syscall" + + criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" + crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" +- specs "github.com/opencontainers/runtime-spec/specs-go" +- "github.com/sirupsen/logrus" +- + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/config" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" + vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" + dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" ++ specs "github.com/opencontainers/runtime-spec/specs-go" ++ "github.com/sirupsen/logrus" + ) + + type annotationContainerType struct { +@@ -560,7 +559,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e + return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err) + } + +- numCPUs := goruntime.NumCPU() ++ numCPUs := utils.GetPhysicalCPUNumber() + + if uint32(vcpus) > uint32(numCPUs) { + return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs) +@@ -575,7 +574,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e + return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err) + } + +- numCPUs := goruntime.NumCPU() ++ numCPUs := utils.GetPhysicalCPUNumber() + max := uint32(maxVCPUs) + + if max > uint32(numCPUs) { +diff --git a/virtcontainers/qemu_arm64.go b/virtcontainers/qemu_arm64.go +index 6d089cf0..40fb2a80 100644 +--- a/virtcontainers/qemu_arm64.go ++++ b/virtcontainers/qemu_arm64.go +@@ -8,11 +8,11 @@ package virtcontainers + import ( + "context" + "io/ioutil" +- "runtime" + "strings" + "time" + + govmmQemu "github.com/intel/govmm/qemu" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/sirupsen/logrus" + ) + +@@ -121,7 +121,7 @@ func MaxQemuVCPUs() uint32 { + if hostGICVersion != 0 { + return gicList[hostGICVersion] + } +- return uint32(runtime.NumCPU()) ++ return uint32(utils.GetPhysicalCPUNumber()) + } + + func newQemuArch(config HypervisorConfig) qemuArch { +diff --git a/virtcontainers/qemu_arm64_test.go b/virtcontainers/qemu_arm64_test.go +index 40351158..372fc4a9 100644 +--- a/virtcontainers/qemu_arm64_test.go ++++ b/virtcontainers/qemu_arm64_test.go +@@ -10,10 +10,10 @@ import ( + "io/ioutil" + "os" + "path/filepath" +- "runtime" + "testing" + + govmmQemu "github.com/intel/govmm/qemu" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/stretchr/testify/assert" + ) + +@@ -63,7 +63,7 @@ func TestMaxQemuVCPUs(t *testing.T) { + } + + data := []testData{ +- {"", uint32(runtime.NumCPU())}, ++ {"", uint32(utils.GetPhysicalCPUNumber())}, + {" 1: 0 0 GICv2 25 Level vgic \n", uint32(8)}, + {" 1: 0 0 GICv3 25 Level vgic \n", uint32(123)}, + {" 1: 0 0 GICv4 25 Level vgic \n", uint32(123)}, +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index a318d677..a8522b96 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1875,6 +1875,11 @@ func (s *Sandbox) updateResources() error { + reqMemMB := uint32(sandboxMemoryByte >> utils.MibToBytesShift) + currentMemMB := s.hypervisor.getMemorySize() + ++ // check request sandbox memory size larger than utils.MaxMemorySizInMB or not ++ if reqMemMB > utils.MaxMemorySizeInMB { ++ return fmt.Errorf("updateResources failed! Sandbox memory should <= %d MiB", utils.MaxMemorySizeInMB) ++ } ++ + // If request hotplug memory size larger than utils.MaxHotplugMemMBOnceTime, + // inorder to avoid hotplug memory oom problem, we need to hotplug large memory + // with two steps. First, hotplug utils.MaxHotplugMemMBOnceTime size memory into +@@ -1898,7 +1903,7 @@ func (s *Sandbox) updateResources() error { + return err + } + } +- if err := s.agent.onlineCPUMem(0, false, false); err != nil { ++ if err := s.agent.onlineCPUMem(0, false, true); err != nil { + return err + } + return nil +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 3ae95aef..5d38e594 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -6,6 +6,7 @@ + package utils + + import ( ++ "bufio" + "crypto/rand" + "errors" + "fmt" +@@ -28,11 +29,26 @@ const MibToBytesShift = 20 + // Max Hotplug Memory size at once time, unit is MB + const MaxHotplugMemMBOnceTime = 32 * 1024 + ++const ( ++ // Min needed memory size to start a Kata VM ++ MinMemorySizeInMB = 300 ++ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift ++ ++ // Max support memory size in the Kata VM ++ MaxMemorySizeInMB = 512 * 1024 ++ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift ++) ++ + // MaxSocketPathLen is the effective maximum Unix domain socket length. + // + // See unix(7). + const MaxSocketPathLen = 107 + ++const ( ++ procCPUInfoPath = "/proc/cpuinfo" ++ processorIdentifier = "processor" ++) ++ + // VHostVSockDevicePath path to vhost-vsock device + var VHostVSockDevicePath = "/dev/vhost-vsock" + +@@ -324,3 +340,28 @@ func IsProcessRunning(pid int, processName string, sandboxID string) bool { + + return false + } ++ ++// GetPhysicalCPUNumber return the number of the CPUs in the physical machine ++func GetPhysicalCPUNumber() int { ++ f, err := os.Open(procCPUInfoPath) ++ if err != nil { ++ return 0 ++ } ++ defer f.Close() ++ ++ cpuNum := 0 ++ s := bufio.NewScanner(f) ++ for s.Scan() { ++ if err := s.Err(); err != nil { ++ return 0 ++ } ++ ++ fields := strings.Fields(s.Text()) ++ if len(fields) > 0 { ++ if fields[0] == processorIdentifier { ++ cpuNum++ ++ } ++ } ++ } ++ return cpuNum ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch b/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch new file mode 100644 index 0000000000000000000000000000000000000000..d7f0d10c25f2713fa8829da986234bffcf5003f7 --- /dev/null +++ b/runtime/patches/0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch @@ -0,0 +1,50 @@ +From c785f8f744050155102664d56de5bfb55e91915d Mon Sep 17 00:00:00 2001 +From: Evan Foster +Date: Mon, 13 Jul 2020 12:53:40 -0600 +Subject: [PATCH 18/50] sandbox: Stop and clean up containers that fail to + create + +A container that is created and added to a sandbox can still fail +the final creation steps. In this case, the container must be stopped +and have its resources cleaned up to prevent leaking sandbox mounts. + +Fixes #2816 + +cherry-pick from: https://github.com/kata-containers/runtime/pull/2826 + +Signed-off-by: Evan Foster +Signed-off-by: jiangpengfei +--- + virtcontainers/sandbox.go | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index a8522b96..3dbf640e 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1,4 +1,5 @@ + // Copyright (c) 2016 Intel Corporation ++// Copyright (c) 2020 Adobe Inc. + // + // SPDX-License-Identifier: Apache-2.0 + // +@@ -1172,6 +1173,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro + defer func() { + // Rollback if error happens. + if err != nil { ++ logger := s.Logger().WithFields(logrus.Fields{"container-id": c.id, "sandox-id": s.id, "rollback": true}) ++ ++ logger.Warning("Cleaning up partially created container") ++ ++ if err2 := c.stop(true); err2 != nil { ++ logger.WithError(err2).Warning("Could not delete container") ++ } ++ ++ logger.Debug("Removing stopped container from sandbox store") ++ + s.removeContainer(c.id) + } + }() +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch b/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..1bee93bae88859fa0c2ebae89c647787c68228c8 --- /dev/null +++ b/runtime/patches/0019-virtcontainers-fix-delete-sandbox-failed-problem.patch @@ -0,0 +1,33 @@ +From d20cb25c8a145e1d3e64eefa69242d87b7a67f92 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sat, 8 Aug 2020 16:00:45 -0400 +Subject: [PATCH 19/50] virtcontainers: fix delete sandbox failed problem + +fixes: #2882 + +reason: If error happens after container create and before sandbox +updateResouce in the `CreateContainer()`, then delete sandbox +forcefully will return error because s.config.Containers config not +flushed into persist store. + +Signed-off-by: jiangpengfei +--- + virtcontainers/sandbox.go | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 3dbf640e..7322ef58 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1156,6 +1156,8 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro + if len(s.config.Containers) > 0 { + // delete container config + s.config.Containers = s.config.Containers[:len(s.config.Containers)-1] ++ // need to flush change to persist storage ++ _ = s.storeSandbox() + } + } + }() +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch b/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch new file mode 100644 index 0000000000000000000000000000000000000000..b1591883ced811ab270c7e65d7956230761c6683 --- /dev/null +++ b/runtime/patches/0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch @@ -0,0 +1,188 @@ +From 75141529674545a2f84b01c730f614901ad2265e Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 10 Aug 2020 10:08:47 +0800 +Subject: [PATCH 20/50] virtcontainers: add enable_cpu_memory_hotplug config in + the configuration.toml + +reason: add enable_cpu_memory_hotplug config to control whether enable hotplug +memory and cpu resource into the Kata VM. If we can calculate the whole sandbox +resource usage already, we just need to pass the value of resouces by annotation +without hotplugging. + +Signed-off-by: jiangpengfei +--- + cli/config/configuration-qemu.toml.in | 4 ++++ + pkg/katautils/config.go | 2 ++ + virtcontainers/container.go | 6 ++++-- + virtcontainers/hypervisor.go | 3 +++ + virtcontainers/persist.go | 2 ++ + virtcontainers/persist/api/config.go | 3 +++ + virtcontainers/sandbox.go | 38 +++++++++++++++++++++-------------- + 7 files changed, 41 insertions(+), 17 deletions(-) + +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index 82461732..b44e84d8 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -100,6 +100,10 @@ default_memory = @DEFMEMSZ@ + # Default false + #enable_reuse_cpu_memory = false + ++# If enabled, the runtime will support cpu and memory hot plug ++# (default: disabled) ++#enable_cpu_memory_hotplug = true ++ + # Disable block device from being used for a container's rootfs. + # In case of a storage driver like devicemapper where a container's + # root file system is backed by a block device, the block device is passed +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index b9b08469..9a99b9d4 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -112,6 +112,7 @@ type hypervisor struct { + MemSlots uint32 `toml:"memory_slots"` + MemOffset uint32 `toml:"memory_offset"` + EnableCPUMemoryReuse bool `toml:"enable_reuse_cpu_memory"` ++ EnableCPUMemoryHotPlug bool `toml:"enable_cpu_memory_hotplug"` + DefaultBridges uint32 `toml:"default_bridges"` + Msize9p uint32 `toml:"msize_9p"` + PCIeRootPort uint32 `toml:"pcie_root_port"` +@@ -675,6 +676,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + MemSlots: h.defaultMemSlots(), + MemOffset: h.defaultMemOffset(), + EnableCPUMemoryReuse: h.EnableCPUMemoryReuse, ++ EnableCPUMemoryHotPlug: h.EnableCPUMemoryHotPlug, + VirtioMem: h.VirtioMem, + EntropySource: h.GetEntropySource(), + DefaultBridges: h.defaultBridges(), +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 75f590eb..1b89f6ac 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1273,8 +1273,10 @@ func (c *Container) update(resources specs.LinuxResources) error { + c.config.Resources.Memory.Limit = mem.Limit + } + +- if err := c.sandbox.updateResources(); err != nil { +- return err ++ if c.sandbox.config.HypervisorConfig.EnableCPUMemoryHotPlug { ++ if err := c.sandbox.updateResources(); err != nil { ++ return err ++ } + } + + if !c.sandbox.config.SandboxCgroupOnly { +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index 9cd685ad..c0723daa 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -253,6 +253,9 @@ type HypervisorConfig struct { + // Enable hypervisor cpu and memory resource share with container + EnableCPUMemoryReuse bool + ++ // Enable hotplug cpu and memory resource into container ++ EnableCPUMemoryHotPlug bool ++ + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + +diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go +index aef4d18d..6bd09a0b 100644 +--- a/virtcontainers/persist.go ++++ b/virtcontainers/persist.go +@@ -215,6 +215,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + MemSlots: sconfig.HypervisorConfig.MemSlots, + MemOffset: sconfig.HypervisorConfig.MemOffset, + EnableCPUMemoryReuse: sconfig.HypervisorConfig.EnableCPUMemoryReuse, ++ EnableCPUMemoryHotPlug: sconfig.HypervisorConfig.EnableCPUMemoryHotPlug, + VirtioMem: sconfig.HypervisorConfig.VirtioMem, + VirtioFSCacheSize: sconfig.HypervisorConfig.VirtioFSCacheSize, + KernelPath: sconfig.HypervisorConfig.KernelPath, +@@ -505,6 +506,7 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { + MemSlots: hconf.MemSlots, + MemOffset: hconf.MemOffset, + EnableCPUMemoryReuse: hconf.EnableCPUMemoryReuse, ++ EnableCPUMemoryHotPlug: hconf.EnableCPUMemoryHotPlug, + VirtioMem: hconf.VirtioMem, + VirtioFSCacheSize: hconf.VirtioFSCacheSize, + KernelPath: hconf.KernelPath, +diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go +index a3c6ec91..cfbee849 100644 +--- a/virtcontainers/persist/api/config.go ++++ b/virtcontainers/persist/api/config.go +@@ -38,6 +38,9 @@ type HypervisorConfig struct { + // Enable hypervisor cpu and memory resource share with container + EnableCPUMemoryReuse bool + ++ // Enable hotplug cpu and memory resource into container ++ EnableCPUMemoryHotPlug bool ++ + // VirtioFSCacheSize is the DAX cache size in MiB + VirtioFSCacheSize uint32 + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 7322ef58..8fcd92d4 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1189,12 +1189,16 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro + } + }() + +- // Sandbox is reponsable to update VM resources needed by Containers +- // Update resources after having added containers to the sandbox, since +- // container status is requiered to know if more resources should be added. +- err = s.updateResources() +- if err != nil { +- return nil, err ++ // update sandbox resource only when enable_cpu_memory_hotplug config set true ++ // in the config.toml file ++ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { ++ // Sandbox is reponsable to update VM resources needed by Containers ++ // Update resources after having added containers to the sandbox, since ++ // container status is requiered to know if more resources should be added. ++ err = s.updateResources() ++ if err != nil { ++ return nil, err ++ } + } + + if err = s.cgroupsUpdate(); err != nil { +@@ -1228,11 +1232,13 @@ func (s *Sandbox) StartContainer(containerID string) (VCContainer, error) { + + s.Logger().Info("Container is started") + +- // Update sandbox resources in case a stopped container +- // is started +- err = s.updateResources() +- if err != nil { +- return nil, err ++ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { ++ // Update sandbox resources in case a stopped container ++ // is started ++ err = s.updateResources() ++ if err != nil { ++ return nil, err ++ } + } + + return c, nil +@@ -1503,10 +1509,12 @@ func (s *Sandbox) createContainers() error { + } + } + +- // Update resources after having added containers to the sandbox, since +- // container status is requiered to know if more resources should be added. +- if err := s.updateResources(); err != nil { +- return err ++ if s.config.HypervisorConfig.EnableCPUMemoryHotPlug { ++ // Update resources after having added containers to the sandbox, since ++ // container status is requiered to know if more resources should be added. ++ if err := s.updateResources(); err != nil { ++ return err ++ } + } + + if err := s.cgroupsUpdate(); err != nil { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch b/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch new file mode 100644 index 0000000000000000000000000000000000000000..eae49d4e5da9c0f829c94a15a432e23005d9f274 --- /dev/null +++ b/runtime/patches/0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch @@ -0,0 +1,195 @@ +From 79cf2f5a52af51d8a62353a99e894808281769e2 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 10 Aug 2020 11:25:02 +0800 +Subject: [PATCH 21/50] kata-runtime: add sandbox_cpu and sandbox_mem + annotations + +reason: add sandbox_cpu and sandbox_men annotations to set +Kata VM vCPU number and memory size. + +Signed-off-by: jiangpengfei +--- + virtcontainers/pkg/annotations/annotations.go | 6 +++++ + virtcontainers/pkg/oci/utils.go | 39 +++++++++++++++++++++++++-- + virtcontainers/sandbox.go | 32 ++++++++++++++++++++++ + virtcontainers/utils/utils.go | 14 ++++++++++ + 4 files changed, 89 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go +index 10ce7833..903c7f03 100644 +--- a/virtcontainers/pkg/annotations/annotations.go ++++ b/virtcontainers/pkg/annotations/annotations.go +@@ -251,6 +251,12 @@ const ( + ContainerPipeSizeKernelParam = "agent." + ContainerPipeSizeOption + ) + ++// iSula self defined annotations ++const ( ++ StaticCPUTypeKey = kataAnnotationsPrefix + "sandbox_cpu" ++ StaticMemTypeKey = kataAnnotationsPrefix + "sandbox_mem" ++) ++ + const ( + // SHA512 is the SHA-512 (64) hash algorithm + SHA512 string = "sha512" +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 0a6f08c7..36c730b7 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -16,6 +16,7 @@ import ( + + criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" + crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" ++ "github.com/docker/go-units" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/config" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" +@@ -34,7 +35,10 @@ type annotationContainerType struct { + + type annotationHandler func(value string) error + +-var annotationHandlerList = map[string]annotationHandler{} ++var annotationHandlerList = map[string]annotationHandler{ ++ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, ++ vcAnnotations.StaticMemTypeKey: validateSandboxMem, ++} + + var ( + // ErrNoLinux is an error for missing Linux sections in the OCI configuration file. +@@ -1036,7 +1040,10 @@ func validateOtherSandboxAnnotations(annotation, value string) error { + + // addOtherSandboxAnnotation add self defined annotation for sandbox + func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { +- otherSandboxAnnotationsKey := []string{} ++ otherSandboxAnnotationsKey := []string{ ++ vcAnnotations.StaticCPUTypeKey, ++ vcAnnotations.StaticMemTypeKey, ++ } + + for _, a := range otherSandboxAnnotationsKey { + value, ok := ocispec.Annotations[a] +@@ -1053,3 +1060,31 @@ func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e + + return nil + } ++ ++func validateSandboxCPU(value string) error { ++ // check min cpu value ++ cpus, err := utils.RoundVCPUNumber(value) ++ if err != nil { ++ return fmt.Errorf("valiate sandbox_cpu annotation fail: %v", err) ++ } ++ ++ maxPhysicalCPUs := utils.GetPhysicalCPUNumber() ++ if cpus > maxPhysicalCPUs { ++ return fmt.Errorf("sandbox_cpu annotation value exceed the machine max CPU number: %d", cpus) ++ } ++ ++ return nil ++} ++ ++func validateSandboxMem(value string) error { ++ memSizeInBytes, err := units.RAMInBytes(value) ++ if err != nil { ++ return fmt.Errorf("parse sandbox_mem value: %d fail: %v", memSizeInBytes, err) ++ } ++ ++ if memSizeInBytes < utils.MinMemorySizeInByte || memSizeInBytes > utils.MaxMemorySizeInByte { ++ return fmt.Errorf("invalid sandbox_mem value size in bytes: %v", memSizeInBytes) ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 8fcd92d4..ba704249 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -20,6 +20,7 @@ import ( + + "github.com/containerd/cgroups" + "github.com/containernetworking/plugins/pkg/ns" ++ "github.com/docker/go-units" + "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +@@ -479,6 +480,10 @@ func createSandbox(ctx context.Context, sandboxConfig SandboxConfig, factory Fac + return nil, err + } + ++ if err := updateStaticSandboxResources(&sandboxConfig); err != nil { ++ return nil, err ++ } ++ + s, err := newSandbox(ctx, sandboxConfig, factory) + if err != nil { + return nil, err +@@ -2359,3 +2364,30 @@ func (s *Sandbox) setContainersState(state types.StateString) error { + + return nil + } ++ ++// updateStaticSandboxResources update sandbox's cpu and memory resource passed by ++// sandbox_cpu and sandbox_mem annotations ++func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { ++ // update cpu resource ++ if cpuNumVal, ok := sandboxConfig.Annotations[annotations.StaticCPUTypeKey]; ok { ++ cpuNum, err := utils.RoundVCPUNumber(cpuNumVal) ++ if err != nil { ++ return err ++ } ++ ++ sandboxConfig.HypervisorConfig.NumVCPUs = (uint32)(cpuNum) ++ } ++ ++ // update mem resource ++ if memVal, ok := sandboxConfig.Annotations[annotations.StaticMemTypeKey]; ok { ++ memSizeInBytes, err := units.RAMInBytes(memVal) ++ if err != nil { ++ return err ++ } ++ ++ memSizeInMB := memSizeInBytes >> utils.MibToBytesShift ++ sandboxConfig.HypervisorConfig.MemorySize = (uint32)(memSizeInMB) ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 5d38e594..9490faa1 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -11,6 +11,7 @@ import ( + "errors" + "fmt" + "io/ioutil" ++ "math" + "os" + "os/exec" + "path/filepath" +@@ -30,6 +31,9 @@ const MibToBytesShift = 20 + const MaxHotplugMemMBOnceTime = 32 * 1024 + + const ( ++ // minCPUs is allowed minimum CPU ++ minCPUs = 0.25 ++ + // Min needed memory size to start a Kata VM + MinMemorySizeInMB = 300 + MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift +@@ -365,3 +369,13 @@ func GetPhysicalCPUNumber() int { + } + return cpuNum + } ++ ++func RoundVCPUNumber(value string) (int, error) { ++ cpuNum, err := strconv.ParseFloat(value, 64) ++ if err != nil || cpuNum < minCPUs { ++ return 0, fmt.Errorf("invalid sandbox cpu number: %v", cpuNum) ++ } ++ ++ cpus := int(math.Ceil(cpuNum)) ++ return cpus, nil ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch b/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch new file mode 100644 index 0000000000000000000000000000000000000000..025f3ff52bfcaef31f075adc837bc99a81ecbbf6 --- /dev/null +++ b/runtime/patches/0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch @@ -0,0 +1,39 @@ +From e5e3232f7268110f7e3e3c4814eab31a6704b672 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 10 Aug 2020 20:11:07 +0800 +Subject: [PATCH 22/50] kata-runtime: skip go version check and do not build + containerd-shim-v2 + +reason: skip go version check and do not build containerd-shim-v2 +because iSulad current not support shimV2 + +Signed-off-by: jiangpengfei +--- + Makefile | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Makefile b/Makefile +index 14a0ea47..d5e4bbe1 100644 +--- a/Makefile ++++ b/Makefile +@@ -12,7 +12,7 @@ for file in /etc/os-release /usr/lib/os-release; do \ + fi \ + done) + +-SKIP_GO_VERSION_CHECK= ++SKIP_GO_VERSION_CHECK=y + include golang.mk + + #Get ARCH. +@@ -503,7 +503,7 @@ define SHOW_ARCH + $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)","")) + endef + +-all: runtime containerd-shim-v2 netmon ++all: runtime netmon + + # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a + # git revision string. They will only be rebuilt if the revision string +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch b/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch new file mode 100644 index 0000000000000000000000000000000000000000..ac1c9a11de02f803f2a0e8533d9d9ca86dfd1f9b --- /dev/null +++ b/runtime/patches/0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch @@ -0,0 +1,42 @@ +From 226c3336dcc70bd17e3471ff98106a2f8dee9ac5 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 11 Aug 2020 22:32:49 +0800 +Subject: [PATCH 23/50] kata-runtime: set PCIBridgeMaxCapacity limit to 25 + +reason: set PCIBridgeMaxCapacity limit to 25. + +Signed-off-by: jiangpengfei +--- + virtcontainers/qemu_arch_base_test.go | 2 +- + virtcontainers/types/bridges.go | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go +index 169e002e..8219f3c5 100644 +--- a/virtcontainers/qemu_arch_base_test.go ++++ b/virtcontainers/qemu_arch_base_test.go +@@ -175,7 +175,7 @@ func TestQemuAddDeviceToBridge(t *testing.T) { + } + + // fail to add device to bridge cause no more available bridge slot +- _, _, err := q.addDeviceToBridge("qemu-bridge-31", types.PCI) ++ _, _, err := q.addDeviceToBridge("qemu-bridge-26", types.PCI) + exceptErr := errors.New("no more bridge slots available") + assert.Equal(exceptErr.Error(), err.Error()) + +diff --git a/virtcontainers/types/bridges.go b/virtcontainers/types/bridges.go +index cb15a88f..c3538ce4 100644 +--- a/virtcontainers/types/bridges.go ++++ b/virtcontainers/types/bridges.go +@@ -10,7 +10,7 @@ import "fmt" + // Type represents a type of bus and bridge. + type Type string + +-const PCIBridgeMaxCapacity = 30 ++const PCIBridgeMaxCapacity = 25 + + const ( + // PCI represents a PCI bus and bridge +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch b/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch new file mode 100644 index 0000000000000000000000000000000000000000..bdf95376ae9398f48137c71837d364c4ee9c6adc --- /dev/null +++ b/runtime/patches/0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch @@ -0,0 +1,462 @@ +From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 13 Aug 2020 11:24:58 +0800 +Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata + VM + +reason: support hotplug exist tap interface or a new created tap +interface into kata VM. + +Signed-off-by: jiangpengfei +--- + cli/network.go | 45 ++++++++++++++--- + virtcontainers/kata_agent.go | 12 ++++- + virtcontainers/network.go | 100 +++++++++++++++++++++++--------------- + virtcontainers/pkg/types/types.go | 16 +++--- + virtcontainers/qemu.go | 11 +++-- + virtcontainers/sandbox.go | 24 +++++++-- + virtcontainers/tap_endpoint.go | 23 +++++++-- + 7 files changed, 161 insertions(+), 70 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index 881a2358..7e7791f1 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -26,6 +26,8 @@ const ( + routeType + ) + ++const defaultLinkType = "tap" ++ + var kataNetworkCLICommand = cli.Command{ + Name: "kata-network", + Usage: "manage interfaces and routes for container", +@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{ + } + + var addIfaceCommand = cli.Command{ +- Name: "add-iface", +- Usage: "add an interface to a container", +- ArgsUsage: `add-iface file or - for stdin`, +- Flags: []cli.Flag{}, ++ Name: "add-iface", ++ Usage: "add an interface to a container", ++ ArgsUsage: `add-iface file or - for stdin ++ file or stdin for example: ++ { ++ "device":"", ++ "name":"", ++ "IPAddresses":[{"address":"","mask":""}], ++ "mtu":, ++ "hwAddr":"", ++ "linkType":"tap", ++ "vhostUserSocket":"" ++ } ++ device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional. ++ `, ++ Flags: []cli.Flag{}, + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { +@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{ + } + + var delIfaceCommand = cli.Command{ +- Name: "del-iface", +- Usage: "delete an interface from a container", +- ArgsUsage: `del-iface file or - for stdin`, +- Flags: []cli.Flag{}, ++ Name: "del-iface", ++ Usage: "delete an interface from a container", ++ ArgsUsage: `del-iface file or - for stdin ++ file or stdin for example: ++ { ++ "device":"", ++ "name":"", ++ "IPAddresses":[], ++ "mtu":0, ++ "hwAddr":"" ++ } ++ Only the "name" field is required. ++ `, ++ Flags: []cli.Flag{}, + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { +@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + if err = json.NewDecoder(f).Decode(&inf); err != nil { + return err + } ++ ++ if len(inf.LinkType) == 0 { ++ inf.LinkType = defaultLinkType ++ } ++ + if add { + resultingInf, err = vci.AddInterface(ctx, sandboxID, inf) + if err != nil { +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 8e073339..dfdd263b 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, + "resulting-interface": fmt.Sprintf("%+v", resultingInterface), + }).WithError(err).Error("update interface request failed") + } +- if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok { +- return resultInterface, err ++ if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok { ++ iface := &vcTypes.Interface{ ++ Device: resultInterface.Device, ++ Name: resultInterface.Name, ++ IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses), ++ Mtu: resultInterface.Mtu, ++ HwAddr: resultInterface.HwAddr, ++ PciAddr: resultInterface.PciAddr, ++ } ++ return iface, err + } + return nil, err + } +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index d70c5360..e909a822 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -117,6 +117,7 @@ type NetlinkIface struct { + // NetworkInfo gathers all information related to a network interface. + // It can be used to store the description of the underlying network. + type NetworkInfo struct { ++ Device string + Iface NetlinkIface + Addrs []netlink.Addr + Routes []netlink.Route +@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin + var newLink netlink.Link + var fds []*os.File + ++ // check if tapname exists, if true, use existing one instead of create new one ++ if name != "" { ++ retLink, err := netlink.LinkByName(name) ++ // link exist, use it ++ if err == nil { ++ networkLogger().Debugf("exist tap device is found, instead of creating new one") ++ tuntapLink, ok := retLink.(*netlink.Tuntap) ++ if ok { ++ fds = tuntapLink.Fds ++ } ++ return retLink, nil, nil ++ } ++ } ++ + switch expectedLink.Type() { + case (&netlink.Tuntap{}).Type(): + flags := netlink.TUNTAP_VNET_HDR +@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + + // Check if interface is a physical interface. Do not create + // tap interface/bridge if it is. +- isPhysical, err := isPhysicalIface(netInfo.Iface.Name) ++ isPhysical, err := isPhysicalIface(netInfo.Device) + if err != nil { + return nil, err + } +@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + if isPhysical { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found") + endpoint, err = createPhysicalEndpoint(netInfo) +- } else { +- var socketPath string ++ return endpoint, err ++ } + +- // Check if this is a dummy interface which has a vhost-user socket associated with it +- socketPath, err = vhostUserSocketPath(netInfo) +- if err != nil { +- return nil, err +- } ++ // Check if this is a dummy interface which has a vhost-user socket associated with it ++ socketPath, err := vhostUserSocketPath(netInfo) ++ if err != nil { ++ return nil, err ++ } + +- if socketPath != "" { +- networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") +- endpoint, err = createVhostUserEndpoint(netInfo, socketPath) +- } else if netInfo.Iface.Type == "macvlan" { +- networkLogger().Infof("macvlan interface found") +- endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) +- } else if netInfo.Iface.Type == "macvtap" { +- networkLogger().Infof("macvtap interface found") +- endpoint, err = createMacvtapNetworkEndpoint(netInfo) +- } else if netInfo.Iface.Type == "tap" { +- networkLogger().Info("tap interface found") +- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name) +- } else if netInfo.Iface.Type == "tuntap" { +- if link != nil { +- switch link.(*netlink.Tuntap).Mode { +- case 0: +- // mount /sys/class/net to get links +- return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") +- case 1: +- return nil, fmt.Errorf("tun networking device not yet supported") +- case 2: +- networkLogger().Info("tuntap tap interface found") +- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) +- default: +- return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) +- } ++ if socketPath != "" { ++ networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") ++ endpoint, err = createVhostUserEndpoint(netInfo, socketPath) ++ return endpoint, err ++ } ++ ++ // We should create tap interface/bridge of other interface type. ++ networkLogger().Infof("%s interface found", netInfo.Iface.Type) ++ switch netInfo.Iface.Type { ++ case "macvlan": ++ networkLogger().Infof("macvlan interface found") ++ endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model) ++ case "macvtap": ++ networkLogger().Infof("macvtap interface found") ++ endpoint, err = createMacvtapNetworkEndpoint(netInfo) ++ case "tap": ++ networkLogger().Info("tap interface found") ++ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device) ++ case "tuntap": ++ if link != nil { ++ switch link.(*netlink.Tuntap).Mode { ++ case 0: ++ // mount /sys/class/net to get links ++ return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller") ++ case 1: ++ return nil, fmt.Errorf("tun networking device not yet supported") ++ case 2: ++ networkLogger().Info("tuntap tap interface found") ++ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) ++ default: ++ return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) + } +- } else if netInfo.Iface.Type == "veth" { +- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) +- } else if netInfo.Iface.Type == "ipvlan" { +- endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) +- } else { +- return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type) + } ++ case "veth": ++ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) ++ case "ipvlan": ++ endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) ++ default: ++ err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type) + } + + return endpoint, err +diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go +index 0d4a9cfa..fcc63d84 100644 +--- a/virtcontainers/pkg/types/types.go ++++ b/virtcontainers/pkg/types/types.go +@@ -14,21 +14,21 @@ type IPAddress struct { + + // Interface describes a network interface. + type Interface struct { +- Device string +- Name string +- IPAddresses []*IPAddress +- Mtu uint64 +- RawFlags uint32 +- HwAddr string ++ Device string `json:"device,omitempty"` ++ Name string `json:"name,omitempty"` ++ IPAddresses []*IPAddress `json:"IPAddresses,omitempty"` ++ Mtu uint64 `json:"mtu,omitempty"` ++ RawFlags uint32 `json:"rawFlags,omitempty"` ++ HwAddr string `json:"hwAddr,omitempty"` + // pciAddr is the PCI address in the format "bridgeAddr/deviceAddr". + // Here, bridgeAddr is the address at which the bridge is attached on the root bus, + // while deviceAddr is the address at which the network device is attached on the bridge. +- PciAddr string ++ PciAddr string `json:"pciAddr,omitempty"` + // LinkType defines the type of interface described by this structure. + // The expected values are the one that are defined by the netlink + // library, regarding each type of link. Here is a non exhaustive + // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... +- LinkType string ++ LinkType string `json:"linkType,omitempty"` + } + + // Route describes a network route. +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index 7bae3278..bb83b1bb 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro + return nil + } + +-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error { ++func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error { + var ( + VMFdNames []string + VhostFdNames []string +@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File + VhostFd.Close() + VhostFdNames = append(VhostFdNames, fdName) + } +- return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) ++ ++ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { ++ return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) ++ } ++ ++ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0) + } + + func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { +@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { + + devID := "virtio-" + tap.ID + if op == addDevice { +- if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { ++ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { + return err + } + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index ba704249..c8981a41 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { + } + + return NetworkInfo{ ++ Device: inf.Device, + Iface: NetlinkIface{ + LinkAttrs: netlink.LinkAttrs{ + Name: inf.Name, +@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { + } + + // AddInterface adds new nic to the sandbox. +-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) { + netInfo, err := s.generateNetInfo(inf) + if err != nil { + return nil, err +@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro + } + + endpoint.SetProperties(netInfo) +- if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { ++ if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") + return endpoint.HotAttach(s.hypervisor) + }); err != nil { + return nil, err + } + ++ defer func() { ++ if err != nil { ++ if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil { ++ s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed") ++ } ++ } ++ }() ++ + // Update the sandbox storage + s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) +- if err := s.Save(); err != nil { ++ if err = s.Save(); err != nil { + return nil, err + } + + // Add network for vm + inf.PciAddr = endpoint.PciAddr() +- return s.agent.updateInterface(inf) ++ grpcIf, err = s.agent.updateInterface(inf) ++ if err != nil { ++ return nil, err ++ } ++ ++ return + } + + // RemoveInterface removes a nic of the sandbox. + func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { + for i, endpoint := range s.networkNS.Endpoints { +- if endpoint.HardwareAddr() == inf.HwAddr { ++ if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name { + s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint") + if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { + return inf, err +diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go +index cb441b87..7d33d5a2 100644 +--- a/virtcontainers/tap_endpoint.go ++++ b/virtcontainers/tap_endpoint.go +@@ -7,6 +7,7 @@ package virtcontainers + + import ( + "fmt" ++ "os" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/vishvananda/netlink" +@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat + return nil + } + +-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { ++func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) { + if idx < 0 { + return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } +@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) { + endpoint.TapInterface.Name = ifName + } + ++ if tapIfName != "" { ++ endpoint.TapInterface.TAPIface.Name = tapIfName ++ } ++ + return endpoint, nil + } + +@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err + if err != nil { + return fmt.Errorf("Could not create TAP interface: %s", err) + } ++ ++ defer func() { ++ if err != nil { ++ if errDel := netHandle.LinkDel(tapLink); errDel != nil { ++ networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link") ++ } ++ } ++ }() ++ + endpoint.TapInterface.VMFds = fds + if !disableVhostNet { +- vhostFds, err := createVhostFds(int(numCPUs)) ++ var vhostFds []*os.File ++ vhostFds, err = createVhostFds(int(numCPUs)) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err) + } +@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err + // bridge created by the network plugin on the host actually expects + // to see traffic from this MAC address and not another one. + endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String() +- if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { ++ if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil { + return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err) + } +- if err := netHandle.LinkSetUp(tapLink); err != nil { ++ if err = netHandle.LinkSetUp(tapLink); err != nil { + return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err) + } + return nil +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch b/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch new file mode 100644 index 0000000000000000000000000000000000000000..8f630dc3b84c80278e948c0290011ca0b2a3a8ad --- /dev/null +++ b/runtime/patches/0025-network-keep-list-ifaces-result-compatible-with-cni.patch @@ -0,0 +1,260 @@ +From be8153f21c0b81d2b194075ecd654501bc708577 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Thu, 13 Aug 2020 18:54:49 +0800 +Subject: [PATCH 25/50] network: keep list-ifaces result compatible with cni + +reason: community list-ifaces command will return the all +interfaces info in the Kata VM, however we may just want +to get the interfaces that we hotplug, so just return the +hotplugged interfaces and convert the interface info to +be compatible with cni. + +Signed-off-by: jiangpengfei +--- + cli/network.go | 29 ++++++++++++++++++++++- + virtcontainers/api.go | 4 +++- + virtcontainers/endpoint.go | 44 +++++++++++++++++++++++++++++++++++ + virtcontainers/network.go | 27 +++++++++++++++++++++ + virtcontainers/persist/api/network.go | 24 +++++++++++++++++++ + virtcontainers/tap_endpoint.go | 9 +++++++ + 6 files changed, 135 insertions(+), 2 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index 7e7791f1..66955725 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -28,6 +28,13 @@ const ( + + const defaultLinkType = "tap" + ++type compatInterface struct { ++ Name string `json:"name,omitempty"` ++ Mac string `json:"mac,omitempty"` ++ IP []string `json:"ip,omitempty"` ++ Mtu int `json:"mtu,omitempty"` ++} ++ + var kataNetworkCLICommand = cli.Command{ + Name: "kata-network", + Usage: "manage interfaces and routes for container", +@@ -244,7 +251,8 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT + kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). + WithError(err).Error("list interfaces failed") + } +- json.NewEncoder(file).Encode(interfaces) ++ compatInfs := convertCompatInterfaces(interfaces) ++ json.NewEncoder(file).Encode(compatInfs) + case routeType: + var routes []*vcTypes.Route + routes, err = vci.ListRoutes(ctx, sandboxID) +@@ -256,3 +264,22 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT + } + return err + } ++ ++func convertCompatInterfaces(interfaces []*vcTypes.Interface) []compatInterface { ++ var infs []compatInterface ++ for _, i := range interfaces { ++ var addrs []string ++ for _, a := range i.IPAddresses { ++ addrs = append(addrs, fmt.Sprintf("%s/%s", a.Address, a.Mask)) ++ } ++ ++ infs = append(infs, compatInterface{ ++ Name: i.Name, ++ Mac: i.HwAddr, ++ IP: addrs, ++ Mtu: int(i.Mtu), ++ }) ++ } ++ ++ return infs ++} +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 5e8c9c9e..eb5b4995 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -949,7 +949,9 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface + } + defer s.releaseStatelessSandbox() + +- return s.ListInterfaces() ++ // get interfaces info from persist.json file ++ // instead of by s.ListInterfaces() ++ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil + } + + // UpdateRoutes is the virtcontainers update routes entry point. +diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go +index 01b5e77f..7efcf49c 100644 +--- a/virtcontainers/endpoint.go ++++ b/virtcontainers/endpoint.go +@@ -132,6 +132,28 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface { + } + } + ++func saveTapEndpointProperties(networkInfo *NetworkInfo) *persistapi.NetworkProperties { ++ if networkInfo == nil { ++ return nil ++ } ++ ++ return &persistapi.NetworkProperties{ ++ Device: networkInfo.Device, ++ Iface: persistapi.NetlinkIface{ ++ LinkAttrs: networkInfo.Iface.LinkAttrs, ++ Type: networkInfo.Iface.Type, ++ }, ++ Addrs: networkInfo.Addrs, ++ Routes: networkInfo.Routes, ++ DNS: persistapi.DNSInfo{ ++ Servers: networkInfo.DNS.Servers, ++ Domain: networkInfo.DNS.Domain, ++ Searches: networkInfo.DNS.Searches, ++ Options: networkInfo.DNS.Options, ++ }, ++ } ++} ++ + func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { + if tapif == nil { + return nil +@@ -148,6 +170,28 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { + } + } + ++func loadTapEndpointProperties(endpointProperties *persistapi.NetworkProperties) *NetworkInfo { ++ if endpointProperties == nil { ++ return nil ++ } ++ ++ return &NetworkInfo{ ++ Device: endpointProperties.Device, ++ Iface: NetlinkIface{ ++ LinkAttrs: endpointProperties.Iface.LinkAttrs, ++ Type: endpointProperties.Iface.Type, ++ }, ++ Addrs: endpointProperties.Addrs, ++ Routes: endpointProperties.Routes, ++ DNS: DNSInfo{ ++ Servers: endpointProperties.DNS.Servers, ++ Domain: endpointProperties.DNS.Domain, ++ Searches: endpointProperties.DNS.Searches, ++ Options: endpointProperties.DNS.Options, ++ }, ++ } ++} ++ + func saveNetIfPair(pair *NetworkInterfacePair) *persistapi.NetworkInterfacePair { + if pair == nil { + return nil +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index e909a822..bf7f9336 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -1340,3 +1340,30 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor h + + return nil + } ++ ++// convertCompatInterfaces convert Endpoint info to vcTypes.Interface ++func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { ++ var infs []*vcTypes.Interface ++ for _, e := range *es { ++ var addrs []*vcTypes.IPAddress ++ for _, a := range e.Properties().Addrs { ++ m, _ := a.Mask.Size() ++ addr := &vcTypes.IPAddress{ ++ Address: fmt.Sprintf("%s", a.IP), ++ Mask: fmt.Sprintf("%d", m), ++ } ++ addrs = append(addrs, addr) ++ } ++ inf := &vcTypes.Interface{ ++ LinkType: string(e.Type()), ++ Name: e.Name(), ++ Mtu: uint64(e.Properties().Iface.MTU), ++ HwAddr: e.HardwareAddr(), ++ IPAddresses: addrs, ++ } ++ ++ infs = append(infs, inf) ++ } ++ ++ return infs ++} +diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go +index 69610c67..53c6de44 100644 +--- a/virtcontainers/persist/api/network.go ++++ b/virtcontainers/persist/api/network.go +@@ -11,6 +11,27 @@ import ( + ) + + // ============= sandbox level resources ============= ++// DNSInfo describes the DNS setup related to a network interface. ++type DNSInfo struct { ++ Servers []string ++ Domain string ++ Searches []string ++ Options []string ++} ++ ++// NetlinkIface describes fully a network interface. ++type NetlinkIface struct { ++ netlink.LinkAttrs ++ Type string ++} ++ ++type NetworkProperties struct { ++ Device string ++ Iface NetlinkIface ++ Addrs []netlink.Addr ++ Routes []netlink.Route ++ DNS DNSInfo ++} + + type NetworkInterface struct { + Name string +@@ -91,6 +112,9 @@ type NetworkEndpoint struct { + Tap *TapEndpoint `json:",omitempty"` + IPVlan *IPVlanEndpoint `json:",omitempty"` + Tuntap *TuntapEndpoint `json:",omitempty"` ++ ++ // store the endpoint properties info ++ EndPointProperties *NetworkProperties `json:",omitempty"` + } + + // NetworkInfo contains network information of sandbox +diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go +index 7d33d5a2..c897670e 100644 +--- a/virtcontainers/tap_endpoint.go ++++ b/virtcontainers/tap_endpoint.go +@@ -206,12 +206,15 @@ func unTapNetwork(name string) error { + + func (endpoint *TapEndpoint) save() persistapi.NetworkEndpoint { + tapif := saveTapIf(&endpoint.TapInterface) ++ // save tap endpoint network properties into persist storage ++ properties := saveTapEndpointProperties(&endpoint.EndpointProperties) + + return persistapi.NetworkEndpoint{ + Type: string(endpoint.Type()), + Tap: &persistapi.TapEndpoint{ + TapInterface: *tapif, + }, ++ EndPointProperties: properties, + } + } + func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) { +@@ -221,4 +224,10 @@ func (endpoint *TapEndpoint) load(s persistapi.NetworkEndpoint) { + tapif := loadTapIf(&s.Tap.TapInterface) + endpoint.TapInterface = *tapif + } ++ ++ if s.EndPointProperties != nil { ++ // restore tap endpoint network properties from persist storage ++ properties := loadTapEndpointProperties(s.EndPointProperties) ++ endpoint.EndpointProperties = *properties ++ } + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch b/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch new file mode 100644 index 0000000000000000000000000000000000000000..7578b55925a60bf14c0253b472eaebff18966f9b --- /dev/null +++ b/runtime/patches/0026-network-add-enable_compat_old_cni-config.patch @@ -0,0 +1,285 @@ +From eeca1e47e9a6422d89d08275864f2c1b15e54941 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Fri, 14 Aug 2020 19:14:35 +0800 +Subject: [PATCH 26/50] network: add enable_compat_old_cni config + +reason: old version kata-network list-ifaces output result different +from the community version, inorder to compatible with the old version +list-ifaces command output format, add enable_compat_old_cni to control +the list-ifaces command output format. + +Signed-off-by: jiangpengfei +--- + cli/config/configuration-qemu.toml.in | 4 ++++ + cli/network.go | 17 +++++++++++++++-- + cli/network_test.go | 6 ++++++ + pkg/katautils/config.go | 2 ++ + virtcontainers/api.go | 10 +++++++--- + virtcontainers/interfaces.go | 1 + + virtcontainers/network.go | 11 ++++++----- + virtcontainers/persist.go | 18 ++++++++++-------- + virtcontainers/persist/api/config.go | 9 +++++---- + virtcontainers/pkg/oci/utils.go | 4 ++++ + virtcontainers/pkg/vcmock/sandbox.go | 5 +++++ + virtcontainers/sandbox.go | 5 +++++ + 12 files changed, 70 insertions(+), 22 deletions(-) + +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index b44e84d8..46f8b632 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -453,6 +453,10 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ + # (default: false) + #disable_new_netns = true + ++# If enabled, the kata-network will return the old interface format info to be compatible with ++# old version CNI plugin ++enable_compat_old_cni = true ++ + # if enabled, the runtime will add all the kata processes inside one dedicated cgroup. + # The container cgroups in the host are not created, just one single cgroup per sandbox. + # The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. +diff --git a/cli/network.go b/cli/network.go +index 66955725..a1a24425 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -251,8 +251,21 @@ func networkListCommand(ctx context.Context, containerID string, opType networkT + kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). + WithError(err).Error("list interfaces failed") + } +- compatInfs := convertCompatInterfaces(interfaces) +- json.NewEncoder(file).Encode(compatInfs) ++ ++ sandbox, err := vci.FetchSandbox(ctx, sandboxID) ++ if err != nil { ++ kataLog.WithField("existing-interfaces", fmt.Sprintf("%+v", interfaces)). ++ WithError(err).Error("fetch sandbox failed") ++ } ++ ++ // If sandbox network config need to be compatible with old CNI, ++ // convert the interface format to old version format. ++ if sandbox.IsCompatOldCNI() { ++ compatInfs := convertCompatInterfaces(interfaces) ++ json.NewEncoder(file).Encode(compatInfs) ++ } else { ++ json.NewEncoder(file).Encode(interfaces) ++ } + case routeType: + var routes []*vcTypes.Route + routes, err = vci.ListRoutes(ctx, sandboxID) +diff --git a/cli/network_test.go b/cli/network_test.go +index 4e3d943d..95fd2749 100644 +--- a/cli/network_test.go ++++ b/cli/network_test.go +@@ -17,6 +17,7 @@ import ( + + vc "github.com/kata-containers/runtime/virtcontainers" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" ++ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" + "github.com/kata-containers/runtime/virtcontainers/types" + ) + +@@ -59,6 +60,10 @@ func TestNetworkCliFunction(t *testing.T) { + return newSingleContainerStatus(testContainerID, state, map[string]string{}, &specs.Spec{}), nil + } + ++ testingImpl.FetchSandboxFunc = func(ctx context.Context, id string) (vc.VCSandbox, error) { ++ return &vcmock.Sandbox{}, nil ++ } ++ + defer func() { + testingImpl.AddInterfaceFunc = nil + testingImpl.RemoveInterfaceFunc = nil +@@ -66,6 +71,7 @@ func TestNetworkCliFunction(t *testing.T) { + testingImpl.UpdateRoutesFunc = nil + testingImpl.ListRoutesFunc = nil + testingImpl.StatusContainerFunc = nil ++ testingImpl.FetchSandboxFunc = nil + }() + + set := flag.NewFlagSet("", 0) +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 9a99b9d4..94c916a0 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -141,6 +141,7 @@ type runtime struct { + Debug bool `toml:"enable_debug"` + Tracing bool `toml:"enable_tracing"` + DisableNewNetNs bool `toml:"disable_new_netns"` ++ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` + DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` + SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + Experimental []string `toml:"experimental"` +@@ -1235,6 +1236,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag + + config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly + config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs ++ config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI + for _, f := range tomlConf.Runtime.Experimental { + feature := exp.Get(f) + if feature == nil { +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index eb5b4995..fb044fe1 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -949,9 +949,13 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface + } + defer s.releaseStatelessSandbox() + +- // get interfaces info from persist.json file +- // instead of by s.ListInterfaces() +- return convertToCompatInterfaces(&s.networkNS.Endpoints), nil ++ // If enable_compat_old_cni is enabled, get interfaces info from ++ // persist.json file instead of by s.ListInterfaces() ++ if s.config.NetworkConfig.EnableCompatOldCNI { ++ return convertToCompatInterfaces(&s.networkNS.Endpoints), nil ++ } ++ ++ return s.ListInterfaces() + } + + // UpdateRoutes is the virtcontainers update routes entry point. +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index fa6b584e..499b386e 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -98,6 +98,7 @@ type VCSandbox interface { + ListInterfaces() ([]*vcTypes.Interface, error) + UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) + ListRoutes() ([]*vcTypes.Route, error) ++ IsCompatOldCNI() bool + } + + // VCContainer is the Container interface +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index bf7f9336..db235cf6 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -155,11 +155,12 @@ type NetworkInterfacePair struct { + + // NetworkConfig is the network configuration related to a network. + type NetworkConfig struct { +- NetNSPath string +- NetNsCreated bool +- DisableNewNetNs bool +- NetmonConfig NetmonConfig +- InterworkingModel NetInterworkingModel ++ NetNSPath string ++ NetNsCreated bool ++ DisableNewNetNs bool ++ EnableCompatOldCNI bool ++ NetmonConfig NetmonConfig ++ InterworkingModel NetInterworkingModel + } + + func networkLogger() *logrus.Entry { +diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go +index 6bd09a0b..fe00bf9a 100644 +--- a/virtcontainers/persist.go ++++ b/virtcontainers/persist.go +@@ -187,10 +187,11 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + }, + ShimType: string(sconfig.ShimType), + NetworkConfig: persistapi.NetworkConfig{ +- NetNSPath: sconfig.NetworkConfig.NetNSPath, +- NetNsCreated: sconfig.NetworkConfig.NetNsCreated, +- DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, +- InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), ++ NetNSPath: sconfig.NetworkConfig.NetNSPath, ++ NetNsCreated: sconfig.NetworkConfig.NetNsCreated, ++ DisableNewNetNs: sconfig.NetworkConfig.DisableNewNetNs, ++ EnableCompatOldCNI: sconfig.NetworkConfig.EnableCompatOldCNI, ++ InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), + }, + + ShmSize: sconfig.ShmSize, +@@ -477,10 +478,11 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { + }, + ShimType: ShimType(savedConf.ShimType), + NetworkConfig: NetworkConfig{ +- NetNSPath: savedConf.NetworkConfig.NetNSPath, +- NetNsCreated: savedConf.NetworkConfig.NetNsCreated, +- DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, +- InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), ++ NetNSPath: savedConf.NetworkConfig.NetNSPath, ++ NetNsCreated: savedConf.NetworkConfig.NetNsCreated, ++ DisableNewNetNs: savedConf.NetworkConfig.DisableNewNetNs, ++ EnableCompatOldCNI: savedConf.NetworkConfig.EnableCompatOldCNI, ++ InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), + }, + + ShmSize: savedConf.ShmSize, +diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go +index cfbee849..3a2df32b 100644 +--- a/virtcontainers/persist/api/config.go ++++ b/virtcontainers/persist/api/config.go +@@ -210,10 +210,11 @@ type ShimConfig struct { + + // NetworkConfig is the network configuration related to a network. + type NetworkConfig struct { +- NetNSPath string +- NetNsCreated bool +- DisableNewNetNs bool +- InterworkingModel int ++ NetNSPath string ++ NetNsCreated bool ++ DisableNewNetNs bool ++ EnableCompatOldCNI bool ++ InterworkingModel int + } + + type ContainerConfig struct { +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 36c730b7..948bd3cb 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -130,6 +130,9 @@ type RuntimeConfig struct { + //Determines if create a netns for hypervisor process + DisableNewNetNs bool + ++ // Determines if compatible with old CNI plugin ++ EnableCompatOldCNI bool ++ + //Determines kata processes are managed only in sandbox cgroup + SandboxCgroupOnly bool + +@@ -275,6 +278,7 @@ func networkConfig(ocispec specs.Spec, config RuntimeConfig) (vc.NetworkConfig, + } + netConf.InterworkingModel = config.InterNetworkModel + netConf.DisableNewNetNs = config.DisableNewNetNs ++ netConf.EnableCompatOldCNI = config.EnableCompatOldCNI + + netConf.NetmonConfig = vc.NetmonConfig{ + Path: config.NetmonConfig.Path, +diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go +index 677457ef..11b83ccd 100644 +--- a/virtcontainers/pkg/vcmock/sandbox.go ++++ b/virtcontainers/pkg/vcmock/sandbox.go +@@ -212,3 +212,8 @@ func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error + func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { + return nil, nil + } ++ ++// IsCompatOldCNI return the whether enable compatible with old CNI ++func (s *Sandbox) IsCompatOldCNI() bool { ++ return false ++} +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index c8981a41..6a643a12 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -2379,6 +2379,11 @@ func (s *Sandbox) setContainersState(state types.StateString) error { + return nil + } + ++// IsCompatOldCNI return the whether enable compatible with old CNI ++func (s *Sandbox) IsCompatOldCNI() bool { ++ return s.config.NetworkConfig.EnableCompatOldCNI ++} ++ + // updateStaticSandboxResources update sandbox's cpu and memory resource passed by + // sandbox_cpu and sandbox_mem annotations + func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch b/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch new file mode 100644 index 0000000000000000000000000000000000000000..d0b0eded35e5502ae025a41d30609f2a6bc6d14d --- /dev/null +++ b/runtime/patches/0027-network-add-more-strict-check-for-input-network-inte.patch @@ -0,0 +1,182 @@ +From ec15337fc816767ca0e8183576405499080b9b1e Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sun, 16 Aug 2020 16:41:18 +0800 +Subject: [PATCH 27/50] network: add more strict check for input network + interface + +reason: kata-network add-iface command will receive the network +interface info from untrust user, so we need to add more strict +check for input network interface info. + +Signed-off-by: jiangpengfei +--- + virtcontainers/network.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++ + virtcontainers/sandbox.go | 11 +++++ + 2 files changed, 128 insertions(+) + +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index db235cf6..a1676ccd 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -13,8 +13,10 @@ import ( + "math/rand" + "net" + "os" ++ "regexp" + "runtime" + "sort" ++ "strings" + "time" + + "github.com/containernetworking/plugins/pkg/ns" +@@ -55,6 +57,17 @@ const ( + NetXConnectInvalidModel + ) + ++const ( ++ maxIPAddrLen = 18 ++ maxInterfaceLen = 15 ++ minMTUVal = 46 ++ maxMTUVal = 9600 ++) ++ ++var ( ++ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) ++) ++ + //IsValid checks if a model is valid + func (n NetInterworkingModel) IsValid() bool { + return 0 <= int(n) && int(n) < int(NetXConnectInvalidModel) +@@ -1368,3 +1381,107 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { + + return infs + } ++ ++// verifyInterfaceName verifies the interface name valid or not ++func verifyInterfaceName(name string) error { ++ // verify `Name` before `Tapname` because of the strict rules ++ name = strings.TrimSpace(name) ++ if len(name) > maxInterfaceLen { ++ return fmt.Errorf("interface name can't be longer than 15") ++ } else if len(name) == 0 || name == "\n" { ++ return fmt.Errorf("interface name can't be empty") ++ } ++ ++ // QMP rules of `Name` ++ chk := regInfName.FindAllString(name, -1) ++ if chk == nil { ++ return fmt.Errorf("invalid input of interface name, please check the rules") ++ } ++ ++ return nil ++} ++ ++// verifyIPAndMask verifies CIDR notation IP address and mask like "192.0.2.0/24" ++func verifyIPAndMask(ip string) (nlIpMask *net.IPNet, err error) { ++ ip = strings.TrimSpace(ip) ++ ++ if len(ip) > maxIPAddrLen { ++ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen) ++ } ++ ++ if nlIpMask, err = netlink.ParseIPNet(ip); err != nil { ++ return nil, fmt.Errorf("invalid input of IP : %v", err) ++ } ++ ++ return nlIpMask, nil ++} ++ ++// verifyMac verifies MAC address ++func verifyMac(mac string) error { ++ mac = strings.TrimSpace(mac) ++ if _, err := net.ParseMAC(mac); err != nil { ++ return fmt.Errorf("invalid input of Mac : %v", err) ++ } ++ ++ return nil ++} ++ ++// verifyMtu verifies MTU value of interface ++func verifyMtu(mtu uint64) error { ++ if mtu < minMTUVal || mtu > maxMTUVal { ++ return fmt.Errorf("invalid input of MTU : %v", mtu) ++ } ++ return nil ++} ++ ++// verifyIP verifies the IP address ++func verifyIP(ip string) (*net.IP, error) { ++ ip = strings.TrimSpace(ip) ++ ++ if len(ip) > maxIPAddrLen { ++ return nil, fmt.Errorf("the length of IP address is too long, max ip len :%d", maxIPAddrLen) ++ } ++ ++ var netIP net.IP ++ if netIP = net.ParseIP(ip); netIP == nil { ++ return nil, fmt.Errorf("invalid IP: %s", ip) ++ } ++ ++ return &netIP, nil ++} ++ ++// validInterface check the input interface valid or not ++func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { ++ if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil { ++ return fmt.Errorf("device name should not be empty when enable_compat_old_cni config enabled") ++ } ++ ++ if inf.Name == "" || inf.Mtu == 0 || inf.HwAddr == "" { ++ return fmt.Errorf("name/mtu/hwaddr of interface must be specified") ++ } ++ ++ if err := verifyInterfaceName(inf.Name); err != nil { ++ return err ++ } ++ ++ if err := verifyMac(inf.HwAddr); err != nil { ++ return err ++ } ++ ++ if err := verifyMtu(inf.Mtu); err != nil { ++ return err ++ } ++ ++ // Currently, only one IP address can be passed, which reduces the test entry and fault injection. ++ if len(inf.IPAddresses) > 0 { ++ if len(inf.IPAddresses) != 1 { ++ return fmt.Errorf("only one IP address is supported currently") ++ } ++ _, err := verifyIP(inf.IPAddresses[0].Address) ++ if err != nil { ++ return err ++ } ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 6a643a12..f6826812 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -952,6 +952,17 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { + + // AddInterface adds new nic to the sandbox. + func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) { ++ err = validInterface(inf, s.config.NetworkConfig.EnableCompatOldCNI) ++ if err != nil { ++ return nil, err ++ } ++ ++ for _, ep := range s.networkNS.Endpoints { ++ if ep.Name() == inf.Name { ++ return nil, fmt.Errorf("interface %s is already exist", inf.Name) ++ } ++ } ++ + netInfo, err := s.generateNetInfo(inf) + if err != nil { + return nil, err +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch b/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch new file mode 100644 index 0000000000000000000000000000000000000000..8b636a48b8dfeb2e7cb3cccd33ae8f63613f8fd6 --- /dev/null +++ b/runtime/patches/0028-network-add-kata-network-add-route-subcommand.patch @@ -0,0 +1,1026 @@ +From 4d76ace47b422fedd9b6ae2f38ee09fc6d4c4bd4 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Sun, 16 Aug 2020 17:43:41 +0800 +Subject: [PATCH 28/50] network: add kata-network add-route subcommand + +reason: add kata-network add-route subcommand to support add one route +operation. And keep compatible with community update-routes subcommand. + +Signed-off-by: jiangpengfei +--- + cli/network.go | 60 ++- + .../agent/protocols/grpc/agent.pb.go | 402 ++++++++++++--------- + virtcontainers/agent.go | 3 +- + virtcontainers/api.go | 4 +- + virtcontainers/api_test.go | 2 +- + virtcontainers/implementation.go | 4 +- + virtcontainers/interfaces.go | 4 +- + virtcontainers/kata_agent.go | 5 +- + virtcontainers/kata_agent_test.go | 2 +- + virtcontainers/network.go | 162 +++++++++ + virtcontainers/noop_agent.go | 4 +- + virtcontainers/noop_agent_test.go | 2 +- + virtcontainers/pkg/types/types.go | 14 + + virtcontainers/pkg/vcmock/mock.go | 2 +- + virtcontainers/pkg/vcmock/sandbox.go | 2 +- + virtcontainers/sandbox.go | 29 +- + 16 files changed, 492 insertions(+), 209 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index a1a24425..2265f54b 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -44,6 +44,7 @@ var kataNetworkCLICommand = cli.Command{ + listIfacesCommand, + updateRoutesCommand, + listRoutesCommand, ++ addRoutesCommand, + }, + Action: func(context *cli.Context) error { + return cli.ShowSubcommandHelp(context) +@@ -73,7 +74,7 @@ var addIfaceCommand = cli.Command{ + return err + } + +- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, true) ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpAdd) + }, + } + +@@ -98,7 +99,7 @@ var delIfaceCommand = cli.Command{ + return err + } + +- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, false) ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpRemove) + }, + } + +@@ -128,7 +129,29 @@ var updateRoutesCommand = cli.Command{ + return err + } + +- return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, true) ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpUpdate) ++ }, ++} ++ ++var addRoutesCommand = cli.Command{ ++ Name: "add-route", ++ Usage: "add one route for a container", ++ ArgsUsage: `add-route file or - for stdin ++ file or stdin for example: ++ { ++ "dest":"<[[/mask] | "default" ]>", ++ "gateway":"[ip]", ++ "device":"", ++ } ++ `, ++ Flags: []cli.Flag{}, ++ Action: func(context *cli.Context) error { ++ ctx, err := cliContextToContext(context) ++ if err != nil { ++ return err ++ } ++ ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpAdd) + }, + } + +@@ -147,7 +170,7 @@ var listRoutesCommand = cli.Command{ + }, + } + +-func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, add bool) (err error) { ++func networkModifyCommand(ctx context.Context, containerID, input string, opType networkType, op vcTypes.NetworkOp) (err error) { + status, sandboxID, err := getExistingContainerInfo(ctx, containerID) + if err != nil { + return err +@@ -188,30 +211,45 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + return err + } + +- if len(inf.LinkType) == 0 { ++ if op != vcTypes.NetworkOpUpdate && inf.LinkType == "" { + inf.LinkType = defaultLinkType + } + +- if add { ++ switch op { ++ case vcTypes.NetworkOpAdd: + resultingInf, err = vci.AddInterface(ctx, sandboxID, inf) + if err != nil { + kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). + WithError(err).Error("add interface failed") + } +- } else { ++ case vcTypes.NetworkOpRemove: + resultingInf, err = vci.RemoveInterface(ctx, sandboxID, inf) + if err != nil { + kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). + WithError(err).Error("delete interface failed") + } + } ++ + json.NewEncoder(output).Encode(resultingInf) + case routeType: +- var routes, resultingRoutes []*vcTypes.Route +- if err = json.NewDecoder(f).Decode(&routes); err != nil { +- return err ++ var ( ++ route *vcTypes.Route ++ routes []*vcTypes.Route ++ resultingRoutes []*vcTypes.Route ++ ) ++ ++ if op == vcTypes.NetworkOpUpdate { ++ if err = json.NewDecoder(f).Decode(&routes); err != nil { ++ return err ++ } ++ } else { ++ if err = json.NewDecoder(f).Decode(&route); err != nil { ++ return err ++ } ++ routes = append(routes, route) + } +- resultingRoutes, err = vci.UpdateRoutes(ctx, sandboxID, routes) ++ ++ resultingRoutes, err = vci.UpdateRoutes(ctx, sandboxID, routes, op) + json.NewEncoder(output).Encode(resultingRoutes) + if err != nil { + kataLog.WithField("resulting-routes", fmt.Sprintf("%+v", resultingRoutes)). +diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +index 77e6d1bc..1b887e55 100644 +--- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go ++++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +@@ -1303,7 +1303,8 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + } + + type UpdateRoutesRequest struct { +- Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` ++ Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` ++ Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` + } + + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } +@@ -1318,6 +1319,13 @@ func (m *UpdateRoutesRequest) GetRoutes() *Routes { + return nil + } + ++func (m *UpdateRoutesRequest) GetIncrement() bool { ++ if m != nil { ++ return m.Increment ++ } ++ return false ++} ++ + type ListInterfacesRequest struct { + } + +@@ -4476,6 +4484,16 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + } + i += n20 + } ++ if m.Increment { ++ dAtA[i] = 0x10 ++ i++ ++ if m.Increment { ++ dAtA[i] = 1 ++ } else { ++ dAtA[i] = 0 ++ } ++ i++ ++ } + return i, nil + } + +@@ -5751,6 +5769,9 @@ func (m *UpdateRoutesRequest) Size() (n int) { + l = m.Routes.Size() + n += 1 + l + sovAgent(uint64(l)) + } ++ if m.Increment { ++ n += 2 ++ } + return n + } + +@@ -10964,6 +10985,26 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { + return err + } + iNdEx = postIndex ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Increment", wireType) ++ } ++ var v int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ v |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ m.Increment = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipAgent(dAtA[iNdEx:]) +@@ -12852,184 +12893,185 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2862 bytes of a gzipped FileDescriptorProto ++ // 2876 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x5b, 0x9f, 0x3c, 0xb6, +- 0x65, 0xfa, 0x73, 0xbc, 0xb4, 0x65, 0x23, 0x7e, 0xc1, 0x11, 0xc4, 0x47, 0x44, 0xc6, 0x56, 0xc4, +- 0x0c, 0x45, 0x38, 0x40, 0x10, 0x0c, 0x86, 0x33, 0xcd, 0x65, 0x9b, 0x3b, 0xd3, 0xe3, 0x9e, 0x1e, +- 0x8a, 0xeb, 0x00, 0x39, 0x26, 0xb7, 0x5c, 0x02, 0xe4, 0x96, 0x3f, 0x10, 0xe4, 0x96, 0x63, 0xae, +- 0x39, 0x18, 0x39, 0xe5, 0x17, 0x04, 0x81, 0x7f, 0x42, 0x7e, 0x41, 0xd0, 0xaf, 0x79, 0xec, 0x0e, +- 0x29, 0x84, 0x20, 0x90, 0xcb, 0xa2, 0xab, 0xba, 0xba, 0x5e, 0xdd, 0x55, 0x53, 0x55, 0x0b, 0x6d, +- 0x77, 0x82, 0x43, 0x3e, 0x8e, 0x18, 0xe5, 0x14, 0xd5, 0x27, 0x2c, 0xf2, 0x46, 0x2d, 0xea, 0x11, +- 0x85, 0x18, 0xfd, 0x70, 0x42, 0xf8, 0x69, 0x72, 0x3c, 0xf6, 0x68, 0xb0, 0x79, 0xe6, 0x72, 0xf7, +- 0x5d, 0x8f, 0x86, 0xdc, 0x25, 0x21, 0x66, 0xf1, 0xa6, 0x3c, 0xb8, 0x19, 0x9d, 0x4d, 0x36, 0xf9, +- 0x2c, 0xc2, 0xb1, 0xfa, 0xd5, 0xe7, 0xee, 0x4e, 0x28, 0x9d, 0x4c, 0xf1, 0xa6, 0x84, 0x8e, 0x93, +- 0x93, 0x4d, 0x1c, 0x44, 0x7c, 0xa6, 0x36, 0xad, 0x3f, 0x56, 0x61, 0x7d, 0x9b, 0x61, 0x97, 0xe3, +- 0x6d, 0xc3, 0xcd, 0xc6, 0xdf, 0x24, 0x38, 0xe6, 0xe8, 0x35, 0xe8, 0xa4, 0x12, 0x1c, 0xe2, 0x0f, +- 0x2b, 0xf7, 0x2b, 0x1b, 0x2d, 0xbb, 0x9d, 0xe2, 0xf6, 0x7d, 0x74, 0x1b, 0x96, 0xf1, 0x05, 0xf6, +- 0xc4, 0x6e, 0x55, 0xee, 0x36, 0x04, 0xb8, 0xef, 0xa3, 0xf7, 0xa1, 0x1d, 0x73, 0x46, 0xc2, 0x89, +- 0x93, 0xc4, 0x98, 0x0d, 0x6b, 0xf7, 0x2b, 0x1b, 0xed, 0x87, 0x2b, 0x63, 0x61, 0xd2, 0xf8, 0x50, +- 0x6e, 0x1c, 0xc5, 0x98, 0xd9, 0x10, 0xa7, 0x6b, 0xf4, 0x00, 0x96, 0x7d, 0x7c, 0x4e, 0x3c, 0x1c, +- 0x0f, 0xeb, 0xf7, 0x6b, 0x1b, 0xed, 0x87, 0x1d, 0x45, 0xbe, 0x23, 0x91, 0xb6, 0xd9, 0x44, 0x6f, +- 0x43, 0x33, 0xe6, 0x94, 0xb9, 0x13, 0x1c, 0x0f, 0x97, 0x24, 0x61, 0xd7, 0xf0, 0x95, 0x58, 0x3b, +- 0xdd, 0x46, 0xaf, 0x40, 0xed, 0xd9, 0xf6, 0xfe, 0xb0, 0x21, 0xa5, 0x83, 0xa6, 0x8a, 0xb0, 0x67, +- 0x0b, 0x34, 0x7a, 0x1d, 0xba, 0xb1, 0x1b, 0xfa, 0xc7, 0xf4, 0xc2, 0x89, 0x88, 0x1f, 0xc6, 0xc3, +- 0xe5, 0xfb, 0x95, 0x8d, 0xa6, 0xdd, 0xd1, 0xc8, 0x03, 0x81, 0xb3, 0x3e, 0x85, 0x5b, 0x87, 0xdc, +- 0x65, 0xfc, 0x1a, 0xde, 0xb1, 0x8e, 0x60, 0xdd, 0xc6, 0x01, 0x3d, 0xbf, 0x96, 0x6b, 0x87, 0xb0, +- 0xcc, 0x49, 0x80, 0x69, 0xc2, 0xa5, 0x6b, 0xbb, 0xb6, 0x01, 0xad, 0x3f, 0x57, 0x00, 0xed, 0x5e, +- 0x60, 0xef, 0x80, 0x51, 0x0f, 0xc7, 0xf1, 0xff, 0xe8, 0xba, 0xde, 0x82, 0xe5, 0x48, 0x29, 0x30, +- 0xac, 0x4b, 0x72, 0x7d, 0x0b, 0x46, 0x2b, 0xb3, 0x6b, 0x7d, 0x0d, 0x6b, 0x87, 0x64, 0x12, 0xba, +- 0xd3, 0x1b, 0xd4, 0x77, 0x1d, 0x1a, 0xb1, 0xe4, 0x29, 0x55, 0xed, 0xda, 0x1a, 0xb2, 0x0e, 0x00, +- 0x7d, 0xe5, 0x12, 0x7e, 0x73, 0x92, 0xac, 0x77, 0x61, 0xb5, 0xc0, 0x31, 0x8e, 0x68, 0x18, 0x63, +- 0xa9, 0x00, 0x77, 0x79, 0x12, 0x4b, 0x66, 0x4b, 0xb6, 0x86, 0x2c, 0x0c, 0x6b, 0x5f, 0x92, 0xd8, +- 0x90, 0xe3, 0xff, 0x46, 0x85, 0x75, 0x68, 0x9c, 0x50, 0x16, 0xb8, 0xdc, 0x68, 0xa0, 0x20, 0x84, +- 0xa0, 0xee, 0xb2, 0x49, 0x3c, 0xac, 0xdd, 0xaf, 0x6d, 0xb4, 0x6c, 0xb9, 0x16, 0xaf, 0x72, 0x4e, +- 0x8c, 0xd6, 0xeb, 0x35, 0xe8, 0x68, 0xbf, 0x3b, 0x53, 0x12, 0x73, 0x29, 0xa7, 0x63, 0xb7, 0x35, +- 0x4e, 0x9c, 0xb1, 0x28, 0xac, 0x1f, 0x45, 0xfe, 0x35, 0x03, 0xfe, 0x21, 0xb4, 0x18, 0x8e, 0x69, +- 0xc2, 0x44, 0x98, 0x56, 0xe5, 0xbd, 0xaf, 0xa9, 0x7b, 0xff, 0x92, 0x84, 0xc9, 0x85, 0x6d, 0xf6, +- 0xec, 0x8c, 0x4c, 0x87, 0x10, 0x8f, 0xaf, 0x13, 0x42, 0x9f, 0xc2, 0xad, 0x03, 0x37, 0x89, 0xaf, +- 0xa3, 0xab, 0xf5, 0x99, 0x08, 0xbf, 0x38, 0x09, 0xae, 0x75, 0xf8, 0x4f, 0x15, 0x68, 0x6e, 0x47, +- 0xc9, 0x51, 0xec, 0x4e, 0x30, 0xfa, 0x3f, 0x68, 0x73, 0xca, 0xdd, 0xa9, 0x93, 0x08, 0x50, 0x92, +- 0xd7, 0x6d, 0x90, 0x28, 0x45, 0x20, 0xdc, 0x8e, 0x99, 0x17, 0x25, 0x9a, 0xa2, 0x7a, 0xbf, 0xb6, +- 0x51, 0xb7, 0xdb, 0x0a, 0xa7, 0x48, 0xc6, 0xb0, 0x2a, 0xf7, 0x1c, 0x12, 0x3a, 0x67, 0x98, 0x85, +- 0x78, 0x1a, 0x50, 0x1f, 0xcb, 0xf7, 0x5b, 0xb7, 0x07, 0x72, 0x6b, 0x3f, 0xfc, 0x22, 0xdd, 0x40, +- 0xff, 0x0f, 0x83, 0x94, 0x5e, 0x04, 0xa5, 0xa4, 0xae, 0x4b, 0xea, 0xbe, 0xa6, 0x3e, 0xd2, 0x68, +- 0xeb, 0xd7, 0xd0, 0x7b, 0x7e, 0xca, 0x28, 0xe7, 0x53, 0x12, 0x4e, 0x76, 0x5c, 0xee, 0x8a, 0xec, +- 0x11, 0x61, 0x46, 0xa8, 0x1f, 0x6b, 0x6d, 0x0d, 0x88, 0xde, 0x81, 0x01, 0x57, 0xb4, 0xd8, 0x77, +- 0x0c, 0x4d, 0x55, 0xd2, 0xac, 0xa4, 0x1b, 0x07, 0x9a, 0xf8, 0x4d, 0xe8, 0x65, 0xc4, 0x22, 0xff, +- 0x68, 0x7d, 0xbb, 0x29, 0xf6, 0x39, 0x09, 0xb0, 0x75, 0x2e, 0x7d, 0x25, 0x2f, 0x19, 0xbd, 0x03, +- 0xad, 0xcc, 0x0f, 0x15, 0xf9, 0x42, 0x7a, 0xea, 0x85, 0x18, 0x77, 0xda, 0xcd, 0xd4, 0x29, 0x9f, +- 0x43, 0x9f, 0xa7, 0x8a, 0x3b, 0xbe, 0xcb, 0xdd, 0xe2, 0xa3, 0x2a, 0x5a, 0x65, 0xf7, 0x78, 0x01, +- 0xb6, 0x3e, 0x83, 0xd6, 0x01, 0xf1, 0x63, 0x25, 0x78, 0x08, 0xcb, 0x5e, 0xc2, 0x18, 0x0e, 0xb9, +- 0x31, 0x59, 0x83, 0x68, 0x0d, 0x96, 0xa6, 0x24, 0x20, 0x5c, 0x9b, 0xa9, 0x00, 0x8b, 0x02, 0x3c, +- 0xc5, 0x01, 0x65, 0x33, 0xe9, 0xb0, 0x35, 0x58, 0xca, 0x5f, 0xae, 0x02, 0xd0, 0x5d, 0x68, 0x05, +- 0xee, 0x45, 0x7a, 0xa9, 0x62, 0xa7, 0x19, 0xb8, 0x17, 0x4a, 0xf9, 0x21, 0x2c, 0x9f, 0xb8, 0x64, +- 0xea, 0x85, 0x5c, 0x7b, 0xc5, 0x80, 0x99, 0xc0, 0x7a, 0x5e, 0xe0, 0xdf, 0xaa, 0xd0, 0x56, 0x12, +- 0x95, 0xc2, 0x6b, 0xb0, 0xe4, 0xb9, 0xde, 0x69, 0x2a, 0x52, 0x02, 0xe8, 0x81, 0x51, 0xa4, 0x9a, +- 0x4f, 0xc2, 0x99, 0xa6, 0x46, 0xb5, 0x4d, 0x80, 0xf8, 0x85, 0x1b, 0x69, 0xdd, 0x6a, 0x97, 0x10, +- 0xb7, 0x04, 0x8d, 0x52, 0xf7, 0x03, 0xe8, 0xa8, 0x77, 0xa7, 0x8f, 0xd4, 0x2f, 0x39, 0xd2, 0x56, +- 0x54, 0xea, 0xd0, 0xeb, 0xd0, 0x4d, 0x62, 0xec, 0x9c, 0x12, 0xcc, 0x5c, 0xe6, 0x9d, 0xce, 0x86, +- 0x4b, 0xea, 0x1b, 0x99, 0xc4, 0x78, 0xcf, 0xe0, 0xd0, 0x43, 0x58, 0x12, 0xe9, 0x2f, 0x1e, 0x36, +- 0xe4, 0xe7, 0xf8, 0x95, 0x3c, 0x4b, 0x69, 0xea, 0x58, 0xfe, 0xee, 0x86, 0x9c, 0xcd, 0x6c, 0x45, +- 0x3a, 0xfa, 0x18, 0x20, 0x43, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0x99, 0x8e, 0x43, 0xb1, 0x14, 0xce, +- 0x39, 0x77, 0xa7, 0x89, 0xf1, 0xba, 0x02, 0x3e, 0xad, 0x7e, 0x5c, 0xb1, 0x3c, 0xe8, 0x6f, 0x4d, +- 0xcf, 0x08, 0xcd, 0x1d, 0x5f, 0x83, 0xa5, 0xc0, 0xfd, 0x9a, 0x32, 0xe3, 0x49, 0x09, 0x48, 0x2c, +- 0x09, 0x29, 0x33, 0x2c, 0x24, 0x80, 0x7a, 0x50, 0xa5, 0x91, 0xf4, 0x57, 0xcb, 0xae, 0xd2, 0x28, +- 0x13, 0x54, 0xcf, 0x09, 0xb2, 0xfe, 0x59, 0x07, 0xc8, 0xa4, 0x20, 0x1b, 0x46, 0x84, 0x3a, 0x31, +- 0x66, 0xa2, 0x04, 0x71, 0x8e, 0x67, 0x1c, 0xc7, 0x0e, 0xc3, 0x5e, 0xc2, 0x62, 0x72, 0x2e, 0xee, +- 0x4f, 0x98, 0x7d, 0x4b, 0x99, 0x3d, 0xa7, 0x9b, 0x7d, 0x9b, 0xd0, 0x43, 0x75, 0x6e, 0x4b, 0x1c, +- 0xb3, 0xcd, 0x29, 0xb4, 0x0f, 0xb7, 0x32, 0x9e, 0x7e, 0x8e, 0x5d, 0xf5, 0x2a, 0x76, 0xab, 0x29, +- 0x3b, 0x3f, 0x63, 0xb5, 0x0b, 0xab, 0x84, 0x3a, 0xdf, 0x24, 0x38, 0x29, 0x30, 0xaa, 0x5d, 0xc5, +- 0x68, 0x40, 0xe8, 0xcf, 0xe4, 0x81, 0x8c, 0xcd, 0x01, 0xdc, 0xc9, 0x59, 0x29, 0xc2, 0x3d, 0xc7, +- 0xac, 0x7e, 0x15, 0xb3, 0xf5, 0x54, 0x2b, 0x91, 0x0f, 0x32, 0x8e, 0x3f, 0x81, 0x75, 0x42, 0x9d, +- 0x17, 0x2e, 0xe1, 0xf3, 0xec, 0x96, 0x5e, 0x62, 0xa4, 0xf8, 0xe8, 0x16, 0x79, 0x29, 0x23, 0x03, +- 0xcc, 0x26, 0x05, 0x23, 0x1b, 0x2f, 0x31, 0xf2, 0xa9, 0x3c, 0x90, 0xb1, 0x79, 0x0c, 0x03, 0x42, +- 0xe7, 0xb5, 0x59, 0xbe, 0x8a, 0x49, 0x9f, 0xd0, 0xa2, 0x26, 0x5b, 0x30, 0x88, 0xb1, 0xc7, 0x29, +- 0xcb, 0x3f, 0x82, 0xe6, 0x55, 0x2c, 0x56, 0x34, 0x7d, 0xca, 0xc3, 0xfa, 0x05, 0x74, 0xf6, 0x92, +- 0x09, 0xe6, 0xd3, 0xe3, 0x34, 0x19, 0xdc, 0x58, 0xfe, 0xb1, 0xfe, 0x5d, 0x85, 0xf6, 0xf6, 0x84, +- 0xd1, 0x24, 0x2a, 0xe4, 0x64, 0x15, 0xa4, 0xf3, 0x39, 0x59, 0x92, 0xc8, 0x9c, 0xac, 0x88, 0x3f, +- 0x84, 0x4e, 0x20, 0x43, 0x57, 0xd3, 0xab, 0x3c, 0x34, 0x58, 0x08, 0x6a, 0xbb, 0x1d, 0xe4, 0x92, +- 0xd9, 0x18, 0x20, 0x22, 0x7e, 0xac, 0xcf, 0xa8, 0x74, 0xd4, 0xd7, 0x15, 0xa1, 0x49, 0xd1, 0x76, +- 0x2b, 0x4a, 0xb3, 0xf5, 0xfb, 0xd0, 0x3e, 0x16, 0x4e, 0xd2, 0x07, 0x0a, 0xc9, 0x28, 0xf3, 0x9e, +- 0x0d, 0xc7, 0x59, 0x10, 0xee, 0x41, 0xf7, 0x54, 0xb9, 0x4c, 0x1f, 0x52, 0x6f, 0xe8, 0x75, 0x6d, +- 0x49, 0x66, 0xef, 0x38, 0xef, 0x59, 0x75, 0x01, 0x9d, 0xd3, 0x1c, 0x6a, 0x74, 0x08, 0x83, 0x05, +- 0x92, 0x92, 0x1c, 0xb4, 0x91, 0xcf, 0x41, 0xed, 0x87, 0x48, 0x09, 0xca, 0x9f, 0xcc, 0xe7, 0xa5, +- 0xdf, 0x55, 0xa1, 0xf3, 0x53, 0xcc, 0x5f, 0x50, 0x76, 0xa6, 0xf4, 0x45, 0x50, 0x0f, 0xdd, 0x00, +- 0x6b, 0x8e, 0x72, 0x8d, 0xee, 0x40, 0x93, 0x5d, 0xa8, 0x04, 0xa2, 0xef, 0x73, 0x99, 0x5d, 0xc8, +- 0xc4, 0x80, 0x5e, 0x05, 0x60, 0x17, 0x4e, 0xe4, 0x7a, 0x67, 0x58, 0x7b, 0xb0, 0x6e, 0xb7, 0xd8, +- 0xc5, 0x81, 0x42, 0x88, 0xa7, 0xc0, 0x2e, 0x1c, 0xcc, 0x18, 0x65, 0xb1, 0xce, 0x55, 0x4d, 0x76, +- 0xb1, 0x2b, 0x61, 0x7d, 0xd6, 0x67, 0x34, 0x8a, 0xb0, 0x2f, 0x73, 0xb4, 0x3c, 0xbb, 0xa3, 0x10, +- 0x42, 0x2a, 0x37, 0x52, 0x1b, 0x4a, 0x2a, 0xcf, 0xa4, 0xf2, 0x4c, 0xea, 0xb2, 0x3a, 0xc9, 0xf3, +- 0x52, 0x79, 0x2a, 0xb5, 0xa9, 0xa4, 0xf2, 0x9c, 0x54, 0x9e, 0x49, 0x6d, 0x99, 0xb3, 0x5a, 0xaa, +- 0xf5, 0xdb, 0x0a, 0xac, 0xcf, 0x17, 0x7e, 0xba, 0x4c, 0xfd, 0x10, 0x3a, 0x9e, 0xbc, 0xaf, 0xc2, +- 0x9b, 0x1c, 0x2c, 0xdc, 0xa4, 0xdd, 0xf6, 0x72, 0xcf, 0xf8, 0x23, 0xe8, 0x86, 0xca, 0xc1, 0xe9, +- 0xd3, 0xac, 0x65, 0xf7, 0x92, 0xf7, 0xbd, 0xdd, 0x09, 0x73, 0x90, 0xe5, 0x03, 0xfa, 0x8a, 0x11, +- 0x8e, 0x0f, 0x39, 0xc3, 0x6e, 0x70, 0x13, 0x0d, 0x08, 0x82, 0xba, 0xac, 0x56, 0x6a, 0xb2, 0xbe, +- 0x96, 0x6b, 0xeb, 0x2d, 0x58, 0x2d, 0x48, 0xd1, 0xb6, 0xae, 0x40, 0x6d, 0x8a, 0x43, 0xc9, 0xbd, +- 0x6b, 0x8b, 0xa5, 0xe5, 0xc2, 0xc0, 0xc6, 0xae, 0x7f, 0x73, 0xda, 0x68, 0x11, 0xb5, 0x4c, 0xc4, +- 0x06, 0xa0, 0xbc, 0x08, 0xad, 0x8a, 0xd1, 0xba, 0x92, 0xd3, 0xfa, 0x19, 0x0c, 0xb6, 0xa7, 0x34, +- 0xc6, 0x87, 0xdc, 0x27, 0xe1, 0x4d, 0x74, 0x4c, 0xbf, 0x82, 0xd5, 0xe7, 0x7c, 0xf6, 0x95, 0x60, +- 0x16, 0x93, 0x6f, 0xf1, 0x0d, 0xd9, 0xc7, 0xe8, 0x0b, 0x63, 0x1f, 0xa3, 0x2f, 0x44, 0xb3, 0xe4, +- 0xd1, 0x69, 0x12, 0x84, 0x32, 0x14, 0xba, 0xb6, 0x86, 0xac, 0x2d, 0xe8, 0xa8, 0x1a, 0xfa, 0x29, +- 0xf5, 0x93, 0x29, 0x2e, 0x8d, 0xc1, 0x7b, 0x00, 0x91, 0xcb, 0xdc, 0x00, 0x73, 0xcc, 0xd4, 0x1b, +- 0x6a, 0xd9, 0x39, 0x8c, 0xf5, 0x87, 0x2a, 0xac, 0xa9, 0x91, 0xc8, 0xa1, 0x9a, 0x04, 0x18, 0x13, +- 0x46, 0xd0, 0x3c, 0xa5, 0x31, 0xcf, 0x31, 0x4c, 0x61, 0xa1, 0xa2, 0x1f, 0x1a, 0x6e, 0x62, 0x59, +- 0x98, 0x53, 0xd4, 0xae, 0x9e, 0x53, 0x2c, 0x4c, 0x22, 0xea, 0x8b, 0x93, 0x08, 0x11, 0x6d, 0x86, +- 0x88, 0xa8, 0x18, 0x6f, 0xd9, 0x2d, 0x8d, 0xd9, 0xf7, 0xd1, 0x03, 0xe8, 0x4f, 0x84, 0x96, 0xce, +- 0x29, 0xa5, 0x67, 0x4e, 0xe4, 0xf2, 0x53, 0x19, 0xea, 0x2d, 0xbb, 0x2b, 0xd1, 0x7b, 0x94, 0x9e, +- 0x1d, 0xb8, 0xfc, 0x14, 0x7d, 0x02, 0x3d, 0x5d, 0x06, 0x06, 0xd2, 0x45, 0xb1, 0xfe, 0xf8, 0xe9, +- 0x28, 0xca, 0x7b, 0xcf, 0xee, 0x9e, 0xe5, 0xa0, 0xd8, 0xba, 0x0d, 0xb7, 0x76, 0x70, 0xcc, 0x19, +- 0x9d, 0x15, 0x1d, 0x63, 0xfd, 0x08, 0x60, 0x3f, 0xe4, 0x98, 0x9d, 0xb8, 0x1e, 0x8e, 0xd1, 0x7b, +- 0x79, 0x48, 0x17, 0x47, 0x2b, 0x63, 0x35, 0x91, 0x4a, 0x37, 0xec, 0x1c, 0x8d, 0x35, 0x86, 0x86, +- 0x4d, 0x13, 0x91, 0x8e, 0xde, 0x30, 0x2b, 0x7d, 0xae, 0xa3, 0xcf, 0x49, 0xa4, 0xad, 0xf7, 0xac, +- 0x3d, 0xd3, 0xc2, 0x66, 0xec, 0xf4, 0x15, 0x8d, 0xa1, 0x45, 0x0c, 0x4e, 0x67, 0x95, 0x45, 0xd1, +- 0x19, 0x89, 0xf5, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x06, 0x34, 0x98, 0x51, 0xa3, +- 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x13, 0xfe, 0x10, 0x1d, 0x75, 0x66, 0x88, 0xf1, 0xc7, 0x2a, +- 0x0c, 0xc4, 0x46, 0x81, 0xa7, 0xf5, 0x4b, 0x58, 0x7d, 0x16, 0x4e, 0x49, 0x88, 0xb7, 0x0f, 0x8e, +- 0x9e, 0xe2, 0x34, 0xee, 0x11, 0xd4, 0x45, 0x7d, 0x24, 0x05, 0x35, 0x6d, 0xb9, 0x16, 0x81, 0x10, +- 0x1e, 0x3b, 0x5e, 0x94, 0xc4, 0x7a, 0xf6, 0xd3, 0x08, 0x8f, 0xb7, 0xa3, 0x24, 0x16, 0x89, 0x5c, +- 0x7c, 0xc8, 0x69, 0x38, 0x9d, 0xc9, 0x68, 0x68, 0xda, 0xcb, 0x5e, 0x94, 0x3c, 0x0b, 0xa7, 0x33, +- 0xeb, 0x07, 0xb2, 0xdb, 0xc5, 0xd8, 0xb7, 0xdd, 0xd0, 0xa7, 0xc1, 0x0e, 0x3e, 0xcf, 0x49, 0x48, +- 0x3b, 0x2b, 0x13, 0xf5, 0xdf, 0x55, 0xa0, 0xf3, 0x78, 0x82, 0x43, 0xbe, 0x83, 0xb9, 0x4b, 0xa6, +- 0xb2, 0x7b, 0x3a, 0xc7, 0x2c, 0x26, 0x34, 0xd4, 0x4f, 0xdb, 0x80, 0xa2, 0xf9, 0x25, 0x21, 0xe1, +- 0x8e, 0xef, 0xe2, 0x80, 0x86, 0x92, 0x4b, 0xd3, 0x06, 0x81, 0xda, 0x91, 0x18, 0xf4, 0x16, 0xf4, +- 0xd5, 0x6c, 0xce, 0x39, 0x75, 0x43, 0x7f, 0x2a, 0x82, 0x4a, 0xcd, 0x2a, 0x7a, 0x0a, 0xbd, 0xa7, +- 0xb1, 0xe8, 0x6d, 0x58, 0xd1, 0x4f, 0x3e, 0xa3, 0xac, 0x4b, 0xca, 0xbe, 0xc6, 0x17, 0x48, 0x93, +- 0x28, 0xa2, 0x8c, 0xc7, 0x4e, 0x8c, 0x3d, 0x8f, 0x06, 0x91, 0x6e, 0x3d, 0xfa, 0x06, 0x7f, 0xa8, +- 0xd0, 0xd6, 0x04, 0x56, 0x9f, 0x08, 0x3b, 0xb5, 0x25, 0xd9, 0x15, 0xf6, 0x02, 0x1c, 0x38, 0xc7, +- 0x53, 0xea, 0x9d, 0x39, 0x22, 0x11, 0x69, 0x0f, 0x8b, 0xe2, 0x66, 0x4b, 0x20, 0x0f, 0xc9, 0xb7, +- 0xb2, 0xcb, 0x16, 0x54, 0xa7, 0x94, 0x47, 0xd3, 0x64, 0xe2, 0x44, 0x8c, 0x1e, 0x63, 0x6d, 0x62, +- 0x3f, 0xc0, 0xc1, 0x9e, 0xc2, 0x1f, 0x08, 0xb4, 0xf5, 0xd7, 0x0a, 0xac, 0x15, 0x25, 0xe9, 0xb4, +- 0xba, 0x09, 0x6b, 0x45, 0x51, 0xfa, 0x53, 0xab, 0x4a, 0xb9, 0x41, 0x5e, 0xa0, 0xfa, 0xe8, 0x7e, +- 0x04, 0x5d, 0x39, 0xb0, 0x75, 0x7c, 0xc5, 0xa9, 0x58, 0x60, 0xe4, 0xef, 0xc5, 0xee, 0xb8, 0xf9, +- 0x5b, 0xfa, 0x04, 0xee, 0x68, 0xf3, 0x9d, 0x45, 0xb5, 0xd5, 0x83, 0x58, 0xd7, 0x04, 0x4f, 0xe7, +- 0xb4, 0xff, 0x12, 0x86, 0x19, 0x6a, 0x6b, 0x26, 0x91, 0xc6, 0x57, 0xef, 0xc1, 0xea, 0x9c, 0xb1, +- 0x8f, 0x7d, 0x9f, 0xc9, 0x10, 0xac, 0xdb, 0x65, 0x5b, 0xd6, 0x23, 0xb8, 0x7d, 0x88, 0xb9, 0xf2, +- 0x86, 0xcb, 0x75, 0xd5, 0xaf, 0x98, 0xad, 0x40, 0xed, 0x10, 0x7b, 0xd2, 0xf8, 0x9a, 0x2d, 0x96, +- 0xe2, 0x01, 0x1e, 0xc5, 0xd8, 0x93, 0x56, 0xd6, 0x6c, 0xb9, 0xb6, 0xfe, 0x52, 0x81, 0x65, 0x9d, +- 0x08, 0x45, 0x32, 0xf7, 0x19, 0x39, 0xc7, 0x4c, 0x3f, 0x3d, 0x0d, 0xa1, 0x37, 0xa1, 0xa7, 0x56, +- 0x0e, 0x8d, 0x38, 0xa1, 0x69, 0x7a, 0xed, 0x2a, 0xec, 0x33, 0x85, 0x94, 0xb3, 0x38, 0x39, 0x6a, +- 0xd2, 0x5d, 0x9d, 0x86, 0xe4, 0x40, 0x2d, 0x16, 0xb1, 0x2f, 0xd3, 0x69, 0xcb, 0xd6, 0x90, 0x78, +- 0xea, 0x86, 0xdf, 0x92, 0xe4, 0x67, 0x40, 0xf1, 0xd4, 0x03, 0x9a, 0x84, 0xdc, 0x89, 0x28, 0x09, +- 0xb9, 0xce, 0x9f, 0x20, 0x51, 0x07, 0x02, 0x63, 0xfd, 0xa6, 0x02, 0x0d, 0x35, 0x8f, 0x16, 0x7d, +- 0x64, 0xfa, 0x15, 0xab, 0x12, 0x59, 0x11, 0x48, 0x59, 0xea, 0xcb, 0x25, 0xd7, 0x22, 0x8e, 0xcf, +- 0x03, 0x95, 0x8b, 0xb5, 0x6a, 0xe7, 0x81, 0x4c, 0xc2, 0x6f, 0x42, 0x2f, 0xfb, 0x18, 0xca, 0x7d, +- 0xa5, 0x62, 0x37, 0xc5, 0x4a, 0xb2, 0x4b, 0x35, 0xb5, 0x7e, 0x2e, 0xda, 0xe7, 0x74, 0x16, 0xbb, +- 0x02, 0xb5, 0x24, 0x55, 0x46, 0x2c, 0x05, 0x66, 0x92, 0x7e, 0x46, 0xc5, 0x12, 0x3d, 0x80, 0x9e, +- 0xeb, 0xfb, 0x44, 0x1c, 0x77, 0xa7, 0x4f, 0x88, 0x9f, 0x06, 0x69, 0x11, 0x6b, 0xfd, 0xbd, 0x02, +- 0xfd, 0x6d, 0x1a, 0xcd, 0x7e, 0x4c, 0xa6, 0x38, 0x97, 0x41, 0xa4, 0x92, 0xfa, 0x2b, 0x2a, 0xd6, +- 0xa2, 0x32, 0x3c, 0x21, 0x53, 0xac, 0x42, 0x4b, 0xdd, 0x6c, 0x53, 0x20, 0x64, 0x58, 0x99, 0xcd, +- 0x74, 0xc4, 0xd5, 0x55, 0x9b, 0x4f, 0xa9, 0x2f, 0x6b, 0x60, 0x9f, 0x30, 0x27, 0x1d, 0x68, 0x75, +- 0xed, 0x65, 0x9f, 0x30, 0xb9, 0xa5, 0x0d, 0x59, 0x92, 0x33, 0xd5, 0xbc, 0x21, 0x0d, 0x85, 0x11, +- 0x86, 0xac, 0x43, 0x83, 0x9e, 0x9c, 0xc4, 0x98, 0xcb, 0x6a, 0xb5, 0x66, 0x6b, 0x28, 0x4d, 0x73, +- 0xcd, 0x5c, 0x9a, 0xbb, 0x05, 0xab, 0x72, 0x7a, 0xff, 0x9c, 0xb9, 0x1e, 0x09, 0x27, 0x26, 0x15, +- 0xaf, 0x01, 0x3a, 0xe4, 0x34, 0x2a, 0x62, 0x1f, 0xfe, 0x7e, 0x45, 0xe7, 0x44, 0xdd, 0xca, 0xa2, +- 0x27, 0xd0, 0x9f, 0xfb, 0x6b, 0x04, 0xe9, 0xd9, 0x46, 0xf9, 0x3f, 0x26, 0xa3, 0xf5, 0xb1, 0xfa, +- 0xab, 0x65, 0x6c, 0xfe, 0x6a, 0x19, 0xef, 0x06, 0x11, 0x9f, 0xa1, 0x5d, 0xe8, 0x15, 0xff, 0x44, +- 0x40, 0x77, 0x4d, 0x29, 0x50, 0xf2, 0xd7, 0xc2, 0xa5, 0x6c, 0x9e, 0x40, 0x7f, 0xee, 0xff, 0x04, +- 0xa3, 0x4f, 0xf9, 0xdf, 0x0c, 0x97, 0x32, 0x7a, 0x04, 0xed, 0xdc, 0x1f, 0x08, 0x68, 0xa8, 0x98, +- 0x2c, 0xfe, 0xa7, 0x70, 0x29, 0x83, 0x6d, 0xe8, 0x16, 0x66, 0xfa, 0x68, 0xa4, 0xed, 0x29, 0x19, +- 0xf4, 0x5f, 0xca, 0x64, 0x0b, 0xda, 0xb9, 0xd1, 0xba, 0xd1, 0x62, 0x71, 0x7e, 0x3f, 0xba, 0x53, +- 0xb2, 0xa3, 0x53, 0xef, 0x1e, 0x74, 0x0b, 0x83, 0x70, 0xa3, 0x48, 0xd9, 0x10, 0x7e, 0x74, 0xb7, +- 0x74, 0x4f, 0x73, 0x7a, 0x02, 0xfd, 0xb9, 0xb1, 0xb8, 0x71, 0x6e, 0xf9, 0xb4, 0xfc, 0x52, 0xb3, +- 0xbe, 0x90, 0x97, 0x9d, 0xeb, 0x7a, 0x72, 0x97, 0xbd, 0x38, 0x04, 0x1f, 0xbd, 0x52, 0xbe, 0xa9, +- 0xb5, 0xda, 0x85, 0x5e, 0x71, 0xfe, 0x6d, 0x98, 0x95, 0x4e, 0xc5, 0xaf, 0x7e, 0x39, 0x85, 0x51, +- 0x78, 0xf6, 0x72, 0xca, 0x26, 0xe4, 0x97, 0x32, 0x7a, 0x0c, 0xa0, 0x7b, 0x1c, 0x9f, 0x84, 0xe9, +- 0x95, 0x2d, 0xf4, 0x56, 0xe9, 0x95, 0x95, 0xf4, 0x43, 0x8f, 0x00, 0x54, 0x6b, 0xe2, 0xd3, 0x84, +- 0xa3, 0xdb, 0x46, 0x8d, 0xb9, 0x7e, 0x68, 0x34, 0x5c, 0xdc, 0x58, 0x60, 0x80, 0x19, 0xbb, 0x0e, +- 0x83, 0xcf, 0x01, 0xb2, 0x96, 0xc7, 0x30, 0x58, 0x68, 0x82, 0xae, 0xf0, 0x41, 0x27, 0xdf, 0xe0, +- 0x20, 0x6d, 0x6b, 0x49, 0xd3, 0x73, 0x05, 0x8b, 0xfe, 0x5c, 0x01, 0x5b, 0x7c, 0x6c, 0xf3, 0x75, +- 0xed, 0x68, 0xa1, 0x88, 0x45, 0x1f, 0x41, 0x27, 0x5f, 0xb9, 0x1a, 0x2d, 0x4a, 0xaa, 0xd9, 0x51, +- 0xa1, 0x7a, 0x45, 0x8f, 0xa0, 0x57, 0xac, 0x5a, 0x51, 0x2e, 0x2e, 0x16, 0x6a, 0xd9, 0x91, 0x9e, +- 0xc9, 0xe4, 0xc8, 0x3f, 0x00, 0xc8, 0xaa, 0x5b, 0xe3, 0xbe, 0x85, 0x7a, 0x77, 0x4e, 0xea, 0x63, +- 0xe8, 0xe4, 0x33, 0xb1, 0x51, 0xb7, 0x24, 0x3b, 0x5f, 0x95, 0xb5, 0x72, 0x59, 0xdb, 0x3c, 0xbe, +- 0xc5, 0x44, 0x7e, 0x55, 0xd6, 0x2a, 0xf4, 0x75, 0x26, 0x59, 0x94, 0x35, 0x7b, 0x57, 0xe5, 0xf2, +- 0x62, 0x13, 0x64, 0xdc, 0x57, 0xda, 0x1a, 0x5d, 0xf5, 0x88, 0xf2, 0xdd, 0x80, 0xf1, 0x47, 0x49, +- 0x87, 0xf0, 0x92, 0xa0, 0xce, 0x57, 0xfc, 0xb9, 0xa0, 0x2e, 0x69, 0x04, 0x2e, 0x65, 0xb4, 0x07, +- 0xfd, 0x27, 0xa6, 0x98, 0xd3, 0x85, 0xa6, 0x56, 0xa7, 0xa4, 0xb0, 0x1e, 0x8d, 0xca, 0xb6, 0x74, +- 0x64, 0x7d, 0x01, 0x83, 0x85, 0x22, 0x13, 0xdd, 0x4b, 0x47, 0x87, 0xa5, 0xd5, 0xe7, 0xa5, 0x6a, +- 0xed, 0xc3, 0xca, 0x7c, 0x8d, 0x89, 0x5e, 0xd5, 0x97, 0x5e, 0x5e, 0x7b, 0x5e, 0xca, 0xea, 0x13, +- 0x68, 0x9a, 0x9a, 0x06, 0xe9, 0x11, 0xed, 0x5c, 0x8d, 0x73, 0xd9, 0xd1, 0xad, 0xce, 0x77, 0xdf, +- 0xdf, 0xab, 0xfc, 0xe3, 0xfb, 0x7b, 0x95, 0x7f, 0x7d, 0x7f, 0xaf, 0x72, 0xdc, 0x90, 0xbb, 0x1f, +- 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xac, 0x85, 0x1d, 0xaa, 0x21, 0x00, 0x00, ++ 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, ++ 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, ++ 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, ++ 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, ++ 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, ++ 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, ++ 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, ++ 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, ++ 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, ++ 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, ++ 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, ++ 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, ++ 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, ++ 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, ++ 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, ++ 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, ++ 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, ++ 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, ++ 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, ++ 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, ++ 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, ++ 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, ++ 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, ++ 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, ++ 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, ++ 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, ++ 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, ++ 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, ++ 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, ++ 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, ++ 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, ++ 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, ++ 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, ++ 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, ++ 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, ++ 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, ++ 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, ++ 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, ++ 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, ++ 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, ++ 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, ++ 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, ++ 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, ++ 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, ++ 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, ++ 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, ++ 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, ++ 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, ++ 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, ++ 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, ++ 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, ++ 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, ++ 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, ++ 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, ++ 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, ++ 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, ++ 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, ++ 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, ++ 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, ++ 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, ++ 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, ++ 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, ++ 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, ++ 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, ++ 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, ++ 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, ++ 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, ++ 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, ++ 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, ++ 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, ++ 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, ++ 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, ++ 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, ++ 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, ++ 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, ++ 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, ++ 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, ++ 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, ++ 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, ++ 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, ++ 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, ++ 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, ++ 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, ++ 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, ++ 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, ++ 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, ++ 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, ++ 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, ++ 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, ++ 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, ++ 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, ++ 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, ++ 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, ++ 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, ++ 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, ++ 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, ++ 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, ++ 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, ++ 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, ++ 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, ++ 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, ++ 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, ++ 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, ++ 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, ++ 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, ++ 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, ++ 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, ++ 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, ++ 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, ++ 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, ++ 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, ++ 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, ++ 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, ++ 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, ++ 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, ++ 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, ++ 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, ++ 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, ++ 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, ++ 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, ++ 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, ++ 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, ++ 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, ++ 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, ++ 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, ++ 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, ++ 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, ++ 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, ++ 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, ++ 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, ++ 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, ++ 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, ++ 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, ++ 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, ++ 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, ++ 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, ++ 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, ++ 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, ++ 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, ++ 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, ++ 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, ++ 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, ++ 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, ++ 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, ++ 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, ++ 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, ++ 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, ++ 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, ++ 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, ++ 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, ++ 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, ++ 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, ++ 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, ++ 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, ++ 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, ++ 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, ++ 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, ++ 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, ++ 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, ++ 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, ++ 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, ++ 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, ++ 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, ++ 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, ++ 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, ++ 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, ++ 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, ++ 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, ++ 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, ++ 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, ++ 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, ++ 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, ++ 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, ++ 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, ++ 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, ++ 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, ++ 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, ++ 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, ++ 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, + } +diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go +index b1dea816..c22ed39c 100644 +--- a/virtcontainers/agent.go ++++ b/virtcontainers/agent.go +@@ -235,7 +235,8 @@ type agent interface { + listInterfaces() ([]*vcTypes.Interface, error) + + // updateRoutes will tell the agent to update route table for an existed Sandbox. +- updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) ++ // increment specifies update route with increment mode ++ updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) + + // listRoutes will tell the agent to list routes of an existed Sandbox + listRoutes() ([]*vcTypes.Route, error) +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index fb044fe1..2331eb94 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -959,7 +959,7 @@ func ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface + } + + // UpdateRoutes is the virtcontainers update routes entry point. +-func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { ++func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + span, ctx := trace(ctx, "UpdateRoutes") + defer span.Finish() + +@@ -979,7 +979,7 @@ func UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route + } + defer s.releaseStatelessSandbox() + +- return s.UpdateRoutes(routes) ++ return s.UpdateRoutes(routes, op) + } + + // ListRoutes is the virtcontainers list routes entry point. +diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go +index 5fb8ff4d..c01d47b8 100644 +--- a/virtcontainers/api_test.go ++++ b/virtcontainers/api_test.go +@@ -1699,7 +1699,7 @@ func TestNetworkOperation(t *testing.T) { + _, err = ListInterfaces(ctx, s.ID()) + assert.NoError(err) + +- _, err = UpdateRoutes(ctx, s.ID(), nil) ++ _, err = UpdateRoutes(ctx, s.ID(), nil, vcTypes.NetworkOpUpdate) + assert.NoError(err) + + _, err = ListRoutes(ctx, s.ID()) +diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go +index 079afa0a..92cc4c12 100644 +--- a/virtcontainers/implementation.go ++++ b/virtcontainers/implementation.go +@@ -163,8 +163,8 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc + } + + // UpdateRoutes implements the VC function of the same name. +-func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { +- return UpdateRoutes(ctx, sandboxID, routes) ++func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { ++ return UpdateRoutes(ctx, sandboxID, routes, op) + } + + // ListRoutes implements the VC function of the same name. +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index 499b386e..6bb9ea22 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -51,7 +51,7 @@ type VC interface { + AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) +- UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) ++ UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) + ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + + CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error +@@ -96,7 +96,7 @@ type VCSandbox interface { + AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) + RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) + ListInterfaces() ([]*vcTypes.Interface, error) +- UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) ++ UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) + ListRoutes() ([]*vcTypes.Route, error) + IsCompatOldCNI() bool + } +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index dfdd263b..16662949 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -623,12 +623,13 @@ func (k *kataAgent) updateInterfaces(interfaces []*vcTypes.Interface) error { + return nil + } + +-func (k *kataAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { ++func (k *kataAgent) updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) { + if routes != nil { + routesReq := &grpc.UpdateRoutesRequest{ + Routes: &grpc.Routes{ + Routes: k.convertToKataAgentRoutes(routes), + }, ++ Increment: increment, + } + resultingRoutes, err := k.sendReq(routesReq) + if err != nil { +@@ -858,7 +859,7 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { + if err = k.updateInterfaces(interfaces); err != nil { + return err + } +- if _, err = k.updateRoutes(routes); err != nil { ++ if _, err = k.updateRoutes(routes, false); err != nil { + return err + } + +diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go +index 2a2ddada..18a5a0a6 100644 +--- a/virtcontainers/kata_agent_test.go ++++ b/virtcontainers/kata_agent_test.go +@@ -943,7 +943,7 @@ func TestAgentNetworkOperation(t *testing.T) { + _, err = k.listInterfaces() + assert.Nil(err) + +- _, err = k.updateRoutes([]*vcTypes.Route{}) ++ _, err = k.updateRoutes([]*vcTypes.Route{}, false) + assert.Nil(err) + + _, err = k.listRoutes() +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index a1676ccd..f3757f84 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -81,6 +81,8 @@ const ( + tcFilterNetModelStr = "tcfilter" + + noneNetModelStr = "none" ++ ++ localHostDeviceName = "lo" + ) + + //SetModel change the model string value +@@ -1485,3 +1487,163 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { + + return nil + } ++ ++// verifyRouteDest verifies the route is valid or not ++func verifyRouteDest(ipMask *string) (nlIpMask *net.IPNet, err error) { ++ // set the defaule mask "32", just like the command `ip route add ${ip}`, ++ // it will give the "255.255.255.255"(/32) by default ++ if !strings.Contains(*ipMask, "/") { ++ *ipMask = *ipMask + "/32" ++ } ++ nlIpNet, err := verifyIPAndMask(*ipMask) ++ if err != nil { ++ return nil, err ++ } ++ ++ return nlIpNet, nil ++} ++ ++// generateAddRoute transform input vcTypes.Route info into netlink.Route info ++func generateAddRoute(route *vcTypes.Route) (r *netlink.Route, err error) { ++ // we must specify the device and destination ++ if route.Device == "" || route.Dest == "" { ++ return nil, fmt.Errorf("device and destination must be specified.") ++ } ++ ++ err = verifyInterfaceName(route.Device) ++ if err != nil { ++ return nil, err ++ } ++ ++ r = &netlink.Route{} ++ if route.Dest != "default" { ++ nlIpNet, err := verifyRouteDest(&route.Dest) ++ if err != nil { ++ return nil, err ++ } ++ r.Dst = nlIpNet ++ } else { ++ // if destination of route rule is "default", gateway is needed ++ if route.Gateway == "" { ++ return nil, fmt.Errorf("gateway must be specified when destination is \"default\"") ++ } ++ } ++ ++ if route.Gateway != "" { ++ nIP, err := verifyIP(route.Gateway) ++ if err != nil { ++ return nil, err ++ } ++ r.Gw = *nIP ++ } ++ ++ if route.Source != "" { ++ nIP, err := verifyIP(route.Source) ++ if err != nil { ++ return nil, err ++ } ++ r.Src = *nIP ++ } ++ ++ r.Scope = netlink.Scope(route.Scope) ++ ++ return r, nil ++} ++ ++// isSameRoute checks two input routes is same or not ++func isSameRoute(existing, r *netlink.Route, fuzzy bool) bool { ++ var ( ++ existDst string ++ inputDst string ++ ) ++ if existing.Dst != nil && existing.Dst.String() != "" { ++ existDst = existing.Dst.String() ++ } ++ if r.Dst != nil && r.Dst.String() != "" { ++ inputDst = r.Dst.String() ++ } ++ ++ if fuzzy { ++ // fuzzy matching ++ if inputDst != "" && existDst != inputDst { ++ return false ++ } else if inputDst == "" && existDst != "" { ++ return false ++ } ++ if r.Gw.String() != "" && existing.Gw.String() != r.Gw.String() { ++ return false ++ } ++ } else { ++ // exact matching ++ if existing.Dst != nil && r.Dst != nil && existing.Dst.String() != r.Dst.String() { ++ return false ++ } ++ if (existing.Dst == nil && r.Dst != nil) || (existing.Dst != nil && r.Dst == nil) { ++ return false ++ } ++ if existing.Gw.String() != r.Gw.String() { ++ return false ++ } ++ } ++ ++ return true ++} ++ ++func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Route, err error) { ++ add, err := generateAddRoute(route) ++ if err != nil { ++ return nil, err ++ } ++ ++ // add the route for "lo" loopback device ++ if route.Device == localHostDeviceName { ++ added = &vcTypes.Route{ ++ Dest: route.Dest, ++ Gateway: route.Gateway, ++ Device: route.Device, ++ } ++ return added, nil ++ } ++ ++ // add the route for exist network enpoints ++ for _, ep := range ns.Endpoints { ++ if ep.Name() != route.Device { ++ continue ++ } ++ ++ netInfo := ep.Properties() ++ for _, exist := range ep.Properties().Routes { ++ if isSameRoute(&exist, add, false) { ++ return nil, fmt.Errorf("route rule %v already exits", add) ++ } ++ } ++ // flush the netInfo.Routes with new added route ++ netInfo.Routes = append(netInfo.Routes, *add) ++ ep.SetProperties(netInfo) ++ added = &vcTypes.Route{ ++ Dest: route.Dest, ++ Gateway: route.Gateway, ++ Device: route.Device, ++ } ++ break ++ } ++ ++ return added, nil ++} ++ ++func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { ++ var updRoutes []*vcTypes.Route ++ ++ if op == vcTypes.NetworkOpAdd { ++ added, err := addOneRoute(ns, route) ++ if err != nil { ++ return nil, err ++ } ++ if added == nil { ++ return nil, fmt.Errorf("no matching endpoints for route device: %#v", route.Device) ++ } ++ updRoutes = append(updRoutes, added) ++ } ++ ++ return updRoutes, nil ++} +diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go +index 6e211bca..b19705c3 100644 +--- a/virtcontainers/noop_agent.go ++++ b/virtcontainers/noop_agent.go +@@ -117,7 +117,7 @@ func (n *noopAgent) listInterfaces() ([]*vcTypes.Interface, error) { + } + + // updateRoutes is the Noop agent Routes update implementation. It does nothing. +-func (n *noopAgent) updateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { ++func (n *noopAgent) updateRoutes(routes []*vcTypes.Route, increment bool) ([]*vcTypes.Route, error) { + return nil, nil + } + +@@ -239,4 +239,4 @@ func (n *noopAgent) load(s persistapi.AgentState) {} + + func (n *noopAgent) getProxyPid() int { + return -1 +-} +\ No newline at end of file ++} +diff --git a/virtcontainers/noop_agent_test.go b/virtcontainers/noop_agent_test.go +index 97c56f21..63f52e58 100644 +--- a/virtcontainers/noop_agent_test.go ++++ b/virtcontainers/noop_agent_test.go +@@ -201,7 +201,7 @@ func TestNoopAgentListInterfaces(t *testing.T) { + func TestNoopAgentUpdateRoutes(t *testing.T) { + assert := assert.New(t) + n := &noopAgent{} +- _, err := n.updateRoutes(nil) ++ _, err := n.updateRoutes(nil, false) + assert.NoError(err) + } + +diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go +index fcc63d84..71fe7fbb 100644 +--- a/virtcontainers/pkg/types/types.go ++++ b/virtcontainers/pkg/types/types.go +@@ -39,3 +39,17 @@ type Route struct { + Source string + Scope uint32 + } ++ ++//NetworkOp describes network operation ++type NetworkOp string ++ ++const ( ++ //NetworkOpAdd adds the network interface ++ NetworkOpAdd NetworkOp = "add" ++ ++ //NetworkOpRemove removes the network interface ++ NetworkOpRemove NetworkOp = "remove" ++ ++ //NetworkOpUpdate updates the network interface ++ NetworkOpUpdate NetworkOp = "update" ++) +diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go +index 51a2c216..9c50e0e4 100644 +--- a/virtcontainers/pkg/vcmock/mock.go ++++ b/virtcontainers/pkg/vcmock/mock.go +@@ -273,7 +273,7 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp + } + + // UpdateRoutes implements the VC function of the same name. +-func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) { ++func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + if m.UpdateRoutesFunc != nil { + return m.UpdateRoutesFunc(ctx, sandboxID, routes) + } +diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go +index 11b83ccd..7329e1a7 100644 +--- a/virtcontainers/pkg/vcmock/sandbox.go ++++ b/virtcontainers/pkg/vcmock/sandbox.go +@@ -204,7 +204,7 @@ func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { + } + + // UpdateRoutes implements the VCSandbox function of the same name. +-func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { ++func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + return nil, nil + } + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index f6826812..f226af4e 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1031,8 +1031,33 @@ func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { + } + + // UpdateRoutes updates the sandbox route table (e.g. for portmapping support). +-func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route) ([]*vcTypes.Route, error) { +- return s.agent.updateRoutes(routes) ++func (s *Sandbox) UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) (r []*vcTypes.Route, err error) { ++ var allUpdateRoutes []*vcTypes.Route ++ ++ increment := true ++ if op == vcTypes.NetworkOpUpdate { ++ allUpdateRoutes = routes ++ increment = false ++ } else { ++ for _, r := range routes { ++ updRoutes, err := updateRoute(&s.networkNS, r, op) ++ if err != nil { ++ return nil, err ++ } ++ allUpdateRoutes = append(allUpdateRoutes, updRoutes...) ++ } ++ } ++ ++ r, err = s.agent.updateRoutes(allUpdateRoutes, increment) ++ if err != nil { ++ return nil, err ++ } ++ ++ if err = s.Save(); err != nil { ++ return nil, err ++ } ++ ++ return r, nil + } + + // ListRoutes lists all routes and their configurations in the sandbox. +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch b/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch new file mode 100644 index 0000000000000000000000000000000000000000..a4c8cc8349c5092de85258b52c8a0f589497389d --- /dev/null +++ b/runtime/patches/0029-network-add-kata-network-del-route-subcommand.patch @@ -0,0 +1,192 @@ +From 51a7270987557ab12ea735fc9781725d1ce1b0a6 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 17 Aug 2020 15:36:32 +0800 +Subject: [PATCH 29/50] network: add kata-network del-route subcommand + +reason: add kata-network del-route subcommand to delete the +specified route in the Kata VM, del-route is more efficient +than the update-routes subcommand. + +Signed-off-by: jiangpengfei +--- + cli/network.go | 24 ++++++++++ + virtcontainers/network.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 139 insertions(+) + +diff --git a/cli/network.go b/cli/network.go +index 2265f54b..046d0ee9 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -45,6 +45,7 @@ var kataNetworkCLICommand = cli.Command{ + updateRoutesCommand, + listRoutesCommand, + addRoutesCommand, ++ deleteRoutesCommand, + }, + Action: func(context *cli.Context) error { + return cli.ShowSubcommandHelp(context) +@@ -155,6 +156,29 @@ var addRoutesCommand = cli.Command{ + }, + } + ++var deleteRoutesCommand = cli.Command{ ++ Name: "del-route", ++ Usage: "delete one route for a container", ++ ArgsUsage: `del-route file or - for stdin ++ file or stdin for example: ++ { ++ "dest":"<[[/mask] | "default" ]>", ++ "gateway":"[ip]", ++ "device":"[tap-name]", ++ } ++ Only destination is required and others are optional, ++ if device is empty, means search every device`, ++ Flags: []cli.Flag{}, ++ Action: func(context *cli.Context) error { ++ ctx, err := cliContextToContext(context) ++ if err != nil { ++ return err ++ } ++ ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), routeType, vcTypes.NetworkOpRemove) ++ }, ++} ++ + var listRoutesCommand = cli.Command{ + Name: "list-routes", + Usage: "list network routes in a container", +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index f3757f84..c7066a11 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -1631,6 +1631,110 @@ func addOneRoute(ns *NetworkNamespace, route *vcTypes.Route) (added *vcTypes.Rou + return added, nil + } + ++func generateRmRoute(route *vcTypes.Route) (r *netlink.Route, err error) { ++ // destination is required and others are optional ++ if route.Dest == "" { ++ return nil, fmt.Errorf("destination must be specified when remove route.") ++ } ++ ++ if route.Device != "" { ++ err = verifyInterfaceName(route.Device) ++ if err != nil { ++ return nil, err ++ } ++ } ++ ++ r = &netlink.Route{} ++ if route.Dest != "default" { ++ nlIpNet, err := verifyRouteDest(&route.Dest) ++ if err != nil { ++ return nil, err ++ } ++ r.Dst = nlIpNet ++ } ++ ++ if route.Gateway != "" { ++ nIP, err := verifyIP(route.Gateway) ++ if err != nil { ++ return nil, err ++ } ++ r.Gw = *nIP ++ } ++ ++ if route.Source != "" { ++ nIP, err := verifyIP(route.Source) ++ if err != nil { ++ return nil, err ++ } ++ r.Src = *nIP ++ } ++ ++ r.Scope = netlink.Scope(route.Scope) ++ ++ return r, nil ++} ++ ++// parseToGrpcRoute convert the netlink.Route to vcTypes.Route and deleted route.Dest will ++// be added a prefix "-" ++func parseToGrpcRoute(device string, route *netlink.Route, add bool) (r *vcTypes.Route) { ++ r = &vcTypes.Route{ ++ Device: device, ++ } ++ if route.Dst != nil && route.Dst.String() != "" { ++ r.Dest = route.Dst.String() ++ if add == false { ++ r.Dest = "-" + route.Dst.String() ++ } ++ } else if route.Dst == nil { ++ r.Dest = "default" ++ if add == false { ++ r.Dest = "-default" ++ } ++ } ++ if route.Gw != nil && route.Gw.String() != "" { ++ r.Gateway = route.Gw.String() ++ } ++ ++ return r ++} ++ ++func removeRoutes(ns *NetworkNamespace, route *vcTypes.Route) (removed []*vcTypes.Route, err error) { ++ del, err := generateRmRoute(route) ++ if err != nil { ++ return nil, err ++ } ++ ++ // remove the lo device related routes ++ if route.Device == localHostDeviceName { ++ removed = append(removed, parseToGrpcRoute(localHostDeviceName, del, false)) ++ ++ return removed, nil ++ } ++ ++ for _, ep := range ns.Endpoints { ++ // if device is empty, means search every device ++ if route.Device != "" && ep.Name() != route.Device { ++ continue ++ } ++ ++ netInfo := ep.Properties() ++ for i, exist := range ep.Properties().Routes { ++ if isSameRoute(&exist, del, true) { ++ // need remove ++ netInfo.Routes = append(netInfo.Routes[:i], netInfo.Routes[i+1:]...) ++ ep.SetProperties(netInfo) ++ dev := route.Device ++ if route.Device == "" { ++ dev = netInfo.Iface.Name ++ } ++ removed = append(removed, parseToGrpcRoute(dev, del, false)) ++ } ++ } ++ } ++ ++ return removed, nil ++} ++ + func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + var updRoutes []*vcTypes.Route + +@@ -1645,5 +1749,16 @@ func updateRoute(ns *NetworkNamespace, route *vcTypes.Route, op vcTypes.NetworkO + updRoutes = append(updRoutes, added) + } + ++ if op == vcTypes.NetworkOpRemove { ++ removed, err := removeRoutes(ns, route) ++ if err != nil { ++ return nil, err ++ } ++ if len(removed) <= 0 { ++ return nil, fmt.Errorf("route of device %s with destination %s is not found", route.Device, route.Dest) ++ } ++ updRoutes = removed ++ } ++ + return updRoutes, nil + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch b/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch new file mode 100644 index 0000000000000000000000000000000000000000..7d628272fd56341da690a90c5df0af7911dfba3a --- /dev/null +++ b/runtime/patches/0030-network-kata-network-list-routes-support-display-com.patch @@ -0,0 +1,104 @@ +From 9cf769178b8f73c7fd624895589e918d6c7b0645 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 17 Aug 2020 16:29:17 +0800 +Subject: [PATCH 30/50] network: kata-network list-routes support display + compatible format + +reason: kata-network list-routes subcommand support display compatible +format when enable_compat_old_cni config is enabled. + +Signed-off-by: jiangpengfei +--- + virtcontainers/api.go | 13 ++++++++++++- + virtcontainers/network.go | 29 +++++++++++++++++++++++++++++ + virtcontainers/pkg/types/types.go | 10 +++++----- + 3 files changed, 46 insertions(+), 6 deletions(-) + +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 2331eb94..a4bf41bb 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -1003,7 +1003,18 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + } + defer s.releaseStatelessSandbox() + +- return s.ListRoutes() ++ routes, err := s.ListRoutes() ++ if err != nil { ++ return nil, err ++ } ++ ++ // If enable_compat_old_cni is enabled, convert routes info to ++ // compatible display format ++ if s.config.NetworkConfig.EnableCompatOldCNI { ++ return convertToDisplayRoutes(&routes), nil ++ } ++ ++ return routes, nil + } + + // CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index c7066a11..488bd00c 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -1384,6 +1384,35 @@ func convertToCompatInterfaces(es *[]Endpoint) []*vcTypes.Interface { + return infs + } + ++// convertToDisplayRoutes convert the default route format to more simple ++// route display format, it is called when enable_compat_old_cni config ++// is enabled. ++func convertToDisplayRoutes(routes *[]*vcTypes.Route) []*vcTypes.Route { ++ var displayRoutes []*vcTypes.Route ++ if routes == nil { ++ return displayRoutes ++ } ++ for _, r := range *routes { ++ // we don't support IPV6 temporarily, so we need to filter ":" which ++ // is the characteristics of IPV6 for those default routes ++ if strings.Contains(r.Dest, ":") { ++ continue ++ } ++ ++ defaultDest := r.Dest ++ if r.Dest == "" { ++ defaultDest = "default" ++ } ++ displayRoutes = append(displayRoutes, &vcTypes.Route{ ++ Dest: defaultDest, ++ Gateway: r.Gateway, ++ Device: r.Device, ++ }) ++ } ++ ++ return displayRoutes ++} ++ + // verifyInterfaceName verifies the interface name valid or not + func verifyInterfaceName(name string) error { + // verify `Name` before `Tapname` because of the strict rules +diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go +index 71fe7fbb..b41b0c75 100644 +--- a/virtcontainers/pkg/types/types.go ++++ b/virtcontainers/pkg/types/types.go +@@ -33,11 +33,11 @@ type Interface struct { + + // Route describes a network route. + type Route struct { +- Dest string +- Gateway string +- Device string +- Source string +- Scope uint32 ++ Dest string `json:"dest,omitempty"` ++ Gateway string `json:"gateway,omitempty"` ++ Device string `json:"device,omitempty"` ++ Source string `json:"source,omitempty"` ++ Scope uint32 `json:"scope,omitempty"` + } + + //NetworkOp describes network operation +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch b/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch new file mode 100644 index 0000000000000000000000000000000000000000..3784e247b13d7a58be4b2bdd4836c977ec41ebfd --- /dev/null +++ b/runtime/patches/0031-device_mangaer-check-VFIO-when-create-device.patch @@ -0,0 +1,41 @@ +From 326e90f97cf5ace73dff95257f7b4faa43c56f99 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Mon, 17 Aug 2020 16:11:35 +0800 +Subject: [PATCH 31/50] device_mangaer: check VFIO when create device + +reason: check VFIO when create device, block device can be +reused and VFIO device can not + +Signed-off-by: yangfeiyu +--- + virtcontainers/device/manager/manager.go | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/device/manager/manager.go b/virtcontainers/device/manager/manager.go +index 6a2b665a..d24a29ab 100644 +--- a/virtcontainers/device/manager/manager.go ++++ b/virtcontainers/device/manager/manager.go +@@ -9,6 +9,7 @@ package manager + import ( + "encoding/hex" + "errors" ++ "fmt" + "sync" + + "github.com/sirupsen/logrus" +@@ -115,7 +116,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device + }() + + if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil { +- return existingDev, nil ++ if isVFIO(devInfo.HostPath) { ++ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath) ++ } else { ++ return existingDev, nil ++ } + } + + // device ID must be generated by manager instead of device itself +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch b/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch new file mode 100644 index 0000000000000000000000000000000000000000..51f5b9273036e04d07db491e4d696279074c1203 --- /dev/null +++ b/runtime/patches/0032-network-add-more-detail-usage-for-update-routes-subc.patch @@ -0,0 +1,46 @@ +From 77711b531867be899df5d2c59a525ea1b7f0e6ac Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 17 Aug 2020 17:14:35 +0800 +Subject: [PATCH 32/50] network: add more detail usage for update-routes + subcommand + +reason: add more detail usage for update-routes subcommand +to explain how to use update-routes subcommand + +Signed-off-by: jiangpengfei +--- + cli/network.go | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index 046d0ee9..3dd0971e 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -120,10 +120,20 @@ var listIfacesCommand = cli.Command{ + } + + var updateRoutesCommand = cli.Command{ +- Name: "update-routes", +- Usage: "update routes of a container", +- ArgsUsage: `update-routes file or - for stdin`, +- Flags: []cli.Flag{}, ++ Name: "update-routes", ++ Usage: "update routes of a container", ++ ArgsUsage: `update-routes file or - for stdin ++ file or stdin for example: ++ [ ++ { ++ "dest":"<[[/mask] | "default" ]>", ++ "gateway":"[ip]", ++ "device":"[tap-name]", ++ "source": "[source]", ++ "scope":[scope] ++ } ++ ]`, ++ Flags: []cli.Flag{}, + Action: func(context *cli.Context) error { + ctx, err := cliContextToContext(context) + if err != nil { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch b/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch new file mode 100644 index 0000000000000000000000000000000000000000..c6a30ffcac0f6648529b68e7bdb0d0e434946604 --- /dev/null +++ b/runtime/patches/0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch @@ -0,0 +1,80 @@ +From 7ab7ff54efa3925a8d372f7830d31b87f8d01ea8 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 17 Aug 2020 17:39:18 +0800 +Subject: [PATCH 33/50] network: do not delete the exist tap device in the host + +reason: If hotplug a exist tap device into Kata VM, kata-runtime +should not delete this exist tap device, because this tap device +is controlled by other network components. + +Signed-off-by: jiangpengfei +--- + virtcontainers/tap_endpoint.go | 16 +++++++++++++--- + virtcontainers/veth_endpoint.go | 3 +++ + 2 files changed, 16 insertions(+), 3 deletions(-) + +diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go +index c897670e..2cf70dce 100644 +--- a/virtcontainers/tap_endpoint.go ++++ b/virtcontainers/tap_endpoint.go +@@ -77,7 +77,7 @@ func (endpoint *TapEndpoint) Detach(netNsCreated bool, netNsPath string) error { + + networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint") + return doNetNS(netNsPath, func(_ ns.NetNS) error { +- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) ++ return unTapNetwork(endpoint) + }) + } + +@@ -91,6 +91,9 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { + + if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error attach tap ep") ++ if errUnTap := unTapNetwork(endpoint); errUnTap != nil { ++ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name) ++ } + return err + } + return nil +@@ -100,7 +103,7 @@ func (endpoint *TapEndpoint) HotAttach(h hypervisor) error { + func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + networkLogger().Info("Hot detaching tap endpoint") + if err := doNetNS(netNsPath, func(_ ns.NetNS) error { +- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) ++ return unTapNetwork(endpoint) + }); err != nil { + networkLogger().WithError(err).Warn("Error un-bridging tap ep") + } +@@ -185,7 +188,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err + return nil + } + +-func unTapNetwork(name string) error { ++func unTapNetwork(endpoint *TapEndpoint) error { ++ // length of VMFDs == 0 means that the endpoint is already exist in the host, ++ // no created by kata, so we don't need to remove it when detach ++ if len(endpoint.TapInterface.VMFds) == 0 { ++ return nil ++ } ++ ++ name := endpoint.TapInterface.TAPIface.Name + netHandle, err := netlink.NewHandle() + if err != nil { + return err +diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go +index 9ece6a74..0f2ec9ba 100644 +--- a/virtcontainers/veth_endpoint.go ++++ b/virtcontainers/veth_endpoint.go +@@ -119,6 +119,9 @@ func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { + + if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error attach virtual ep") ++ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil { ++ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep") ++ } + return err + } + return nil +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch b/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch new file mode 100644 index 0000000000000000000000000000000000000000..407e129be6d263ecad85aa154016ad961ce3187f --- /dev/null +++ b/runtime/patches/0034-kata-runtime-add-kata-ipvs-command.patch @@ -0,0 +1,1273 @@ +From 4113462fe4bef73cd1f3a9c74f1cf0b2a1b643bd Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 18 Aug 2020 03:26:32 +0800 +Subject: [PATCH 34/50] kata-runtime: add kata-ipvs command + +reason: add kata-ipvs command to update IPVS rules +in VM. + +Signed-off-by: xiadanni1 +--- + cli/ipvsadm.go | 239 ++++++++ + cli/main.go | 1 + + .../agent/protocols/grpc/agent.pb.go | 663 +++++++++++++++------ + virtcontainers/agent.go | 3 + + virtcontainers/api.go | 25 + + virtcontainers/implementation.go | 5 + + virtcontainers/interfaces.go | 3 + + virtcontainers/kata_agent.go | 24 + + virtcontainers/noop_agent.go | 4 + + virtcontainers/pkg/vcmock/mock.go | 8 + + virtcontainers/pkg/vcmock/sandbox.go | 5 + + virtcontainers/pkg/vcmock/types.go | 2 + + virtcontainers/sandbox.go | 5 + + 13 files changed, 806 insertions(+), 181 deletions(-) + create mode 100644 cli/ipvsadm.go + +diff --git a/cli/ipvsadm.go b/cli/ipvsadm.go +new file mode 100644 +index 00000000..51e4bf41 +--- /dev/null ++++ b/cli/ipvsadm.go +@@ -0,0 +1,239 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: IPVS related common functions ++// Author: xiadanni ++// Create: 2020-08-01 ++ ++package main ++ ++import ( ++ "bufio" ++ "bytes" ++ "fmt" ++ "io" ++ "os" ++ "regexp" ++ "strconv" ++ "strings" ++ ++ "github.com/kata-containers/agent/protocols/grpc" ++ "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/urfave/cli" ++) ++ ++const ( ++ ruleLimitMax = 20000 ++ restoreHead = "restore" ++) ++ ++var kataIPVSCLICommand = cli.Command{ ++ Name: "kata-ipvs", ++ Usage: "manage IPVS service for container", ++ Subcommands: []cli.Command{ ++ IPVSadmCommand, ++ CleanupCommand, ++ }, ++ Action: func(context *cli.Context) error { ++ return cli.ShowSubcommandHelp(context) ++ }, ++} ++ ++var IPVSadmCommand = cli.Command{ ++ Name: "ipvsadm", ++ Usage: "manage virtual server and real server for container", ++ ArgsUsage: ``, ++ Flags: []cli.Flag{ ++ cli.StringFlag{ ++ Name: "parameters", ++ Usage: "parameters of IPVSadm command", ++ }, ++ cli.StringFlag{ ++ Name: "restore", ++ Usage: "restore rules from stdin, - for stdin", ++ }, ++ }, ++ Action: func(context *cli.Context) error { ++ ctx, err := cliContextToContext(context) ++ if err != nil { ++ return err ++ } ++ ++ status, sandboxID, err := getExistingContainerInfo(ctx, context.Args().First()) ++ if err != nil { ++ return err ++ } ++ ++ if status.State.State != types.StateReady && ++ status.State.State != types.StateRunning { ++ return fmt.Errorf("container with id %s is not running", status.ID) ++ } ++ ++ var IPVSRule string ++ input := context.String("restore") ++ if input != "" { ++ if input != "-" { ++ return fmt.Errorf("invalid kata-IPVS command, please use --restore - for stdin") ++ } ++ ruleFile, ruleCnt, err := getRestoreFile(os.Stdin) ++ if err != nil { ++ return err ++ } ++ // restore format: restore|ruleCnt|ruleFileContext ++ IPVSRule = restoreHead + "|" + strconv.Itoa(ruleCnt) + "|" + ruleFile ++ } else { ++ parameters := context.String("parameters") ++ IPVSRule = "ipvsadm " + parameters ++ IPVSRule, err = checkIPVSRule(IPVSRule) ++ if err != nil { ++ return err ++ } ++ ++ // we just use NAT mode ++ if strings.Contains(IPVSRule, "add-server") || strings.Contains(IPVSRule, "edit-server") || ++ strings.Contains(IPVSRule, " -a ") || strings.Contains(IPVSRule, " -e ") { ++ IPVSRule = IPVSRule + " -m" ++ } ++ } ++ ++ IPVSReq := &grpc.UpdateIPVSRequest{ ++ IPVSReq: IPVSRule, ++ } ++ resultingIPVS, err := vci.UpdateIPVSRule(ctx, sandboxID, IPVSReq) ++ if err != nil { ++ kataLog.WithField("resulting-IPVSadm", fmt.Sprintf("%+v", resultingIPVS)). ++ WithError(err).Error("exec IPVSadm failed") ++ return err ++ } ++ ++ if resultingIPVS.IPVSRes != "" { ++ fmt.Printf(resultingIPVS.IPVSRes) ++ } ++ ++ return nil ++ }, ++} ++ ++var CleanupCommand = cli.Command{ ++ Name: "cleanup", ++ Usage: "delete conntrack of destination address", ++ ArgsUsage: ``, ++ Flags: []cli.Flag{ ++ cli.StringFlag{ ++ Name: "parameters", ++ Usage: "parameters of cleanup command", ++ }, ++ }, ++ Action: func(context *cli.Context) error { ++ ctx, err := cliContextToContext(context) ++ if err != nil { ++ return err ++ } ++ ++ status, sandboxID, err := getExistingContainerInfo(ctx, context.Args().First()) ++ if err != nil { ++ return err ++ } ++ ++ if status.State.State != types.StateReady && ++ status.State.State != types.StateRunning { ++ return fmt.Errorf("container with id %s is not running", status.ID) ++ } ++ ++ parameters := context.String("parameters") ++ IPVSRule := "conntrack -D " + parameters ++ ++ _, err = checkIPVSRule(IPVSRule) ++ if err != nil { ++ return err ++ } ++ ++ IPVSReq := &grpc.UpdateIPVSRequest{ ++ IPVSReq: IPVSRule, ++ } ++ resultingIPVS, err := vci.UpdateIPVSRule(ctx, sandboxID, IPVSReq) ++ if err != nil { ++ kataLog.WithField("resulting-conntrack", fmt.Sprintf("%+v", resultingIPVS)). ++ WithError(err).Error("exec conntrack failed") ++ return err ++ } ++ ++ fmt.Printf(resultingIPVS.IPVSRes) ++ ++ return nil ++ }, ++} ++ ++var ( ++ match = regexp.MustCompile ++ ipReg = `((2[0-4]\d|25[0-5]|[01]?\d\d?)\.){3}(2[0-4]\d|25[0-5]|[01]?\d\d?)` ++ portReg = `:(6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]|[1-5]\d{4}|[1-9]\d{0,3})\b` ++ ++ addOrEditServiceReg = match(`(--(add|edit)-service|-A|-E)\s+`) ++ serverTopReg = match(`(--(add|edit|delete)-server|-a|-e|-d)\s+`) ++ schedulerReg = match(`(--scheduler|-s)(\s+|=)(rr|wrr|lc|wlc|lblc|lblcr|dh|sh|sed|nq)\b`) ++ persistentReg = match(`(--persistent|-p)(\s+|=)\d{1,7}\b`) ++ persistentZeroReg = match(`(--persistent|-p)\s+0\b`) ++ serviceReg = match(`(--(tcp|udp)-service|-t|-u)(\s+|=)` + ipReg + portReg) ++ serverReg = match(`(--real-server|-r)(\s+|=)` + ipReg + portReg) ++ delServiceReg = match(`(--delete-service|-D)\s+` + serviceReg.String()) ++ conntrackReg = match(`conntrack\s+-D\s+(--orig-dst|-d)\s+` + ipReg + `\s+(--protonum|-p)\s+(tcp|udp)\s*$`) ++ otherReg = match(`((--list|-L|-l|--clear|-C|--set).*)|` + conntrackReg.String()) ++) ++ ++func checkIPVSRule(rule string) (string, error) { ++ switch { ++ // match add-service or edit-service rule format ++ case addOrEditServiceReg.MatchString(rule) && serviceReg.MatchString(rule) && ++ schedulerReg.MatchString(rule) && persistentReg.MatchString(rule): ++ rule = persistentZeroReg.ReplaceAllString(rule, "") ++ return rule, nil ++ // match delete-service rule format ++ case delServiceReg.MatchString(rule): ++ return rule, nil ++ // match add/edit/delete-server rule format ++ case serverTopReg.MatchString(rule) && serviceReg.MatchString(rule) && serverReg.MatchString(rule): ++ return rule, nil ++ // match cleanup/list/set/clear rule format ++ case otherReg.MatchString(rule): ++ return rule, nil ++ default: ++ return "", fmt.Errorf("invalid input of IPVS rule") ++ } ++} ++ ++func getRestoreFile(stdin io.Reader) (string, int, error) { ++ var ruleContent bytes.Buffer ++ firstFlag := true ++ ruleCnt := 0 ++ scanner := bufio.NewScanner(stdin) ++ for scanner.Scan() { ++ if err := scanner.Err(); err != nil { ++ return "", 0, err ++ } ++ newLine := scanner.Text() ++ ++ _, err := checkIPVSRule(newLine) ++ if err != nil { ++ return "", 0, err ++ } ++ ++ ruleCnt++ ++ ++ if ruleCnt > ruleLimitMax { ++ return "", 0, fmt.Errorf("Rules exceed limit %v", ruleLimitMax) ++ } ++ ++ if firstFlag { ++ firstFlag = false ++ } else { ++ ruleContent.WriteString("\n") ++ } ++ ruleContent.WriteString(newLine) ++ } ++ ++ if ruleCnt == 0 { ++ return "", 0, fmt.Errorf("Rule file has no effective rule.") ++ } ++ ++ return ruleContent.String(), ruleCnt, nil ++} +diff --git a/cli/main.go b/cli/main.go +index 5eb2fb19..95a0786b 100644 +--- a/cli/main.go ++++ b/cli/main.go +@@ -146,6 +146,7 @@ var runtimeCommands = []cli.Command{ + kataNetworkCLICommand, + kataOverheadCLICommand, + factoryCLICommand, ++ kataIPVSCLICommand, + } + + // runtimeBeforeSubcommands is the function to run before command-line +diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +index 1b887e55..04d0ee52 100644 +--- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go ++++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +@@ -63,6 +63,8 @@ + CopyFileRequest + StartTracingRequest + StopTracingRequest ++ UpdateIPVSRequest ++ IPVSResponse + CheckRequest + HealthCheckResponse + VersionCheckResponse +@@ -1842,6 +1844,40 @@ func (m *StopTracingRequest) String() string { return proto.CompactTe + func (*StopTracingRequest) ProtoMessage() {} + func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + ++type UpdateIPVSRequest struct { ++ // IPVS_req is the IPVS rule message needed to update ++ IPVSReq string `protobuf:"bytes,1,opt,name=IPVS_req,json=IPVSReq,proto3" json:"IPVS_req,omitempty"` ++} ++ ++func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } ++func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } ++func (*UpdateIPVSRequest) ProtoMessage() {} ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++ ++func (m *UpdateIPVSRequest) GetIPVSReq() string { ++ if m != nil { ++ return m.IPVSReq ++ } ++ return "" ++} ++ ++type IPVSResponse struct { ++ // IPVS_res is the response of IPVS updating ++ IPVSRes string `protobuf:"bytes,1,opt,name=IPVS_res,json=IPVSRes,proto3" json:"IPVS_res,omitempty"` ++} ++ ++func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } ++func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } ++func (*IPVSResponse) ProtoMessage() {} ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++ ++func (m *IPVSResponse) GetIPVSRes() string { ++ if m != nil { ++ return m.IPVSRes ++ } ++ return "" ++} ++ + func init() { + proto.RegisterType((*CreateContainerRequest)(nil), "grpc.CreateContainerRequest") + proto.RegisterType((*StartContainerRequest)(nil), "grpc.StartContainerRequest") +@@ -1896,6 +1932,8 @@ func init() { + proto.RegisterType((*CopyFileRequest)(nil), "grpc.CopyFileRequest") + proto.RegisterType((*StartTracingRequest)(nil), "grpc.StartTracingRequest") + proto.RegisterType((*StopTracingRequest)(nil), "grpc.StopTracingRequest") ++ proto.RegisterType((*UpdateIPVSRequest)(nil), "grpc.UpdateIPVSRequest") ++ proto.RegisterType((*IPVSResponse)(nil), "grpc.IPVSResponse") + } + + // Reference imports to suppress errors if they are not otherwise used. +@@ -1938,6 +1976,7 @@ type AgentServiceClient interface { + UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) + ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) + ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) ++ UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) + // tracing + StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) + StopTracing(ctx context.Context, in *StopTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) +@@ -2140,6 +2179,15 @@ func (c *agentServiceClient) ListRoutes(ctx context.Context, in *ListRoutesReque + return out, nil + } + ++func (c *agentServiceClient) UpdateIPVSRule(ctx context.Context, in *UpdateIPVSRequest, opts ...grpc1.CallOption) (*IPVSResponse, error) { ++ out := new(IPVSResponse) ++ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateIPVSRule", in, out, c.cc, opts...) ++ if err != nil { ++ return nil, err ++ } ++ return out, nil ++} ++ + func (c *agentServiceClient) StartTracing(ctx context.Context, in *StartTracingRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) { + out := new(google_protobuf2.Empty) + err := grpc1.Invoke(ctx, "/grpc.AgentService/StartTracing", in, out, c.cc, opts...) +@@ -2262,6 +2310,7 @@ type AgentServiceServer interface { + UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) + ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) + ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) ++ UpdateIPVSRule(context.Context, *UpdateIPVSRequest) (*IPVSResponse, error) + // tracing + StartTracing(context.Context, *StartTracingRequest) (*google_protobuf2.Empty, error) + StopTracing(context.Context, *StopTracingRequest) (*google_protobuf2.Empty, error) +@@ -2640,6 +2689,24 @@ func _AgentService_ListRoutes_Handler(srv interface{}, ctx context.Context, dec + return interceptor(ctx, in, info, handler) + } + ++func _AgentService_UpdateIPVSRule_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { ++ in := new(UpdateIPVSRequest) ++ if err := dec(in); err != nil { ++ return nil, err ++ } ++ if interceptor == nil { ++ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, in) ++ } ++ info := &grpc1.UnaryServerInfo{ ++ Server: srv, ++ FullMethod: "/grpc.AgentService/UpdateIPVSRule", ++ } ++ handler := func(ctx context.Context, req interface{}) (interface{}, error) { ++ return srv.(AgentServiceServer).UpdateIPVSRule(ctx, req.(*UpdateIPVSRequest)) ++ } ++ return interceptor(ctx, in, info, handler) ++} ++ + func _AgentService_StartTracing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { + in := new(StartTracingRequest) + if err := dec(in); err != nil { +@@ -2904,6 +2971,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ + MethodName: "ListRoutes", + Handler: _AgentService_ListRoutes_Handler, + }, ++ { ++ MethodName: "UpdateIPVSRule", ++ Handler: _AgentService_UpdateIPVSRule_Handler, ++ }, + { + MethodName: "StartTracing", + Handler: _AgentService_StartTracing_Handler, +@@ -5088,6 +5159,54 @@ func (m *StopTracingRequest) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + ++func (m *UpdateIPVSRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *UpdateIPVSRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.IPVSReq) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSReq))) ++ i += copy(dAtA[i:], m.IPVSReq) ++ } ++ return i, nil ++} ++ ++func (m *IPVSResponse) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *IPVSResponse) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.IPVSRes) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.IPVSRes))) ++ i += copy(dAtA[i:], m.IPVSRes) ++ } ++ return i, nil ++} ++ + func encodeVarintAgent(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) +@@ -6019,6 +6138,26 @@ func (m *StopTracingRequest) Size() (n int) { + return n + } + ++func (m *UpdateIPVSRequest) Size() (n int) { ++ var l int ++ _ = l ++ l = len(m.IPVSReq) ++ if l > 0 { ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ ++func (m *IPVSResponse) Size() (n int) { ++ var l int ++ _ = l ++ l = len(m.IPVSRes) ++ if l > 0 { ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func sovAgent(x uint64) (n int) { + for { + n++ +@@ -12785,6 +12924,164 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { + } + return nil + } ++func (m *UpdateIPVSRequest) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UpdateIPVSRequest: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UpdateIPVSRequest: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field IPVSReq", wireType) ++ } ++ var stringLen uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ stringLen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ intStringLen := int(stringLen) ++ if intStringLen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + intStringLen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.IPVSReq = string(dAtA[iNdEx:postIndex]) ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *IPVSResponse) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: IPVSResponse: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: IPVSResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field IPVSRes", wireType) ++ } ++ var stringLen uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ stringLen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ intStringLen := int(stringLen) ++ if intStringLen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + intStringLen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.IPVSRes = string(dAtA[iNdEx:postIndex]) ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} + func skipAgent(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 +@@ -12893,185 +13190,189 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2876 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0xd1, 0xd8, 0x07, 0x97, 0xbb, 0xb5, 0x2f, 0x6e, 0x93, 0xa2, 0x56, 0x2b, 0x59, 0x9f, 0x3c, 0xb6, +- 0x65, 0xfa, 0xf3, 0xe7, 0xa5, 0x2d, 0x1b, 0x9f, 0x5f, 0x70, 0x04, 0xf1, 0x11, 0x91, 0xb1, 0x15, +- 0x31, 0x43, 0x11, 0x4e, 0x10, 0x04, 0x83, 0xe1, 0x4c, 0x73, 0xd9, 0xe6, 0xce, 0xf4, 0xb8, 0xa7, +- 0x87, 0xe2, 0x3a, 0x40, 0x8e, 0xc9, 0x2d, 0x97, 0x00, 0xb9, 0xe5, 0x0f, 0x04, 0xb9, 0xe5, 0x98, +- 0x6b, 0x0e, 0x46, 0x4e, 0xf9, 0x05, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xf4, 0x6b, 0x1e, 0xbb, +- 0x43, 0x1a, 0x21, 0x08, 0xe4, 0xb2, 0xe8, 0xaa, 0xae, 0xae, 0x57, 0x77, 0xd5, 0x54, 0xd5, 0x42, +- 0xdb, 0x9d, 0xe0, 0x90, 0x8f, 0x23, 0x46, 0x39, 0x45, 0xf5, 0x09, 0x8b, 0xbc, 0x51, 0x8b, 0x7a, +- 0x44, 0x21, 0x46, 0xff, 0x3f, 0x21, 0xfc, 0x34, 0x39, 0x1e, 0x7b, 0x34, 0xd8, 0x3c, 0x73, 0xb9, +- 0xfb, 0x8e, 0x47, 0x43, 0xee, 0x92, 0x10, 0xb3, 0x78, 0x53, 0x1e, 0xdc, 0x8c, 0xce, 0x26, 0x9b, +- 0x7c, 0x16, 0xe1, 0x58, 0xfd, 0xea, 0x73, 0x77, 0x27, 0x94, 0x4e, 0xa6, 0x78, 0x53, 0x42, 0xc7, +- 0xc9, 0xc9, 0x26, 0x0e, 0x22, 0x3e, 0x53, 0x9b, 0xd6, 0x1f, 0xaa, 0xb0, 0xbe, 0xcd, 0xb0, 0xcb, +- 0xf1, 0xb6, 0xe1, 0x66, 0xe3, 0xaf, 0x13, 0x1c, 0x73, 0xf4, 0x2a, 0x74, 0x52, 0x09, 0x0e, 0xf1, +- 0x87, 0x95, 0x07, 0x95, 0x8d, 0x96, 0xdd, 0x4e, 0x71, 0xfb, 0x3e, 0xba, 0x0d, 0xcb, 0xf8, 0x02, +- 0x7b, 0x62, 0xb7, 0x2a, 0x77, 0x1b, 0x02, 0xdc, 0xf7, 0xd1, 0x7b, 0xd0, 0x8e, 0x39, 0x23, 0xe1, +- 0xc4, 0x49, 0x62, 0xcc, 0x86, 0xb5, 0x07, 0x95, 0x8d, 0xf6, 0xa3, 0x95, 0xb1, 0x30, 0x69, 0x7c, +- 0x28, 0x37, 0x8e, 0x62, 0xcc, 0x6c, 0x88, 0xd3, 0x35, 0x7a, 0x08, 0xcb, 0x3e, 0x3e, 0x27, 0x1e, +- 0x8e, 0x87, 0xf5, 0x07, 0xb5, 0x8d, 0xf6, 0xa3, 0x8e, 0x22, 0xdf, 0x91, 0x48, 0xdb, 0x6c, 0xa2, +- 0xb7, 0xa0, 0x19, 0x73, 0xca, 0xdc, 0x09, 0x8e, 0x87, 0x4b, 0x92, 0xb0, 0x6b, 0xf8, 0x4a, 0xac, +- 0x9d, 0x6e, 0xa3, 0x7b, 0x50, 0x7b, 0xbe, 0xbd, 0x3f, 0x6c, 0x48, 0xe9, 0xa0, 0xa9, 0x22, 0xec, +- 0xd9, 0x02, 0x8d, 0x5e, 0x83, 0x6e, 0xec, 0x86, 0xfe, 0x31, 0xbd, 0x70, 0x22, 0xe2, 0x87, 0xf1, +- 0x70, 0xf9, 0x41, 0x65, 0xa3, 0x69, 0x77, 0x34, 0xf2, 0x40, 0xe0, 0xac, 0x4f, 0xe0, 0xd6, 0x21, +- 0x77, 0x19, 0xbf, 0x86, 0x77, 0xac, 0x23, 0x58, 0xb7, 0x71, 0x40, 0xcf, 0xaf, 0xe5, 0xda, 0x21, +- 0x2c, 0x73, 0x12, 0x60, 0x9a, 0x70, 0xe9, 0xda, 0xae, 0x6d, 0x40, 0xeb, 0x4f, 0x15, 0x40, 0xbb, +- 0x17, 0xd8, 0x3b, 0x60, 0xd4, 0xc3, 0x71, 0xfc, 0x5f, 0xba, 0xae, 0x37, 0x61, 0x39, 0x52, 0x0a, +- 0x0c, 0xeb, 0x92, 0x5c, 0xdf, 0x82, 0xd1, 0xca, 0xec, 0x5a, 0x5f, 0xc1, 0xda, 0x21, 0x99, 0x84, +- 0xee, 0xf4, 0x06, 0xf5, 0x5d, 0x87, 0x46, 0x2c, 0x79, 0x4a, 0x55, 0xbb, 0xb6, 0x86, 0xac, 0x03, +- 0x40, 0x5f, 0xba, 0x84, 0xdf, 0x9c, 0x24, 0xeb, 0x1d, 0x58, 0x2d, 0x70, 0x8c, 0x23, 0x1a, 0xc6, +- 0x58, 0x2a, 0xc0, 0x5d, 0x9e, 0xc4, 0x92, 0xd9, 0x92, 0xad, 0x21, 0x0b, 0xc3, 0xda, 0x17, 0x24, +- 0x36, 0xe4, 0xf8, 0x3f, 0x51, 0x61, 0x1d, 0x1a, 0x27, 0x94, 0x05, 0x2e, 0x37, 0x1a, 0x28, 0x08, +- 0x21, 0xa8, 0xbb, 0x6c, 0x12, 0x0f, 0x6b, 0x0f, 0x6a, 0x1b, 0x2d, 0x5b, 0xae, 0xc5, 0xab, 0x9c, +- 0x13, 0xa3, 0xf5, 0x7a, 0x15, 0x3a, 0xda, 0xef, 0xce, 0x94, 0xc4, 0x5c, 0xca, 0xe9, 0xd8, 0x6d, +- 0x8d, 0x13, 0x67, 0x2c, 0x0a, 0xeb, 0x47, 0x91, 0x7f, 0xcd, 0x80, 0x7f, 0x04, 0x2d, 0x86, 0x63, +- 0x9a, 0x30, 0x11, 0xa6, 0x55, 0x79, 0xef, 0x6b, 0xea, 0xde, 0xbf, 0x20, 0x61, 0x72, 0x61, 0x9b, +- 0x3d, 0x3b, 0x23, 0xd3, 0x21, 0xc4, 0xe3, 0xeb, 0x84, 0xd0, 0x27, 0x70, 0xeb, 0xc0, 0x4d, 0xe2, +- 0xeb, 0xe8, 0x6a, 0x7d, 0x2a, 0xc2, 0x2f, 0x4e, 0x82, 0x6b, 0x1d, 0xfe, 0x63, 0x05, 0x9a, 0xdb, +- 0x51, 0x72, 0x14, 0xbb, 0x13, 0x8c, 0xfe, 0x07, 0xda, 0x9c, 0x72, 0x77, 0xea, 0x24, 0x02, 0x94, +- 0xe4, 0x75, 0x1b, 0x24, 0x4a, 0x11, 0x08, 0xb7, 0x63, 0xe6, 0x45, 0x89, 0xa6, 0xa8, 0x3e, 0xa8, +- 0x6d, 0xd4, 0xed, 0xb6, 0xc2, 0x29, 0x92, 0x31, 0xac, 0xca, 0x3d, 0x87, 0x84, 0xce, 0x19, 0x66, +- 0x21, 0x9e, 0x06, 0xd4, 0xc7, 0xf2, 0xfd, 0xd6, 0xed, 0x81, 0xdc, 0xda, 0x0f, 0x3f, 0x4f, 0x37, +- 0xd0, 0xff, 0xc2, 0x20, 0xa5, 0x17, 0x41, 0x29, 0xa9, 0xeb, 0x92, 0xba, 0xaf, 0xa9, 0x8f, 0x34, +- 0xda, 0xfa, 0x15, 0xf4, 0x5e, 0x9c, 0x32, 0xca, 0xf9, 0x94, 0x84, 0x93, 0x1d, 0x97, 0xbb, 0x22, +- 0x7b, 0x44, 0x98, 0x11, 0xea, 0xc7, 0x5a, 0x5b, 0x03, 0xa2, 0xb7, 0x61, 0xc0, 0x15, 0x2d, 0xf6, +- 0x1d, 0x43, 0x53, 0x95, 0x34, 0x2b, 0xe9, 0xc6, 0x81, 0x26, 0x7e, 0x03, 0x7a, 0x19, 0xb1, 0xc8, +- 0x3f, 0x5a, 0xdf, 0x6e, 0x8a, 0x7d, 0x41, 0x02, 0x6c, 0x9d, 0x4b, 0x5f, 0xc9, 0x4b, 0x46, 0x6f, +- 0x43, 0x2b, 0xf3, 0x43, 0x45, 0xbe, 0x90, 0x9e, 0x7a, 0x21, 0xc6, 0x9d, 0x76, 0x33, 0x75, 0xca, +- 0x67, 0xd0, 0xe7, 0xa9, 0xe2, 0x8e, 0xef, 0x72, 0xb7, 0xf8, 0xa8, 0x8a, 0x56, 0xd9, 0x3d, 0x5e, +- 0x80, 0xad, 0x4f, 0xa1, 0x75, 0x40, 0xfc, 0x58, 0x09, 0x1e, 0xc2, 0xb2, 0x97, 0x30, 0x86, 0x43, +- 0x6e, 0x4c, 0xd6, 0x20, 0x5a, 0x83, 0xa5, 0x29, 0x09, 0x08, 0xd7, 0x66, 0x2a, 0xc0, 0xa2, 0x00, +- 0xcf, 0x70, 0x40, 0xd9, 0x4c, 0x3a, 0x6c, 0x0d, 0x96, 0xf2, 0x97, 0xab, 0x00, 0x74, 0x17, 0x5a, +- 0x81, 0x7b, 0x91, 0x5e, 0xaa, 0xd8, 0x69, 0x06, 0xee, 0x85, 0x52, 0x7e, 0x08, 0xcb, 0x27, 0x2e, +- 0x99, 0x7a, 0x21, 0xd7, 0x5e, 0x31, 0x60, 0x26, 0xb0, 0x9e, 0x17, 0xf8, 0xd7, 0x2a, 0xb4, 0x95, +- 0x44, 0xa5, 0xf0, 0x1a, 0x2c, 0x79, 0xae, 0x77, 0x9a, 0x8a, 0x94, 0x00, 0x7a, 0x68, 0x14, 0xa9, +- 0xe6, 0x93, 0x70, 0xa6, 0xa9, 0x51, 0x6d, 0x13, 0x20, 0x7e, 0xe9, 0x46, 0x5a, 0xb7, 0xda, 0x25, +- 0xc4, 0x2d, 0x41, 0xa3, 0xd4, 0x7d, 0x1f, 0x3a, 0xea, 0xdd, 0xe9, 0x23, 0xf5, 0x4b, 0x8e, 0xb4, +- 0x15, 0x95, 0x3a, 0xf4, 0x1a, 0x74, 0x93, 0x18, 0x3b, 0xa7, 0x04, 0x33, 0x97, 0x79, 0xa7, 0xb3, +- 0xe1, 0x92, 0xfa, 0x46, 0x26, 0x31, 0xde, 0x33, 0x38, 0xf4, 0x08, 0x96, 0x44, 0xfa, 0x8b, 0x87, +- 0x0d, 0xf9, 0x39, 0xbe, 0x97, 0x67, 0x29, 0x4d, 0x1d, 0xcb, 0xdf, 0xdd, 0x90, 0xb3, 0x99, 0xad, +- 0x48, 0x47, 0x1f, 0x01, 0x64, 0x48, 0xb4, 0x02, 0xb5, 0x33, 0x3c, 0xd3, 0x71, 0x28, 0x96, 0xc2, +- 0x39, 0xe7, 0xee, 0x34, 0x31, 0x5e, 0x57, 0xc0, 0x27, 0xd5, 0x8f, 0x2a, 0x96, 0x07, 0xfd, 0xad, +- 0xe9, 0x19, 0xa1, 0xb9, 0xe3, 0x6b, 0xb0, 0x14, 0xb8, 0x5f, 0x51, 0x66, 0x3c, 0x29, 0x01, 0x89, +- 0x25, 0x21, 0x65, 0x86, 0x85, 0x04, 0x50, 0x0f, 0xaa, 0x34, 0x92, 0xfe, 0x6a, 0xd9, 0x55, 0x1a, +- 0x65, 0x82, 0xea, 0x39, 0x41, 0xd6, 0x3f, 0xea, 0x00, 0x99, 0x14, 0x64, 0xc3, 0x88, 0x50, 0x27, +- 0xc6, 0x4c, 0x94, 0x20, 0xce, 0xf1, 0x8c, 0xe3, 0xd8, 0x61, 0xd8, 0x4b, 0x58, 0x4c, 0xce, 0xc5, +- 0xfd, 0x09, 0xb3, 0x6f, 0x29, 0xb3, 0xe7, 0x74, 0xb3, 0x6f, 0x13, 0x7a, 0xa8, 0xce, 0x6d, 0x89, +- 0x63, 0xb6, 0x39, 0x85, 0xf6, 0xe1, 0x56, 0xc6, 0xd3, 0xcf, 0xb1, 0xab, 0x5e, 0xc5, 0x6e, 0x35, +- 0x65, 0xe7, 0x67, 0xac, 0x76, 0x61, 0x95, 0x50, 0xe7, 0xeb, 0x04, 0x27, 0x05, 0x46, 0xb5, 0xab, +- 0x18, 0x0d, 0x08, 0xfd, 0x89, 0x3c, 0x90, 0xb1, 0x39, 0x80, 0x3b, 0x39, 0x2b, 0x45, 0xb8, 0xe7, +- 0x98, 0xd5, 0xaf, 0x62, 0xb6, 0x9e, 0x6a, 0x25, 0xf2, 0x41, 0xc6, 0xf1, 0x47, 0xb0, 0x4e, 0xa8, +- 0xf3, 0xd2, 0x25, 0x7c, 0x9e, 0xdd, 0xd2, 0xf7, 0x18, 0x29, 0x3e, 0xba, 0x45, 0x5e, 0xca, 0xc8, +- 0x00, 0xb3, 0x49, 0xc1, 0xc8, 0xc6, 0xf7, 0x18, 0xf9, 0x4c, 0x1e, 0xc8, 0xd8, 0x3c, 0x81, 0x01, +- 0xa1, 0xf3, 0xda, 0x2c, 0x5f, 0xc5, 0xa4, 0x4f, 0x68, 0x51, 0x93, 0x2d, 0x18, 0xc4, 0xd8, 0xe3, +- 0x94, 0xe5, 0x1f, 0x41, 0xf3, 0x2a, 0x16, 0x2b, 0x9a, 0x3e, 0xe5, 0x61, 0xfd, 0x1c, 0x3a, 0x7b, +- 0xc9, 0x04, 0xf3, 0xe9, 0x71, 0x9a, 0x0c, 0x6e, 0x2c, 0xff, 0x58, 0xff, 0xaa, 0x42, 0x7b, 0x7b, +- 0xc2, 0x68, 0x12, 0x15, 0x72, 0xb2, 0x0a, 0xd2, 0xf9, 0x9c, 0x2c, 0x49, 0x64, 0x4e, 0x56, 0xc4, +- 0x1f, 0x40, 0x27, 0x90, 0xa1, 0xab, 0xe9, 0x55, 0x1e, 0x1a, 0x2c, 0x04, 0xb5, 0xdd, 0x0e, 0x72, +- 0xc9, 0x6c, 0x0c, 0x10, 0x11, 0x3f, 0xd6, 0x67, 0x54, 0x3a, 0xea, 0xeb, 0x8a, 0xd0, 0xa4, 0x68, +- 0xbb, 0x15, 0xa5, 0xd9, 0xfa, 0x3d, 0x68, 0x1f, 0x0b, 0x27, 0xe9, 0x03, 0x85, 0x64, 0x94, 0x79, +- 0xcf, 0x86, 0xe3, 0x2c, 0x08, 0xf7, 0xa0, 0x7b, 0xaa, 0x5c, 0xa6, 0x0f, 0xa9, 0x37, 0xf4, 0x9a, +- 0xb6, 0x24, 0xb3, 0x77, 0x9c, 0xf7, 0xac, 0xba, 0x80, 0xce, 0x69, 0x0e, 0x35, 0x3a, 0x84, 0xc1, +- 0x02, 0x49, 0x49, 0x0e, 0xda, 0xc8, 0xe7, 0xa0, 0xf6, 0x23, 0xa4, 0x04, 0xe5, 0x4f, 0xe6, 0xf3, +- 0xd2, 0x6f, 0xab, 0xd0, 0xf9, 0x31, 0xe6, 0x2f, 0x29, 0x3b, 0x53, 0xfa, 0x22, 0xa8, 0x87, 0x6e, +- 0x80, 0x35, 0x47, 0xb9, 0x46, 0x77, 0xa0, 0xc9, 0x2e, 0x54, 0x02, 0xd1, 0xf7, 0xb9, 0xcc, 0x2e, +- 0x64, 0x62, 0x40, 0xaf, 0x00, 0xb0, 0x0b, 0x27, 0x72, 0xbd, 0x33, 0xac, 0x3d, 0x58, 0xb7, 0x5b, +- 0xec, 0xe2, 0x40, 0x21, 0xc4, 0x53, 0x60, 0x17, 0x0e, 0x66, 0x8c, 0xb2, 0x58, 0xe7, 0xaa, 0x26, +- 0xbb, 0xd8, 0x95, 0xb0, 0x3e, 0xeb, 0x33, 0x1a, 0x45, 0xd8, 0x97, 0x39, 0x5a, 0x9e, 0xdd, 0x51, +- 0x08, 0x21, 0x95, 0x1b, 0xa9, 0x0d, 0x25, 0x95, 0x67, 0x52, 0x79, 0x26, 0x75, 0x59, 0x9d, 0xe4, +- 0x79, 0xa9, 0x3c, 0x95, 0xda, 0x54, 0x52, 0x79, 0x4e, 0x2a, 0xcf, 0xa4, 0xb6, 0xcc, 0x59, 0x2d, +- 0xd5, 0xfa, 0x4d, 0x05, 0xd6, 0xe7, 0x0b, 0x3f, 0x5d, 0xa6, 0x7e, 0x00, 0x1d, 0x4f, 0xde, 0x57, +- 0xe1, 0x4d, 0x0e, 0x16, 0x6e, 0xd2, 0x6e, 0x7b, 0xb9, 0x67, 0xfc, 0x21, 0x74, 0x43, 0xe5, 0xe0, +- 0xf4, 0x69, 0xd6, 0xb2, 0x7b, 0xc9, 0xfb, 0xde, 0xee, 0x84, 0x39, 0xc8, 0xf2, 0x01, 0x7d, 0xc9, +- 0x08, 0xc7, 0x87, 0x9c, 0x61, 0x37, 0xb8, 0x89, 0x06, 0x04, 0x41, 0x5d, 0x56, 0x2b, 0x35, 0x59, +- 0x5f, 0xcb, 0xb5, 0xf5, 0x26, 0xac, 0x16, 0xa4, 0x68, 0x5b, 0x57, 0xa0, 0x36, 0xc5, 0xa1, 0xe4, +- 0xde, 0xb5, 0xc5, 0xd2, 0x72, 0x61, 0x60, 0x63, 0xd7, 0xbf, 0x39, 0x6d, 0xb4, 0x88, 0x5a, 0x26, +- 0x62, 0x03, 0x50, 0x5e, 0x84, 0x56, 0xc5, 0x68, 0x5d, 0xc9, 0x69, 0xfd, 0x1c, 0x06, 0xdb, 0x53, +- 0x1a, 0xe3, 0x43, 0xee, 0x93, 0xf0, 0x26, 0x3a, 0xa6, 0x5f, 0xc2, 0xea, 0x0b, 0x3e, 0xfb, 0x52, +- 0x30, 0x8b, 0xc9, 0x37, 0xf8, 0x86, 0xec, 0x63, 0xf4, 0xa5, 0xb1, 0x8f, 0xd1, 0x97, 0xa2, 0x59, +- 0xf2, 0xe8, 0x34, 0x09, 0x42, 0x19, 0x0a, 0x5d, 0x5b, 0x43, 0xd6, 0x16, 0x74, 0x54, 0x0d, 0xfd, +- 0x8c, 0xfa, 0xc9, 0x14, 0x97, 0xc6, 0xe0, 0x7d, 0x80, 0xc8, 0x65, 0x6e, 0x80, 0x39, 0x66, 0xea, +- 0x0d, 0xb5, 0xec, 0x1c, 0xc6, 0xfa, 0x7d, 0x15, 0xd6, 0xd4, 0x48, 0xe4, 0x50, 0x4d, 0x02, 0x8c, +- 0x09, 0x23, 0x68, 0x9e, 0xd2, 0x98, 0xe7, 0x18, 0xa6, 0xb0, 0x50, 0xd1, 0x0f, 0x0d, 0x37, 0xb1, +- 0x2c, 0xcc, 0x29, 0x6a, 0x57, 0xcf, 0x29, 0x16, 0x26, 0x11, 0xf5, 0xc5, 0x49, 0x84, 0x88, 0x36, +- 0x43, 0x44, 0x54, 0x8c, 0xb7, 0xec, 0x96, 0xc6, 0xec, 0xfb, 0xe8, 0x21, 0xf4, 0x27, 0x42, 0x4b, +- 0xe7, 0x94, 0xd2, 0x33, 0x27, 0x72, 0xf9, 0xa9, 0x0c, 0xf5, 0x96, 0xdd, 0x95, 0xe8, 0x3d, 0x4a, +- 0xcf, 0x0e, 0x5c, 0x7e, 0x8a, 0x3e, 0x86, 0x9e, 0x2e, 0x03, 0x03, 0xe9, 0xa2, 0x58, 0x7f, 0xfc, +- 0x74, 0x14, 0xe5, 0xbd, 0x67, 0x77, 0xcf, 0x72, 0x50, 0x6c, 0xdd, 0x86, 0x5b, 0x3b, 0x38, 0xe6, +- 0x8c, 0xce, 0x8a, 0x8e, 0xb1, 0x7e, 0x00, 0xb0, 0x1f, 0x72, 0xcc, 0x4e, 0x5c, 0x0f, 0xc7, 0xe8, +- 0xdd, 0x3c, 0xa4, 0x8b, 0xa3, 0x95, 0xb1, 0x9a, 0x48, 0xa5, 0x1b, 0x76, 0x8e, 0xc6, 0x1a, 0x43, +- 0xc3, 0xa6, 0x89, 0x48, 0x47, 0xaf, 0x9b, 0x95, 0x3e, 0xd7, 0xd1, 0xe7, 0x24, 0xd2, 0xd6, 0x7b, +- 0xd6, 0x9e, 0x69, 0x61, 0x33, 0x76, 0xfa, 0x8a, 0xc6, 0xd0, 0x22, 0x06, 0xa7, 0xb3, 0xca, 0xa2, +- 0xe8, 0x8c, 0xc4, 0xfa, 0x19, 0xac, 0x2a, 0x4e, 0x8a, 0xb3, 0x61, 0xf3, 0x3a, 0x34, 0x98, 0x51, +- 0xa3, 0x92, 0x8d, 0xa2, 0x34, 0x91, 0xde, 0x43, 0xf7, 0x84, 0x30, 0x8f, 0xe1, 0x40, 0xf4, 0x1c, +- 0x55, 0x79, 0x65, 0x19, 0x42, 0x78, 0x4b, 0xf4, 0xdb, 0x99, 0x99, 0xc6, 0x5b, 0xab, 0x30, 0x10, +- 0x1b, 0x05, 0x89, 0xd6, 0x2f, 0x60, 0xf5, 0x79, 0x38, 0x25, 0x21, 0xde, 0x3e, 0x38, 0x7a, 0x86, +- 0xd3, 0xac, 0x80, 0xa0, 0x2e, 0xaa, 0x27, 0xa9, 0x46, 0xd3, 0x96, 0x6b, 0x11, 0x26, 0xe1, 0xb1, +- 0xe3, 0x45, 0x49, 0xac, 0x27, 0x43, 0x8d, 0xf0, 0x78, 0x3b, 0x4a, 0x62, 0x91, 0xe6, 0xc5, 0x67, +- 0x9e, 0x86, 0xd3, 0x99, 0x8c, 0x95, 0xa6, 0xbd, 0xec, 0x45, 0xc9, 0xf3, 0x70, 0x3a, 0xb3, 0xfe, +- 0x4f, 0xf6, 0xc2, 0x18, 0xfb, 0xb6, 0x1b, 0xfa, 0x34, 0xd8, 0xc1, 0xe7, 0x39, 0x09, 0x69, 0xdf, +- 0x65, 0x72, 0xc2, 0xb7, 0x15, 0xe8, 0x3c, 0x99, 0xe0, 0x90, 0xef, 0x60, 0xee, 0x92, 0xa9, 0xec, +- 0xad, 0xce, 0x31, 0x8b, 0x09, 0x0d, 0xf5, 0xc3, 0x37, 0xa0, 0x68, 0x8d, 0x49, 0x48, 0xb8, 0xe3, +- 0xbb, 0x38, 0xa0, 0xa1, 0xf6, 0x02, 0x08, 0xd4, 0x8e, 0xc4, 0xa0, 0x37, 0xa1, 0xaf, 0x26, 0x77, +- 0xce, 0xa9, 0x1b, 0xfa, 0x53, 0x11, 0x72, 0x6a, 0x92, 0xd1, 0x53, 0xe8, 0x3d, 0x8d, 0x45, 0x6f, +- 0xc1, 0x8a, 0x0e, 0x88, 0x8c, 0xb2, 0x2e, 0x29, 0xfb, 0x1a, 0x5f, 0x20, 0x4d, 0xa2, 0x88, 0x32, +- 0x1e, 0x3b, 0x31, 0xf6, 0x3c, 0x1a, 0x44, 0xba, 0x31, 0xe9, 0x1b, 0xfc, 0xa1, 0x42, 0x5b, 0x13, +- 0x58, 0x7d, 0x2a, 0xec, 0xd4, 0x96, 0x64, 0x17, 0xdc, 0x0b, 0x70, 0xe0, 0x1c, 0x4f, 0xa9, 0x77, +- 0xe6, 0x88, 0x34, 0xa5, 0x3d, 0x2c, 0x4a, 0x9f, 0x2d, 0x81, 0x3c, 0x24, 0xdf, 0xc8, 0x1e, 0x5c, +- 0x50, 0x9d, 0x52, 0x1e, 0x4d, 0x93, 0x89, 0x13, 0x31, 0x7a, 0x8c, 0xb5, 0x89, 0xfd, 0x00, 0x07, +- 0x7b, 0x0a, 0x7f, 0x20, 0xd0, 0xd6, 0x5f, 0x2a, 0xb0, 0x56, 0x94, 0xa4, 0x93, 0xee, 0x26, 0xac, +- 0x15, 0x45, 0xe9, 0x0f, 0xb1, 0x2a, 0xf4, 0x06, 0x79, 0x81, 0xea, 0x93, 0xfc, 0x21, 0x74, 0xe5, +- 0x38, 0xd7, 0xf1, 0x15, 0xa7, 0x62, 0xf9, 0x91, 0xbf, 0x17, 0xbb, 0xe3, 0xe6, 0x6f, 0xe9, 0x63, +- 0xb8, 0xa3, 0xcd, 0x77, 0x16, 0xd5, 0x56, 0x0f, 0x62, 0x5d, 0x13, 0x3c, 0x9b, 0xd3, 0xfe, 0x0b, +- 0x18, 0x66, 0xa8, 0xad, 0x99, 0x44, 0x1a, 0x5f, 0xbd, 0x0b, 0xab, 0x73, 0xc6, 0x3e, 0xf1, 0x7d, +- 0x26, 0x03, 0xb4, 0x6e, 0x97, 0x6d, 0x59, 0x8f, 0xe1, 0xf6, 0x21, 0xe6, 0xca, 0x1b, 0x2e, 0xd7, +- 0x3d, 0x81, 0x62, 0xb6, 0x02, 0xb5, 0x43, 0xec, 0x49, 0xe3, 0x6b, 0xb6, 0x58, 0x8a, 0x07, 0x78, +- 0x14, 0x63, 0x4f, 0x5a, 0x59, 0xb3, 0xe5, 0xda, 0xfa, 0x73, 0x05, 0x96, 0x75, 0x9a, 0x14, 0xa9, +- 0xde, 0x67, 0xe4, 0x1c, 0x33, 0xfd, 0xf4, 0x34, 0x84, 0xde, 0x80, 0x9e, 0x5a, 0x39, 0x34, 0xe2, +- 0x84, 0xa6, 0xc9, 0xb7, 0xab, 0xb0, 0xcf, 0x15, 0x52, 0x4e, 0xea, 0xe4, 0x20, 0x4a, 0xf7, 0x7c, +- 0x1a, 0x92, 0xe3, 0xb6, 0x58, 0x64, 0x06, 0x99, 0x6c, 0x5b, 0xb6, 0x86, 0xc4, 0x53, 0x37, 0xfc, +- 0x96, 0x24, 0x3f, 0x03, 0x8a, 0xa7, 0x1e, 0xd0, 0x24, 0xe4, 0x4e, 0x44, 0x49, 0xc8, 0x75, 0x76, +- 0x05, 0x89, 0x3a, 0x10, 0x18, 0xeb, 0xd7, 0x15, 0x68, 0xa8, 0x69, 0xb5, 0xe8, 0x32, 0xd3, 0x6f, +- 0x5c, 0x95, 0xc8, 0x7a, 0x41, 0xca, 0x52, 0xdf, 0x35, 0xb9, 0x16, 0x71, 0x7c, 0x1e, 0xa8, 0x4c, +- 0xad, 0x55, 0x3b, 0x0f, 0x64, 0x8a, 0x7e, 0x03, 0x7a, 0xd9, 0xa7, 0x52, 0xee, 0x2b, 0x15, 0xbb, +- 0x29, 0x56, 0x92, 0x5d, 0xaa, 0xa9, 0xf5, 0x53, 0xd1, 0x5c, 0xa7, 0x93, 0xda, 0x15, 0xa8, 0x25, +- 0xa9, 0x32, 0x62, 0x29, 0x30, 0x93, 0xf4, 0x23, 0x2b, 0x96, 0xe8, 0x21, 0xf4, 0x5c, 0xdf, 0x27, +- 0xe2, 0xb8, 0x3b, 0x7d, 0x4a, 0xfc, 0x34, 0x48, 0x8b, 0x58, 0xeb, 0x6f, 0x15, 0xe8, 0x6f, 0xd3, +- 0x68, 0xf6, 0x43, 0x32, 0xc5, 0xb9, 0x0c, 0x22, 0x95, 0xd4, 0xdf, 0x58, 0xb1, 0x16, 0x75, 0xe3, +- 0x09, 0x99, 0x62, 0x15, 0x5a, 0xea, 0x66, 0x9b, 0x02, 0x21, 0xc3, 0xca, 0x6c, 0xa6, 0x03, 0xb0, +- 0xae, 0xda, 0x7c, 0x46, 0x7d, 0x59, 0x21, 0xfb, 0x84, 0x39, 0xe9, 0xb8, 0xab, 0x6b, 0x2f, 0xfb, +- 0x84, 0xc9, 0x2d, 0x6d, 0xc8, 0x92, 0x9c, 0xb8, 0xe6, 0x0d, 0x69, 0x28, 0x8c, 0x30, 0x64, 0x1d, +- 0x1a, 0xf4, 0xe4, 0x24, 0xc6, 0x5c, 0xd6, 0xb2, 0x35, 0x5b, 0x43, 0x69, 0x9a, 0x6b, 0xe6, 0xd2, +- 0xdc, 0x2d, 0x58, 0x95, 0xb3, 0xfd, 0x17, 0xcc, 0xf5, 0x48, 0x38, 0x31, 0xa9, 0x78, 0x0d, 0xd0, +- 0x21, 0xa7, 0x51, 0x11, 0xfb, 0xe8, 0x77, 0x2b, 0x3a, 0x27, 0xea, 0x46, 0x17, 0x3d, 0x85, 0xfe, +- 0xdc, 0x1f, 0x27, 0x48, 0x4f, 0x3e, 0xca, 0xff, 0x4f, 0x19, 0xad, 0x8f, 0xd5, 0x1f, 0x31, 0x63, +- 0xf3, 0x47, 0xcc, 0x78, 0x37, 0x88, 0xf8, 0x0c, 0xed, 0x42, 0xaf, 0xf8, 0x17, 0x03, 0xba, 0x6b, +- 0x0a, 0x85, 0x92, 0x3f, 0x1e, 0x2e, 0x65, 0xf3, 0x14, 0xfa, 0x73, 0xff, 0x36, 0x18, 0x7d, 0xca, +- 0xff, 0x84, 0xb8, 0x94, 0xd1, 0x63, 0x68, 0xe7, 0xfe, 0x5e, 0x40, 0x43, 0xc5, 0x64, 0xf1, 0x1f, +- 0x87, 0x4b, 0x19, 0x6c, 0x43, 0xb7, 0x30, 0xf1, 0x47, 0x23, 0x6d, 0x4f, 0xc9, 0xdf, 0x00, 0x97, +- 0x32, 0xd9, 0x82, 0x76, 0x6e, 0xf0, 0x6e, 0xb4, 0x58, 0x9c, 0xee, 0x8f, 0xee, 0x94, 0xec, 0xe8, +- 0xd4, 0xbb, 0x07, 0xdd, 0xc2, 0x98, 0xdc, 0x28, 0x52, 0x36, 0xa2, 0x1f, 0xdd, 0x2d, 0xdd, 0xd3, +- 0x9c, 0x9e, 0x42, 0x7f, 0x6e, 0x68, 0x6e, 0x9c, 0x5b, 0x3e, 0x4b, 0xbf, 0xd4, 0xac, 0xcf, 0xe5, +- 0x65, 0xe7, 0x7a, 0xa2, 0xdc, 0x65, 0x2f, 0x8e, 0xc8, 0x47, 0xf7, 0xca, 0x37, 0xb5, 0x56, 0xbb, +- 0xd0, 0x2b, 0x4e, 0xc7, 0x0d, 0xb3, 0xd2, 0x99, 0xf9, 0xd5, 0x2f, 0xa7, 0x30, 0x28, 0xcf, 0x5e, +- 0x4e, 0xd9, 0xfc, 0xfc, 0x52, 0x46, 0x4f, 0x00, 0x74, 0x07, 0xe4, 0x93, 0x30, 0xbd, 0xb2, 0x85, +- 0xce, 0x2b, 0xbd, 0xb2, 0x92, 0x6e, 0xe9, 0x31, 0x80, 0x6a, 0x5c, 0x7c, 0x9a, 0x70, 0x74, 0xdb, +- 0xa8, 0x31, 0xd7, 0x2d, 0x8d, 0x86, 0x8b, 0x1b, 0x0b, 0x0c, 0x30, 0x63, 0xd7, 0x61, 0xf0, 0x19, +- 0x40, 0xd6, 0x10, 0x19, 0x06, 0x0b, 0x2d, 0xd2, 0x15, 0x3e, 0xe8, 0xe4, 0xdb, 0x1f, 0xa4, 0x6d, +- 0x2d, 0x69, 0x89, 0xae, 0x60, 0xd1, 0x9f, 0x2b, 0x6f, 0x8b, 0x8f, 0x6d, 0xbe, 0xea, 0x1d, 0x2d, +- 0x94, 0xb8, 0xe8, 0x43, 0xe8, 0xe4, 0xeb, 0x5a, 0xa3, 0x45, 0x49, 0xad, 0x3b, 0x2a, 0xd4, 0xb6, +- 0xe8, 0x31, 0xf4, 0x8a, 0x55, 0x2b, 0xca, 0xc5, 0xc5, 0x42, 0x2d, 0x3b, 0xd2, 0x13, 0x9b, 0x1c, +- 0xf9, 0xfb, 0x00, 0x59, 0x75, 0x6b, 0xdc, 0xb7, 0x50, 0xef, 0xce, 0x49, 0x7d, 0x02, 0x9d, 0x7c, +- 0x26, 0x36, 0xea, 0x96, 0x64, 0xe7, 0xab, 0xb2, 0x56, 0x2e, 0x6b, 0x9b, 0xc7, 0xb7, 0x98, 0xc8, +- 0xaf, 0xca, 0x5a, 0x85, 0xae, 0xcf, 0x24, 0x8b, 0xb2, 0x56, 0xf0, 0xaa, 0x5c, 0x5e, 0x6c, 0x91, +- 0x8c, 0xfb, 0x4a, 0x1b, 0xa7, 0xab, 0x1e, 0x51, 0xbe, 0x1b, 0x30, 0xfe, 0x28, 0xe9, 0x10, 0xbe, +- 0x27, 0xa8, 0xf3, 0x15, 0x7f, 0x2e, 0xa8, 0x4b, 0x1a, 0x81, 0x4b, 0x19, 0xed, 0x41, 0xff, 0xa9, +- 0x29, 0xe6, 0x74, 0xa1, 0xa9, 0xd5, 0x29, 0x29, 0xac, 0x47, 0xa3, 0xb2, 0x2d, 0x1d, 0x59, 0x9f, +- 0xc3, 0x60, 0xa1, 0xc8, 0x44, 0xf7, 0xd3, 0xc1, 0x62, 0x69, 0xf5, 0x79, 0xa9, 0x5a, 0xfb, 0xb0, +- 0x32, 0x5f, 0x63, 0xa2, 0x57, 0xf4, 0xa5, 0x97, 0xd7, 0x9e, 0x97, 0xb2, 0xfa, 0x18, 0x9a, 0xa6, +- 0xa6, 0x41, 0x7a, 0x80, 0x3b, 0x57, 0xe3, 0x5c, 0x76, 0x74, 0xab, 0xf3, 0xed, 0x77, 0xf7, 0x2b, +- 0x7f, 0xff, 0xee, 0x7e, 0xe5, 0x9f, 0xdf, 0xdd, 0xaf, 0x1c, 0x37, 0xe4, 0xee, 0xfb, 0xff, 0x0e, +- 0x00, 0x00, 0xff, 0xff, 0x8d, 0x89, 0xaa, 0x73, 0xc8, 0x21, 0x00, 0x00, ++ // 2931 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, ++ 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, ++ 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, ++ 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, ++ 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, ++ 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, ++ 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, ++ 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, ++ 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, ++ 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, ++ 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, ++ 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, ++ 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, ++ 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, ++ 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, ++ 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, ++ 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, ++ 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, ++ 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, ++ 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, ++ 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, ++ 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, ++ 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, ++ 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, ++ 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, ++ 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, ++ 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, ++ 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, ++ 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, ++ 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, ++ 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, ++ 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, ++ 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, ++ 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, ++ 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, ++ 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, ++ 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, ++ 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, ++ 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, ++ 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, ++ 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, ++ 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, ++ 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, ++ 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, ++ 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, ++ 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, ++ 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, ++ 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, ++ 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, ++ 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, ++ 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, ++ 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, ++ 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, ++ 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, ++ 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, ++ 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, ++ 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, ++ 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, ++ 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, ++ 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, ++ 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, ++ 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, ++ 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, ++ 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, ++ 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, ++ 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, ++ 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, ++ 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, ++ 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, ++ 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, ++ 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, ++ 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, ++ 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, ++ 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, ++ 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, ++ 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, ++ 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, ++ 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, ++ 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, ++ 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, ++ 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, ++ 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, ++ 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, ++ 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, ++ 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, ++ 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, ++ 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, ++ 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, ++ 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, ++ 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, ++ 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, ++ 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, ++ 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, ++ 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, ++ 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, ++ 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, ++ 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, ++ 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, ++ 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, ++ 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, ++ 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, ++ 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, ++ 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, ++ 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, ++ 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, ++ 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, ++ 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, ++ 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, ++ 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, ++ 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, ++ 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, ++ 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, ++ 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, ++ 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, ++ 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, ++ 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, ++ 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, ++ 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, ++ 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, ++ 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, ++ 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, ++ 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, ++ 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, ++ 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, ++ 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, ++ 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, ++ 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, ++ 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, ++ 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, ++ 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, ++ 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, ++ 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, ++ 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, ++ 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, ++ 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, ++ 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, ++ 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, ++ 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, ++ 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, ++ 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, ++ 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, ++ 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, ++ 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, ++ 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, ++ 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, ++ 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, ++ 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, ++ 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, ++ 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, ++ 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, ++ 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, ++ 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, ++ 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, ++ 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, ++ 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, ++ 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, ++ 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, ++ 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, ++ 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, ++ 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, ++ 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, ++ 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, ++ 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, ++ 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, ++ 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, ++ 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, ++ 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, ++ 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, ++ 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, ++ 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, ++ 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, ++ 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, ++ 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, ++ 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, ++ 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, ++ 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, ++ 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, ++ 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, ++ 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, ++ 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, ++ 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, ++ 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, ++ 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, ++ 0x22, 0x00, 0x00, + } +diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go +index c22ed39c..3f22d9f0 100644 +--- a/virtcontainers/agent.go ++++ b/virtcontainers/agent.go +@@ -264,4 +264,7 @@ type agent interface { + + // get proxy process pid + getProxyPid() int ++ ++ // updateIPVSRule updates IPVS rules in VM ++ updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) + } +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index a4bf41bb..7036df8c 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -13,6 +13,7 @@ import ( + "strings" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + deviceApi "github.com/kata-containers/runtime/virtcontainers/device/api" + deviceConfig "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/persist" +@@ -1017,6 +1018,30 @@ func ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + return routes, nil + } + ++// UpdateIPVSRule adds IPVS rule ++func UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ span, ctx := trace(ctx, "UpdateIPVSRule") ++ defer span.Finish() ++ ++ if sandboxID == "" { ++ return nil, vcTypes.ErrNeedSandboxID ++ } ++ ++ unlock, err := rwLockSandbox(sandboxID) ++ if err != nil { ++ return nil, err ++ } ++ defer unlock() ++ ++ s, err := fetchSandbox(ctx, sandboxID) ++ if err != nil { ++ return nil, err ++ } ++ defer s.releaseStatelessSandbox() ++ ++ return s.UpdateIPVSRule(IPVSRule) ++} ++ + // CleanupContaienr is used by shimv2 to stop and delete a container exclusively, once there is no container + // in the sandbox left, do stop the sandbox and delete it. Those serial operations will be done exclusively by + // locking the sandbox. +diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go +index 92cc4c12..0265d1ed 100644 +--- a/virtcontainers/implementation.go ++++ b/virtcontainers/implementation.go +@@ -13,6 +13,7 @@ import ( + "context" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" +@@ -178,3 +179,7 @@ func (impl *VCImpl) ListRoutes(ctx context.Context, sandboxID string) ([]*vcType + func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error { + return CleanupContainer(ctx, sandboxID, containerID, force) + } ++ ++func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return UpdateIPVSRule(ctx, sandboxID, IPVSRule) ++} +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index 6bb9ea22..d8f0be69 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -10,6 +10,7 @@ import ( + "io" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" +@@ -53,6 +54,7 @@ type VC interface { + ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) + UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) + ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) ++ UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) + + CleanupContainer(ctx context.Context, sandboxID, containerID string, force bool) error + } +@@ -99,6 +101,7 @@ type VCSandbox interface { + UpdateRoutes(routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) + ListRoutes() ([]*vcTypes.Route, error) + IsCompatOldCNI() bool ++ UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) + } + + // VCContainer is the Container interface +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 16662949..29287685 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -128,6 +128,7 @@ const ( + grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" + grpcStartTracingRequest = "grpc.StartTracingRequest" + grpcStopTracingRequest = "grpc.StopTracingRequest" ++ grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" + ) + + // The function is declared this way for mocking in unit tests +@@ -2067,6 +2068,10 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { + k.reqHandlers[grpcStopTracingRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.StopTracing(ctx, req.(*grpc.StopTracingRequest), opts...) + } ++ k.reqHandlers[grpcUpdateIPVSRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { ++ return k.client.UpdateIPVSRule(ctx, req.(*grpc.UpdateIPVSRequest), opts...) ++ } ++ + } + + func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel context.CancelFunc) { +@@ -2402,3 +2407,22 @@ func (k *kataAgent) load(s persistapi.AgentState) { + func (k *kataAgent) getProxyPid() int { + return k.state.ProxyPid + } ++ ++func (k *kataAgent) updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ if IPVSRule == nil { ++ return nil, nil ++ } ++ resultingIPVS, err := k.sendReq(IPVSRule) ++ if err != nil { ++ k.Logger().WithFields(logrus.Fields{ ++ "IPVS-requested": fmt.Sprintf("%+v", IPVSRule), ++ "resulting-IPVS": fmt.Sprintf("%+v", resultingIPVS), ++ }).WithError(err).Error("update IPVS rule request failed") ++ } ++ resultIPVS, ok := resultingIPVS.(*grpc.IPVSResponse) ++ if ok && resultIPVS != nil { ++ return resultIPVS, err ++ } ++ ++ return nil, err ++} +diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go +index b19705c3..d174623b 100644 +--- a/virtcontainers/noop_agent.go ++++ b/virtcontainers/noop_agent.go +@@ -240,3 +240,7 @@ func (n *noopAgent) load(s persistapi.AgentState) {} + func (n *noopAgent) getProxyPid() int { + return -1 + } ++ ++func (n *noopAgent) updateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return nil, nil ++} +diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go +index 9c50e0e4..aa9bae9a 100644 +--- a/virtcontainers/pkg/vcmock/mock.go ++++ b/virtcontainers/pkg/vcmock/mock.go +@@ -20,6 +20,7 @@ import ( + "fmt" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +@@ -296,3 +297,10 @@ func (m *VCMock) CleanupContainer(ctx context.Context, sandboxID, containerID st + } + return fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) + } ++ ++func (m *VCMock) UpdateIPVSRule(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ if m.UpdateIPVSRuleFunc != nil { ++ return m.UpdateIPVSRuleFunc(ctx, sandboxID, ipvs) ++ } ++ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) ++} +diff --git a/virtcontainers/pkg/vcmock/sandbox.go b/virtcontainers/pkg/vcmock/sandbox.go +index 7329e1a7..063c9eca 100644 +--- a/virtcontainers/pkg/vcmock/sandbox.go ++++ b/virtcontainers/pkg/vcmock/sandbox.go +@@ -9,6 +9,7 @@ import ( + "io" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +@@ -217,3 +218,7 @@ func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { + func (s *Sandbox) IsCompatOldCNI() bool { + return false + } ++ ++func (s *Sandbox) UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return nil, nil ++} +diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go +index 8cf0e0b7..43247ef9 100644 +--- a/virtcontainers/pkg/vcmock/types.go ++++ b/virtcontainers/pkg/vcmock/types.go +@@ -9,6 +9,7 @@ import ( + "context" + "syscall" + ++ "github.com/kata-containers/agent/protocols/grpc" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/device/api" + "github.com/kata-containers/runtime/virtcontainers/device/config" +@@ -75,4 +76,5 @@ type VCMock struct { + UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) + ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error ++ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error) + } +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index f226af4e..0d599267 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1065,6 +1065,11 @@ func (s *Sandbox) ListRoutes() ([]*vcTypes.Route, error) { + return s.agent.listRoutes() + } + ++// UpdateIPVSRule manages IPVS rule in the sandbox ++func (s *Sandbox) UpdateIPVSRule(IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return s.agent.updateIPVSRule(IPVSRule) ++} ++ + // startVM starts the VM. + func (s *Sandbox) startVM() (err error) { + span, ctx := s.trace("startVM") +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch b/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch new file mode 100644 index 0000000000000000000000000000000000000000..143d6b5aff5b0abb2c31da9540438bafb3820ba1 --- /dev/null +++ b/runtime/patches/0035-device-mount-blockdevices-in-the-guest-VM.patch @@ -0,0 +1,234 @@ +From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Mon, 17 Aug 2020 21:44:57 +0800 +Subject: [PATCH 35/50] device: mount blockdevices in the guest VM + +reason: support mount blockdevices in the guest VM like +"-v /dev/blockdevice:/home/test" + +Signed-off-by: yangfeiyu +--- + cli/config/configuration-qemu.toml.in | 5 ++++ + pkg/katautils/config.go | 6 +++++ + virtcontainers/kata_agent.go | 27 ++++++++++++++++--- + virtcontainers/kata_agent_test.go | 2 +- + virtcontainers/utils/utils.go | 50 +++++++++++++++++++++++++++++++++++ + virtcontainers/vm_test.go | 2 +- + 6 files changed, 86 insertions(+), 6 deletions(-) + +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index 46f8b632..aa11b38f 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -391,6 +391,11 @@ path = "@SHIMPATH@" + # + kernel_modules=[] + ++# If enabled, when we pass a block device to the guest VM ++# through -v, such as `docker run -v /dev/loop100:/foo/bar`, ++# agent will create a directory in guest VM and mount the ++# file system on `/dev/loop100` inside the container ++enable_blk_mount = true + + [netmon] + # If enabled, the network monitoring process gets started when the +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 94c916a0..51120311 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -160,6 +160,7 @@ type agent struct { + TraceMode string `toml:"trace_mode"` + TraceType string `toml:"trace_type"` + KernelModules []string `toml:"kernel_modules"` ++ MountBlkInVM bool `toml:"enable_blk_mount"` + } + + type netmon struct { +@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) + return + } + ++func (a agent) mountBlkDevInVM() bool { ++ return a.MountBlkInVM ++} ++ + func (p proxy) path() (string, error) { + path := p.Path + if path == "" { +@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc + TraceMode: agent.traceMode(), + TraceType: agent.traceType(), + KernelModules: agent.kernelModules(), ++ MountBlkInVM: agent.mountBlkDevInVM(), + } + default: + return fmt.Errorf("%s agent type is not supported", k) +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 16662949..b0f88c15 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -30,6 +30,7 @@ import ( + ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" + "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" +@@ -64,6 +65,7 @@ var ( + errorMissingOCISpec = errors.New("Missing OCI specification") + defaultKataHostSharedDir = "/run/kata-containers/shared/sandboxes/" + defaultKataGuestSharedDir = "/run/kata-containers/shared/containers/" ++ kataGuestStorageDir = "/run/kata-containers/storage/containers/" + mountGuestTag = "kataShared" + defaultKataGuestSandboxDir = "/run/kata-containers/sandbox/" + type9pFs = "9p" +@@ -199,6 +201,7 @@ type KataAgentConfig struct { + TraceMode string + TraceType string + KernelModules []string ++ MountBlkInVM bool + } + + // KataAgentState is the structure describing the data stored from this +@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages + // Create a temporary location to mount the Storage. Mounting to the correct location + // will be handled by the OCI mount structure. + filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) +- path := filepath.Join(kataGuestSandboxStorageDir(), filename) ++ var path string ++ if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType { ++ path = filepath.Join(kataGuestStorageDir, filename) ++ } else { ++ path = filepath.Join(kataGuestSandboxStorageDir(), filename) ++ } + + k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) + ociMounts[index].Source = path +@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, + // Note this call modifies the list of container devices to make sure + // all hotplugged devices are unplugged, so this needs be done + // after devices passed with --device are handled. +- volumeStorages, err := k.handleBlockVolumes(c) ++ volumeStorages, err := k.handleBlockVolumes(sandbox, c) + if err != nil { + return nil, err + } +@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (* + + // handleBlockVolumes handles volumes that are block devices files + // by passing the block devices as Storage to the agent. +-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { ++func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) { + + var volumeStorages []*grpc.Storage + +@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { + } + + vol.MountPoint = m.Destination +- if vol.Fstype == "" { ++ ++ ac, _ := sandbox.config.AgentConfig.(KataAgentConfig) ++ if ac.MountBlkInVM { ++ // Ensure the block device is formatted, for the devices here are specified as volumes ++ fsType, gerr := utils.GetDevFormat(m.Source) ++ if gerr != nil || fsType == "" { ++ k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed") ++ return nil, gerr ++ } ++ vol.Fstype = fmt.Sprintf("bind-%s", fsType) ++ } else { + vol.Fstype = "bind" + } ++ + if len(vol.Options) == 0 { + vol.Options = []string{"bind"} + } +diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go +index 18a5a0a6..68caaab2 100644 +--- a/virtcontainers/kata_agent_test.go ++++ b/virtcontainers/kata_agent_test.go +@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) { + containers[c.id].sandbox = &sandbox + containers[c.id].mounts = mounts + +- volumeStorages, err := k.handleBlockVolumes(c) ++ volumeStorages, err := k.handleBlockVolumes(&sandbox, c) + assert.Nil(t, err, "Error while handling block volumes") + + vStorage := &pb.Storage{ +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 9490faa1..36ac67a7 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) { + cpus := int(math.Ceil(cpuNum)) + return cpus, nil + } ++ ++// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc. ++func GetDevFormat(disk string) (string, error) { ++ // refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512 ++ args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk} ++ dataOut, err := exec.Command("blkid", args...).Output() ++ output := string(dataOut) ++ ++ if err != nil { ++ if strings.Contains(err.Error(), "exit status 2") { ++ // Disk device is unformatted. ++ // For `blkid`, if the specified token (TYPE/PTTYPE, etc) was ++ // not found, or no (specified) devices could be identified, an ++ // exit code of 2 is returned. ++ return "", nil ++ } ++ return "", err ++ } ++ ++ var fstype string ++ ++ lines := strings.Split(output, "\n") ++ for _, l := range lines { ++ if len(l) <= 0 { ++ // Ignore empty line. ++ continue ++ } ++ // if we use busybox as rootfs,the output of command ++ // `busybox blkid` is different with original`blkid`, ++ // so we should make a compatible parse ++ subLine := strings.Split(l, " ") ++ for _, sl := range subLine { ++ if len(subLine) <= 0 { ++ continue ++ } ++ cs := strings.Split(sl, "=") ++ if len(cs) != 2 { ++ continue ++ } ++ if cs[0] == "TYPE" { ++ fstype = cs[1] ++ if strings.Contains(fstype, `"`) { ++ fstype = strings.Replace(fstype, `"`, "", -1) ++ } ++ } ++ } ++ } ++ ++ return fstype, nil ++} +diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go +index 36fd5c2f..d2414232 100644 +--- a/virtcontainers/vm_test.go ++++ b/virtcontainers/vm_test.go +@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) { + HypervisorType: QemuHypervisor, + HypervisorConfig: newQemuConfig(), + AgentType: KataContainersAgent, +- AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}}, ++ AgentConfig: KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false }, + ProxyType: NoopProxyType, + } + +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch b/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch new file mode 100644 index 0000000000000000000000000000000000000000..51431e899621fef913f30dee66d5a338d54f0ff3 --- /dev/null +++ b/runtime/patches/0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch @@ -0,0 +1,131 @@ +From 448bb661d6759c1e20c18084604b150bff08ada4 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Tue, 18 Aug 2020 11:49:26 +0800 +Subject: [PATCH 36/50] mount: limit the maximum number of virtio-scsi bus + slots + +reason: +1. add SCSIBus functions for add or remove device about ScsiBus +2. limit the maximum number of virtio-scsi bus slots to 25 + +Signed-off-by: yangfeiyu +--- + virtcontainers/qemu.go | 20 +++++++++++++++++ + virtcontainers/types/scsi.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ + 2 files changed, 73 insertions(+) + create mode 100644 virtcontainers/types/scsi.go + +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index bb83b1bb..c2b65376 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -65,6 +65,7 @@ type CPUDevice struct { + // QemuState keeps Qemu's state + type QemuState struct { + Bridges []types.Bridge ++ ScsiBus *types.SCSIBus + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + HotpluggedMemory int +@@ -411,6 +412,13 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I + + var ioThread *govmmQemu.IOThread + if q.config.BlockDeviceDriver == config.VirtioSCSI { ++ if q.state.ScsiBus == nil { ++ // only use `scsi0.0` ++ q.state.ScsiBus = &types.SCSIBus{ ++ Address: make(map[uint32]int), ++ ID: fmt.Sprintf("%s.0", scsiControllerID), ++ } ++ } + return q.arch.appendSCSIController(devices, q.config.EnableIOThreads) + } + +@@ -1168,6 +1176,16 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev + + // Bus exposed by the SCSI Controller + bus := scsiControllerID + ".0" ++ var err error ++ if err = q.state.ScsiBus.AddDevToScsiBus(drive.Index); err != nil { ++ return err ++ } ++ ++ defer func() { ++ if err != nil { ++ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index) ++ } ++ }() + + // Get SCSI-id and LUN based on the order of attaching drives. + scsiID, lun, err := utils.GetSCSIIdLun(drive.Index) +@@ -1234,6 +1252,8 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error + if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { + return err + } ++ } else if q.config.BlockDeviceDriver == config.VirtioSCSI { ++ q.state.ScsiBus.RemoveDevFromScsiBus(drive.Index) + } + + if err := q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { +diff --git a/virtcontainers/types/scsi.go b/virtcontainers/types/scsi.go +new file mode 100644 +index 00000000..f4d489f0 +--- /dev/null ++++ b/virtcontainers/types/scsi.go +@@ -0,0 +1,53 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: leizhongkai ++// Create: 2019-06-30 ++ ++package types ++ ++import ( ++ "fmt" ++) ++ ++const ( ++ scsiMaxCapacity = 25 ++) ++ ++type SCSIBus struct { ++ // Address contains information about devices plugged and number limit ++ Address map[uint32]int ++ ++ // ID is used to identify the bus exposed by the SCSI Controller ++ ID string ++} ++ ++func (sb *SCSIBus) AddDevToScsiBus(index int) error { ++ var addr uint32 ++ ++ // looking for the first available address ++ for i := uint32(1); i <= scsiMaxCapacity; i++ { ++ if _, ok := sb.Address[i]; !ok { ++ addr = i ++ break ++ } ++ } ++ ++ if addr == 0 { ++ return fmt.Errorf("Scsi bus capacity limited.") ++ } ++ ++ // save address and device ++ sb.Address[addr] = index ++ ++ return nil ++} ++ ++func (sb *SCSIBus) RemoveDevFromScsiBus(index int) { ++ for addr, i := range sb.Address { ++ if i == index { ++ delete(sb.Address, addr) ++ return ++ } ++ } ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0037-runtime-add-IPVS-test.patch b/runtime/patches/0037-runtime-add-IPVS-test.patch new file mode 100644 index 0000000000000000000000000000000000000000..9c43970ba7529c8fc37601d700d6b32f0f23b501 --- /dev/null +++ b/runtime/patches/0037-runtime-add-IPVS-test.patch @@ -0,0 +1,875 @@ +From d083f0e0247fbded92a0ae2a0e71da4176baed95 Mon Sep 17 00:00:00 2001 +From: xiadanni +Date: Tue, 18 Aug 2020 17:08:23 +0800 +Subject: [PATCH 37/50] runtime: add IPVS test + +Signed-off-by: xiadanni +--- + cli/ipvsadm_test.go | 775 +++++++++++++++++++++++++++++++++++++ + virtcontainers/api_test.go | 36 ++ + virtcontainers/kata_agent_test.go | 4 + + virtcontainers/pkg/vcmock/types.go | 2 +- + 4 files changed, 816 insertions(+), 1 deletion(-) + create mode 100644 cli/ipvsadm_test.go + +diff --git a/cli/ipvsadm_test.go b/cli/ipvsadm_test.go +new file mode 100644 +index 00000000..92958aee +--- /dev/null ++++ b/cli/ipvsadm_test.go +@@ -0,0 +1,775 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: IPVS common functions test ++// Author: xiadanni ++// Create: 2020-08-01 ++ ++package main ++ ++import ( ++ "context" ++ "flag" ++ "fmt" ++ "os" ++ "strings" ++ "testing" ++ ++ "github.com/kata-containers/agent/protocols/grpc" ++ vc "github.com/kata-containers/runtime/virtcontainers" ++ vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ++ "github.com/kata-containers/runtime/virtcontainers/pkg/vcmock" ++ "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/stretchr/testify/assert" ++ "github.com/urfave/cli" ++) ++ ++func TestKataIPVSCliAction(t *testing.T) { ++ assert := assert.New(t) ++ ++ actionFunc, ok := kataIPVSCLICommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ flagSet := flag.NewFlagSet("kata-IPVS", flag.ContinueOnError) ++ ctx := createCLIContext(flagSet) ++ ++ err := actionFunc(ctx) ++ assert.NoError(err) ++} ++ ++func TestIPVSadmCliAction(t *testing.T) { ++ assert := assert.New(t) ++ ++ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ ctx := createCLIContext(flagSet) ++ ++ err := actionFunc(ctx) ++ assert.Error(err, "Missing container ID") ++} ++ ++func TestIPVSadmCLISuccessful(t *testing.T) { ++ assert := assert.New(t) ++ ++ sandbox := &vcmock.Sandbox{ ++ MockID: testContainerID, ++ } ++ ++ sandbox.MockContainers = []*vcmock.Container{ ++ { ++ MockID: sandbox.ID(), ++ MockSandbox: sandbox, ++ }, ++ } ++ ++ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { ++ return vc.ContainerStatus{ ++ ID: sandbox.ID(), ++ Annotations: map[string]string{ ++ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), ++ }, ++ State: types.ContainerState{ ++ State: types.StateRunning, ++ }, ++ }, nil ++ } ++ ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, nil ++ } ++ ++ defer func() { ++ testingImpl.StatusContainerFunc = nil ++ testingImpl.UpdateIPVSRuleFunc = nil ++ }() ++ ++ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ defer os.RemoveAll(path) ++ ++ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-a -t 192.168.0.7:80 -r 192.168.0.4:80", "") ++ ctx := createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.NoError(err) ++ ++ // result not nil ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{IPVSRes: "IPVS rule updating success"}, nil ++ } ++ ++ err = actionFunc(ctx) ++ assert.NoError(err) ++} ++ ++func TestIPVSadmCLIError(t *testing.T) { ++ assert := assert.New(t) ++ ++ sandbox := &vcmock.Sandbox{ ++ MockID: testContainerID, ++ } ++ ++ sandbox.MockContainers = []*vcmock.Container{ ++ { ++ MockID: sandbox.ID(), ++ MockSandbox: sandbox, ++ }, ++ } ++ ++ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { ++ return vc.ContainerStatus{ ++ ID: sandbox.ID(), ++ Annotations: map[string]string{ ++ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), ++ }, ++ State: types.ContainerState{ ++ State: types.StateRunning, ++ }, ++ }, nil ++ } ++ ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, nil ++ } ++ ++ defer func() { ++ testingImpl.StatusContainerFunc = nil ++ testingImpl.UpdateIPVSRuleFunc = nil ++ }() ++ ++ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ defer os.RemoveAll(path) ++ ++ actionFunc, ok := IPVSadmCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ // no stdin rule file ++ flagSet := flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("restore", "-", "") ++ ctx := createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++ ++ // restore parameter error ++ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("restore", "abc", "") ++ ctx = createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++ ++ // checkrule returns error ++ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p -3000", "") ++ ctx = createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++ ++ // updatefunction returns error ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm test error") ++ } ++ ++ flagSet = flag.NewFlagSet("IPVSadm", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-A -t 192.168.0.7:80 -s rr -p 3000", "") ++ ctx = createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++} ++ ++func TestCleanupCliAction(t *testing.T) { ++ assert := assert.New(t) ++ ++ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) ++ ctx := createCLIContext(flagSet) ++ ++ err := actionFunc(ctx) ++ assert.Error(err, "Missing container ID") ++} ++ ++func TestCleanupCLISuccessful(t *testing.T) { ++ assert := assert.New(t) ++ ++ sandbox := &vcmock.Sandbox{ ++ MockID: testContainerID, ++ } ++ ++ sandbox.MockContainers = []*vcmock.Container{ ++ { ++ MockID: sandbox.ID(), ++ MockSandbox: sandbox, ++ }, ++ } ++ ++ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { ++ return vc.ContainerStatus{ ++ ID: sandbox.ID(), ++ Annotations: map[string]string{ ++ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), ++ }, ++ State: types.ContainerState{ ++ State: types.StateRunning, ++ }, ++ }, nil ++ } ++ ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, nil ++ } ++ ++ defer func() { ++ testingImpl.StatusContainerFunc = nil ++ testingImpl.UpdateIPVSRuleFunc = nil ++ }() ++ ++ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ defer os.RemoveAll(path) ++ ++ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "") ++ ctx := createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.NoError(err) ++} ++ ++func TestCleanupCLIError(t *testing.T) { ++ assert := assert.New(t) ++ ++ sandbox := &vcmock.Sandbox{ ++ MockID: testContainerID, ++ } ++ ++ sandbox.MockContainers = []*vcmock.Container{ ++ { ++ MockID: sandbox.ID(), ++ MockSandbox: sandbox, ++ }, ++ } ++ ++ testingImpl.StatusContainerFunc = func(ctx context.Context, sandboxID, containerID string) (vc.ContainerStatus, error) { ++ return vc.ContainerStatus{ ++ ID: sandbox.ID(), ++ Annotations: map[string]string{ ++ vcAnnotations.ContainerTypeKey: string(vc.PodContainer), ++ }, ++ State: types.ContainerState{ ++ State: types.StateRunning, ++ }, ++ }, nil ++ } ++ ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, nil ++ } ++ ++ defer func() { ++ testingImpl.StatusContainerFunc = nil ++ testingImpl.UpdateIPVSRuleFunc = nil ++ }() ++ ++ path, err := createTempContainerIDMapping(sandbox.ID(), sandbox.ID()) ++ assert.NoError(err) ++ defer os.RemoveAll(path) ++ ++ actionFunc, ok := CleanupCommand.Action.(func(ctx *cli.Context) error) ++ assert.True(ok) ++ ++ // checkrule returns error ++ flagSet := flag.NewFlagSet("cleanup", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-d 192.168.0.4", "") ++ ctx := createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++ ++ // updatefunction returns error ++ testingImpl.UpdateIPVSRuleFunc = func(ctx context.Context, sandboxID string, IPVS *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { ++ return &grpc.IPVSResponse{}, fmt.Errorf("IPVSadm cleanup test error") ++ } ++ ++ flagSet = flag.NewFlagSet("cleanup", flag.ContinueOnError) ++ flagSet.Parse([]string{testContainerID}) ++ flagSet.String("parameters", "-d 192.168.0.4 -p tcp", "") ++ ctx = createCLIContext(flagSet) ++ err = actionFunc(ctx) ++ assert.Error(err) ++} ++ ++func TestAddServiceSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommandTCP := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommandTCP) ++ assert.NoError(err) ++ addServiceCommandUDP := "IPVSadm --add-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000" ++ _, err = checkIPVSRule(addServiceCommandUDP) ++ assert.NoError(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.NoError(err) ++} ++ ++func TestAddServiceNoService(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -s rr -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServiceNoScheduler(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServiceNoPersistent(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServiceIpError(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.2.2.7:80 -s rr -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServicePortError(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:9999999 -s rr -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServiceSchedulerError(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rrr -p 3000" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServicePersistentError(t *testing.T) { ++ assert := assert.New(t) ++ addServiceCommand := "IPVSadm --add-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999" ++ _, err := checkIPVSRule(addServiceCommand) ++ assert.Error(err) ++ addServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 99999999999" ++ _, err = checkIPVSRule(addServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServiceSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommandTCP := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommandTCP) ++ assert.NoError(err) ++ editServiceCommandUcp := "IPVSadm --edit-service --udp-service 192.168.0.7:80 --scheduler rr --persistent 3000" ++ _, err = checkIPVSRule(editServiceCommandUcp) ++ assert.NoError(err) ++ editServiceCmd := "IPVSadm -A -t 192.168.0.7:80 -s rr -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.NoError(err) ++} ++ ++func TestEditServiceNoService(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -s rr -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServiceNoScheduler(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServiceNoPersistent(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServiceIpError(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.2.2.7:80 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.2.2.7:80 -s rr -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServicePortError(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:9999999 --scheduler rr --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.0.7:9999999 -s rr -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServiceSchedulerError(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rrr --persistent 3000" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rrr -p 3000" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestEditServicePersistentError(t *testing.T) { ++ assert := assert.New(t) ++ editServiceCommand := "IPVSadm --edit-service --tcp-service 192.168.0.7:80 --scheduler rr --persistent 99999999999" ++ _, err := checkIPVSRule(editServiceCommand) ++ assert.Error(err) ++ editServiceCmd := "IPVSadm -E -t 192.168.0.7:80 -s rr -p 99999999999" ++ _, err = checkIPVSRule(editServiceCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServiceSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ deleteServiceCommandTCP := "IPVSadm --delete-service --tcp-service 192.168.0.7:80" ++ _, err := checkIPVSRule(deleteServiceCommandTCP) ++ assert.NoError(err) ++ deleteServiceCommandUDP := "IPVSadm --delete-service --udp-service 192.168.0.7:80" ++ _, err = checkIPVSRule(deleteServiceCommandUDP) ++ assert.NoError(err) ++ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:80" ++ _, err = checkIPVSRule(deleteServiceCmd) ++ assert.NoError(err) ++} ++ ++func TestDeleteServiceNoService(t *testing.T) { ++ assert := assert.New(t) ++ deleteServiceCommand := "IPVSadm --delete-service" ++ _, err := checkIPVSRule(deleteServiceCommand) ++ assert.Error(err) ++ deleteServiceCmd := "IPVSadm -D" ++ _, err = checkIPVSRule(deleteServiceCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServiceIpError(t *testing.T) { ++ assert := assert.New(t) ++ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.2.2.7:80" ++ _, err := checkIPVSRule(deleteServiceCommand) ++ assert.Error(err) ++ deleteServiceCmd := "IPVSadm -D -t 192.168.2.2.7:80" ++ _, err = checkIPVSRule(deleteServiceCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServicePortError(t *testing.T) { ++ assert := assert.New(t) ++ deleteServiceCommand := "IPVSadm --delete-service --tcp-service 192.168.0.7:9999999" ++ _, err := checkIPVSRule(deleteServiceCommand) ++ assert.Error(err) ++ deleteServiceCmd := "IPVSadm -D -t 192.168.0.7:9999999" ++ _, err = checkIPVSRule(deleteServiceCmd) ++ assert.Error(err) ++} ++ ++func TestAddServerSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ addServerCommandTCP := "IPVSadm --add-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" ++ _, err := checkIPVSRule(addServerCommandTCP) ++ assert.NoError(err) ++ addServerCommandUDP := "IPVSadm --add-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" ++ _, err = checkIPVSRule(addServerCommandUDP) ++ assert.NoError(err) ++ addServerCmd := "IPVSadm -a -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100" ++ _, err = checkIPVSRule(addServerCmd) ++ assert.NoError(err) ++} ++ ++func TestAddServerNoService(t *testing.T) { ++ assert := assert.New(t) ++ addServerCommand := "IPVSadm --add-server --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(addServerCommand) ++ assert.Error(err) ++ addServerCmd := "IPVSadm -a -r 192.168.0.4:80" ++ _, err = checkIPVSRule(addServerCmd) ++ assert.Error(err) ++} ++ ++func TestAddServerNoServer(t *testing.T) { ++ assert := assert.New(t) ++ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:80" ++ _, err := checkIPVSRule(addServerCommand) ++ assert.Error(err) ++ addServerCmd := "IPVSadm -a -t 192.168.0.7:80" ++ _, err = checkIPVSRule(addServerCmd) ++ assert.Error(err) ++} ++ ++func TestAddServerIpError(t *testing.T) { ++ assert := assert.New(t) ++ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(addServerCommand) ++ assert.Error(err) ++ addServerCmd := "IPVSadm -a -t 192.168.2.0.7:80 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(addServerCmd) ++ assert.Error(err) ++} ++ ++func TestAddServerPortError(t *testing.T) { ++ assert := assert.New(t) ++ addServerCommand := "IPVSadm --add-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(addServerCommand) ++ assert.Error(err) ++ addServerCmd := "IPVSadm -a -t 192.168.0.7:99999 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(addServerCmd) ++ assert.Error(err) ++} ++ ++func TestEditServerSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ editServerCommandTCP := "IPVSadm --edit-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" ++ _, err := checkIPVSRule(editServerCommandTCP) ++ assert.NoError(err) ++ editServerCommandUDP := "IPVSadm --edit-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80 --weight 100" ++ _, err = checkIPVSRule(editServerCommandUDP) ++ assert.NoError(err) ++ editServerCmd := "IPVSadm -e -t 192.168.0.7:80 -r 192.168.0.4:80 -w 100" ++ _, err = checkIPVSRule(editServerCmd) ++ assert.NoError(err) ++} ++ ++func TestEditServerNoService(t *testing.T) { ++ assert := assert.New(t) ++ editServerCommand := "IPVSadm --edit-server --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(editServerCommand) ++ assert.Error(err) ++ editServerCmd := "IPVSadm -e -r 192.168.0.4:80" ++ _, err = checkIPVSRule(editServerCmd) ++ assert.Error(err) ++} ++ ++func TestEditServerNoServer(t *testing.T) { ++ assert := assert.New(t) ++ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:80" ++ _, err := checkIPVSRule(editServerCommand) ++ assert.Error(err) ++ editServerCmd := "IPVSadm -e -t 192.168.0.7:80" ++ _, err = checkIPVSRule(editServerCmd) ++ assert.Error(err) ++} ++ ++func TestEditServerIpError(t *testing.T) { ++ assert := assert.New(t) ++ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(editServerCommand) ++ assert.Error(err) ++ editServerCmd := "IPVSadm -e -t 192.168.2.0.7:80 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(editServerCmd) ++ assert.Error(err) ++} ++ ++func TestEditServerPortError(t *testing.T) { ++ assert := assert.New(t) ++ editServerCommand := "IPVSadm --edit-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(editServerCommand) ++ assert.Error(err) ++ editServerCmd := "IPVSadm -e -t 192.168.0.7:99999 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(editServerCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServerSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ deleteServerCommandTCP := "IPVSadm --delete-server --tcp-service 192.168.0.7:80 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(deleteServerCommandTCP) ++ assert.NoError(err) ++ deleteServerCommandUDP := "IPVSadm --delete-server --udp-service 192.168.0.7:80 --real-server 192.168.0.4:80" ++ _, err = checkIPVSRule(deleteServerCommandUDP) ++ assert.NoError(err) ++ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(deleteServerCmd) ++ assert.NoError(err) ++} ++ ++func TestDeleteServerNoService(t *testing.T) { ++ assert := assert.New(t) ++ deleteServerCommand := "IPVSadm --delete-server --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(deleteServerCommand) ++ assert.Error(err) ++ deleteServerCmd := "IPVSadm -d -r 192.168.0.4:80" ++ _, err = checkIPVSRule(deleteServerCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServerNoServer(t *testing.T) { ++ assert := assert.New(t) ++ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:80" ++ _, err := checkIPVSRule(deleteServerCommand) ++ assert.Error(err) ++ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:80" ++ _, err = checkIPVSRule(deleteServerCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServerIpError(t *testing.T) { ++ assert := assert.New(t) ++ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.2.0.7:80 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(deleteServerCommand) ++ assert.Error(err) ++ deleteServerCmd := "IPVSadm -d -t 192.168.2.0.7:80 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(deleteServerCmd) ++ assert.Error(err) ++} ++ ++func TestDeleteServerPortError(t *testing.T) { ++ assert := assert.New(t) ++ deleteServerCommand := "IPVSadm --delete-server --tcp-service 192.168.0.7:99999 --real-server 192.168.0.4:80" ++ _, err := checkIPVSRule(deleteServerCommand) ++ assert.Error(err) ++ deleteServerCmd := "IPVSadm -d -t 192.168.0.7:99999 -r 192.168.0.4:80" ++ _, err = checkIPVSRule(deleteServerCmd) ++ assert.Error(err) ++} ++ ++func TestListSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ listCommand := "IPVSadm --list" ++ _, err := checkIPVSRule(listCommand) ++ assert.NoError(err) ++ listCmd := "IPVSadm -L" ++ _, err = checkIPVSRule(listCmd) ++ assert.NoError(err) ++} ++ ++func TestSetSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ setCommand := "IPVSadm --set" ++ _, err := checkIPVSRule(setCommand) ++ assert.NoError(err) ++} ++ ++func TestCleanupSuccessfully(t *testing.T) { ++ assert := assert.New(t) ++ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum tcp" ++ _, err := checkIPVSRule(cleanupCommand) ++ assert.NoError(err) ++ cleanupCmd := "conntrack -D -d 192.168.0.4 -p tcp" ++ _, err = checkIPVSRule(cleanupCmd) ++ assert.NoError(err) ++} ++ ++func TestCleanupNoDestination(t *testing.T) { ++ assert := assert.New(t) ++ cleanupCommand := "conntrack -D --protonum tcp" ++ _, err := checkIPVSRule(cleanupCommand) ++ assert.Error(err) ++ cleanupCmd := "conntrack -D -p tcp" ++ _, err = checkIPVSRule(cleanupCmd) ++ assert.Error(err) ++} ++ ++func TestCleanupNoProtocol(t *testing.T) { ++ assert := assert.New(t) ++ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4" ++ _, err := checkIPVSRule(cleanupCommand) ++ assert.Error(err) ++ cleanupCmd := "conntrack -D -d 192.168.0.4" ++ _, err = checkIPVSRule(cleanupCmd) ++ assert.Error(err) ++} ++ ++func TestCleanupIpError(t *testing.T) { ++ assert := assert.New(t) ++ cleanupCommand := "conntrack -D --orig-dst 192.168.0.2.4 --protonum tcp" ++ _, err := checkIPVSRule(cleanupCommand) ++ assert.Error(err) ++ cleanupCmd := "conntrack -D -d 192.168.0.2.4 -p tcp" ++ _, err = checkIPVSRule(cleanupCmd) ++ assert.Error(err) ++} ++ ++func TestCleanupProtocolError(t *testing.T) { ++ assert := assert.New(t) ++ cleanupCommand := "conntrack -D --orig-dst 192.168.0.4 --protonum ttcp" ++ _, err := checkIPVSRule(cleanupCommand) ++ assert.Error(err) ++ cleanupCmd := "conntrack -D -d 192.168.0.4 -p ttcp" ++ _, err = checkIPVSRule(cleanupCmd) ++ assert.Error(err) ++} ++ ++func TestRestoreFileNil(t *testing.T) { ++ assert := assert.New(t) ++ stdin := strings.NewReader("") ++ fileContext, cnt, err := getRestoreFile(stdin) ++ assert.Equal(cnt, 0) ++ assert.Equal(fileContext, "") ++ assert.Error(err) ++} ++ ++func TestRestoreFileError(t *testing.T) { ++ assert := assert.New(t) ++ stdin := strings.NewReader("aaa") ++ fileContext, cnt, err := getRestoreFile(stdin) ++ assert.Equal(cnt, 0) ++ assert.Equal(fileContext, "") ++ assert.Error(err) ++} ++ ++func TestRestoreFile(t *testing.T) { ++ assert := assert.New(t) ++ stdin := strings.NewReader("-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m") ++ fileContext, cnt, err := getRestoreFile(stdin) ++ assert.Equal(cnt, 2) ++ assert.Equal(fileContext, "-A -t 10.10.11.12:100 -s rr -p 3000\n-a -t 10.10.11.12:100 -r 172.16.0.1:80 -m") ++ assert.NoError(err) ++} +diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go +index c01d47b8..2a5488f9 100644 +--- a/virtcontainers/api_test.go ++++ b/virtcontainers/api_test.go +@@ -15,6 +15,7 @@ import ( + "syscall" + "testing" + ++ "github.com/kata-containers/agent/protocols/grpc" + ktu "github.com/kata-containers/runtime/pkg/katatestutils" + "github.com/kata-containers/runtime/virtcontainers/persist" + "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" +@@ -1745,3 +1746,38 @@ func TestCleanupContainer(t *testing.T) { + t.Fatal(err) + } + } ++ ++func TestUpdateIPVSRule(t *testing.T) { ++ defer cleanUp() ++ assert := assert.New(t) ++ ++ contID := "abc" ++ ctx := context.Background() ++ var ipvs = &grpc.UpdateIPVSRequest{ ++ IPVSReq: "ipvsadm -A -t 192.168.0.7:80 -s rr -p -3000", ++ } ++ ++ config := newTestSandboxConfigNoop() ++ p, _, err := createAndStartSandbox(ctx, config) ++ if p == nil || err != nil { ++ t.Fatal(err) ++ } ++ ++ s, ok := p.(*Sandbox) ++ assert.True(ok) ++ ++ contConfig := newTestContainerConfigNoop(contID) ++ c, err := p.CreateContainer(contConfig) ++ if c == nil || err != nil { ++ t.Fatal(err) ++ } ++ ++ _, err = UpdateIPVSRule(ctx, s.id, ipvs) ++ assert.NoError(err) ++ ++ _, err = UpdateIPVSRule(ctx, "aaa", ipvs) ++ assert.Error(err) ++ ++ _, err = UpdateIPVSRule(ctx, "", ipvs) ++ assert.Error(err) ++} +diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go +index 18a5a0a6..4f9409a0 100644 +--- a/virtcontainers/kata_agent_test.go ++++ b/virtcontainers/kata_agent_test.go +@@ -246,6 +246,10 @@ func (p *gRPCProxy) MemHotplugByProbe(ctx context.Context, req *pb.MemHotplugByP + return &gpb.Empty{}, nil + } + ++func (p *gRPCProxy) UpdateIPVSRule(ctx context.Context, req *pb.UpdateIPVSRequest) (*pb.IPVSResponse, error) { ++ return &pb.IPVSResponse{}, nil ++} ++ + func gRPCRegister(s *grpc.Server, srv interface{}) { + switch g := srv.(type) { + case *gRPCProxy: +diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go +index 43247ef9..610b4602 100644 +--- a/virtcontainers/pkg/vcmock/types.go ++++ b/virtcontainers/pkg/vcmock/types.go +@@ -76,5 +76,5 @@ type VCMock struct { + UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) + ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error +- UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIpvsRequest) (*grpc.IpvsResponse, error) ++ UpdateIPVSRuleFunc func(ctx context.Context, sandboxID string, ipvs *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch b/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch new file mode 100644 index 0000000000000000000000000000000000000000..8f3584fa9cb06818b77bdc740df4cba139a05c04 --- /dev/null +++ b/runtime/patches/0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch @@ -0,0 +1,501 @@ +From 5a220e9be1cfb03316a62aa00d2040638ba1a855 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 15:15:52 +0800 +Subject: [PATCH 38/50] pcie: using pcie-root-port driver to hotplug device in + aarch64 + +reason: Since qemu with "virt" machine type doesn't support hotplug +device in the pcie.0 root bus, so need to use add root port devices +to support hotplug pci device in aarch64. + +we reuse the pcie_root_port config in the configuration.toml file to +set pcie_root_port device number when qemu process start. + +Signed-off-by: jiangpengfei +--- + vendor/github.com/intel/govmm/qemu/qemu.go | 22 ++--- + virtcontainers/persist/api/hypervisor.go | 5 +- + virtcontainers/qemu.go | 130 +++++++++++++++++++++++------ + virtcontainers/qemu_arch_base.go | 9 ++ + virtcontainers/types/pcie.go | 103 +++++++++++++++++++++++ + 5 files changed, 230 insertions(+), 39 deletions(-) + create mode 100644 virtcontainers/types/pcie.go + +diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go +index a5e5dfaf..3e7720b4 100644 +--- a/vendor/github.com/intel/govmm/qemu/qemu.go ++++ b/vendor/github.com/intel/govmm/qemu/qemu.go +@@ -1252,6 +1252,8 @@ type PCIeRootPortDevice struct { + Chassis string // (slot, chassis) pair is mandatory and must be unique for each pcie-root-port, >=0, default is 0x00 + Slot string // >=0, default is 0x00 + ++ Port string // Port number is the device index ++ + Multifunction bool // true => "on", false => "off", default is off + Addr string // >=0, default is 0x00 + +@@ -1277,6 +1279,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { + + deviceParams = append(deviceParams, fmt.Sprintf("%s,id=%s", driver, b.ID)) + ++ if b.Port != "" { ++ deviceParams = append(deviceParams, fmt.Sprintf("port=%s", b.Port)) ++ } ++ + if b.Bus == "" { + b.Bus = "pcie.0" + } +@@ -1287,20 +1293,10 @@ func (b PCIeRootPortDevice) QemuParams(config *Config) []string { + } + deviceParams = append(deviceParams, fmt.Sprintf("chassis=%s", b.Chassis)) + +- if b.Slot == "" { +- b.Slot = "0x00" +- } +- deviceParams = append(deviceParams, fmt.Sprintf("slot=%s", b.Slot)) +- +- multifunction := "off" + if b.Multifunction { +- multifunction = "on" +- if b.Addr == "" { +- b.Addr = "0x00" +- } +- deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr)) ++ deviceParams = append(deviceParams, "multifunction=on") + } +- deviceParams = append(deviceParams, fmt.Sprintf("multifunction=%v", multifunction)) ++ deviceParams = append(deviceParams, fmt.Sprintf("addr=%s", b.Addr)) + + if b.BusReserve != "" { + deviceParams = append(deviceParams, fmt.Sprintf("bus-reserve=%s", b.BusReserve)) +@@ -1337,7 +1333,7 @@ func (b PCIeRootPortDevice) Valid() bool { + if b.Pref64Reserve != "" && b.Pref32Reserve != "" { + return false + } +- if b.ID == "" { ++ if b.ID == "" || b.Port == "" || b.Bus == "" || b.Addr == ""{ + return false + } + return true +diff --git a/virtcontainers/persist/api/hypervisor.go b/virtcontainers/persist/api/hypervisor.go +index 375fd56b..fd61b3c2 100644 +--- a/virtcontainers/persist/api/hypervisor.go ++++ b/virtcontainers/persist/api/hypervisor.go +@@ -5,6 +5,8 @@ + + package persistapi + ++import "github.com/kata-containers/runtime/virtcontainers/types" ++ + // Bridge is a bridge where devices can be hot plugged + type Bridge struct { + // DeviceAddr contains information about devices plugged and its address in the bridge +@@ -35,7 +37,8 @@ type HypervisorState struct { + + // Belows are qemu specific + // Refs: virtcontainers/qemu.go:QemuState +- Bridges []Bridge ++ Bridges []Bridge ++ PCIeRootPortsPool *types.PCIeRootPortPool + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + HotpluggedMemory int +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index c2b65376..a10c66fb 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -64,8 +64,9 @@ type CPUDevice struct { + + // QemuState keeps Qemu's state + type QemuState struct { +- Bridges []types.Bridge +- ScsiBus *types.SCSIBus ++ Bridges []types.Bridge ++ ScsiBus *types.SCSIBus ++ PCIeRootPortsPool *types.PCIeRootPortPool + // HotpluggedCPUs is the list of CPUs that were hot-added + HotpluggedVCPUs []CPUDevice + HotpluggedMemory int +@@ -271,6 +272,9 @@ func (q *qemu) setup(id string, hypervisorConfig *HypervisorConfig) error { + + q.state.HotplugVFIOOnRootBus = q.config.HotplugVFIOOnRootBus + q.state.PCIeRootPort = int(q.config.PCIeRootPort) ++ // init the PCIeRootPortsPool with pcie_root_port config value ++ q.state.PCIeRootPortsPool = &types.PCIeRootPortPool{} ++ q.state.PCIeRootPortsPool.Init(q.state.PCIeRootPort) + + // The path might already exist, but in case of VM templating, + // we have to create it since the sandbox has not created it yet. +@@ -394,9 +398,18 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I + return nil, nil, err + } + +- // Add bridges before any other devices. This way we make sure that +- // bridge gets the first available PCI address i.e bridgePCIStartAddr +- devices = q.arch.appendBridges(devices) ++ machine, err := q.getQemuMachine() ++ if err != nil { ++ return nil, nil, err ++ } ++ switch machine.Type { ++ case QemuVirt: ++ devices = q.arch.appendRootPorts(devices, q.state.PCIeRootPortsPool) ++ default: ++ // Add bridges before any other devices. This way we make sure that ++ // bridge gets the first available PCI address i.e bridgePCIStartAddr ++ devices = q.arch.appendBridges(devices) ++ } + + devices, err = q.arch.appendConsole(devices, console) + if err != nil { +@@ -608,7 +621,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa + // Add PCIe Root Port devices to hypervisor + // The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port. + // For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt +- if hypervisorConfig.PCIeRootPort > 0 { ++ if hypervisorConfig.PCIeRootPort > 0 && hypervisorConfig.HypervisorMachineType == QemuQ35 { + qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort) + } + +@@ -1154,21 +1167,19 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev + } + case q.config.BlockDeviceDriver == config.VirtioBlock: + driver := "virtio-blk-pci" +- addr, bridge, err := q.arch.addDeviceToBridge(drive.ID, types.PCI) ++ ++ addr, bus, pciAddr, err := q.getPciAddress(drive.ID, types.PCI) + if err != nil { + return err + } +- + defer func() { + if err != nil { +- q.arch.removeDeviceFromBridge(drive.ID) ++ q.putPciAddress(drive.ID) + } + }() ++ drive.PCIAddr = pciAddr + +- // PCI address is in the format bridge-addr/device-addr eg. "03/02" +- drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr +- +- if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { ++ if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus, romFile, 0, true, defaultDisableModern); err != nil { + return err + } + case q.config.BlockDeviceDriver == config.VirtioSCSI: +@@ -1249,7 +1260,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error + err = q.hotplugAddBlockDevice(drive, op, devID) + } else { + if q.config.BlockDeviceDriver == config.VirtioBlock { +- if err := q.arch.removeDeviceFromBridge(drive.ID); err != nil { ++ if err := q.putPciAddress(drive.ID); err != nil { + return err + } + } else if q.config.BlockDeviceDriver == config.VirtioSCSI { +@@ -1345,22 +1356,22 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro + } + } + +- addr, bridge, err := q.arch.addDeviceToBridge(devID, types.PCI) ++ addr, bus, _, err := q.getPciAddress(devID, types.PCI) + if err != nil { + return err + } + + defer func() { + if err != nil { +- q.arch.removeDeviceFromBridge(devID) ++ q.putPciAddress(devID) + } + }() + + switch device.Type { + case config.VFIODeviceNormalType: +- return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID, romFile) ++ return q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus, romFile) + case config.VFIODeviceMediatedType: +- return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bridge.ID, romFile) ++ return q.qmpMonitorCh.qmp.ExecutePCIVFIOMediatedDeviceAdd(q.qmpMonitorCh.ctx, devID, device.SysfsDev, addr, bus, romFile) + default: + return fmt.Errorf("Incorrect VFIO device type found") + } +@@ -1368,7 +1379,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro + q.Logger().WithField("dev-id", devID).Info("Start hot-unplug VFIO device") + + if !q.state.HotplugVFIOOnRootBus { +- if err := q.arch.removeDeviceFromBridge(devID); err != nil { ++ if err := q.putPciAddress(devID); err != nil { + return err + } + } +@@ -1439,18 +1450,17 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { + } + }() + +- addr, bridge, err := q.arch.addDeviceToBridge(tap.ID, types.PCI) ++ addr, bus, pciAddr, err := q.getPciAddress(tap.ID, types.PCI) + if err != nil { + return err + } + + defer func() { + if err != nil { +- q.arch.removeDeviceFromBridge(tap.ID) ++ q.putPciAddress(tap.ID) + } + }() + +- pciAddr := fmt.Sprintf("%02x/%s", bridge.Addr, addr) + endpoint.SetPciAddr(pciAddr) + + var machine govmmQemu.Machine +@@ -1459,14 +1469,14 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { + return err + } + if machine.Type == QemuCCWVirtio { +- devNoHotplug := fmt.Sprintf("fe.%x.%x", bridge.Addr, addr) ++ devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr) + return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) + } +- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bridge.ID, romFile, int(q.config.NumVCPUs), defaultDisableModern) ++ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern) + + } + +- if err := q.arch.removeDeviceFromBridge(tap.ID); err != nil { ++ if err := q.putPciAddress(tap.ID); err != nil { + return err + } + +@@ -2042,6 +2052,21 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff + return memory + } + ++func genericAppendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device { ++ for _, rp := range rootPorts.Items { ++ devices = append(devices, govmmQemu.PCIeRootPortDevice{ ++ Port: rp.Port, ++ Bus: rp.Bus, ++ ID: rp.ID, ++ Chassis: strconv.Itoa(int(rp.Chassis)), ++ Multifunction: rp.Multifunction, ++ Addr: fmt.Sprintf("0x%s.0x%s", rp.Slot, rp.Function), ++ }) ++ } ++ ++ return devices ++} ++ + // genericAppendPCIeRootPort appends to devices the given pcie-root-port + func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device { + var ( +@@ -2241,6 +2266,7 @@ func (q *qemu) save() (s persistapi.HypervisorState) { + s.HotpluggedMemory = q.state.HotpluggedMemory + s.HotplugVFIOOnRootBus = q.state.HotplugVFIOOnRootBus + s.PCIeRootPort = q.state.PCIeRootPort ++ s.PCIeRootPortsPool = q.state.PCIeRootPortsPool + + for _, bridge := range q.arch.getBridges() { + s.Bridges = append(s.Bridges, persistapi.Bridge{ +@@ -2265,6 +2291,7 @@ func (q *qemu) load(s persistapi.HypervisorState) { + q.state.HotplugVFIOOnRootBus = s.HotplugVFIOOnRootBus + q.state.VirtiofsdPid = s.VirtiofsdPid + q.state.PCIeRootPort = s.PCIeRootPort ++ q.state.PCIeRootPortsPool = s.PCIeRootPortsPool + + for _, bridge := range s.Bridges { + q.state.Bridges = append(q.state.Bridges, types.NewBridge(types.Type(bridge.Type), bridge.ID, bridge.DeviceAddr, bridge.Addr)) +@@ -2302,3 +2329,56 @@ func (q *qemu) generateSocket(id string, useVsock bool) (interface{}, error) { + func (q *qemu) getMemorySize() uint32 { + return q.config.MemorySize + uint32(q.state.HotpluggedMemory) + } ++ ++// getPciAddress allocate the pci slot to hotplugged device and ++// return the pci slot address ++func (q *qemu) getPciAddress(devID string, t types.Type) (slot, bus, pciAddr string, err error) { ++ machine, err := q.getQemuMachine() ++ if err != nil { ++ return "", "", "", err ++ } ++ ++ switch machine.Type { ++ case QemuVirt: ++ rp, err := q.state.PCIeRootPortsPool.AddDevice(devID) ++ if err != nil { ++ return "", "", "", err ++ } ++ // PCIe Root Port only have one slot ++ slot = "0x0" ++ // pciAddr specifies the slot and function of the Root Port and the slot of the device ++ pciAddr = fmt.Sprintf("%s.%s/00", rp.Slot, rp.Function) ++ bus = rp.ID ++ default: ++ var bridge types.Bridge ++ slot, bridge, err = q.arch.addDeviceToBridge(devID, t) ++ if err != nil { ++ return "", "", "", err ++ } ++ bus = bridge.ID ++ // PCI address is in the format bridge-addr.0/device-addr eg. "03.0/02" ++ pciAddr = fmt.Sprintf("%02x.0", bridge.Addr) + "/" + slot ++ } ++ return slot, bus, pciAddr, nil ++} ++ ++func (q *qemu) putPciAddress(devID string) error { ++ machine, err := q.getQemuMachine() ++ if err != nil { ++ return err ++ } ++ ++ switch machine.Type { ++ case QemuVirt: ++ err := q.state.PCIeRootPortsPool.RemoveDevice(devID) ++ if err != nil { ++ return err ++ } ++ default: ++ if err := q.arch.removeDeviceFromBridge(devID); err != nil { ++ return err ++ } ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go +index 9d72dd09..cb045530 100644 +--- a/virtcontainers/qemu_arch_base.go ++++ b/virtcontainers/qemu_arch_base.go +@@ -130,6 +130,9 @@ type qemuArch interface { + + // appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus + appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device ++ ++ // appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt" ++ appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device + } + + type qemuArchBase struct { +@@ -766,3 +769,9 @@ func (q *qemuArchBase) addBridge(b types.Bridge) { + func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device { + return genericAppendPCIeRootPort(devices, number, q.machineType) + } ++ ++// appendRootPorts appends a pcie-root-port device to devices when qemu machine type is "virt" ++// which is different appendPCIeRootPortDevice function ++func (q *qemuArchBase) appendRootPorts(devices []govmmQemu.Device, rootPorts *types.PCIeRootPortPool) []govmmQemu.Device { ++ return genericAppendRootPorts(devices, rootPorts) ++} +diff --git a/virtcontainers/types/pcie.go b/virtcontainers/types/pcie.go +new file mode 100644 +index 00000000..83eb6944 +--- /dev/null ++++ b/virtcontainers/types/pcie.go +@@ -0,0 +1,103 @@ ++package types ++ ++import ( ++ "fmt" ++) ++ ++const ( ++ maxRootPortsCapacity = 25 ++ slotPerDevice = 7 ++ ++ // PCIeRootBus is "pcie.0" ++ PCIeRootBus = "pcie.0" ++ ++ // startPort specifies the start port of pcie-root-port ++ // for the first slot of "pcie.0" is reserved, so the 1~7 ports are also reserved ++ startPort = 8 ++ funcNumPerSlot = 8 ++) ++ ++// PCIeRootPort describe the PCIe Root Port ++type PCIeRootPort struct { ++ // DeviceID specify the device hotplug on the Root Port ++ DeviceID string ++ ++ // Port number is the Root Port index ++ Port string ++ ++ // Bus number where the Root Port is plugged, typically pcie.0 ++ Bus string ++ ++ // ID is used to identify the pcie-root-port in qemu ++ ID string ++ ++ // Slot specifies slot address of Root Port ++ Slot string ++ ++ // Function specifies function of Root Port ++ Function string ++ ++ // Chassis number ++ Chassis uint32 ++ ++ // Multifunction is used to specify the pcie-root-port is multifunction supported ++ Multifunction bool ++} ++ ++// PCIeRootPortPool describe a set of PCIe Root Ports ++type PCIeRootPortPool struct { ++ // Items contains information about devices plugged and number limit ++ Items []*PCIeRootPort ++} ++ ++// Init Initialized the PCIeRootPortPool instance ++func (rp *PCIeRootPortPool) Init(number int) { ++ if number == 0 || number > maxRootPortsCapacity { ++ number = maxRootPortsCapacity ++ } ++ ++ for i := 0; i < number; i++ { ++ dev := &PCIeRootPort{ ++ DeviceID: "", ++ Port: fmt.Sprintf("0x%x", startPort+i), ++ Bus: PCIeRootBus, ++ ID: fmt.Sprintf("pci.%d", i+1), ++ Chassis: uint32(i + 1), ++ } ++ ++ major := i / funcNumPerSlot ++ minor := i % funcNumPerSlot ++ dev.Multifunction = false ++ if minor == 0 { ++ dev.Multifunction = true ++ } ++ dev.Slot = fmt.Sprintf("%02x", major+1) ++ dev.Function = fmt.Sprintf("%x", minor) ++ ++ rp.Items = append(rp.Items, dev) ++ } ++} ++ ++// AddDevice add a device to the PCIeRootPortPool ++func (rp *PCIeRootPortPool) AddDevice(devID string) (*PCIeRootPort, error) { ++ for _, it := range rp.Items { ++ if it.DeviceID == "" { ++ it.DeviceID = devID ++ return it, nil ++ } ++ } ++ return nil, fmt.Errorf("Unable to hot plug device on Root Ports: there are not empty slots") ++} ++ ++// RemoveDevice remove a device from the PCIeRootPortPool ++func (rp *PCIeRootPortPool) RemoveDevice(devID string) error { ++ for _, it := range rp.Items { ++ if it.DeviceID == devID { ++ // free address to re-use the same slot with other devices ++ it.DeviceID = "" ++ return nil ++ } ++ } ++ ++ return fmt.Errorf("Unable to hot unplug device %s: not present on Root Port", devID) ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch b/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch new file mode 100644 index 0000000000000000000000000000000000000000..701b072725feb0cac70d00aa44e3f7f8859726d0 --- /dev/null +++ b/runtime/patches/0039-storage-add-storage-common-functions-and-structs.patch @@ -0,0 +1,245 @@ +From a36c9857447aaf22628af1ef01406a916436133b Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 22:00:04 +0800 +Subject: [PATCH 39/50] storage: add storage common functions and structs + +reason: +1. add storage.go and storage_spec.go +2. provide funcs for mount nfs and gpath + +Signed-off-by: jiangpengfei +--- + virtcontainers/storage/storage.go | 115 +++++++++++++++++++++++++++++++++ + virtcontainers/storage/storage_spec.go | 98 ++++++++++++++++++++++++++++ + 2 files changed, 213 insertions(+) + create mode 100644 virtcontainers/storage/storage.go + create mode 100644 virtcontainers/storage/storage_spec.go + +diff --git a/virtcontainers/storage/storage.go b/virtcontainers/storage/storage.go +new file mode 100644 +index 00000000..77ef994f +--- /dev/null ++++ b/virtcontainers/storage/storage.go +@@ -0,0 +1,115 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: licuifang ++// Create: 2019-07-14 ++ ++// Package storage provides functions for mounting ++package storage ++ ++import ( ++ "fmt" ++ "path/filepath" ++ "regexp" ++ "strings" ++ ++ "github.com/kata-containers/agent/protocols/grpc" ++ "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++var ( ++ // simple regular expressions for "nfs-server.com:/aaa/bbb/ccc/" or "192.168.1.1:/remote/path" ++ regUrl = regexp.MustCompile(`^((([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6})|((?:(?:25[0-5]|2[0-4]\d|[01]?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d?\d))):/[A-Za-z0-9_./-]{0,1000}$`) ++ ++ defaultMountOption = "nolock" ++ storageMountOptionKey = "mount_op" ++ storageCustomOptionKey = "custom_op" ++) ++ ++type RemoteStorage struct { ++ Source string ++ Dest string ++ Options map[string][]string ++} ++ ++func (n *RemoteStorage) Validate(volumeType string) error { ++ if n.Source == "" || n.Dest == "" { ++ return fmt.Errorf("the source and dest of storage cannot be empty") ++ } ++ switch volumeType { ++ case NFS: ++ chk := regUrl.FindAllString(n.Source, -1) ++ if chk == nil { ++ return fmt.Errorf("invalid url for nfs") ++ } ++ case GPATHFS: ++ if !filepath.IsAbs(n.Source) { ++ return fmt.Errorf("invalid gpath") ++ } ++ } ++ ++ return nil ++} ++ ++func (n *RemoteStorage) GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage { ++ var ( ++ grpcStorag *grpc.Storage ++ vmPath string ++ ) ++ switch volumeType { ++ case NFS: ++ // source of sfs like remote-nfs.com:/share-53ee51a1 ++ // source of sfs with subpath like remote-nfs.com:/share-53ee51a1/aaa/bbb ++ // source of sfs-turbo like ip:/ ++ // source of sfs-turbo with subpath like ip://aaa or ip://aaa/bbb ++ // here we only get the origin source such as remote-nfs.com:/share-53ee51a1 or ip:/ as the Source of grpcStorage ++ item := strings.Split(n.Source, ":") ++ if len(item) != 2 { ++ return nil ++ } ++ vmPath = filepath.Join(vmBasePath, sandboxId, item[0], item[1]) ++ parts := strings.SplitAfter(item[1], "/") ++ if len(parts) > 2 { ++ n.Source = item[0] + ":" + parts[0] + parts[1] ++ } ++ ++ grpcStorag = &grpc.Storage{ ++ Driver: NFS, ++ Source: n.Source, ++ MountPoint: vmPath, ++ } ++ grpcStorag.Options = append(grpcStorag.Options, defaultMountOption) ++ case GPATHFS: ++ vmPath = n.Source ++ grpcStorag = &grpc.Storage{ ++ Driver: GPATHFS, ++ Source: vmPath, ++ MountPoint: n.Dest, ++ } ++ } ++ if custumOpts, ok := n.Options[storageCustomOptionKey]; ok { ++ for _, custumOpt := range custumOpts { ++ grpcStorag.Options = append(grpcStorag.Options, custumOpt) ++ } ++ } ++ ++ mnt := specs.Mount{ ++ Source: vmPath, ++ Destination: n.Dest, ++ } ++ if mountOpts, ok := n.Options[storageMountOptionKey]; ok { ++ for _, mountOpt := range mountOpts { ++ if mountOpt == "shared" { ++ // in order to support "shared" propagation for guestpath mounting to container directory ++ // we should set rootfspropagation shared ++ spec.Linux.RootfsPropagation = "shared" ++ } ++ mnt.Options = append(mnt.Options, mountOpt) ++ } ++ } ++ mnt.Options = append(mnt.Options, "rbind") ++ mnt.Type = "bind" ++ spec.Mounts = append(spec.Mounts, mnt) ++ ++ return grpcStorag ++} +diff --git a/virtcontainers/storage/storage_spec.go b/virtcontainers/storage/storage_spec.go +new file mode 100644 +index 00000000..8e866b8d +--- /dev/null ++++ b/virtcontainers/storage/storage_spec.go +@@ -0,0 +1,98 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// Description: common functions ++// Author: caihaomin c00416947 ++// Create: 2019-05-05 ++ ++package storage ++ ++import ( ++ "encoding/json" ++ "fmt" ++ ++ "github.com/kata-containers/agent/protocols/grpc" ++ "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++const ( ++ NFS = "nfs" ++ GPATHFS = "gpath" ++) ++ ++// STORAGES specifies the storage type and is supported ++var STORAGES = map[string]bool{ ++ NFS: true, ++ GPATHFS: true, ++} ++ ++type StorageSpec struct { ++ // StorageType specifies the type of storage passing down ++ StorageType string `json:"storage_type,omitempty"` ++ ++ Source string `json:"source,omitempty"` ++ Destination string `json:"dest,omitempty"` ++ ++ // Options specifies the options of storage type ++ Options map[string][]string `json:"options,omitempty"` ++} ++ ++func (s *StorageSpec) generateInstances() StorageOperation { ++ var storageOpt StorageOperation ++ storageOpt = &RemoteStorage{ ++ Source: s.Source, ++ Dest: s.Destination, ++ Options: s.Options, ++ } ++ ++ return storageOpt ++} ++ ++func ValidateStorageValue(value string) error { ++ var storageSpecs []StorageSpec ++ if err := json.Unmarshal([]byte(value), &storageSpecs); err != nil { ++ return err ++ } ++ ++ var storageOpt []StorageOperation ++ for _, item := range storageSpecs { ++ if supported, ok := STORAGES[item.StorageType]; !ok || !supported { ++ return fmt.Errorf("type %s of storage is not supported", item.StorageType) ++ } ++ storageItem := item.generateInstances() ++ if err := storageItem.Validate(item.StorageType); err != nil { ++ return err ++ } ++ storageOpt = append(storageOpt, storageItem) ++ } ++ ++ return nil ++} ++ ++func GetGrpcStorageAndAppendMount(vmBasePath, value string, spec *specs.Spec, sandboxId string) []*grpc.Storage { ++ var ( ++ grpcStorages []*grpc.Storage ++ storageSpec []StorageSpec ++ ) ++ ++ // the value has been validated before, and there is no need to judge the return value here ++ json.Unmarshal([]byte(value), &storageSpec) ++ for _, item := range storageSpec { ++ storageItem := item.generateInstances() ++ grpcStorages = append(grpcStorages, storageItem.GetGrpcStorageAndAppendMount(item.StorageType, vmBasePath, spec, sandboxId)) ++ } ++ return grpcStorages ++} ++ ++type StorageOperation interface { ++ Validate(volumeType string) error ++ GetGrpcStorageAndAppendMount(volumeType string, vmBasePath string, spec *specs.Spec, sandboxId string) *grpc.Storage ++} ++ ++type FakeStorage struct{} ++ ++func (f *FakeStorage) Validate() error { ++ return nil ++} ++ ++func (f *FakeStorage) GetGrpcStorageAndAppendMount(vmBasePath string, spec *specs.Spec) *grpc.Storage { ++ return &grpc.Storage{} ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0040-storage-add-go-tests-for-storage.patch b/runtime/patches/0040-storage-add-go-tests-for-storage.patch new file mode 100644 index 0000000000000000000000000000000000000000..e08f71da0810fe685b75df5db342b24f7a792da0 --- /dev/null +++ b/runtime/patches/0040-storage-add-go-tests-for-storage.patch @@ -0,0 +1,221 @@ +From 2e32e2c156a134605de42c53ef77366ac73f2614 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 22:01:58 +0800 +Subject: [PATCH 40/50] storage: add go tests for storage + +reason: add go tests file storage_test.go and storage_spec_test.go + +Signed-off-by: jiangpengfei +--- + virtcontainers/storage/storage_spec_test.go | 151 ++++++++++++++++++++++++++++ + virtcontainers/storage/storage_test.go | 40 ++++++++ + 2 files changed, 191 insertions(+) + create mode 100644 virtcontainers/storage/storage_spec_test.go + create mode 100644 virtcontainers/storage/storage_test.go + +diff --git a/virtcontainers/storage/storage_spec_test.go b/virtcontainers/storage/storage_spec_test.go +new file mode 100644 +index 00000000..f638245a +--- /dev/null ++++ b/virtcontainers/storage/storage_spec_test.go +@@ -0,0 +1,151 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++Description: common functions test ++Author: yangfeiyu ++Create: 2019-07-24 ++*/ ++ ++package storage ++ ++import ( ++ "encoding/json" ++ "testing" ++ ++ "github.com/stretchr/testify/assert" ++ "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++func TestValidateStorageValue(t *testing.T) { ++ assert := assert.New(t) ++ //invalid storage type,not NFS or GPATHS ++ spec := &StorageSpec{ ++ StorageType: "invalidStorage", ++ Source: "sfs.com:/remote/path", ++ Destination: "/opt/", ++ } ++ data, err := json.Marshal(spec) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.Error(err, "StorageType is not NFS or GPATHS,it should return error") ++ ++ //case 1: NFS ++ specNFS := []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "sfs.com:/remote/path", ++ Destination: "/opt/"}, ++ } ++ data, err = json.Marshal(specNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.NoError(err) ++ ++ // case2: invalid source domain address ++ specNFS = []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "sfs..com:/remote/path", ++ Destination: "/opt/"}, ++ } ++ data, err = json.Marshal(specNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.Error(err) ++ ++ // case 3: nfs source is valid ip address ++ ipSpecNFS := []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "192.168.18.147:/remote/path", ++ Destination: "/tmp/sfsturbo0", ++ }, ++ } ++ ++ data, err = json.Marshal(ipSpecNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.NoError(err) ++ ++ // case 4: invalid ip address for nfs source ++ ipSpecNFS = []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "192.168.18.300:/remote/path", ++ Destination: "/tmp/sfsturbo0", ++ }, ++ } ++ ++ data, err = json.Marshal(ipSpecNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.Error(err) ++ ++ // case 5: validate ip address and source remote path is / ++ ipSpecNFS = []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "192.168.18.3:/", ++ Destination: "/tmp/sfsturbo0", ++ }, ++ } ++ ++ data, err = json.Marshal(ipSpecNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.NoError(err) ++ ++ // case 6: invalid ip address 192.168.18.147.11 ++ ipSpecNFS = []StorageSpec{ ++ { ++ StorageType: NFS, ++ Source: "192.168.18.147.11:/", ++ Destination: "/tmp/sfsturbo0", ++ }, ++ } ++ ++ data, err = json.Marshal(ipSpecNFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ assert.Error(err) ++ ++ //GPATHS ++ specGPATHFS := []StorageSpec{ ++ { ++ StorageType: GPATHFS, ++ Source: "/remote/path", ++ Destination: "/opt/"}, ++ } ++ data, err = json.Marshal(specGPATHFS) ++ assert.NoError(err) ++ ++ err = ValidateStorageValue(string(data)) ++ if STORAGES[GPATHFS] == true { ++ assert.NoError(err, "StorageType GPATHFS is valid,it should return no error") ++ } else { ++ assert.Error(err, "StorageType GPATHFS is invalid,it should return error") ++ } ++} ++ ++func TestGetGrpcStorageAndAppendMount(t *testing.T) { ++ assert := assert.New(t) ++ var spec specs.Spec ++ sandboxId := "7f3b5e32cc7fe757e1b69f7a005ca1971def5011e730c82535783630fe24b318" ++ vmBasePath := "/tmp" ++ ++ // storageType nfs with legal source and dest will succeed ++ storageSpec := "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com:/remote/path\",\"dest\":\"/opt/nfs\"}]" ++ grpcStorages := GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId) ++ assert.NotNil(grpcStorages[0]) ++ ++ // storageType nfs with illegal source and dest will fail ++ storageSpec = "[{\"storage_type\":\"nfs\",\"source\":\"sfs.com\",\"dest\":\"/opt/nfs\"}]" ++ grpcStorages = GetGrpcStorageAndAppendMount(vmBasePath,storageSpec,&spec,sandboxId) ++ assert.Nil(grpcStorages[0]) ++} +diff --git a/virtcontainers/storage/storage_test.go b/virtcontainers/storage/storage_test.go +new file mode 100644 +index 00000000..c9ca4926 +--- /dev/null ++++ b/virtcontainers/storage/storage_test.go +@@ -0,0 +1,40 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++Description: common functions test ++Author: yangfeiyu ++Create: 2019-07-24 ++*/ ++ ++package storage ++ ++import ( ++ "testing" ++ ++ "github.com/stretchr/testify/assert" ++) ++ ++func TestValidate(t *testing.T) { ++ //NFS valid ++ assert := assert.New(t) ++ remoteStorage := &RemoteStorage{ ++ Source: "sfs.com:/remote/path", ++ Dest: "/opt/", ++ } ++ err := remoteStorage.Validate(NFS) ++ assert.NoError(err, "NFS url is valid,it should return no error") ++ ++ //NFS invalid ++ remoteStorage.Source = "sfs.com/../remote/path" ++ err = remoteStorage.Validate(NFS) ++ assert.Error(err, "NFS url is invalid,it should return error") ++ ++ //GPATHS valid ++ remoteStorage.Source = "/path/in/vm" ++ err = remoteStorage.Validate(GPATHFS) ++ assert.NoError(err, "GPATHFS is valid,it should return no error") ++ ++ //GPATHS invalid ++ remoteStorage.Source = "./../path/in/../vm" ++ err = remoteStorage.Validate(GPATHFS) ++ assert.Error(err, "GPATHFS is invalid,it should return error") ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch b/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch new file mode 100644 index 0000000000000000000000000000000000000000..b63c2d761417a5672b3dc0effa5ad348c0ae954d --- /dev/null +++ b/runtime/patches/0041-storage-mount-nfs-and-gpath-with-given-annotation.patch @@ -0,0 +1,115 @@ +From 31e07f1b6cbf361783c4d7adf9e4b8da30c67384 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 22:05:25 +0800 +Subject: [PATCH 41/50] storage: mount nfs and gpath with given annotation + +reason: when run container with annotation about storage spec, +prepare basic info in kata-runtime + +Signed-off-by: jiangpengfei +--- + virtcontainers/kata_agent.go | 14 +++++++++++++- + virtcontainers/pkg/annotations/annotations.go | 3 +++ + virtcontainers/pkg/oci/utils.go | 16 ++++++++++++++++ + 3 files changed, 32 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index d82a7f2d..ac64817a 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -30,10 +30,11 @@ import ( + ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter" + "github.com/kata-containers/runtime/virtcontainers/pkg/rootless" + vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" +- "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" ++ "github.com/kata-containers/runtime/virtcontainers/storage" + "github.com/kata-containers/runtime/virtcontainers/store" + "github.com/kata-containers/runtime/virtcontainers/types" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + "github.com/opencontainers/runtime-spec/specs-go" + opentracing "github.com/opentracing/opentracing-go" + "github.com/sirupsen/logrus" +@@ -1427,6 +1428,9 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, + localStorages := k.handleLocalStorage(ociSpec.Mounts, sandbox.id, c.rootfsSuffix) + ctrStorages = append(ctrStorages, localStorages...) + ++ remoteStoragtes := k.handleRemoteStorage(ociSpec, sandbox.id) ++ ctrStorages = append(ctrStorages, remoteStoragtes...) ++ + // We replace all OCI mount sources that match our container mount + // with the right source path (The guest one). + if err = k.replaceOCIMountSource(ociSpec, newMounts); err != nil { +@@ -1510,6 +1514,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, + k.state.URL, consoleURL, c.config.Cmd, createNSList, enterNSList) + } + ++func (k *kataAgent) handleRemoteStorage(spec *specs.Spec, sandboxId string) []*grpc.Storage { ++ if value, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok { ++ return storage.GetGrpcStorageAndAppendMount(kataGuestStorageDir, value, spec, sandboxId) ++ } ++ ++ return []*grpc.Storage{} ++} ++ + // handleEphemeralStorage handles ephemeral storages by + // creating a Storage from corresponding source of the mount point + func (k *kataAgent) handleEphemeralStorage(mounts []specs.Mount) []*grpc.Storage { +diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go +index 903c7f03..e50a697c 100644 +--- a/virtcontainers/pkg/annotations/annotations.go ++++ b/virtcontainers/pkg/annotations/annotations.go +@@ -68,6 +68,9 @@ const ( + // AssetHashType is the hash type used for assets verification + AssetHashType = kataAnnotationsPrefix + "asset_hash_type" + ++ // StorageSpecTypeKey is the annotation key to fetch storage_spec ++ StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec" ++ + // + // Generic annotations + // +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 948bd3cb..d032227e 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -22,6 +22,7 @@ import ( + exp "github.com/kata-containers/runtime/virtcontainers/experimental" + vcAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" + dockershimAnnotations "github.com/kata-containers/runtime/virtcontainers/pkg/annotations/dockershim" ++ "github.com/kata-containers/runtime/virtcontainers/storage" + "github.com/kata-containers/runtime/virtcontainers/types" + "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" +@@ -340,6 +341,17 @@ func SandboxID(spec specs.Spec) (string, error) { + return "", fmt.Errorf("Could not find sandbox ID") + } + ++func validateStorageSpec(spec specs.Spec) error { ++ if storageSpec, ok := spec.Annotations[vcAnnotations.StorageSpecTypeKey]; ok { ++ err := storage.ValidateStorageValue(storageSpec) ++ if err != nil { ++ return err ++ } ++ } ++ ++ return nil ++} ++ + func addAnnotations(ocispec specs.Spec, config *vc.SandboxConfig) error { + addAssetAnnotations(ocispec, config) + if err := addHypervisorConfigOverrides(ocispec, config); err != nil { +@@ -873,6 +885,10 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c + // ContainerConfig converts an OCI compatible runtime configuration + // file to a virtcontainers container configuration structure. + func ContainerConfig(ocispec specs.Spec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) { ++ err := validateStorageSpec(ocispec) ++ if err != nil { ++ return vc.ContainerConfig{}, err ++ } + rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true} + if !filepath.IsAbs(rootfs.Target) { + rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path) +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch b/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch new file mode 100644 index 0000000000000000000000000000000000000000..e5bc286027646fd4ddff5d31c99d128c2e2d02bf --- /dev/null +++ b/runtime/patches/0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch @@ -0,0 +1,38 @@ +From 37372be961528471d418bbe066d88a010ad0513c Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 22:12:59 +0800 +Subject: [PATCH 42/50] kata-runtime: do not ignore updateInterface return + error + +reason: If send UpdateInterfaceRequest to kata-agent and return the +error back, we should process it correctly, should not ignore the +error, which may casue deference nil pointer problem. + +Signed-off-by: jiangpengfei +--- + virtcontainers/kata_agent.go | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index ac64817a..fee4215f 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -604,7 +604,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, + "interface-requested": fmt.Sprintf("%+v", ifc), + "resulting-interface": fmt.Sprintf("%+v", resultingInterface), + }).WithError(err).Error("update interface request failed") ++ return nil, err ++ } ++ ++ // need to judege resultingInterface is not not, otherwise may cause ++ // deference nil pointer panic problem ++ if resultingInterface == nil { ++ return nil, fmt.Errorf("resultingInterface should not be nil") + } ++ + if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok { + iface := &vcTypes.Interface{ + Device: resultInterface.Device, +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch b/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch new file mode 100644 index 0000000000000000000000000000000000000000..6543ee92f26a548dc8d00a12cfc870395a17dbcd --- /dev/null +++ b/runtime/patches/0043-kata-runtime-support-add-hypervisor-global-parameter.patch @@ -0,0 +1,162 @@ +From d9738cfd6500ced30efde9e747eb73e5076e73ed Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Tue, 18 Aug 2020 22:47:13 +0800 +Subject: [PATCH 43/50] kata-runtime: support add hypervisor global parameters + in config file + +reason: support add hypervisor global parameters in config file +with defaultHypervisorParams config option. + +Signed-off-by: jiangpengfei +--- + Makefile | 2 ++ + cli/config/configuration-qemu.toml.in | 7 +++++++ + pkg/katautils/config-settings.go.in | 1 + + pkg/katautils/config.go | 13 ++++++++++++- + vendor/github.com/intel/govmm/qemu/qemu.go | 6 +++--- + virtcontainers/qemu.go | 2 +- + 6 files changed, 26 insertions(+), 5 deletions(-) + +diff --git a/Makefile b/Makefile +index d5e4bbe1..b62e64b0 100644 +--- a/Makefile ++++ b/Makefile +@@ -400,6 +400,7 @@ USER_VARS += FIRMWAREPATH + USER_VARS += MACHINEACCELERATORS + USER_VARS += DEFMACHINETYPE_CLH + USER_VARS += KERNELPARAMS ++USER_VARS += HYPERVISORPARAMS + USER_VARS += LIBEXECDIR + USER_VARS += LOCALSTATEDIR + USER_VARS += PKGDATADIR +@@ -607,6 +608,7 @@ $(GENERATED_FILES): %: %.in $(MAKEFILE_LIST) VERSION .git-commit + -e "s|@FIRMWAREPATH_CLH@|$(FIRMWAREPATH_CLH)|g" \ + -e "s|@DEFMACHINETYPE_CLH@|$(DEFMACHINETYPE_CLH)|g" \ + -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ ++ -e "s|@HYPERVISORPARAMS@|$(HYPERVISORPARAMS)|g" \ + -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ + -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ + -e "s|@PKGRUNDIR@|$(PKGRUNDIR)|g" \ +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index aa11b38f..e57a954c 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -29,6 +29,13 @@ machine_type = "@MACHINETYPE@" + # container and look for 'default-kernel-parameters' log entries. + kernel_params = "@KERNELPARAMS@" + ++# Optional space-separated list of options to pass to the hypervisor. ++# For example, use `hypervisor_params = "kvm-pit.lost_tick_policy=discard"` ++# ++# WARNING: - any parameter specified here will take priority over the default ++# parameter value of the same name used to start the hypervisor. ++hypervisor_params = "@HYPERVISORPARAMS@" ++ + # Path to the firmware. + # If you want that qemu uses the default firmware leave this option empty + firmware = "@FIRMWAREPATH@" +diff --git a/pkg/katautils/config-settings.go.in b/pkg/katautils/config-settings.go.in +index aaf78cc3..b2dfdfa1 100644 +--- a/pkg/katautils/config-settings.go.in ++++ b/pkg/katautils/config-settings.go.in +@@ -20,6 +20,7 @@ var defaultShimPath = "/usr/libexec/kata-containers/kata-shim" + var systemdUnitName = "kata-containers.target" + + const defaultKernelParams = "" ++const defaultHypervisorParams = "kvm-pit.lost_tick_policy=discard" + const defaultMachineType = "pc" + + const defaultVCPUCount uint32 = 1 +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 51120311..3365b3f5 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -130,6 +130,7 @@ type hypervisor struct { + HotplugVFIOOnRootBus bool `toml:"hotplug_vfio_on_root_bus"` + DisableVhostNet bool `toml:"disable_vhost_net"` + GuestHookPath string `toml:"guest_hook_path"` ++ HypervisorParams string `toml:"hypervisor_params"` + } + + type proxy struct { +@@ -270,6 +271,14 @@ func (h hypervisor) kernelParams() string { + return h.KernelParams + } + ++func (h hypervisor) hypervisorParams() string { ++ if h.HypervisorParams == "" { ++ return defaultHypervisorParams ++ } ++ ++ return h.HypervisorParams ++} ++ + func (h hypervisor) machineType() string { + if h.MachineType == "" { + return defaultMachineType +@@ -632,6 +641,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + + machineAccelerators := h.machineAccelerators() + kernelParams := h.kernelParams() ++ hypervisorParams := h.hypervisorParams() + machineType := h.machineType() + + blockDriver, err := h.blockDeviceDriver() +@@ -675,6 +685,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { + FirmwarePath: firmware, + MachineAccelerators: machineAccelerators, + KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), ++ HypervisorParams: vc.DeserializeParams(strings.Fields(hypervisorParams)), + HypervisorMachineType: machineType, + NumVCPUs: h.defaultVCPUs(), + DefaultMaxVCPUs: h.defaultMaxVCPUs(), +@@ -983,7 +994,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc + TraceMode: agent.traceMode(), + TraceType: agent.traceType(), + KernelModules: agent.kernelModules(), +- MountBlkInVM: agent.mountBlkDevInVM(), ++ MountBlkInVM: agent.mountBlkDevInVM(), + } + default: + return fmt.Errorf("%s agent type is not supported", k) +diff --git a/vendor/github.com/intel/govmm/qemu/qemu.go b/vendor/github.com/intel/govmm/qemu/qemu.go +index 3e7720b4..68f8d2b0 100644 +--- a/vendor/github.com/intel/govmm/qemu/qemu.go ++++ b/vendor/github.com/intel/govmm/qemu/qemu.go +@@ -2096,7 +2096,7 @@ type Config struct { + SMP SMP + + // GlobalParam is the -global parameter. +- GlobalParam string ++ GlobalParam []string + + // Knobs is a set of qemu boolean settings. + Knobs Knobs +@@ -2285,9 +2285,9 @@ func (config *Config) appendRTC() { + } + + func (config *Config) appendGlobalParam() { +- if config.GlobalParam != "" { ++ for _, param := range config.GlobalParam { + config.qemuParams = append(config.qemuParams, "-global") +- config.qemuParams = append(config.qemuParams, config.GlobalParam) ++ config.qemuParams = append(config.qemuParams, param) + } + } + +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index a10c66fb..657b6be7 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -600,7 +600,7 @@ func (q *qemu) createSandbox(ctx context.Context, id string, networkNS NetworkNa + Knobs: knobs, + Incoming: incoming, + VGA: "none", +- GlobalParam: "kvm-pit.lost_tick_policy=discard", ++ GlobalParam: SerializeParams(hypervisorConfig.HypervisorParams, "="), + Bios: firmwarePath, + PidFile: filepath.Join(q.store.RunVMStoragePath(), q.id, "pid"), + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch b/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch new file mode 100644 index 0000000000000000000000000000000000000000..cf800a7499a20bf105bce72ee3b8fb69ec2c7ba9 --- /dev/null +++ b/runtime/patches/0044-network-support-dpdk-vhost_user-net-device.patch @@ -0,0 +1,375 @@ +From dbbf8c5deb14d8033b2e863fb0eb731523af2a47 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 09:58:07 +0800 +Subject: [PATCH 44/50] network: support dpdk vhost_user net device + +reason: support dpdk vhost_user net device + +Signed-off-by: jiangpengfei +--- + vendor/github.com/intel/govmm/qemu/qmp.go | 3 +- + virtcontainers/hypervisor.go | 3 ++ + virtcontainers/network.go | 49 ++++++++++++++------ + virtcontainers/pkg/types/types.go | 2 + + virtcontainers/qemu.go | 75 +++++++++++++++++++++++++++++++ + virtcontainers/sandbox.go | 7 ++- + virtcontainers/vhostuser_endpoint.go | 18 ++++++-- + virtcontainers/vhostuser_endpoint_test.go | 7 ++- + 8 files changed, 143 insertions(+), 21 deletions(-) + +diff --git a/vendor/github.com/intel/govmm/qemu/qmp.go b/vendor/github.com/intel/govmm/qemu/qmp.go +index a64039de..0cb82ffa 100644 +--- a/vendor/github.com/intel/govmm/qemu/qmp.go ++++ b/vendor/github.com/intel/govmm/qemu/qmp.go +@@ -970,11 +970,12 @@ func (q *QMP) ExecuteNetdevAdd(ctx context.Context, netdevType, netdevID, ifname + // ExecuteNetdevChardevAdd adds a Net device to a QEMU instance + // using the netdev_add command. netdevID is the id of the device to add. + // Must be valid QMP identifier. +-func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, queues int) error { ++func (q *QMP) ExecuteNetdevChardevAdd(ctx context.Context, netdevType, netdevID, chardev string, vhostforce bool, queues int) error { + args := map[string]interface{}{ + "type": netdevType, + "id": netdevID, + "chardev": chardev, ++ "vhostforce": vhostforce, + } + if queues > 1 { + args["queues"] = queues +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index c0723daa..60f1d190 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -132,6 +132,9 @@ const ( + // vhostuserDev is a Vhost-user device type + vhostuserDev + ++ // vhostUserNetDev is a Vhost-user net device type ++ vhostUserNetDev ++ + // CPUDevice is CPU device type + cpuDev + +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index 488bd00c..a0c64356 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -13,6 +13,7 @@ import ( + "math/rand" + "net" + "os" ++ "path/filepath" + "regexp" + "runtime" + "sort" +@@ -65,7 +66,8 @@ const ( + ) + + var ( +- regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) ++ regInfName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_\-.]*$`) ++ regVhostName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9\-.]{0,127}$`) + ) + + //IsValid checks if a model is valid +@@ -132,11 +134,12 @@ type NetlinkIface struct { + // NetworkInfo gathers all information related to a network interface. + // It can be used to store the description of the underlying network. + type NetworkInfo struct { +- Device string +- Iface NetlinkIface +- Addrs []netlink.Addr +- Routes []netlink.Route +- DNS DNSInfo ++ Device string ++ Iface NetlinkIface ++ Addrs []netlink.Addr ++ Routes []netlink.Route ++ DNS DNSInfo ++ VhostUserSocket string + } + + // NetworkInterface defines a network interface. +@@ -1198,15 +1201,11 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + return endpoint, err + } + +- // Check if this is a dummy interface which has a vhost-user socket associated with it +- socketPath, err := vhostUserSocketPath(netInfo) +- if err != nil { +- return nil, err +- } +- +- if socketPath != "" { ++ // Currently, we only accept the vhost-user socket paas from user input ++ // interface info json file ++ if netInfo.VhostUserSocket != "" { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") +- endpoint, err = createVhostUserEndpoint(netInfo, socketPath) ++ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket) + return endpoint, err + } + +@@ -1481,6 +1480,24 @@ func verifyIP(ip string) (*net.IP, error) { + return &netIP, nil + } + ++// verifyVhostSocket verifies the vhost socket is valid or not ++func verifyVhostSocket(vhostSocket string) error { ++ if vhostSocket == "" { ++ networkLogger().Debug("no dpdk network") ++ return nil ++ } ++ if regVhostName.FindAllString(filepath.Base(vhostSocket), -1) == nil { ++ return fmt.Errorf("invalid input of vhostSocket name, please check the rules") ++ } ++ info, err := os.Stat(vhostSocket) ++ if err != nil || info.IsDir() || info.Mode()&os.ModeSocket == 0 { ++ return fmt.Errorf("invalid vhost socket: %v", vhostSocket) ++ } ++ ++ networkLogger().WithField("vhostSocket", vhostSocket).Infof("using dpdk network, socket file is: %s ", vhostSocket) ++ return nil ++} ++ + // validInterface check the input interface valid or not + func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { + if enableCompatOldCNI && verifyInterfaceName(inf.Device) != nil { +@@ -1503,6 +1520,10 @@ func validInterface(inf *vcTypes.Interface, enableCompatOldCNI bool) error { + return err + } + ++ if err := verifyVhostSocket(inf.VhostUserSocket); err != nil { ++ return err ++ } ++ + // Currently, only one IP address can be passed, which reduces the test entry and fault injection. + if len(inf.IPAddresses) > 0 { + if len(inf.IPAddresses) != 1 { +diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go +index b41b0c75..dccd92f8 100644 +--- a/virtcontainers/pkg/types/types.go ++++ b/virtcontainers/pkg/types/types.go +@@ -29,6 +29,8 @@ type Interface struct { + // library, regarding each type of link. Here is a non exhaustive + // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... + LinkType string `json:"linkType,omitempty"` ++ // VhostUserSocket is DPDK-backed vHost user ports. ++ VhostUserSocket string `json:"vhostUserSocket,omitempty"` + } + + // Route describes a network route. +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index 657b6be7..84797e0d 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -1279,6 +1279,78 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error + return err + } + ++func (q *qemu) hotplugVhostUserNetDevice(endpoint Endpoint, op operation) error { ++ err := q.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ device := endpoint.(*VhostUserEndpoint) ++ ++ charDevID := utils.MakeNameID("char", device.ID, maxDevIDSize) ++ devID := utils.MakeNameID("virtio", device.ID, maxDevIDSize) ++ ++ if op == addDevice { ++ // Add chardev specifying the appropriate socket. ++ if err = q.qmpMonitorCh.qmp.ExecuteCharDevUnixSocketAdd(q.qmpMonitorCh.ctx, charDevID, device.SocketPath, true, false); err != nil { ++ return err ++ } ++ ++ defer func() { ++ if err != nil { ++ errRb := q.qmpMonitorCh.qmp.ExecuteChardevDel(q.qmpMonitorCh.ctx, charDevID) ++ if errRb != nil { ++ q.Logger().Errorf("deletes a net device roll back failed. charDev id:%s", charDevID) ++ } ++ } ++ }() ++ ++ netDevType := "vhost-user" ++ // Add netdev of type vhost-user. ++ if err = q.qmpMonitorCh.qmp.ExecuteNetdevChardevAdd(q.qmpMonitorCh.ctx, netDevType, devID, charDevID, true, 0); err != nil { ++ return err ++ } ++ ++ defer func() { ++ if err != nil { ++ errRb := q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID) ++ if errRb != nil { ++ q.Logger().Errorf("deletes a net device roll back failed. netDev id:%s", devID) ++ } ++ } ++ }() ++ ++ var addr, bus, pciAddr string ++ addr, bus, pciAddr, err = q.getPciAddress(device.ID, types.PCI) ++ if err != nil { ++ return nil ++ } ++ defer func() { ++ if err != nil { ++ q.putPciAddress(device.ID) ++ } ++ }() ++ ++ endpoint.SetPciAddr(pciAddr) ++ ++ // Add virtio-net-pci device. ++ err = q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, devID, devID, device.HardAddr, addr, bus, romFile, 0, q.arch.runNested()) ++ return err ++ } ++ ++ if err = q.putPciAddress(device.ID); err != nil { ++ return err ++ } ++ if err = q.qmpMonitorCh.qmp.ExecuteDeviceDel(q.qmpMonitorCh.ctx, devID); err != nil { ++ return err ++ } ++ if err = q.qmpMonitorCh.qmp.ExecuteNetdevDel(q.qmpMonitorCh.ctx, devID); err != nil { ++ return err ++ } ++ ++ return nil ++} ++ + func (q *qemu) hotplugVhostUserDevice(vAttr *config.VhostUserDeviceAttrs, op operation) error { + err := q.qmpSetup() + if err != nil { +@@ -1510,6 +1582,9 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati + case vhostuserDev: + vAttr := devInfo.(*config.VhostUserDeviceAttrs) + return nil, q.hotplugVhostUserDevice(vAttr, op) ++ case vhostUserNetDev: ++ device := devInfo.(Endpoint) ++ return nil, q.hotplugVhostUserNetDevice(device, op) + default: + return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) + } +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 0d599267..e6f155a3 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -946,7 +946,8 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { + }, + Type: inf.LinkType, + }, +- Addrs: addrs, ++ Addrs: addrs, ++ VhostUserSocket: inf.VhostUserSocket, + }, nil + } + +@@ -961,6 +962,10 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interfac + if ep.Name() == inf.Name { + return nil, fmt.Errorf("interface %s is already exist", inf.Name) + } ++ ++ if ep.Properties().VhostUserSocket != "" && inf.VhostUserSocket != "" { ++ return nil, fmt.Errorf("sandbox %s only support one dpdk socket", s.ID()) ++ } + } + + netInfo, err := s.generateNetInfo(inf) +diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go +index bb4a67be..2fc3d837 100644 +--- a/virtcontainers/vhostuser_endpoint.go ++++ b/virtcontainers/vhostuser_endpoint.go +@@ -12,6 +12,7 @@ import ( + + "github.com/kata-containers/runtime/virtcontainers/device/config" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ++ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/kata-containers/runtime/virtcontainers/utils" + ) + +@@ -23,6 +24,7 @@ const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" + + // VhostUserEndpoint represents a vhost-user socket based network interface + type VhostUserEndpoint struct { ++ ID string + // Path to the vhost-user socket on the host system + SocketPath string + // MAC address of the interface +@@ -99,18 +101,28 @@ func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) e + + // HotAttach for vhostuser endpoint not supported yet + func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { +- return fmt.Errorf("VhostUserEndpoint does not support Hot attach") ++ networkLogger().Debug("Hot attaching vhost-user endpoint") ++ if _, err := h.hotplugAddDevice(endpoint, vhostUserNetDev); err != nil { ++ return err ++ } ++ return nil + } + + // HotDetach for vhostuser endpoint not supported yet + func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { +- return fmt.Errorf("VhostUserEndpoint does not support Hot detach") ++ networkLogger().Debug("Hot detaching vhost-user endpoint") ++ if _, err := h.hotplugRemoveDevice(endpoint, vhostUserNetDev); err != nil { ++ networkLogger().WithError(err).Errorf("Error detach vhostUserSocket") ++ return err ++ } ++ return nil + } + + // Create a vhostuser endpoint + func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { +- ++ uniqueID := uuid.Generate().String() + vhostUserEndpoint := &VhostUserEndpoint{ ++ ID: uniqueID, + SocketPath: socket, + HardAddr: netInfo.Iface.HardwareAddr.String(), + IfaceName: netInfo.Iface.Name, +diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go +index ad013e12..584490cc 100644 +--- a/virtcontainers/vhostuser_endpoint_test.go ++++ b/virtcontainers/vhostuser_endpoint_test.go +@@ -11,6 +11,7 @@ import ( + "os" + "testing" + ++ "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + ) +@@ -95,7 +96,7 @@ func TestVhostUserEndpoint_HotAttach(t *testing.T) { + h := &mockHypervisor{} + + err := v.HotAttach(h) +- assert.Error(err) ++ assert.NoError(err) + } + + func TestVhostUserEndpoint_HotDetach(t *testing.T) { +@@ -109,10 +110,11 @@ func TestVhostUserEndpoint_HotDetach(t *testing.T) { + h := &mockHypervisor{} + + err := v.HotDetach(h, true, "") +- assert.Error(err) ++ assert.NoError(err) + } + + func TestCreateVhostUserEndpoint(t *testing.T) { ++ uniqueID := uuid.Generate().String() + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48} + ifcName := "vhost-deadbeef" + socket := "/tmp/vhu_192.168.0.1" +@@ -128,6 +130,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) { + } + + expected := &VhostUserEndpoint{ ++ ID: uniqueID, + SocketPath: socket, + HardAddr: macAddr.String(), + IfaceName: ifcName, +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0045-network-support-set-dns.patch b/runtime/patches/0045-network-support-set-dns.patch new file mode 100644 index 0000000000000000000000000000000000000000..2d039ff6791b474513069f2eadc0b877e35a0407 --- /dev/null +++ b/runtime/patches/0045-network-support-set-dns.patch @@ -0,0 +1,115 @@ +From e104f084ed4f10049f45d9473faed229371a1c6c Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 09:59:52 +0800 +Subject: [PATCH 45/50] network: support set dns + +reason: when running a sandbox with annotation about +dns spec, overload k.getDNS(sandbox) + +Signed-off-by: yangfeiyu +--- + virtcontainers/kata_agent.go | 11 +++++++++++ + virtcontainers/network.go | 8 ++++---- + virtcontainers/pkg/annotations/annotations.go | 3 +++ + virtcontainers/pkg/oci/utils.go | 19 +++++++++++++++++-- + 4 files changed, 35 insertions(+), 6 deletions(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index fee4215f..d28d8cce 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -890,6 +890,17 @@ func (k *kataAgent) startSandbox(sandbox *Sandbox) error { + KernelModules: kmodules, + } + ++ if value, ok := sandbox.config.Annotations[vcAnnotations.SandboxDNSTypeKey]; ok { ++ var sandboxDns *DNSInfo ++ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil { ++ return fmt.Errorf("get sandbox dns failed %v", err) ++ } ++ ++ if sandboxDns != nil { ++ req.Dns = sandboxDns.Servers ++ } ++ } ++ + _, err = k.sendReq(req) + if err != nil { + return err +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index 488bd00c..68eda4a6 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -117,10 +117,10 @@ const ( + + // DNSInfo describes the DNS setup related to a network interface. + type DNSInfo struct { +- Servers []string +- Domain string +- Searches []string +- Options []string ++ Servers []string `json:"Servers"` ++ Domain string `json:"Domain,omitempty"` ++ Searches []string `json:"Searches,omitempty"` ++ Options []string `json:"Options,omitempty"` + } + + // NetlinkIface describes fully a network interface. +diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go +index e50a697c..528dfa66 100644 +--- a/virtcontainers/pkg/annotations/annotations.go ++++ b/virtcontainers/pkg/annotations/annotations.go +@@ -71,6 +71,9 @@ const ( + // StorageSpecTypeKey is the annotation key to fetch storage_spec + StorageSpecTypeKey = kataAnnotationsPrefix + "storage_spec" + ++ // SandboxDNSTypeKey is the annotation key to fetch sandbox dns options ++ SandboxDNSTypeKey = kataAnnotationsPrefix + "sandbox_dns" ++ + // + // Generic annotations + // +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index d032227e..3b2af753 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -7,6 +7,7 @@ package oci + + import ( + "context" ++ "encoding/json" + "errors" + "fmt" + "path/filepath" +@@ -37,8 +38,9 @@ type annotationContainerType struct { + type annotationHandler func(value string) error + + var annotationHandlerList = map[string]annotationHandler{ +- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, +- vcAnnotations.StaticMemTypeKey: validateSandboxMem, ++ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, ++ vcAnnotations.StaticMemTypeKey: validateSandboxMem, ++ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, + } + + var ( +@@ -1108,3 +1110,16 @@ func validateSandboxMem(value string) error { + + return nil + } ++ ++func validateSandboxDNS(value string) error { ++ if value == "" { ++ return fmt.Errorf("--annotation %s value should not be empty", vcAnnotations.SandboxDNSTypeKey) ++ } ++ ++ var sandboxDns *vc.DNSInfo ++ if err := json.Unmarshal([]byte(value), &sandboxDns); err != nil { ++ return fmt.Errorf("invalid value passed by --annotation %s, error: %v", vcAnnotations.SandboxDNSTypeKey, err) ++ } ++ ++ return nil ++} +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch b/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch new file mode 100644 index 0000000000000000000000000000000000000000..ac07aec7679d05aeb455403e32a29178b0dfe0a5 --- /dev/null +++ b/runtime/patches/0046-network-support-multiqueue-when-inserting-interface-.patch @@ -0,0 +1,489 @@ +From e9c1fb235d0e69d5db72497a8779df707c499d3b Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 20:36:40 +0800 +Subject: [PATCH 46/50] network: support multiqueue when inserting interface + dynamically + +reason: support multiqueue when inserting interface dynamically + +Signed-off-by: jiangpengfei +--- + .../kata-containers/agent/pkg/types/types.pb.go | 82 ++++++++++++++++------ + virtcontainers/bridgedmacvlan_endpoint.go | 2 +- + virtcontainers/ipvlan_endpoint.go | 2 +- + virtcontainers/kata_agent.go | 2 + + virtcontainers/network.go | 13 ++-- + virtcontainers/network_test.go | 2 +- + virtcontainers/pkg/types/types.go | 2 + + virtcontainers/qemu.go | 10 +-- + virtcontainers/sandbox.go | 1 + + virtcontainers/tap_endpoint.go | 6 +- + virtcontainers/tuntap_endpoint.go | 4 +- + virtcontainers/veth_endpoint.go | 4 +- + virtcontainers/veth_endpoint_test.go | 6 +- + virtcontainers/vhostuser_endpoint.go | 4 +- + virtcontainers/vhostuser_endpoint_test.go | 2 +- + 15 files changed, 96 insertions(+), 46 deletions(-) + +diff --git a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go +index 7ea63e3c..8b7e2a5d 100644 +--- a/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go ++++ b/vendor/github.com/kata-containers/agent/pkg/types/types.pb.go +@@ -100,6 +100,7 @@ type Interface struct { + // list: "veth", "macvtap", "vlan", "macvlan", "tap", ... + Type string `protobuf:"bytes,7,opt,name=type,proto3" json:"type,omitempty"` + RawFlags uint32 `protobuf:"varint,8,opt,name=raw_flags,json=rawFlags,proto3" json:"raw_flags,omitempty"` ++ Queues uint32 `protobuf:"varint,9,opt,name=Queues,proto3" json:"Queues,omitempty"` + } + + func (m *Interface) Reset() { *m = Interface{} } +@@ -163,6 +164,13 @@ func (m *Interface) GetRawFlags() uint32 { + return 0 + } + ++func (m *Interface) GetQueues() uint32 { ++ if m != nil { ++ return m.Queues ++ } ++ return 0 ++} ++ + type Route struct { + Dest string `protobuf:"bytes,1,opt,name=dest,proto3" json:"dest,omitempty"` + Gateway string `protobuf:"bytes,2,opt,name=gateway,proto3" json:"gateway,omitempty"` +@@ -319,6 +327,11 @@ func (m *Interface) MarshalTo(dAtA []byte) (int, error) { + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.RawFlags)) + } ++ if m.Queues != 0 { ++ dAtA[i] = 0x48 ++ i++ ++ i = encodeVarintTypes(dAtA, i, uint64(m.Queues)) ++ } + return i, nil + } + +@@ -430,6 +443,9 @@ func (m *Interface) Size() (n int) { + if m.RawFlags != 0 { + n += 1 + sovTypes(uint64(m.RawFlags)) + } ++ if m.Queues != 0 { ++ n += 1 + sovTypes(uint64(m.Queues)) ++ } + return n + } + +@@ -841,6 +857,25 @@ func (m *Interface) Unmarshal(dAtA []byte) error { + break + } + } ++ case 9: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) ++ } ++ m.Queues = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowTypes ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Queues |= (uint32(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) +@@ -1155,27 +1190,28 @@ var ( + func init() { proto.RegisterFile("pkg/types/types.proto", fileDescriptorTypes) } + + var fileDescriptorTypes = []byte{ +- // 345 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x91, 0xd1, 0x6a, 0xfa, 0x30, +- 0x14, 0xc6, 0xff, 0xb1, 0xb6, 0xda, 0xe3, 0xdf, 0xad, 0x84, 0x4d, 0xc2, 0x06, 0x52, 0xbc, 0x59, +- 0xd9, 0x85, 0x03, 0x37, 0x76, 0xbf, 0x5d, 0x08, 0xde, 0x49, 0x5e, 0x60, 0xc4, 0x36, 0x3a, 0xd1, +- 0xda, 0xd2, 0x44, 0x8b, 0xec, 0x05, 0x77, 0xb9, 0x47, 0x10, 0x9f, 0x64, 0xe4, 0x24, 0x8a, 0xbb, +- 0xd1, 0xef, 0x97, 0x93, 0xd3, 0xef, 0x7c, 0x27, 0x70, 0x5b, 0xae, 0x16, 0x4f, 0x7a, 0x5f, 0x4a, +- 0x65, 0x7f, 0x87, 0x65, 0x55, 0xe8, 0x82, 0xfa, 0x08, 0x83, 0x19, 0x84, 0x93, 0xe9, 0x5b, 0x96, +- 0x55, 0x52, 0x29, 0xfa, 0x00, 0xc1, 0x5c, 0xe4, 0xcb, 0xf5, 0x9e, 0x91, 0x98, 0x24, 0x57, 0xa3, +- 0xeb, 0xa1, 0xed, 0x98, 0x4c, 0xc7, 0x78, 0xcc, 0x5d, 0x99, 0x32, 0x68, 0x09, 0xdb, 0xc3, 0x1a, +- 0x31, 0x49, 0x42, 0x7e, 0x42, 0x4a, 0xa1, 0x99, 0x0b, 0xb5, 0x62, 0x1e, 0x1e, 0xa3, 0x1e, 0x1c, +- 0x08, 0x84, 0x93, 0x8d, 0x96, 0xd5, 0x5c, 0xa4, 0x92, 0xf6, 0x20, 0xc8, 0xe4, 0x6e, 0x99, 0x4a, +- 0x34, 0x09, 0xb9, 0x23, 0xd3, 0xb9, 0x11, 0xb9, 0x74, 0x1f, 0x44, 0x4d, 0x47, 0xd0, 0x39, 0x4f, +- 0x27, 0x15, 0xf3, 0x62, 0x2f, 0xe9, 0x8c, 0xa2, 0xf3, 0x54, 0xae, 0xc2, 0x2f, 0x2f, 0xd1, 0x08, +- 0xbc, 0x5c, 0x6f, 0x59, 0x33, 0x26, 0x49, 0x93, 0x1b, 0x69, 0x1c, 0x3f, 0x6b, 0x73, 0x81, 0xf9, +- 0xd6, 0xd1, 0x92, 0x49, 0x51, 0xa6, 0x4b, 0x2c, 0x04, 0x36, 0x85, 0x43, 0x33, 0x8b, 0xf1, 0x60, +- 0x2d, 0x3b, 0x8b, 0xd1, 0xf4, 0x1e, 0xc2, 0x4a, 0xd4, 0x1f, 0xf3, 0xb5, 0x58, 0x28, 0xd6, 0x8e, +- 0x49, 0xd2, 0xe5, 0xed, 0x4a, 0xd4, 0x63, 0xc3, 0x83, 0x2f, 0xf0, 0x79, 0xb1, 0xd5, 0x98, 0x22, +- 0x93, 0x4a, 0xbb, 0x6c, 0xa8, 0x8d, 0xcf, 0x42, 0x68, 0x59, 0x8b, 0xfd, 0x69, 0x5b, 0x0e, 0x2f, +- 0x76, 0xe1, 0xfd, 0xd9, 0x45, 0x0f, 0x02, 0x55, 0x6c, 0xab, 0x54, 0x62, 0x8c, 0x90, 0x3b, 0xa2, +- 0x37, 0xe0, 0xab, 0xb4, 0x28, 0x25, 0x06, 0xe9, 0x72, 0x0b, 0x8f, 0x77, 0xd0, 0x3e, 0xbd, 0x10, +- 0x0d, 0xa0, 0xb1, 0x7b, 0x89, 0xfe, 0xe1, 0xff, 0x6b, 0x44, 0xde, 0xff, 0x7f, 0x1f, 0xfb, 0xe4, +- 0xe7, 0xd8, 0x27, 0x87, 0x63, 0x9f, 0xcc, 0x02, 0x7c, 0xfb, 0xe7, 0xdf, 0x00, 0x00, 0x00, 0xff, +- 0xff, 0xb5, 0x52, 0x37, 0xf2, 0x14, 0x02, 0x00, 0x00, ++ // 356 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x92, 0xc1, 0x4e, 0xc2, 0x40, ++ 0x10, 0x86, 0x5d, 0x4a, 0x0b, 0x1d, 0x44, 0x9b, 0x8d, 0x92, 0x8d, 0x26, 0xa4, 0xe1, 0x62, 0xe3, ++ 0x01, 0x13, 0x34, 0xde, 0xf5, 0x40, 0xc2, 0x0d, 0xf7, 0x05, 0xcc, 0xd2, 0x2e, 0x48, 0xa0, 0xb4, ++ 0xe9, 0xb6, 0x34, 0xc4, 0x17, 0xf4, 0xe8, 0x23, 0x18, 0x9e, 0xc2, 0xa3, 0xd9, 0xd9, 0x85, 0xe0, ++ 0x05, 0xfe, 0x6f, 0x67, 0xa7, 0xff, 0xfc, 0xd3, 0xc2, 0x75, 0xbe, 0x5a, 0x3c, 0x94, 0xbb, 0x5c, ++ 0x2a, 0xf3, 0x3b, 0xcc, 0x8b, 0xac, 0xcc, 0xa8, 0x8b, 0x30, 0x98, 0x81, 0x3f, 0x99, 0xbe, 0x24, ++ 0x49, 0x21, 0x95, 0xa2, 0x77, 0xe0, 0xcd, 0x45, 0xba, 0x5c, 0xef, 0x18, 0x09, 0x49, 0x74, 0x31, ++ 0xba, 0x1c, 0x9a, 0x8e, 0xc9, 0x74, 0x8c, 0xc7, 0xdc, 0x96, 0x29, 0x83, 0x96, 0x30, 0x3d, 0xac, ++ 0x11, 0x92, 0xc8, 0xe7, 0x07, 0xa4, 0x14, 0x9a, 0xa9, 0x50, 0x2b, 0xe6, 0xe0, 0x31, 0xea, 0xc1, ++ 0x2f, 0x01, 0x7f, 0xb2, 0x29, 0x65, 0x31, 0x17, 0xb1, 0xa4, 0x3d, 0xf0, 0x12, 0xb9, 0x5d, 0xc6, ++ 0x12, 0x4d, 0x7c, 0x6e, 0x49, 0x77, 0x6e, 0x44, 0x2a, 0xed, 0x03, 0x51, 0xd3, 0x11, 0x74, 0x8e, ++ 0xd3, 0x49, 0xc5, 0x9c, 0xd0, 0x89, 0x3a, 0xa3, 0xe0, 0x38, 0x95, 0xad, 0xf0, 0xd3, 0x4b, 0x34, ++ 0x00, 0x27, 0x2d, 0x2b, 0xd6, 0x0c, 0x49, 0xd4, 0xe4, 0x5a, 0x6a, 0xc7, 0x8f, 0x5a, 0x5f, 0x60, ++ 0xae, 0x71, 0x34, 0xa4, 0x53, 0xe4, 0xf1, 0x12, 0x0b, 0x9e, 0x49, 0x61, 0x51, 0xcf, 0xa2, 0x3d, ++ 0x58, 0xcb, 0xcc, 0xa2, 0x35, 0xbd, 0x05, 0xbf, 0x10, 0xf5, 0xfb, 0x7c, 0x2d, 0x16, 0x8a, 0xb5, ++ 0x43, 0x12, 0x75, 0x79, 0xbb, 0x10, 0xf5, 0x58, 0xb3, 0xb6, 0x78, 0xab, 0x64, 0x25, 0x15, 0xf3, ++ 0xb1, 0x62, 0x69, 0xf0, 0x09, 0x2e, 0xcf, 0xaa, 0x12, 0xd3, 0x25, 0x52, 0x95, 0x36, 0x33, 0x6a, ++ 0xed, 0xbf, 0x10, 0xa5, 0xac, 0xc5, 0xee, 0xb0, 0x45, 0x8b, 0x27, 0x3b, 0x72, 0xfe, 0xed, 0xa8, ++ 0x07, 0x9e, 0xca, 0xaa, 0x22, 0x96, 0x18, 0xcf, 0xe7, 0x96, 0xe8, 0x15, 0xb8, 0x2a, 0xce, 0x72, ++ 0x89, 0x01, 0xbb, 0xdc, 0xc0, 0xfd, 0x0d, 0xb4, 0x0f, 0x6f, 0x8e, 0x7a, 0xd0, 0xd8, 0x3e, 0x05, ++ 0x67, 0xf8, 0xff, 0x1c, 0x90, 0xd7, 0xf3, 0xaf, 0x7d, 0x9f, 0x7c, 0xef, 0xfb, 0xe4, 0x67, 0xdf, ++ 0x27, 0x33, 0x0f, 0xbf, 0x89, 0xc7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x57, 0x70, 0xc8, ++ 0x2c, 0x02, 0x00, 0x00, + } +diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go +index 700aea34..9179bbbe 100644 +--- a/virtcontainers/bridgedmacvlan_endpoint.go ++++ b/virtcontainers/bridgedmacvlan_endpoint.go +@@ -25,7 +25,7 @@ func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingMod + return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + +- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) ++ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0) + if err != nil { + return nil, err + } +diff --git a/virtcontainers/ipvlan_endpoint.go b/virtcontainers/ipvlan_endpoint.go +index 6e924a74..a6ef8179 100644 +--- a/virtcontainers/ipvlan_endpoint.go ++++ b/virtcontainers/ipvlan_endpoint.go +@@ -28,7 +28,7 @@ func createIPVlanNetworkEndpoint(idx int, ifName string) (*IPVlanEndpoint, error + // Use tc filtering for ipvlan, since the other inter networking models will + // not work for ipvlan. + interworkingModel := NetXConnectTCFilterModel +- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) ++ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, 0) + if err != nil { + return nil, err + } +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index d28d8cce..38e9a204 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -621,6 +621,7 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, + Mtu: resultInterface.Mtu, + HwAddr: resultInterface.HwAddr, + PciAddr: resultInterface.PciAddr, ++ Queues: resultInterface.Queues, + } + return iface, err + } +@@ -2295,6 +2296,7 @@ func (k *kataAgent) convertToKataAgentInterface(iface *vcTypes.Interface) *aType + RawFlags: iface.RawFlags, + HwAddr: iface.HwAddr, + PciAddr: iface.PciAddr, ++ Queues: iface.Queues, + } + } + +diff --git a/virtcontainers/network.go b/virtcontainers/network.go +index 46703656..15eb7906 100644 +--- a/virtcontainers/network.go ++++ b/virtcontainers/network.go +@@ -140,6 +140,7 @@ type NetworkInfo struct { + Routes []netlink.Route + DNS DNSInfo + VhostUserSocket string ++ Queues uint32 + } + + // NetworkInterface defines a network interface. +@@ -156,6 +157,7 @@ type TapInterface struct { + TAPIface NetworkInterface + VMFds []*os.File + VhostFds []*os.File ++ Queues uint32 + } + + // TuntapInterface defines a tap interface +@@ -1047,7 +1049,7 @@ func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*vcTypes.Interfa + return ifaces, routes, nil + } + +-func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel) (NetworkInterfacePair, error) { ++func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (NetworkInterfacePair, error) { + uniqueID := uuid.Generate().String() + + randomMacAddr, err := generateRandomPrivateMacAddr() +@@ -1062,6 +1064,7 @@ func createNetworkInterfacePair(idx int, ifName string, interworkingModel NetInt + TAPIface: NetworkInterface{ + Name: fmt.Sprintf("tap%d_kata", idx), + }, ++ Queues: queues, + }, + VirtIface: NetworkInterface{ + Name: fmt.Sprintf("eth%d", idx), +@@ -1205,7 +1208,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + // interface info json file + if netInfo.VhostUserSocket != "" { + networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found") +- endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket) ++ endpoint, err = createVhostUserEndpoint(netInfo, netInfo.VhostUserSocket, netInfo.Queues) + return endpoint, err + } + +@@ -1220,7 +1223,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + endpoint, err = createMacvtapNetworkEndpoint(netInfo) + case "tap": + networkLogger().Info("tap interface found") +- endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device) ++ endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device, netInfo.Queues) + case "tuntap": + if link != nil { + switch link.(*netlink.Tuntap).Mode { +@@ -1231,13 +1234,13 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li + return nil, fmt.Errorf("tun networking device not yet supported") + case 2: + networkLogger().Info("tuntap tap interface found") +- endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model) ++ endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model, netInfo.Queues) + default: + return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode) + } + } + case "veth": +- endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) ++ endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model, netInfo.Queues) + case "ipvlan": + endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name) + default: +diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go +index b86f679f..c52c7452 100644 +--- a/virtcontainers/network_test.go ++++ b/virtcontainers/network_test.go +@@ -265,7 +265,7 @@ func TestTcRedirectNetwork(t *testing.T) { + err = netlink.LinkAdd(veth) + assert.NoError(err) + +- endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel) ++ endpoint, err := createVethNetworkEndpoint(1, vethName, NetXConnectTCFilterModel, 0) + assert.NoError(err) + + link, err := netlink.LinkByName(vethName) +diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go +index dccd92f8..6502259b 100644 +--- a/virtcontainers/pkg/types/types.go ++++ b/virtcontainers/pkg/types/types.go +@@ -31,6 +31,8 @@ type Interface struct { + LinkType string `json:"linkType,omitempty"` + // VhostUserSocket is DPDK-backed vHost user ports. + VhostUserSocket string `json:"vhostUserSocket,omitempty"` ++ // Queues specifies the interface queue number ++ Queues uint32 `json:"queues,omitempty"` + } + + // Route describes a network route. +diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go +index 84797e0d..be6e33b9 100644 +--- a/virtcontainers/qemu.go ++++ b/virtcontainers/qemu.go +@@ -1464,7 +1464,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro + return nil + } + +-func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error { ++func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File, queues uint32) error { + var ( + VMFdNames []string + VhostFdNames []string +@@ -1489,7 +1489,7 @@ func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFd + return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames) + } + +- return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0) ++ return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", int(queues)) + } + + func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { +@@ -1512,7 +1512,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { + + devID := "virtio-" + tap.ID + if op == addDevice { +- if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil { ++ if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds, tap.Queues); err != nil { + return err + } + +@@ -1542,9 +1542,9 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) { + } + if machine.Type == QemuCCWVirtio { + devNoHotplug := fmt.Sprintf("fe.%x.%x", bus, addr) +- return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(q.config.NumVCPUs)) ++ return q.qmpMonitorCh.qmp.ExecuteNetCCWDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), devNoHotplug, int(tap.Queues)) + } +- return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(q.config.NumVCPUs), defaultDisableModern) ++ return q.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(q.qmpMonitorCh.ctx, tap.Name, devID, endpoint.HardwareAddr(), addr, bus, romFile, int(tap.Queues), defaultDisableModern) + + } + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index e6f155a3..d8ab6c1a 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -948,6 +948,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) { + }, + Addrs: addrs, + VhostUserSocket: inf.VhostUserSocket, ++ Queues: inf.Queues, + }, nil + } + +diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go +index 2cf70dce..5a3e7f7e 100644 +--- a/virtcontainers/tap_endpoint.go ++++ b/virtcontainers/tap_endpoint.go +@@ -115,7 +115,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat + return nil + } + +-func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) { ++func createTapNetworkEndpoint(idx int, ifName string, tapIfName string, queues uint32) (*TapEndpoint, error) { + if idx < 0 { + return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } +@@ -139,6 +139,10 @@ func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEnd + endpoint.TapInterface.TAPIface.Name = tapIfName + } + ++ if queues > 0 { ++ endpoint.TapInterface.Queues = queues ++ } ++ + return endpoint, nil + } + +diff --git a/virtcontainers/tuntap_endpoint.go b/virtcontainers/tuntap_endpoint.go +index b076d694..827d8852 100644 +--- a/virtcontainers/tuntap_endpoint.go ++++ b/virtcontainers/tuntap_endpoint.go +@@ -117,12 +117,12 @@ func (endpoint *TuntapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNs + return nil + } + +-func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel) (*TuntapEndpoint, error) { ++func createTuntapNetworkEndpoint(idx int, ifName string, hwName net.HardwareAddr, internetworkingModel NetInterworkingModel, queues uint32) (*TuntapEndpoint, error) { + if idx < 0 { + return &TuntapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + +- netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel) ++ netPair, err := createNetworkInterfacePair(idx, ifName, internetworkingModel, queues) + if err != nil { + return nil, err + } +diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go +index 0f2ec9ba..554b9b22 100644 +--- a/virtcontainers/veth_endpoint.go ++++ b/virtcontainers/veth_endpoint.go +@@ -20,12 +20,12 @@ type VethEndpoint struct { + PCIAddr string + } + +-func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) { ++func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel, queues uint32) (*VethEndpoint, error) { + if idx < 0 { + return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + +- netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) ++ netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel, queues) + if err != nil { + return nil, err + } +diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go +index 9649b82e..23f1876d 100644 +--- a/virtcontainers/veth_endpoint_test.go ++++ b/virtcontainers/veth_endpoint_test.go +@@ -34,7 +34,7 @@ func TestCreateVethNetworkEndpoint(t *testing.T) { + EndpointType: VethEndpointType, + } + +- result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel) ++ result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel, 0) + assert.NoError(err) + + // the resulting ID will be random - so let's overwrite to test the rest of the flow +@@ -68,7 +68,7 @@ func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) { + EndpointType: VethEndpointType, + } + +- result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) ++ result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel, 0) + assert.NoError(err) + + // the resulting ID will be random - so let's overwrite to test the rest of the flow +@@ -95,7 +95,7 @@ func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { + } + + for _, d := range failingValues { +- _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) ++ _, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel, 0) + assert.Error(err) + } + } +diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go +index 2fc3d837..85ef67b4 100644 +--- a/virtcontainers/vhostuser_endpoint.go ++++ b/virtcontainers/vhostuser_endpoint.go +@@ -33,6 +33,7 @@ type VhostUserEndpoint struct { + EndpointProperties NetworkInfo + EndpointType EndpointType + PCIAddr string ++ Queues uint32 + } + + // Properties returns the properties of the interface. +@@ -119,7 +120,7 @@ func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, ne + } + + // Create a vhostuser endpoint +-func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { ++func createVhostUserEndpoint(netInfo NetworkInfo, socket string, queues uint32) (*VhostUserEndpoint, error) { + uniqueID := uuid.Generate().String() + vhostUserEndpoint := &VhostUserEndpoint{ + ID: uniqueID, +@@ -127,6 +128,7 @@ func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndp + HardAddr: netInfo.Iface.HardwareAddr.String(), + IfaceName: netInfo.Iface.Name, + EndpointType: VhostUserEndpointType, ++ Queues: queues, + } + return vhostUserEndpoint, nil + } +diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go +index 584490cc..046ca229 100644 +--- a/virtcontainers/vhostuser_endpoint_test.go ++++ b/virtcontainers/vhostuser_endpoint_test.go +@@ -137,7 +137,7 @@ func TestCreateVhostUserEndpoint(t *testing.T) { + EndpointType: VhostUserEndpointType, + } + +- result, err := createVhostUserEndpoint(netinfo, socket) ++ result, err := createVhostUserEndpoint(netinfo, socket, 0) + assert.NoError(err) + assert.Exactly(result, expected) + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch b/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch new file mode 100644 index 0000000000000000000000000000000000000000..bbb179f7c309dc0f172322935e809e8e7ff06bec --- /dev/null +++ b/runtime/patches/0047-network-add-support-to-get-network-stats-of-vm-throu.patch @@ -0,0 +1,2511 @@ +From 292df0bbfd4cb529eb7febdd8216859682b27bf2 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 22:06:31 +0800 +Subject: [PATCH 47/50] network: add support to get network stats of vm through + events interface + +reason: add support to get network stats of vm through events interface + +Signed-off-by: jiangpengfei +--- + cli/events.go | 29 +- + .../agent/protocols/grpc/agent.pb.go | 1749 ++++++++++++++++---- + virtcontainers/container.go | 89 +- + virtcontainers/kata_agent.go | 18 +- + 4 files changed, 1516 insertions(+), 369 deletions(-) + +diff --git a/cli/events.go b/cli/events.go +index 9e9200a6..75554665 100644 +--- a/cli/events.go ++++ b/cli/events.go +@@ -13,10 +13,9 @@ import ( + "sync" + "time" + ++ "github.com/kata-containers/runtime/pkg/katautils" + vc "github.com/kata-containers/runtime/virtcontainers" + "github.com/kata-containers/runtime/virtcontainers/types" +- +- "github.com/kata-containers/runtime/pkg/katautils" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" + ) +@@ -29,12 +28,17 @@ type event struct { + + // stats is the runc specific stats structure for stability when encoding and decoding stats. + type stats struct { +- CPU cpu `json:"cpu"` +- Memory memory `json:"memory"` +- Pids pids `json:"pids"` +- Blkio blkio `json:"blkio"` +- Hugetlb map[string]hugetlb `json:"hugetlb"` +- IntelRdt intelRdt `json:"intel_rdt"` ++ CPU cpu `json:"cpu"` ++ Memory memory `json:"memory"` ++ Pids pids `json:"pids"` ++ Blkio blkio `json:"blkio"` ++ Hugetlb map[string]hugetlb `json:"hugetlb"` ++ IntelRdt intelRdt `json:"intel_rdt"` ++ Interfaces []vc.InterfaceStats `json:"interfaces"` ++ Tcp vc.TcpStat `json:"tcp"` ++ Tcp6 vc.TcpStat `json:"tcp6"` ++ Udp vc.UdpStat `json:"udp"` ++ Udp6 vc.UdpStat `json:"udp6"` + } + + type hugetlb struct { +@@ -225,7 +229,8 @@ information is displayed once every 5 seconds.`, + + func convertVirtcontainerStats(containerStats *vc.ContainerStats) *stats { + cg := containerStats.CgroupStats +- if cg == nil { ++ net := containerStats.NetworkStats ++ if cg == nil && net == nil { + return nil + } + var s stats +@@ -261,6 +266,12 @@ func convertVirtcontainerStats(containerStats *vc.ContainerStats) *stats { + s.Hugetlb[k] = convertHugtlb(v) + } + ++ s.Interfaces = net.Interfaces ++ s.Tcp = net.Tcp ++ s.Tcp6 = net.Tcp6 ++ s.Udp = net.Udp ++ s.Udp6 = net.Udp6 ++ + return &s + } + +diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +index 04d0ee52..c50ecb58 100644 +--- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go ++++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +@@ -33,6 +33,9 @@ + BlkioStats + HugetlbStats + CgroupStats ++ InterfaceStats ++ TcpStat ++ UdpStat + NetworkStats + StatsContainerResponse + WriteStreamRequest +@@ -883,7 +886,7 @@ func (m *CgroupStats) GetHugetlbStats() map[string]*HugetlbStats { + return nil + } + +-type NetworkStats struct { ++type InterfaceStats struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` + RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` +@@ -895,83 +898,267 @@ type NetworkStats struct { + TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` + } + +-func (m *NetworkStats) Reset() { *m = NetworkStats{} } +-func (m *NetworkStats) String() string { return proto.CompactTextString(m) } +-func (*NetworkStats) ProtoMessage() {} +-func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } ++func (m *InterfaceStats) Reset() { *m = InterfaceStats{} } ++func (m *InterfaceStats) String() string { return proto.CompactTextString(m) } ++func (*InterfaceStats) ProtoMessage() {} ++func (*InterfaceStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{23} } + +-func (m *NetworkStats) GetName() string { ++func (m *InterfaceStats) GetName() string { + if m != nil { + return m.Name + } + return "" + } + +-func (m *NetworkStats) GetRxBytes() uint64 { ++func (m *InterfaceStats) GetRxBytes() uint64 { + if m != nil { + return m.RxBytes + } + return 0 + } + +-func (m *NetworkStats) GetRxPackets() uint64 { ++func (m *InterfaceStats) GetRxPackets() uint64 { + if m != nil { + return m.RxPackets + } + return 0 + } + +-func (m *NetworkStats) GetRxErrors() uint64 { ++func (m *InterfaceStats) GetRxErrors() uint64 { + if m != nil { + return m.RxErrors + } + return 0 + } + +-func (m *NetworkStats) GetRxDropped() uint64 { ++func (m *InterfaceStats) GetRxDropped() uint64 { + if m != nil { + return m.RxDropped + } + return 0 + } + +-func (m *NetworkStats) GetTxBytes() uint64 { ++func (m *InterfaceStats) GetTxBytes() uint64 { + if m != nil { + return m.TxBytes + } + return 0 + } + +-func (m *NetworkStats) GetTxPackets() uint64 { ++func (m *InterfaceStats) GetTxPackets() uint64 { + if m != nil { + return m.TxPackets + } + return 0 + } + +-func (m *NetworkStats) GetTxErrors() uint64 { ++func (m *InterfaceStats) GetTxErrors() uint64 { + if m != nil { + return m.TxErrors + } + return 0 + } + +-func (m *NetworkStats) GetTxDropped() uint64 { ++func (m *InterfaceStats) GetTxDropped() uint64 { + if m != nil { + return m.TxDropped + } + return 0 + } + ++type TcpStat struct { ++ Established uint64 `protobuf:"varint,1,opt,name=established,proto3" json:"established,omitempty"` ++ SynSent uint64 `protobuf:"varint,2,opt,name=syn_sent,json=synSent,proto3" json:"syn_sent,omitempty"` ++ SynRecv uint64 `protobuf:"varint,3,opt,name=syn_recv,json=synRecv,proto3" json:"syn_recv,omitempty"` ++ FinWait1 uint64 `protobuf:"varint,4,opt,name=fin_wait1,json=finWait1,proto3" json:"fin_wait1,omitempty"` ++ FinWait2 uint64 `protobuf:"varint,5,opt,name=fin_wait2,json=finWait2,proto3" json:"fin_wait2,omitempty"` ++ TimeWait uint64 `protobuf:"varint,6,opt,name=time_wait,json=timeWait,proto3" json:"time_wait,omitempty"` ++ Close uint64 `protobuf:"varint,7,opt,name=close,proto3" json:"close,omitempty"` ++ CloseWait uint64 `protobuf:"varint,8,opt,name=close_wait,json=closeWait,proto3" json:"close_wait,omitempty"` ++ LastAck uint64 `protobuf:"varint,9,opt,name=last_ack,json=lastAck,proto3" json:"last_ack,omitempty"` ++ Listen uint64 `protobuf:"varint,10,opt,name=listen,proto3" json:"listen,omitempty"` ++ Closing uint64 `protobuf:"varint,11,opt,name=closing,proto3" json:"closing,omitempty"` ++} ++ ++func (m *TcpStat) Reset() { *m = TcpStat{} } ++func (m *TcpStat) String() string { return proto.CompactTextString(m) } ++func (*TcpStat) ProtoMessage() {} ++func (*TcpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } ++ ++func (m *TcpStat) GetEstablished() uint64 { ++ if m != nil { ++ return m.Established ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetSynSent() uint64 { ++ if m != nil { ++ return m.SynSent ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetSynRecv() uint64 { ++ if m != nil { ++ return m.SynRecv ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetFinWait1() uint64 { ++ if m != nil { ++ return m.FinWait1 ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetFinWait2() uint64 { ++ if m != nil { ++ return m.FinWait2 ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetTimeWait() uint64 { ++ if m != nil { ++ return m.TimeWait ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetClose() uint64 { ++ if m != nil { ++ return m.Close ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetCloseWait() uint64 { ++ if m != nil { ++ return m.CloseWait ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetLastAck() uint64 { ++ if m != nil { ++ return m.LastAck ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetListen() uint64 { ++ if m != nil { ++ return m.Listen ++ } ++ return 0 ++} ++ ++func (m *TcpStat) GetClosing() uint64 { ++ if m != nil { ++ return m.Closing ++ } ++ return 0 ++} ++ ++type UdpStat struct { ++ Listen uint64 `protobuf:"varint,1,opt,name=listen,proto3" json:"listen,omitempty"` ++ Dropped uint64 `protobuf:"varint,2,opt,name=dropped,proto3" json:"dropped,omitempty"` ++ RxQueued uint64 `protobuf:"varint,3,opt,name=rx_queued,json=rxQueued,proto3" json:"rx_queued,omitempty"` ++ TxQueued uint64 `protobuf:"varint,4,opt,name=tx_queued,json=txQueued,proto3" json:"tx_queued,omitempty"` ++} ++ ++func (m *UdpStat) Reset() { *m = UdpStat{} } ++func (m *UdpStat) String() string { return proto.CompactTextString(m) } ++func (*UdpStat) ProtoMessage() {} ++func (*UdpStat) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } ++ ++func (m *UdpStat) GetListen() uint64 { ++ if m != nil { ++ return m.Listen ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetDropped() uint64 { ++ if m != nil { ++ return m.Dropped ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetRxQueued() uint64 { ++ if m != nil { ++ return m.RxQueued ++ } ++ return 0 ++} ++ ++func (m *UdpStat) GetTxQueued() uint64 { ++ if m != nil { ++ return m.TxQueued ++ } ++ return 0 ++} ++ ++type NetworkStats struct { ++ Interfaces []*InterfaceStats `protobuf:"bytes,1,rep,name=interfaces" json:"interfaces,omitempty"` ++ Tcp *TcpStat `protobuf:"bytes,2,opt,name=tcp" json:"tcp,omitempty"` ++ Tcp6 *TcpStat `protobuf:"bytes,3,opt,name=tcp6" json:"tcp6,omitempty"` ++ Udp *UdpStat `protobuf:"bytes,4,opt,name=udp" json:"udp,omitempty"` ++ Udp6 *UdpStat `protobuf:"bytes,5,opt,name=udp6" json:"udp6,omitempty"` ++} ++ ++func (m *NetworkStats) Reset() { *m = NetworkStats{} } ++func (m *NetworkStats) String() string { return proto.CompactTextString(m) } ++func (*NetworkStats) ProtoMessage() {} ++func (*NetworkStats) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } ++ ++func (m *NetworkStats) GetInterfaces() []*InterfaceStats { ++ if m != nil { ++ return m.Interfaces ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetTcp() *TcpStat { ++ if m != nil { ++ return m.Tcp ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetTcp6() *TcpStat { ++ if m != nil { ++ return m.Tcp6 ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetUdp() *UdpStat { ++ if m != nil { ++ return m.Udp ++ } ++ return nil ++} ++ ++func (m *NetworkStats) GetUdp6() *UdpStat { ++ if m != nil { ++ return m.Udp6 ++ } ++ return nil ++} ++ + type StatsContainerResponse struct { +- CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` +- NetworkStats []*NetworkStats `protobuf:"bytes,2,rep,name=network_stats,json=networkStats" json:"network_stats,omitempty"` ++ CgroupStats *CgroupStats `protobuf:"bytes,1,opt,name=cgroup_stats,json=cgroupStats" json:"cgroup_stats,omitempty"` ++ NetworkStats *NetworkStats `protobuf:"bytes,2,opt,name=network_stats,json=networkStats" json:"network_stats,omitempty"` + } + + func (m *StatsContainerResponse) Reset() { *m = StatsContainerResponse{} } + func (m *StatsContainerResponse) String() string { return proto.CompactTextString(m) } + func (*StatsContainerResponse) ProtoMessage() {} +-func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{24} } ++func (*StatsContainerResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } + + func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { + if m != nil { +@@ -980,7 +1167,7 @@ func (m *StatsContainerResponse) GetCgroupStats() *CgroupStats { + return nil + } + +-func (m *StatsContainerResponse) GetNetworkStats() []*NetworkStats { ++func (m *StatsContainerResponse) GetNetworkStats() *NetworkStats { + if m != nil { + return m.NetworkStats + } +@@ -996,7 +1183,7 @@ type WriteStreamRequest struct { + func (m *WriteStreamRequest) Reset() { *m = WriteStreamRequest{} } + func (m *WriteStreamRequest) String() string { return proto.CompactTextString(m) } + func (*WriteStreamRequest) ProtoMessage() {} +-func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{25} } ++func (*WriteStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } + + func (m *WriteStreamRequest) GetContainerId() string { + if m != nil { +@@ -1026,7 +1213,7 @@ type WriteStreamResponse struct { + func (m *WriteStreamResponse) Reset() { *m = WriteStreamResponse{} } + func (m *WriteStreamResponse) String() string { return proto.CompactTextString(m) } + func (*WriteStreamResponse) ProtoMessage() {} +-func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{26} } ++func (*WriteStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } + + func (m *WriteStreamResponse) GetLen() uint32 { + if m != nil { +@@ -1044,7 +1231,7 @@ type ReadStreamRequest struct { + func (m *ReadStreamRequest) Reset() { *m = ReadStreamRequest{} } + func (m *ReadStreamRequest) String() string { return proto.CompactTextString(m) } + func (*ReadStreamRequest) ProtoMessage() {} +-func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{27} } ++func (*ReadStreamRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } + + func (m *ReadStreamRequest) GetContainerId() string { + if m != nil { +@@ -1074,7 +1261,7 @@ type ReadStreamResponse struct { + func (m *ReadStreamResponse) Reset() { *m = ReadStreamResponse{} } + func (m *ReadStreamResponse) String() string { return proto.CompactTextString(m) } + func (*ReadStreamResponse) ProtoMessage() {} +-func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{28} } ++func (*ReadStreamResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } + + func (m *ReadStreamResponse) GetData() []byte { + if m != nil { +@@ -1091,7 +1278,7 @@ type CloseStdinRequest struct { + func (m *CloseStdinRequest) Reset() { *m = CloseStdinRequest{} } + func (m *CloseStdinRequest) String() string { return proto.CompactTextString(m) } + func (*CloseStdinRequest) ProtoMessage() {} +-func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{29} } ++func (*CloseStdinRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } + + func (m *CloseStdinRequest) GetContainerId() string { + if m != nil { +@@ -1117,7 +1304,7 @@ type TtyWinResizeRequest struct { + func (m *TtyWinResizeRequest) Reset() { *m = TtyWinResizeRequest{} } + func (m *TtyWinResizeRequest) String() string { return proto.CompactTextString(m) } + func (*TtyWinResizeRequest) ProtoMessage() {} +-func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{30} } ++func (*TtyWinResizeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } + + func (m *TtyWinResizeRequest) GetContainerId() string { + if m != nil { +@@ -1158,7 +1345,7 @@ type KernelModule struct { + func (m *KernelModule) Reset() { *m = KernelModule{} } + func (m *KernelModule) String() string { return proto.CompactTextString(m) } + func (*KernelModule) ProtoMessage() {} +-func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{31} } ++func (*KernelModule) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } + + func (m *KernelModule) GetName() string { + if m != nil { +@@ -1197,7 +1384,7 @@ type CreateSandboxRequest struct { + func (m *CreateSandboxRequest) Reset() { *m = CreateSandboxRequest{} } + func (m *CreateSandboxRequest) String() string { return proto.CompactTextString(m) } + func (*CreateSandboxRequest) ProtoMessage() {} +-func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{32} } ++func (*CreateSandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } + + func (m *CreateSandboxRequest) GetHostname() string { + if m != nil { +@@ -1254,7 +1441,7 @@ type DestroySandboxRequest struct { + func (m *DestroySandboxRequest) Reset() { *m = DestroySandboxRequest{} } + func (m *DestroySandboxRequest) String() string { return proto.CompactTextString(m) } + func (*DestroySandboxRequest) ProtoMessage() {} +-func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{33} } ++func (*DestroySandboxRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } + + type Interfaces struct { + Interfaces []*types.Interface `protobuf:"bytes,1,rep,name=Interfaces" json:"Interfaces,omitempty"` +@@ -1263,7 +1450,7 @@ type Interfaces struct { + func (m *Interfaces) Reset() { *m = Interfaces{} } + func (m *Interfaces) String() string { return proto.CompactTextString(m) } + func (*Interfaces) ProtoMessage() {} +-func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{34} } ++func (*Interfaces) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } + + func (m *Interfaces) GetInterfaces() []*types.Interface { + if m != nil { +@@ -1279,7 +1466,7 @@ type Routes struct { + func (m *Routes) Reset() { *m = Routes{} } + func (m *Routes) String() string { return proto.CompactTextString(m) } + func (*Routes) ProtoMessage() {} +-func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{35} } ++func (*Routes) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } + + func (m *Routes) GetRoutes() []*types.Route { + if m != nil { +@@ -1295,7 +1482,7 @@ type UpdateInterfaceRequest struct { + func (m *UpdateInterfaceRequest) Reset() { *m = UpdateInterfaceRequest{} } + func (m *UpdateInterfaceRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateInterfaceRequest) ProtoMessage() {} +-func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{36} } ++func (*UpdateInterfaceRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } + + func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + if m != nil { +@@ -1312,7 +1499,7 @@ type UpdateRoutesRequest struct { + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } + func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateRoutesRequest) ProtoMessage() {} +-func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{37} } ++func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } + + func (m *UpdateRoutesRequest) GetRoutes() *Routes { + if m != nil { +@@ -1334,7 +1521,7 @@ type ListInterfacesRequest struct { + func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } + func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } + func (*ListInterfacesRequest) ProtoMessage() {} +-func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{38} } ++func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } + + type ListRoutesRequest struct { + } +@@ -1342,7 +1529,7 @@ type ListRoutesRequest struct { + func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } + func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*ListRoutesRequest) ProtoMessage() {} +-func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{39} } ++func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } + + type OnlineCPUMemRequest struct { + // Wait specifies if the caller waits for the agent to online all resources. +@@ -1358,7 +1545,7 @@ type OnlineCPUMemRequest struct { + func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } + func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } + func (*OnlineCPUMemRequest) ProtoMessage() {} +-func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } ++func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } + + func (m *OnlineCPUMemRequest) GetWait() bool { + if m != nil { +@@ -1389,7 +1576,7 @@ type ReseedRandomDevRequest struct { + func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } + func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } + func (*ReseedRandomDevRequest) ProtoMessage() {} +-func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } ++func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } + + func (m *ReseedRandomDevRequest) GetData() []byte { + if m != nil { +@@ -1416,7 +1603,7 @@ type AgentDetails struct { + func (m *AgentDetails) Reset() { *m = AgentDetails{} } + func (m *AgentDetails) String() string { return proto.CompactTextString(m) } + func (*AgentDetails) ProtoMessage() {} +-func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } ++func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } + + func (m *AgentDetails) GetVersion() string { + if m != nil { +@@ -1467,7 +1654,7 @@ type GuestDetailsRequest struct { + func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } + func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsRequest) ProtoMessage() {} +-func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } ++func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } + + func (m *GuestDetailsRequest) GetMemBlockSize() bool { + if m != nil { +@@ -1493,7 +1680,7 @@ type GuestDetailsResponse struct { + func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } + func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsResponse) ProtoMessage() {} +-func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } ++func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } + + func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { + if m != nil { +@@ -1525,7 +1712,7 @@ type MemHotplugByProbeRequest struct { + func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } + func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } + func (*MemHotplugByProbeRequest) ProtoMessage() {} +-func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } ++func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } + + func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { + if m != nil { +@@ -1544,7 +1731,7 @@ type SetGuestDateTimeRequest struct { + func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } + func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } + func (*SetGuestDateTimeRequest) ProtoMessage() {} +-func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } ++func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } + + func (m *SetGuestDateTimeRequest) GetSec() int64 { + if m != nil { +@@ -1593,7 +1780,7 @@ type Storage struct { + func (m *Storage) Reset() { *m = Storage{} } + func (m *Storage) String() string { return proto.CompactTextString(m) } + func (*Storage) ProtoMessage() {} +-func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } ++func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } + + func (m *Storage) GetDriver() string { + if m != nil { +@@ -1676,7 +1863,7 @@ type Device struct { + func (m *Device) Reset() { *m = Device{} } + func (m *Device) String() string { return proto.CompactTextString(m) } + func (*Device) ProtoMessage() {} +-func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } ++func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } + + func (m *Device) GetId() string { + if m != nil { +@@ -1722,7 +1909,7 @@ type StringUser struct { + func (m *StringUser) Reset() { *m = StringUser{} } + func (m *StringUser) String() string { return proto.CompactTextString(m) } + func (*StringUser) ProtoMessage() {} +-func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } ++func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + + func (m *StringUser) GetUid() string { + if m != nil { +@@ -1770,7 +1957,7 @@ type CopyFileRequest struct { + func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } + func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } + func (*CopyFileRequest) ProtoMessage() {} +-func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } ++func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } + + func (m *CopyFileRequest) GetPath() string { + if m != nil { +@@ -1834,7 +2021,7 @@ type StartTracingRequest struct { + func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } + func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StartTracingRequest) ProtoMessage() {} +-func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } ++func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } + + type StopTracingRequest struct { + } +@@ -1842,7 +2029,7 @@ type StopTracingRequest struct { + func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } + func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StopTracingRequest) ProtoMessage() {} +-func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } ++func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } + + type UpdateIPVSRequest struct { + // IPVS_req is the IPVS rule message needed to update +@@ -1852,7 +2039,7 @@ type UpdateIPVSRequest struct { + func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } + func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateIPVSRequest) ProtoMessage() {} +-func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } + + func (m *UpdateIPVSRequest) GetIPVSReq() string { + if m != nil { +@@ -1869,7 +2056,7 @@ type IPVSResponse struct { + func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } + func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } + func (*IPVSResponse) ProtoMessage() {} +-func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } + + func (m *IPVSResponse) GetIPVSRes() string { + if m != nil { +@@ -1902,6 +2089,9 @@ func init() { + proto.RegisterType((*BlkioStats)(nil), "grpc.BlkioStats") + proto.RegisterType((*HugetlbStats)(nil), "grpc.HugetlbStats") + proto.RegisterType((*CgroupStats)(nil), "grpc.CgroupStats") ++ proto.RegisterType((*InterfaceStats)(nil), "grpc.InterfaceStats") ++ proto.RegisterType((*TcpStat)(nil), "grpc.TcpStat") ++ proto.RegisterType((*UdpStat)(nil), "grpc.UdpStat") + proto.RegisterType((*NetworkStats)(nil), "grpc.NetworkStats") + proto.RegisterType((*StatsContainerResponse)(nil), "grpc.StatsContainerResponse") + proto.RegisterType((*WriteStreamRequest)(nil), "grpc.WriteStreamRequest") +@@ -4008,7 +4198,7 @@ func (m *CgroupStats) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + +-func (m *NetworkStats) Marshal() (dAtA []byte, err error) { ++func (m *InterfaceStats) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4018,7 +4208,7 @@ func (m *NetworkStats) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { ++func (m *InterfaceStats) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int +@@ -4072,7 +4262,7 @@ func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + +-func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { ++func (m *TcpStat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4082,37 +4272,70 @@ func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { ++func (m *TcpStat) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if m.CgroupStats != nil { +- dAtA[i] = 0xa ++ if m.Established != 0 { ++ dAtA[i] = 0x8 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) +- n18, err := m.CgroupStats.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n18 ++ i = encodeVarintAgent(dAtA, i, uint64(m.Established)) + } +- if len(m.NetworkStats) > 0 { +- for _, msg := range m.NetworkStats { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } ++ if m.SynSent != 0 { ++ dAtA[i] = 0x10 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.SynSent)) ++ } ++ if m.SynRecv != 0 { ++ dAtA[i] = 0x18 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.SynRecv)) ++ } ++ if m.FinWait1 != 0 { ++ dAtA[i] = 0x20 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait1)) ++ } ++ if m.FinWait2 != 0 { ++ dAtA[i] = 0x28 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.FinWait2)) ++ } ++ if m.TimeWait != 0 { ++ dAtA[i] = 0x30 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.TimeWait)) ++ } ++ if m.Close != 0 { ++ dAtA[i] = 0x38 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Close)) ++ } ++ if m.CloseWait != 0 { ++ dAtA[i] = 0x40 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.CloseWait)) ++ } ++ if m.LastAck != 0 { ++ dAtA[i] = 0x48 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.LastAck)) ++ } ++ if m.Listen != 0 { ++ dAtA[i] = 0x50 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) ++ } ++ if m.Closing != 0 { ++ dAtA[i] = 0x58 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Closing)) + } + return i, nil + } + +-func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { ++func (m *UdpStat) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4122,33 +4345,35 @@ func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++func (m *UdpStat) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if len(m.ContainerId) > 0 { +- dAtA[i] = 0xa ++ if m.Listen != 0 { ++ dAtA[i] = 0x8 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) +- i += copy(dAtA[i:], m.ContainerId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Listen)) + } +- if len(m.ExecId) > 0 { +- dAtA[i] = 0x12 ++ if m.Dropped != 0 { ++ dAtA[i] = 0x10 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) +- i += copy(dAtA[i:], m.ExecId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Dropped)) + } +- if len(m.Data) > 0 { +- dAtA[i] = 0x1a ++ if m.RxQueued != 0 { ++ dAtA[i] = 0x18 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) +- i += copy(dAtA[i:], m.Data) ++ i = encodeVarintAgent(dAtA, i, uint64(m.RxQueued)) ++ } ++ if m.TxQueued != 0 { ++ dAtA[i] = 0x20 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.TxQueued)) + } + return i, nil + } + +-func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { ++func (m *NetworkStats) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4158,20 +4383,67 @@ func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { ++func (m *NetworkStats) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if m.Len != 0 { +- dAtA[i] = 0x8 ++ if len(m.Interfaces) > 0 { ++ for _, msg := range m.Interfaces { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(msg.Size())) ++ n, err := msg.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n ++ } ++ } ++ if m.Tcp != nil { ++ dAtA[i] = 0x12 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp.Size())) ++ n18, err := m.Tcp.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n18 ++ } ++ if m.Tcp6 != nil { ++ dAtA[i] = 0x1a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Tcp6.Size())) ++ n19, err := m.Tcp6.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n19 ++ } ++ if m.Udp != nil { ++ dAtA[i] = 0x22 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Udp.Size())) ++ n20, err := m.Udp.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n20 ++ } ++ if m.Udp6 != nil { ++ dAtA[i] = 0x2a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Udp6.Size())) ++ n21, err := m.Udp6.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n21 + } + return i, nil + } + +-func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { ++func (m *StatsContainerResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4181,32 +4453,129 @@ func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { + return dAtA[:n], nil + } + +-func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++func (m *StatsContainerResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l +- if len(m.ContainerId) > 0 { ++ if m.CgroupStats != nil { + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) +- i += copy(dAtA[i:], m.ContainerId) ++ i = encodeVarintAgent(dAtA, i, uint64(m.CgroupStats.Size())) ++ n22, err := m.CgroupStats.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n22 + } +- if len(m.ExecId) > 0 { ++ if m.NetworkStats != nil { + dAtA[i] = 0x12 + i++ +- i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) +- i += copy(dAtA[i:], m.ExecId) +- } +- if m.Len != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ i = encodeVarintAgent(dAtA, i, uint64(m.NetworkStats.Size())) ++ n23, err := m.NetworkStats.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n23 + } + return i, nil + } + +-func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { ++func (m *WriteStreamRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *WriteStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.ContainerId) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) ++ i += copy(dAtA[i:], m.ContainerId) ++ } ++ if len(m.ExecId) > 0 { ++ dAtA[i] = 0x12 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) ++ i += copy(dAtA[i:], m.ExecId) ++ } ++ if len(m.Data) > 0 { ++ dAtA[i] = 0x1a ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.Data))) ++ i += copy(dAtA[i:], m.Data) ++ } ++ return i, nil ++} ++ ++func (m *WriteStreamResponse) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *WriteStreamResponse) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if m.Len != 0 { ++ dAtA[i] = 0x8 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ } ++ return i, nil ++} ++ ++func (m *ReadStreamRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *ReadStreamRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if len(m.ContainerId) > 0 { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ContainerId))) ++ i += copy(dAtA[i:], m.ContainerId) ++ } ++ if len(m.ExecId) > 0 { ++ dAtA[i] = 0x12 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(len(m.ExecId))) ++ i += copy(dAtA[i:], m.ExecId) ++ } ++ if m.Len != 0 { ++ dAtA[i] = 0x18 ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Len)) ++ } ++ return i, nil ++} ++ ++func (m *ReadStreamResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) +@@ -4521,11 +4890,11 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) +- n19, err := m.Interface.MarshalTo(dAtA[i:]) ++ n24, err := m.Interface.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n19 ++ i += n24 + } + return i, nil + } +@@ -4549,11 +4918,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) +- n20, err := m.Routes.MarshalTo(dAtA[i:]) ++ n25, err := m.Routes.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n20 ++ i += n25 + } + if m.Increment { + dAtA[i] = 0x10 +@@ -4807,11 +5176,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0x12 + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) +- n21, err := m.AgentDetails.MarshalTo(dAtA[i:]) ++ n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n21 ++ i += n26 + } + if m.SupportMemHotplugProbe { + dAtA[i] = 0x18 +@@ -4842,21 +5211,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { +- dAtA23 := make([]byte, len(m.MemHotplugProbeAddr)*10) +- var j22 int ++ dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) ++ var j27 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { +- dAtA23[j22] = uint8(uint64(num)&0x7f | 0x80) ++ dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 +- j22++ ++ j27++ + } +- dAtA23[j22] = uint8(num) +- j22++ ++ dAtA28[j27] = uint8(num) ++ j27++ + } + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(j22)) +- i += copy(dAtA[i:], dAtA23[:j22]) ++ i = encodeVarintAgent(dAtA, i, uint64(j27)) ++ i += copy(dAtA[i:], dAtA28[:j27]) + } + return i, nil + } +@@ -5648,7 +6017,7 @@ func (m *CgroupStats) Size() (n int) { + return n + } + +-func (m *NetworkStats) Size() (n int) { ++func (m *InterfaceStats) Size() (n int) { + var l int + _ = l + l = len(m.Name) +@@ -5682,6 +6051,91 @@ func (m *NetworkStats) Size() (n int) { + return n + } + ++func (m *TcpStat) Size() (n int) { ++ var l int ++ _ = l ++ if m.Established != 0 { ++ n += 1 + sovAgent(uint64(m.Established)) ++ } ++ if m.SynSent != 0 { ++ n += 1 + sovAgent(uint64(m.SynSent)) ++ } ++ if m.SynRecv != 0 { ++ n += 1 + sovAgent(uint64(m.SynRecv)) ++ } ++ if m.FinWait1 != 0 { ++ n += 1 + sovAgent(uint64(m.FinWait1)) ++ } ++ if m.FinWait2 != 0 { ++ n += 1 + sovAgent(uint64(m.FinWait2)) ++ } ++ if m.TimeWait != 0 { ++ n += 1 + sovAgent(uint64(m.TimeWait)) ++ } ++ if m.Close != 0 { ++ n += 1 + sovAgent(uint64(m.Close)) ++ } ++ if m.CloseWait != 0 { ++ n += 1 + sovAgent(uint64(m.CloseWait)) ++ } ++ if m.LastAck != 0 { ++ n += 1 + sovAgent(uint64(m.LastAck)) ++ } ++ if m.Listen != 0 { ++ n += 1 + sovAgent(uint64(m.Listen)) ++ } ++ if m.Closing != 0 { ++ n += 1 + sovAgent(uint64(m.Closing)) ++ } ++ return n ++} ++ ++func (m *UdpStat) Size() (n int) { ++ var l int ++ _ = l ++ if m.Listen != 0 { ++ n += 1 + sovAgent(uint64(m.Listen)) ++ } ++ if m.Dropped != 0 { ++ n += 1 + sovAgent(uint64(m.Dropped)) ++ } ++ if m.RxQueued != 0 { ++ n += 1 + sovAgent(uint64(m.RxQueued)) ++ } ++ if m.TxQueued != 0 { ++ n += 1 + sovAgent(uint64(m.TxQueued)) ++ } ++ return n ++} ++ ++func (m *NetworkStats) Size() (n int) { ++ var l int ++ _ = l ++ if len(m.Interfaces) > 0 { ++ for _, e := range m.Interfaces { ++ l = e.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ } ++ if m.Tcp != nil { ++ l = m.Tcp.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Tcp6 != nil { ++ l = m.Tcp6.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Udp != nil { ++ l = m.Udp.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ if m.Udp6 != nil { ++ l = m.Udp6.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func (m *StatsContainerResponse) Size() (n int) { + var l int + _ = l +@@ -5689,11 +6143,9 @@ func (m *StatsContainerResponse) Size() (n int) { + l = m.CgroupStats.Size() + n += 1 + l + sovAgent(uint64(l)) + } +- if len(m.NetworkStats) > 0 { +- for _, e := range m.NetworkStats { +- l = e.Size() +- n += 1 + l + sovAgent(uint64(l)) +- } ++ if m.NetworkStats != nil { ++ l = m.NetworkStats.Size() ++ n += 1 + l + sovAgent(uint64(l)) + } + return n + } +@@ -9396,7 +9848,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { + } + return nil + } +-func (m *NetworkStats) Unmarshal(dAtA []byte) error { ++func (m *InterfaceStats) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { +@@ -9419,10 +9871,10 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { +- return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") ++ return fmt.Errorf("proto: InterfaceStats: wiretype end group for non-group") + } + if fieldNum <= 0 { +- return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) ++ return fmt.Errorf("proto: InterfaceStats: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: +@@ -9627,7 +10079,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { + } + return nil + } +-func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { ++func (m *TcpStat) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { +@@ -9650,17 +10102,17 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { +- return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") ++ return fmt.Errorf("proto: TcpStat: wiretype end group for non-group") + } + if fieldNum <= 0 { +- return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ return fmt.Errorf("proto: TcpStat: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Established", wireType) + } +- var msglen int ++ m.Established = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent +@@ -9670,30 +10122,54 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + } + b := dAtA[iNdEx] + iNdEx++ +- msglen |= (int(b) & 0x7F) << shift ++ m.Established |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } +- if msglen < 0 { +- return ErrInvalidLengthAgent ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field SynSent", wireType) + } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF ++ m.SynSent = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.SynSent |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } + } +- if m.CgroupStats == nil { +- m.CgroupStats = &CgroupStats{} ++ case 3: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field SynRecv", wireType) + } +- if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err ++ m.SynRecv = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.SynRecv |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } + } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) ++ case 4: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field FinWait1", wireType) + } +- var msglen int ++ m.FinWait1 = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowAgent +@@ -9703,20 +10179,596 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { + } + b := dAtA[iNdEx] + iNdEx++ +- msglen |= (int(b) & 0x7F) << shift ++ m.FinWait1 |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } +- if msglen < 0 { +- return ErrInvalidLengthAgent ++ case 5: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field FinWait2", wireType) + } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF ++ m.FinWait2 = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.FinWait2 |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 6: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field TimeWait", wireType) ++ } ++ m.TimeWait = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.TimeWait |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 7: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Close", wireType) ++ } ++ m.Close = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Close |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 8: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field CloseWait", wireType) ++ } ++ m.CloseWait = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.CloseWait |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 9: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field LastAck", wireType) ++ } ++ m.LastAck = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.LastAck |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 10: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) ++ } ++ m.Listen = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Listen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 11: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Closing", wireType) ++ } ++ m.Closing = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Closing |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *UdpStat) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UdpStat: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UdpStat: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Listen", wireType) ++ } ++ m.Listen = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Listen |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 2: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Dropped", wireType) ++ } ++ m.Dropped = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.Dropped |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 3: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field RxQueued", wireType) ++ } ++ m.RxQueued = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.RxQueued |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ case 4: ++ if wireType != 0 { ++ return fmt.Errorf("proto: wrong wireType = %d for field TxQueued", wireType) ++ } ++ m.TxQueued = 0 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ m.TxQueued |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *NetworkStats) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: NetworkStats: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: NetworkStats: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Interfaces", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ m.Interfaces = append(m.Interfaces, &InterfaceStats{}) ++ if err := m.Interfaces[len(m.Interfaces)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 2: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Tcp", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Tcp == nil { ++ m.Tcp = &TcpStat{} ++ } ++ if err := m.Tcp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 3: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Tcp6", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Tcp6 == nil { ++ m.Tcp6 = &TcpStat{} ++ } ++ if err := m.Tcp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 4: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Udp", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Udp == nil { ++ m.Udp = &UdpStat{} ++ } ++ if err := m.Udp.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 5: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Udp6", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Udp6 == nil { ++ m.Udp6 = &UdpStat{} ++ } ++ if err := m.Udp6.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} ++func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: StatsContainerResponse: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: StatsContainerResponse: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field CgroupStats", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.CgroupStats == nil { ++ m.CgroupStats = &CgroupStats{} ++ } ++ if err := m.CgroupStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ case 2: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field NetworkStats", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.NetworkStats == nil { ++ m.NetworkStats = &NetworkStats{} + } +- m.NetworkStats = append(m.NetworkStats, &NetworkStats{}) +- if err := m.NetworkStats[len(m.NetworkStats)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ if err := m.NetworkStats.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex +@@ -13190,189 +14242,204 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 2931 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, +- 0xb5, 0x98, 0x07, 0x87, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0xae, 0xdc, 0xb6, +- 0x65, 0xfa, 0xfa, 0x7a, 0x68, 0xcb, 0xc6, 0xf5, 0x0b, 0xbe, 0x82, 0x48, 0xe9, 0x8a, 0x8c, 0xad, +- 0x88, 0xe9, 0x91, 0xe2, 0x04, 0x41, 0xd0, 0x68, 0x76, 0x97, 0x86, 0x65, 0x4e, 0x77, 0xb5, 0xab, +- 0xaa, 0x29, 0x8e, 0x03, 0x64, 0x99, 0xec, 0xb2, 0xcc, 0x2e, 0x3f, 0x10, 0x64, 0x97, 0x65, 0xb6, +- 0x59, 0x18, 0x59, 0x05, 0xf9, 0x80, 0x20, 0xf0, 0x27, 0xe4, 0x0b, 0x82, 0x7a, 0xf5, 0x63, 0x66, +- 0x48, 0x23, 0x82, 0x80, 0x6c, 0x1a, 0x75, 0x4e, 0x9d, 0x3a, 0xaf, 0xaa, 0x3a, 0x75, 0xce, 0x69, +- 0x68, 0xfb, 0x53, 0x1c, 0x8b, 0x71, 0xc2, 0xa8, 0xa0, 0xa8, 0x3e, 0x65, 0x49, 0x30, 0x6a, 0xd1, +- 0x80, 0x68, 0xc4, 0xe8, 0x7f, 0xa7, 0x44, 0x9c, 0xa4, 0xc7, 0xe3, 0x80, 0x46, 0xbb, 0xa7, 0xbe, +- 0xf0, 0xdf, 0x09, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0x77, 0xd5, 0xc2, 0xdd, 0xe4, 0x74, 0xba, +- 0x2b, 0xe6, 0x09, 0xe6, 0xfa, 0x6b, 0xd6, 0x5d, 0x9f, 0x52, 0x3a, 0x9d, 0xe1, 0x5d, 0x05, 0x1d, +- 0xa7, 0xcf, 0x76, 0x71, 0x94, 0x88, 0xb9, 0x9e, 0x74, 0x7e, 0x57, 0x85, 0xed, 0x7d, 0x86, 0x7d, +- 0x81, 0xf7, 0x2d, 0x37, 0x17, 0x7f, 0x9d, 0x62, 0x2e, 0xd0, 0xab, 0xd0, 0xc9, 0x24, 0x78, 0x24, +- 0x1c, 0x56, 0x6e, 0x55, 0x76, 0x5a, 0x6e, 0x3b, 0xc3, 0x1d, 0x86, 0xe8, 0x2a, 0xac, 0xe3, 0x73, +- 0x1c, 0xc8, 0xd9, 0xaa, 0x9a, 0x6d, 0x48, 0xf0, 0x30, 0x44, 0xef, 0x41, 0x9b, 0x0b, 0x46, 0xe2, +- 0xa9, 0x97, 0x72, 0xcc, 0x86, 0xb5, 0x5b, 0x95, 0x9d, 0xf6, 0x9d, 0x8d, 0xb1, 0x34, 0x69, 0x3c, +- 0x51, 0x13, 0x4f, 0x39, 0x66, 0x2e, 0xf0, 0x6c, 0x8c, 0x6e, 0xc3, 0x7a, 0x88, 0xcf, 0x48, 0x80, +- 0xf9, 0xb0, 0x7e, 0xab, 0xb6, 0xd3, 0xbe, 0xd3, 0xd1, 0xe4, 0xf7, 0x15, 0xd2, 0xb5, 0x93, 0xe8, +- 0x2d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc5, 0x7c, 0xb8, 0xa6, 0x08, 0xbb, 0x96, 0xaf, 0xc2, 0xba, +- 0xd9, 0x34, 0xba, 0x01, 0xb5, 0xc7, 0xfb, 0x87, 0xc3, 0x86, 0x92, 0x0e, 0x86, 0x2a, 0xc1, 0x81, +- 0x2b, 0xd1, 0xe8, 0x35, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0xb9, 0x97, 0x90, 0x30, 0xe6, 0xc3, +- 0xf5, 0x5b, 0x95, 0x9d, 0xa6, 0xdb, 0x31, 0xc8, 0x23, 0x89, 0x73, 0x3e, 0x81, 0x2b, 0x13, 0xe1, +- 0x33, 0xf1, 0x02, 0xde, 0x71, 0x9e, 0xc2, 0xb6, 0x8b, 0x23, 0x7a, 0xf6, 0x42, 0xae, 0x1d, 0xc2, +- 0xba, 0x20, 0x11, 0xa6, 0xa9, 0x50, 0xae, 0xed, 0xba, 0x16, 0x74, 0xfe, 0x50, 0x01, 0xf4, 0xe0, +- 0x1c, 0x07, 0x47, 0x8c, 0x06, 0x98, 0xf3, 0xff, 0xd0, 0x76, 0xbd, 0x09, 0xeb, 0x89, 0x56, 0x60, +- 0x58, 0x57, 0xe4, 0x66, 0x17, 0xac, 0x56, 0x76, 0xd6, 0xf9, 0x0a, 0xb6, 0x26, 0x64, 0x1a, 0xfb, +- 0xb3, 0x97, 0xa8, 0xef, 0x36, 0x34, 0xb8, 0xe2, 0xa9, 0x54, 0xed, 0xba, 0x06, 0x72, 0x8e, 0x00, +- 0x7d, 0xe9, 0x13, 0xf1, 0xf2, 0x24, 0x39, 0xef, 0xc0, 0x66, 0x89, 0x23, 0x4f, 0x68, 0xcc, 0xb1, +- 0x52, 0x40, 0xf8, 0x22, 0xe5, 0x8a, 0xd9, 0x9a, 0x6b, 0x20, 0x07, 0xc3, 0xd6, 0x17, 0x84, 0x5b, +- 0x72, 0xfc, 0xef, 0xa8, 0xb0, 0x0d, 0x8d, 0x67, 0x94, 0x45, 0xbe, 0xb0, 0x1a, 0x68, 0x08, 0x21, +- 0xa8, 0xfb, 0x6c, 0xca, 0x87, 0xb5, 0x5b, 0xb5, 0x9d, 0x96, 0xab, 0xc6, 0xf2, 0x54, 0x2e, 0x88, +- 0x31, 0x7a, 0xbd, 0x0a, 0x1d, 0xe3, 0x77, 0x6f, 0x46, 0xb8, 0x50, 0x72, 0x3a, 0x6e, 0xdb, 0xe0, +- 0xe4, 0x1a, 0x87, 0xc2, 0xf6, 0xd3, 0x24, 0x7c, 0xc1, 0x0b, 0x7f, 0x07, 0x5a, 0x0c, 0x73, 0x9a, +- 0x32, 0x79, 0x4d, 0xab, 0x6a, 0xdf, 0xb7, 0xf4, 0xbe, 0x7f, 0x41, 0xe2, 0xf4, 0xdc, 0xb5, 0x73, +- 0x6e, 0x4e, 0x66, 0xae, 0x90, 0xe0, 0x2f, 0x72, 0x85, 0x3e, 0x81, 0x2b, 0x47, 0x7e, 0xca, 0x5f, +- 0x44, 0x57, 0xe7, 0x53, 0x79, 0xfd, 0x78, 0x1a, 0xbd, 0xd0, 0xe2, 0xdf, 0x57, 0xa0, 0xb9, 0x9f, +- 0xa4, 0x4f, 0xb9, 0x3f, 0xc5, 0xe8, 0xbf, 0xa0, 0x2d, 0xa8, 0xf0, 0x67, 0x5e, 0x2a, 0x41, 0x45, +- 0x5e, 0x77, 0x41, 0xa1, 0x34, 0x81, 0x74, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8a, 0xea, 0xad, 0xda, +- 0x4e, 0xdd, 0x6d, 0x6b, 0x9c, 0x26, 0x19, 0xc3, 0xa6, 0x9a, 0xf3, 0x48, 0xec, 0x9d, 0x62, 0x16, +- 0xe3, 0x59, 0x44, 0x43, 0xac, 0xce, 0x6f, 0xdd, 0x1d, 0xa8, 0xa9, 0xc3, 0xf8, 0xf3, 0x6c, 0x02, +- 0xfd, 0x37, 0x0c, 0x32, 0x7a, 0x79, 0x29, 0x15, 0x75, 0x5d, 0x51, 0xf7, 0x0d, 0xf5, 0x53, 0x83, +- 0x76, 0x7e, 0x09, 0xbd, 0x27, 0x27, 0x8c, 0x0a, 0x31, 0x23, 0xf1, 0xf4, 0xbe, 0x2f, 0x7c, 0x19, +- 0x3d, 0x12, 0xcc, 0x08, 0x0d, 0xb9, 0xd1, 0xd6, 0x82, 0xe8, 0x6d, 0x18, 0x08, 0x4d, 0x8b, 0x43, +- 0xcf, 0xd2, 0x54, 0x15, 0xcd, 0x46, 0x36, 0x71, 0x64, 0x88, 0xdf, 0x80, 0x5e, 0x4e, 0x2c, 0xe3, +- 0x8f, 0xd1, 0xb7, 0x9b, 0x61, 0x9f, 0x90, 0x08, 0x3b, 0x67, 0xca, 0x57, 0x6a, 0x93, 0xd1, 0xdb, +- 0xd0, 0xca, 0xfd, 0x50, 0x51, 0x27, 0xa4, 0xa7, 0x4f, 0x88, 0x75, 0xa7, 0xdb, 0xcc, 0x9c, 0xf2, +- 0x19, 0xf4, 0x45, 0xa6, 0xb8, 0x17, 0xfa, 0xc2, 0x2f, 0x1f, 0xaa, 0xb2, 0x55, 0x6e, 0x4f, 0x94, +- 0x60, 0xe7, 0x53, 0x68, 0x1d, 0x91, 0x90, 0x6b, 0xc1, 0x43, 0x58, 0x0f, 0x52, 0xc6, 0x70, 0x2c, +- 0xac, 0xc9, 0x06, 0x44, 0x5b, 0xb0, 0x36, 0x23, 0x11, 0x11, 0xc6, 0x4c, 0x0d, 0x38, 0x14, 0xe0, +- 0x11, 0x8e, 0x28, 0x9b, 0x2b, 0x87, 0x6d, 0xc1, 0x5a, 0x71, 0x73, 0x35, 0x80, 0xae, 0x43, 0x2b, +- 0xf2, 0xcf, 0xb3, 0x4d, 0x95, 0x33, 0xcd, 0xc8, 0x3f, 0xd7, 0xca, 0x0f, 0x61, 0xfd, 0x99, 0x4f, +- 0x66, 0x41, 0x2c, 0x8c, 0x57, 0x2c, 0x98, 0x0b, 0xac, 0x17, 0x05, 0xfe, 0xb9, 0x0a, 0x6d, 0x2d, +- 0x51, 0x2b, 0xbc, 0x05, 0x6b, 0x81, 0x1f, 0x9c, 0x64, 0x22, 0x15, 0x80, 0x6e, 0x5b, 0x45, 0xaa, +- 0xc5, 0x20, 0x9c, 0x6b, 0x6a, 0x55, 0xdb, 0x05, 0xe0, 0xcf, 0xfd, 0xc4, 0xe8, 0x56, 0xbb, 0x80, +- 0xb8, 0x25, 0x69, 0xb4, 0xba, 0xef, 0x43, 0x47, 0x9f, 0x3b, 0xb3, 0xa4, 0x7e, 0xc1, 0x92, 0xb6, +- 0xa6, 0xd2, 0x8b, 0x5e, 0x83, 0x6e, 0xca, 0xb1, 0x77, 0x42, 0x30, 0xf3, 0x59, 0x70, 0x32, 0x1f, +- 0xae, 0xe9, 0x37, 0x32, 0xe5, 0xf8, 0xc0, 0xe2, 0xd0, 0x1d, 0x58, 0x93, 0xe1, 0x8f, 0x0f, 0x1b, +- 0xea, 0x39, 0xbe, 0x51, 0x64, 0xa9, 0x4c, 0x1d, 0xab, 0xef, 0x83, 0x58, 0xb0, 0xb9, 0xab, 0x49, +- 0x47, 0x1f, 0x01, 0xe4, 0x48, 0xb4, 0x01, 0xb5, 0x53, 0x3c, 0x37, 0xf7, 0x50, 0x0e, 0xa5, 0x73, +- 0xce, 0xfc, 0x59, 0x6a, 0xbd, 0xae, 0x81, 0x4f, 0xaa, 0x1f, 0x55, 0x9c, 0x00, 0xfa, 0x7b, 0xb3, +- 0x53, 0x42, 0x0b, 0xcb, 0xb7, 0x60, 0x2d, 0xf2, 0xbf, 0xa2, 0xcc, 0x7a, 0x52, 0x01, 0x0a, 0x4b, +- 0x62, 0xca, 0x2c, 0x0b, 0x05, 0xa0, 0x1e, 0x54, 0x69, 0xa2, 0xfc, 0xd5, 0x72, 0xab, 0x34, 0xc9, +- 0x05, 0xd5, 0x0b, 0x82, 0x9c, 0xbf, 0xd7, 0x01, 0x72, 0x29, 0xc8, 0x85, 0x11, 0xa1, 0x1e, 0xc7, +- 0x4c, 0xa6, 0x20, 0xde, 0xf1, 0x5c, 0x60, 0xee, 0x31, 0x1c, 0xa4, 0x8c, 0x93, 0x33, 0xb9, 0x7f, +- 0xd2, 0xec, 0x2b, 0xda, 0xec, 0x05, 0xdd, 0xdc, 0xab, 0x84, 0x4e, 0xf4, 0xba, 0x3d, 0xb9, 0xcc, +- 0xb5, 0xab, 0xd0, 0x21, 0x5c, 0xc9, 0x79, 0x86, 0x05, 0x76, 0xd5, 0xcb, 0xd8, 0x6d, 0x66, 0xec, +- 0xc2, 0x9c, 0xd5, 0x03, 0xd8, 0x24, 0xd4, 0xfb, 0x3a, 0xc5, 0x69, 0x89, 0x51, 0xed, 0x32, 0x46, +- 0x03, 0x42, 0x7f, 0xa4, 0x16, 0xe4, 0x6c, 0x8e, 0xe0, 0x5a, 0xc1, 0x4a, 0x79, 0xdd, 0x0b, 0xcc, +- 0xea, 0x97, 0x31, 0xdb, 0xce, 0xb4, 0x92, 0xf1, 0x20, 0xe7, 0xf8, 0x03, 0xd8, 0x26, 0xd4, 0x7b, +- 0xee, 0x13, 0xb1, 0xc8, 0x6e, 0xed, 0x7b, 0x8c, 0x94, 0x8f, 0x6e, 0x99, 0x97, 0x36, 0x32, 0xc2, +- 0x6c, 0x5a, 0x32, 0xb2, 0xf1, 0x3d, 0x46, 0x3e, 0x52, 0x0b, 0x72, 0x36, 0xf7, 0x60, 0x40, 0xe8, +- 0xa2, 0x36, 0xeb, 0x97, 0x31, 0xe9, 0x13, 0x5a, 0xd6, 0x64, 0x0f, 0x06, 0x1c, 0x07, 0x82, 0xb2, +- 0xe2, 0x21, 0x68, 0x5e, 0xc6, 0x62, 0xc3, 0xd0, 0x67, 0x3c, 0x9c, 0x9f, 0x41, 0xe7, 0x20, 0x9d, +- 0x62, 0x31, 0x3b, 0xce, 0x82, 0xc1, 0x4b, 0x8b, 0x3f, 0xce, 0x3f, 0xab, 0xd0, 0xde, 0x9f, 0x32, +- 0x9a, 0x26, 0xa5, 0x98, 0xac, 0x2f, 0xe9, 0x62, 0x4c, 0x56, 0x24, 0x2a, 0x26, 0x6b, 0xe2, 0x0f, +- 0xa0, 0x13, 0xa9, 0xab, 0x6b, 0xe8, 0x75, 0x1c, 0x1a, 0x2c, 0x5d, 0x6a, 0xb7, 0x1d, 0x15, 0x82, +- 0xd9, 0x18, 0x20, 0x21, 0x21, 0x37, 0x6b, 0x74, 0x38, 0xea, 0x9b, 0x8c, 0xd0, 0x86, 0x68, 0xb7, +- 0x95, 0x64, 0xd1, 0xfa, 0x3d, 0x68, 0x1f, 0x4b, 0x27, 0x99, 0x05, 0xa5, 0x60, 0x94, 0x7b, 0xcf, +- 0x85, 0xe3, 0xfc, 0x12, 0x1e, 0x40, 0xf7, 0x44, 0xbb, 0xcc, 0x2c, 0xd2, 0x67, 0xe8, 0x35, 0x63, +- 0x49, 0x6e, 0xef, 0xb8, 0xe8, 0x59, 0xbd, 0x01, 0x9d, 0x93, 0x02, 0x6a, 0x34, 0x81, 0xc1, 0x12, +- 0xc9, 0x8a, 0x18, 0xb4, 0x53, 0x8c, 0x41, 0xed, 0x3b, 0x48, 0x0b, 0x2a, 0xae, 0x2c, 0xc6, 0xa5, +- 0xdf, 0x54, 0xa1, 0xf3, 0x43, 0x2c, 0x9e, 0x53, 0x76, 0xaa, 0xf5, 0x45, 0x50, 0x8f, 0xfd, 0x08, +- 0x1b, 0x8e, 0x6a, 0x8c, 0xae, 0x41, 0x93, 0x9d, 0xeb, 0x00, 0x62, 0xf6, 0x73, 0x9d, 0x9d, 0xab, +- 0xc0, 0x80, 0x5e, 0x01, 0x60, 0xe7, 0x5e, 0xe2, 0x07, 0xa7, 0xd8, 0x78, 0xb0, 0xee, 0xb6, 0xd8, +- 0xf9, 0x91, 0x46, 0xc8, 0xa3, 0xc0, 0xce, 0x3d, 0xcc, 0x18, 0x65, 0xdc, 0xc4, 0xaa, 0x26, 0x3b, +- 0x7f, 0xa0, 0x60, 0xb3, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0x62, 0xb4, 0x5a, 0x7b, 0x5f, 0x23, +- 0xa4, 0x54, 0x61, 0xa5, 0x36, 0xb4, 0x54, 0x91, 0x4b, 0x15, 0xb9, 0xd4, 0x75, 0xbd, 0x52, 0x14, +- 0xa5, 0x8a, 0x4c, 0x6a, 0x53, 0x4b, 0x15, 0x05, 0xa9, 0x22, 0x97, 0xda, 0xb2, 0x6b, 0x8d, 0x54, +- 0xe7, 0xd7, 0x15, 0xd8, 0x5e, 0x4c, 0xfc, 0x4c, 0x9a, 0xfa, 0x01, 0x74, 0x02, 0xb5, 0x5f, 0xa5, +- 0x33, 0x39, 0x58, 0xda, 0x49, 0xb7, 0x1d, 0x14, 0x8e, 0xf1, 0x87, 0xd0, 0x8d, 0xb5, 0x83, 0xb3, +- 0xa3, 0x59, 0xcb, 0xf7, 0xa5, 0xe8, 0x7b, 0xb7, 0x13, 0x17, 0x20, 0x27, 0x04, 0xf4, 0x25, 0x23, +- 0x02, 0x4f, 0x04, 0xc3, 0x7e, 0xf4, 0x32, 0x0a, 0x10, 0x04, 0x75, 0x95, 0xad, 0xd4, 0x54, 0x7e, +- 0xad, 0xc6, 0xce, 0x9b, 0xb0, 0x59, 0x92, 0x62, 0x6c, 0xdd, 0x80, 0xda, 0x0c, 0xc7, 0x8a, 0x7b, +- 0xd7, 0x95, 0x43, 0xc7, 0x87, 0x81, 0x8b, 0xfd, 0xf0, 0xe5, 0x69, 0x63, 0x44, 0xd4, 0x72, 0x11, +- 0x3b, 0x80, 0x8a, 0x22, 0x8c, 0x2a, 0x56, 0xeb, 0x4a, 0x41, 0xeb, 0xc7, 0x30, 0xd8, 0x9f, 0x51, +- 0x8e, 0x27, 0x22, 0x24, 0xf1, 0xcb, 0xa8, 0x98, 0x7e, 0x01, 0x9b, 0x4f, 0xc4, 0xfc, 0x4b, 0xc9, +- 0x8c, 0x93, 0x6f, 0xf0, 0x4b, 0xb2, 0x8f, 0xd1, 0xe7, 0xd6, 0x3e, 0x46, 0x9f, 0xcb, 0x62, 0x29, +- 0xa0, 0xb3, 0x34, 0x8a, 0xd5, 0x55, 0xe8, 0xba, 0x06, 0x72, 0xf6, 0xa0, 0xa3, 0x73, 0xe8, 0x47, +- 0x34, 0x4c, 0x67, 0x78, 0xe5, 0x1d, 0xbc, 0x09, 0x90, 0xf8, 0xcc, 0x8f, 0xb0, 0xc0, 0x4c, 0x9f, +- 0xa1, 0x96, 0x5b, 0xc0, 0x38, 0xbf, 0xad, 0xc2, 0x96, 0x6e, 0x89, 0x4c, 0x74, 0x27, 0xc0, 0x9a, +- 0x30, 0x82, 0xe6, 0x09, 0xe5, 0xa2, 0xc0, 0x30, 0x83, 0xa5, 0x8a, 0x61, 0x6c, 0xb9, 0xc9, 0x61, +- 0xa9, 0x4f, 0x51, 0xbb, 0xbc, 0x4f, 0xb1, 0xd4, 0x89, 0xa8, 0x2f, 0x77, 0x22, 0xe4, 0x6d, 0xb3, +- 0x44, 0x44, 0xdf, 0xf1, 0x96, 0xdb, 0x32, 0x98, 0xc3, 0x10, 0xdd, 0x86, 0xfe, 0x54, 0x6a, 0xe9, +- 0x9d, 0x50, 0x7a, 0xea, 0x25, 0xbe, 0x38, 0x51, 0x57, 0xbd, 0xe5, 0x76, 0x15, 0xfa, 0x80, 0xd2, +- 0xd3, 0x23, 0x5f, 0x9c, 0xa0, 0x8f, 0xa1, 0x67, 0xd2, 0xc0, 0x48, 0xb9, 0x88, 0x9b, 0xc7, 0xcf, +- 0xdc, 0xa2, 0xa2, 0xf7, 0xdc, 0xee, 0x69, 0x01, 0xe2, 0xce, 0x55, 0xb8, 0x72, 0x1f, 0x73, 0xc1, +- 0xe8, 0xbc, 0xec, 0x18, 0xe7, 0xff, 0x00, 0x0e, 0x63, 0x81, 0xd9, 0x33, 0x3f, 0xc0, 0x1c, 0xbd, +- 0x5b, 0x84, 0x4c, 0x72, 0xb4, 0x31, 0xd6, 0x1d, 0xa9, 0x6c, 0xc2, 0x2d, 0xd0, 0x38, 0x63, 0x68, +- 0xb8, 0x34, 0x95, 0xe1, 0xe8, 0x75, 0x3b, 0x32, 0xeb, 0x3a, 0x66, 0x9d, 0x42, 0xba, 0x66, 0xce, +- 0x39, 0xb0, 0x25, 0x6c, 0xce, 0xce, 0x6c, 0xd1, 0x18, 0x5a, 0xc4, 0xe2, 0x4c, 0x54, 0x59, 0x16, +- 0x9d, 0x93, 0x38, 0x3f, 0x85, 0x4d, 0xcd, 0x49, 0x73, 0xb6, 0x6c, 0x5e, 0x87, 0x06, 0xb3, 0x6a, +- 0x54, 0xf2, 0x56, 0x94, 0x21, 0x32, 0x73, 0xe8, 0x86, 0x14, 0x16, 0x30, 0x1c, 0xc9, 0x9a, 0xa3, +- 0xaa, 0xb6, 0x2c, 0x47, 0x48, 0x6f, 0xc9, 0x7a, 0x3b, 0x37, 0xd3, 0x7a, 0x6b, 0x13, 0x06, 0x72, +- 0xa2, 0x24, 0xd1, 0xf9, 0x39, 0x6c, 0x3e, 0x8e, 0x67, 0x24, 0xc6, 0xfb, 0x47, 0x4f, 0x1f, 0xe1, +- 0x2c, 0x2a, 0x20, 0xa8, 0xcb, 0xec, 0x49, 0xa9, 0xd1, 0x74, 0xd5, 0x58, 0x5e, 0x93, 0xf8, 0xd8, +- 0x0b, 0x92, 0x94, 0x9b, 0xce, 0x50, 0x23, 0x3e, 0xde, 0x4f, 0x52, 0x2e, 0xc3, 0xbc, 0x7c, 0xe6, +- 0x69, 0x3c, 0x9b, 0xab, 0xbb, 0xd2, 0x74, 0xd7, 0x83, 0x24, 0x7d, 0x1c, 0xcf, 0xe6, 0xce, 0xff, +- 0xa8, 0x5a, 0x18, 0xe3, 0xd0, 0xf5, 0xe3, 0x90, 0x46, 0xf7, 0xf1, 0x59, 0x41, 0x42, 0x56, 0x77, +- 0xd9, 0x98, 0xf0, 0x6d, 0x05, 0x3a, 0xf7, 0xa6, 0x38, 0x16, 0xf7, 0xb1, 0xf0, 0xc9, 0x4c, 0xd5, +- 0x56, 0x67, 0x98, 0x71, 0x42, 0x63, 0x73, 0xf0, 0x2d, 0x28, 0x4b, 0x63, 0x12, 0x13, 0xe1, 0x85, +- 0x3e, 0x8e, 0x68, 0x6c, 0xbc, 0x00, 0x12, 0x75, 0x5f, 0x61, 0xd0, 0x9b, 0xd0, 0xd7, 0x9d, 0x3b, +- 0xef, 0xc4, 0x8f, 0xc3, 0x99, 0xbc, 0x72, 0xba, 0x93, 0xd1, 0xd3, 0xe8, 0x03, 0x83, 0x45, 0x6f, +- 0xc1, 0x86, 0xb9, 0x10, 0x39, 0x65, 0x5d, 0x51, 0xf6, 0x0d, 0xbe, 0x44, 0x9a, 0x26, 0x09, 0x65, +- 0x82, 0x7b, 0x1c, 0x07, 0x01, 0x8d, 0x12, 0x53, 0x98, 0xf4, 0x2d, 0x7e, 0xa2, 0xd1, 0xce, 0x14, +- 0x36, 0x1f, 0x4a, 0x3b, 0x8d, 0x25, 0xf9, 0x06, 0xf7, 0x22, 0x1c, 0x79, 0xc7, 0x33, 0x1a, 0x9c, +- 0x7a, 0x32, 0x4c, 0x19, 0x0f, 0xcb, 0xd4, 0x67, 0x4f, 0x22, 0x27, 0xe4, 0x1b, 0x55, 0x83, 0x4b, +- 0xaa, 0x13, 0x2a, 0x92, 0x59, 0x3a, 0xf5, 0x12, 0x46, 0x8f, 0xb1, 0x31, 0xb1, 0x1f, 0xe1, 0xe8, +- 0x40, 0xe3, 0x8f, 0x24, 0xda, 0xf9, 0x53, 0x05, 0xb6, 0xca, 0x92, 0x4c, 0xd0, 0xdd, 0x85, 0xad, +- 0xb2, 0x28, 0xf3, 0x10, 0xeb, 0x44, 0x6f, 0x50, 0x14, 0xa8, 0x9f, 0xe4, 0x0f, 0xa1, 0xab, 0xda, +- 0xb9, 0x5e, 0xa8, 0x39, 0x95, 0xd3, 0x8f, 0xe2, 0xbe, 0xb8, 0x1d, 0xbf, 0xb8, 0x4b, 0x1f, 0xc3, +- 0x35, 0x63, 0xbe, 0xb7, 0xac, 0xb6, 0x3e, 0x10, 0xdb, 0x86, 0xe0, 0xd1, 0x82, 0xf6, 0x5f, 0xc0, +- 0x30, 0x47, 0xed, 0xcd, 0x15, 0xd2, 0xfa, 0xea, 0x5d, 0xd8, 0x5c, 0x30, 0xf6, 0x5e, 0x18, 0x32, +- 0x75, 0x41, 0xeb, 0xee, 0xaa, 0x29, 0xe7, 0x2e, 0x5c, 0x9d, 0x60, 0xa1, 0xbd, 0xe1, 0x0b, 0x53, +- 0x13, 0x68, 0x66, 0x1b, 0x50, 0x9b, 0xe0, 0x40, 0x19, 0x5f, 0x73, 0xe5, 0x50, 0x1e, 0xc0, 0xa7, +- 0x1c, 0x07, 0xca, 0xca, 0x9a, 0xab, 0xc6, 0xce, 0x1f, 0x2b, 0xb0, 0x6e, 0xc2, 0xa4, 0x0c, 0xf5, +- 0x21, 0x23, 0x67, 0x98, 0x99, 0xa3, 0x67, 0x20, 0xf4, 0x06, 0xf4, 0xf4, 0xc8, 0xa3, 0x89, 0x20, +- 0x34, 0x0b, 0xbe, 0x5d, 0x8d, 0x7d, 0xac, 0x91, 0xaa, 0x53, 0xa7, 0x1a, 0x51, 0xa6, 0xe6, 0x33, +- 0x90, 0x6a, 0xb7, 0x71, 0x19, 0x19, 0x54, 0xb0, 0x6d, 0xb9, 0x06, 0x92, 0x47, 0xdd, 0xf2, 0x5b, +- 0x53, 0xfc, 0x2c, 0x28, 0x8f, 0x7a, 0x44, 0xd3, 0x58, 0x78, 0x09, 0x25, 0xb1, 0x30, 0xd1, 0x15, +- 0x14, 0xea, 0x48, 0x62, 0x9c, 0x5f, 0x55, 0xa0, 0xa1, 0xbb, 0xd5, 0xb2, 0xca, 0xcc, 0xde, 0xb8, +- 0x2a, 0x51, 0xf9, 0x82, 0x92, 0xa5, 0xdf, 0x35, 0x35, 0x96, 0xf7, 0xf8, 0x2c, 0xd2, 0x91, 0xda, +- 0xa8, 0x76, 0x16, 0xa9, 0x10, 0xfd, 0x06, 0xf4, 0xf2, 0xa7, 0x52, 0xcd, 0x6b, 0x15, 0xbb, 0x19, +- 0x56, 0x91, 0x5d, 0xa8, 0xa9, 0xf3, 0x13, 0x59, 0x5c, 0x67, 0x9d, 0xda, 0x0d, 0xa8, 0xa5, 0x99, +- 0x32, 0x72, 0x28, 0x31, 0xd3, 0xec, 0x91, 0x95, 0x43, 0x74, 0x1b, 0x7a, 0x7e, 0x18, 0x12, 0xb9, +- 0xdc, 0x9f, 0x3d, 0x24, 0x61, 0x76, 0x49, 0xcb, 0x58, 0xe7, 0x2f, 0x15, 0xe8, 0xef, 0xd3, 0x64, +- 0xfe, 0xff, 0x64, 0x86, 0x0b, 0x11, 0x44, 0x29, 0x69, 0xde, 0x58, 0x39, 0x96, 0x79, 0xe3, 0x33, +- 0x32, 0xc3, 0xfa, 0x6a, 0xe9, 0x9d, 0x6d, 0x4a, 0x84, 0xba, 0x56, 0x76, 0x32, 0x6b, 0x80, 0x75, +- 0xf5, 0xe4, 0x23, 0x1a, 0xaa, 0x0c, 0x39, 0x24, 0xcc, 0xcb, 0xda, 0x5d, 0x5d, 0x77, 0x3d, 0x24, +- 0x4c, 0x4d, 0x19, 0x43, 0xd6, 0x54, 0xc7, 0xb5, 0x68, 0x48, 0x43, 0x63, 0xa4, 0x21, 0xdb, 0xd0, +- 0xa0, 0xcf, 0x9e, 0x71, 0x2c, 0x54, 0x2e, 0x5b, 0x73, 0x0d, 0x94, 0x85, 0xb9, 0x66, 0x21, 0xcc, +- 0x5d, 0x81, 0x4d, 0xd5, 0xdb, 0x7f, 0xc2, 0xfc, 0x80, 0xc4, 0x53, 0x1b, 0x8a, 0xb7, 0x00, 0x4d, +- 0x04, 0x4d, 0x16, 0xb0, 0x63, 0x18, 0x98, 0x37, 0xe7, 0xe8, 0xc7, 0x13, 0x6b, 0xfa, 0x35, 0x68, +- 0x4a, 0xd0, 0x63, 0xf8, 0x6b, 0x1b, 0x18, 0xcd, 0xb4, 0xf3, 0x16, 0x74, 0xf4, 0xd0, 0x84, 0x81, +- 0x9c, 0x94, 0x97, 0x49, 0xf9, 0x9d, 0xbf, 0x6d, 0x98, 0x70, 0x6b, 0x6a, 0x68, 0xf4, 0x10, 0xfa, +- 0x0b, 0xff, 0x64, 0x90, 0x69, 0xaa, 0xac, 0xfe, 0x55, 0x33, 0xda, 0x1e, 0xeb, 0x7f, 0x3c, 0x63, +- 0xfb, 0x8f, 0x67, 0xfc, 0x20, 0x4a, 0xc4, 0x1c, 0x3d, 0x80, 0x5e, 0xf9, 0xef, 0x05, 0xba, 0x6e, +- 0x73, 0x90, 0x15, 0xff, 0x34, 0x2e, 0x64, 0xf3, 0x10, 0xfa, 0x0b, 0x3f, 0x32, 0xac, 0x3e, 0xab, +- 0xff, 0x6f, 0x5c, 0xc8, 0xe8, 0x2e, 0xb4, 0x0b, 0x7f, 0x2e, 0xd0, 0x50, 0x33, 0x59, 0xfe, 0x99, +- 0x71, 0x21, 0x83, 0x7d, 0xe8, 0x96, 0x7e, 0x26, 0xa0, 0x91, 0xb1, 0x67, 0xc5, 0x1f, 0x86, 0x0b, +- 0x99, 0xec, 0x41, 0xbb, 0xd0, 0xd3, 0xb7, 0x5a, 0x2c, 0xff, 0x38, 0x18, 0x5d, 0x5b, 0x31, 0x63, +- 0xb6, 0xf3, 0x00, 0xba, 0xa5, 0x0e, 0xbc, 0x55, 0x64, 0x55, 0xf7, 0x7f, 0x74, 0x7d, 0xe5, 0x9c, +- 0xe1, 0xf4, 0x10, 0xfa, 0x0b, 0xfd, 0x78, 0xeb, 0xdc, 0xd5, 0x6d, 0xfa, 0x0b, 0xcd, 0xfa, 0x5c, +- 0x6d, 0x76, 0xa1, 0xdc, 0x2a, 0x6c, 0xf6, 0x72, 0xf7, 0x7d, 0x74, 0x63, 0xf5, 0xa4, 0xd1, 0xea, +- 0x01, 0xf4, 0xca, 0x8d, 0x77, 0xcb, 0x6c, 0x65, 0x3b, 0xfe, 0xf2, 0x93, 0x53, 0xea, 0xc1, 0xe7, +- 0x27, 0x67, 0x55, 0x6b, 0xfe, 0x42, 0x46, 0xf7, 0x00, 0x4c, 0x71, 0x15, 0x92, 0x38, 0xdb, 0xb2, +- 0xa5, 0xa2, 0x2e, 0xdb, 0xb2, 0x15, 0x85, 0xd8, 0x5d, 0x00, 0x5d, 0x13, 0x85, 0x34, 0x15, 0xe8, +- 0xaa, 0x55, 0x63, 0xa1, 0x10, 0x1b, 0x0d, 0x97, 0x27, 0x96, 0x18, 0x60, 0xc6, 0x5e, 0x84, 0xc1, +- 0x67, 0x00, 0x79, 0xad, 0x65, 0x19, 0x2c, 0x55, 0x5f, 0x97, 0xf8, 0xa0, 0x53, 0xac, 0xac, 0x90, +- 0xb1, 0x75, 0x45, 0xb5, 0x75, 0x09, 0x8b, 0xfe, 0x42, 0xe6, 0x5c, 0x3e, 0x6c, 0x8b, 0x09, 0xf5, +- 0x68, 0x29, 0x7b, 0x46, 0x1f, 0x42, 0xa7, 0x98, 0x32, 0x5b, 0x2d, 0x56, 0xa4, 0xd1, 0xa3, 0x52, +- 0xda, 0x8c, 0xee, 0x42, 0xaf, 0x9c, 0x10, 0xa3, 0xc2, 0xbd, 0x58, 0x4a, 0x93, 0x47, 0xa6, 0x19, +- 0x54, 0x20, 0x7f, 0x1f, 0x20, 0x4f, 0x9c, 0xad, 0xfb, 0x96, 0x52, 0xe9, 0x05, 0xa9, 0x9f, 0x41, +- 0xaf, 0x10, 0xb7, 0x65, 0x4d, 0x78, 0xb5, 0x64, 0x70, 0x1e, 0xcd, 0x47, 0x26, 0xc3, 0x2a, 0x85, +- 0xed, 0x7b, 0xd0, 0x29, 0xbe, 0x11, 0xd6, 0xda, 0x15, 0xef, 0xc6, 0x65, 0x41, 0xaf, 0xf0, 0x9e, +- 0xd8, 0xb3, 0xbb, 0xfc, 0xc4, 0x5c, 0x16, 0xf4, 0x4a, 0xf5, 0xa8, 0x8d, 0x35, 0xab, 0x8a, 0xd4, +- 0xcb, 0x9e, 0x82, 0x72, 0xf1, 0x66, 0xbd, 0xbf, 0xb2, 0xa4, 0xbb, 0xec, 0x0c, 0x16, 0xeb, 0x14, +- 0xeb, 0x8f, 0x15, 0xb5, 0xcb, 0xf7, 0xc4, 0x84, 0x62, 0x2d, 0x52, 0x88, 0x09, 0x2b, 0x4a, 0x94, +- 0x0b, 0x19, 0x1d, 0x40, 0xff, 0xa1, 0x4d, 0x33, 0x4d, 0x0a, 0x6c, 0xd4, 0x59, 0x91, 0xf2, 0x8f, +- 0x46, 0xab, 0xa6, 0xcc, 0x2e, 0x7f, 0x0e, 0x83, 0xa5, 0xf4, 0x17, 0xdd, 0xcc, 0x5a, 0x9e, 0x2b, +- 0xf3, 0xe2, 0x0b, 0xd5, 0x3a, 0x84, 0x8d, 0xc5, 0xec, 0x17, 0xbd, 0x62, 0x36, 0x7d, 0x75, 0x56, +- 0x7c, 0x21, 0xab, 0x8f, 0xa1, 0x69, 0xb3, 0x2d, 0x64, 0x5a, 0xcb, 0x0b, 0xd9, 0xd7, 0x45, 0x4b, +- 0xf7, 0x3a, 0xdf, 0x7e, 0x77, 0xb3, 0xf2, 0xd7, 0xef, 0x6e, 0x56, 0xfe, 0xf1, 0xdd, 0xcd, 0xca, +- 0x71, 0x43, 0xcd, 0xbe, 0xff, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x45, 0xa8, 0x91, 0xab, 0x62, +- 0x22, 0x00, 0x00, ++ // 3183 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, ++ 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, ++ 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, ++ 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, ++ 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, ++ 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, ++ 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, ++ 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, ++ 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, ++ 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, ++ 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, ++ 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, ++ 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, ++ 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, ++ 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, ++ 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, ++ 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, ++ 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, ++ 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, ++ 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, ++ 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, ++ 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, ++ 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, ++ 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, ++ 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, ++ 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, ++ 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, ++ 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, ++ 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, ++ 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, ++ 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, ++ 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, ++ 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, ++ 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, ++ 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, ++ 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, ++ 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, ++ 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, ++ 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, ++ 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, ++ 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, ++ 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, ++ 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, ++ 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, ++ 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, ++ 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, ++ 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, ++ 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, ++ 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, ++ 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, ++ 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, ++ 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, ++ 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, ++ 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, ++ 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, ++ 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, ++ 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, ++ 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, ++ 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, ++ 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, ++ 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, ++ 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, ++ 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, ++ 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, ++ 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, ++ 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, ++ 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, ++ 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, ++ 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, ++ 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, ++ 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, ++ 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, ++ 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, ++ 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, ++ 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, ++ 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, ++ 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, ++ 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, ++ 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, ++ 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, ++ 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, ++ 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, ++ 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, ++ 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, ++ 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, ++ 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, ++ 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, ++ 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, ++ 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, ++ 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, ++ 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, ++ 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, ++ 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, ++ 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, ++ 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, ++ 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, ++ 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, ++ 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, ++ 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, ++ 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, ++ 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, ++ 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, ++ 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, ++ 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, ++ 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, ++ 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, ++ 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, ++ 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, ++ 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, ++ 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, ++ 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, ++ 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, ++ 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, ++ 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, ++ 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, ++ 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, ++ 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, ++ 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, ++ 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, ++ 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, ++ 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, ++ 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, ++ 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, ++ 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, ++ 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, ++ 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, ++ 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, ++ 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, ++ 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, ++ 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, ++ 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, ++ 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, ++ 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, ++ 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, ++ 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, ++ 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, ++ 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, ++ 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, ++ 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, ++ 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, ++ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, ++ 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, ++ 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, ++ 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, ++ 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, ++ 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, ++ 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, ++ 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, ++ 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, ++ 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, ++ 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, ++ 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, ++ 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, ++ 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, ++ 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, ++ 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, ++ 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, ++ 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, ++ 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, ++ 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, ++ 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, ++ 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, ++ 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, ++ 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, ++ 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, ++ 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, ++ 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, ++ 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, ++ 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, ++ 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, ++ 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, ++ 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, ++ 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, ++ 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, ++ 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, ++ 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, ++ 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, ++ 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, ++ 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, ++ 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, ++ 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, ++ 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, ++ 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, ++ 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, ++ 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, ++ 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, ++ 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, ++ 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, ++ 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, ++ 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, ++ 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, ++ 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, ++ 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, ++ 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, ++ 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, ++ 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, ++ 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, ++ 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, ++ 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, + } +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 1b89f6ac..29e17bab 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -195,25 +195,82 @@ type CgroupStats struct { + HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"` + } + +-// NetworkStats describe all network stats. +-type NetworkStats struct { +- // Name is the name of the network interface. +- Name string `json:"name,omitempty"` +- +- RxBytes uint64 `json:"rx_bytes,omitempty"` +- RxPackets uint64 `json:"rx_packets,omitempty"` +- RxErrors uint64 `json:"rx_errors,omitempty"` +- RxDropped uint64 `json:"rx_dropped,omitempty"` +- TxBytes uint64 `json:"tx_bytes,omitempty"` +- TxPackets uint64 `json:"tx_packets,omitempty"` +- TxErrors uint64 `json:"tx_errors,omitempty"` +- TxDropped uint64 `json:"tx_dropped,omitempty"` +-} +- + // ContainerStats describes a container stats. + type ContainerStats struct { + CgroupStats *CgroupStats +- NetworkStats []*NetworkStats ++ NetworkStats *NetworkStats ++} ++ ++type InterfaceStats struct { ++ // The name of the interface. ++ Name string `json:"name"` ++ // Cumulative count of bytes received. ++ RxBytes uint64 `json:"rx_bytes"` ++ // Cumulative count of packets received. ++ RxPackets uint64 `json:"rx_packets"` ++ // Cumulative count of receive errors encountered. ++ RxErrors uint64 `json:"rx_errors"` ++ // Cumulative count of packets dropped while receiving. ++ RxDropped uint64 `json:"rx_dropped"` ++ // Cumulative count of bytes transmitted. ++ TxBytes uint64 `json:"tx_bytes"` ++ // Cumulative count of packets transmitted. ++ TxPackets uint64 `json:"tx_packets"` ++ // Cumulative count of transmit errors encountered. ++ TxErrors uint64 `json:"tx_errors"` ++ // Cumulative count of packets dropped while transmitting. ++ TxDropped uint64 `json:"tx_dropped"` ++} ++ ++type NetworkStats struct { ++ Interfaces []InterfaceStats `json:"interfaces,omitempty"` ++ // TCP connection stats (Established, Listen...) ++ Tcp TcpStat `json:"tcp,omitempty"` ++ // TCP6 connection stats (Established, Listen...) ++ Tcp6 TcpStat `json:"tcp6,omitempty"` ++ // UDP connection stats ++ Udp UdpStat `json:"udp,omitempty"` ++ // UDP6 connection stats ++ Udp6 UdpStat `json:"udp6,omitempty"` ++} ++ ++type TcpStat struct { ++ // Count of TCP connections in state "Established" ++ Established uint64 `json:"established,omitempty"` ++ // Count of TCP connections in state "Syn_Sent" ++ SynSent uint64 `json:"syn_sent,omitempty"` ++ // Count of TCP connections in state "Syn_Recv" ++ SynRecv uint64 `json:"syn_recv,omitempty"` ++ // Count of TCP connections in state "Fin_Wait1" ++ FinWait1 uint64 `json:"fin_wait1,omitempty"` ++ // Count of TCP connections in state "Fin_Wait2" ++ FinWait2 uint64 `json:"fin_wait2,omitempty"` ++ // Count of TCP connections in state "Time_Wait ++ TimeWait uint64 `json:"time_wait,omitempty"` ++ // Count of TCP connections in state "Close" ++ Close uint64 `json:"close,omitempty"` ++ // Count of TCP connections in state "Close_Wait" ++ CloseWait uint64 `json:"close_wait,omitempty"` ++ // Count of TCP connections in state "Listen_Ack" ++ LastAck uint64 `json:"last_ack,omitempty"` ++ // Count of TCP connections in state "Listen" ++ Listen uint64 `json:"listen,omitempty"` ++ // Count of TCP connections in state "Closing" ++ Closing uint64 `json:"closing,omitempty"` ++} ++ ++type UdpStat struct { ++ // Count of UDP sockets in state "Listen" ++ Listen uint64 `json:"listen,omitempty"` ++ ++ // Count of UDP packets dropped by the IP stack ++ Dropped uint64 `json:"dropped,omitempty"` ++ ++ // Count of packets Queued for Receieve ++ RxQueued uint64 `json:"rx_queued,omitempty"` ++ ++ // Count of packets Queued for Transmit ++ TxQueued uint64 `json:"tx_queued,omitempty"` + } + + // ContainerResources describes container resources +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 38e9a204..0f03c9d9 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -1894,18 +1894,30 @@ func (k *kataAgent) statsContainer(sandbox *Sandbox, c Container) (*ContainerSta + return nil, fmt.Errorf("irregular response container stats") + } + +- data, err := json.Marshal(stats.CgroupStats) ++ cgroupData, err := json.Marshal(stats.CgroupStats) + if err != nil { + return nil, err + } + + var cgroupStats CgroupStats +- err = json.Unmarshal(data, &cgroupStats) ++ err = json.Unmarshal(cgroupData, &cgroupStats) ++ if err != nil { ++ return nil, err ++ } ++ ++ networkData, err := json.Marshal(stats.NetworkStats) ++ if err != nil { ++ return nil, err ++ } ++ ++ var networkStats NetworkStats ++ err = json.Unmarshal(networkData, &networkStats) + if err != nil { + return nil, err + } + containerStats := &ContainerStats{ +- CgroupStats: &cgroupStats, ++ CgroupStats: &cgroupStats, ++ NetworkStats: &networkStats, + } + return containerStats, nil + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0048-console-fix-the-file-resource-leak.patch b/runtime/patches/0048-console-fix-the-file-resource-leak.patch new file mode 100644 index 0000000000000000000000000000000000000000..8cc751c40ca35b7b4bf77812e887800497e1c643 --- /dev/null +++ b/runtime/patches/0048-console-fix-the-file-resource-leak.patch @@ -0,0 +1,34 @@ +From d3b8e21829bd671a1717fed8ab645ad2d447899a Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 22:18:05 +0800 +Subject: [PATCH 48/50] console: fix the file resource leak + +reason: newConsole will open a file, if error occurs in the +middle proccess, the file resource may leak, use defer close +to prevent it + +Signed-off-by: yangfeiyu +--- + cli/console.go | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/cli/console.go b/cli/console.go +index c1555c9f..97dfee99 100644 +--- a/cli/console.go ++++ b/cli/console.go +@@ -47,6 +47,12 @@ func newConsole() (*Console, error) { + if err != nil { + return nil, err + } ++ defer func() { ++ if err != nil { ++ master.Close() ++ } ++ }() ++ + if err := saneTerminal(master); err != nil { + return nil, err + } +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch b/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch new file mode 100644 index 0000000000000000000000000000000000000000..4d8892e5215a9c18cd58201374f240dd282a06b8 --- /dev/null +++ b/runtime/patches/0049-container-fix-the-write-operation-transparently-tran.patch @@ -0,0 +1,58 @@ +From a6fab7014922d85b1105b44fdbb98239b22d3e00 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 19 Aug 2020 22:31:57 +0800 +Subject: [PATCH 49/50] container: fix the write operation transparently + transmitted to the host + +reason:fix the write operation transparently transmitted to the host +when we fullfill the "/etc/hosts","/etc/resolv.conf","/etc/hostname" file in the container, +for example: +```bash +$ docker exec -ti 63 bash + +Signed-off-by: yangfeiyu +--- + virtcontainers/container.go | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 1b89f6ac..6edcb3f2 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -51,6 +51,12 @@ var cdromMajors = map[int64]string{ + 32: "CM206_CDROM_MAJOR", + } + ++var safeCopyFiles = map[string]struct{}{ ++ "resolv.conf": {}, ++ "hostname": {}, ++ "hosts": {}, ++} ++ + // https://github.com/torvalds/linux/blob/master/include/uapi/linux/major.h + // #define FLOPPY_MAJOR 2 + const floppyMajor = int64(2) +@@ -452,12 +458,18 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s + } + + filename := fmt.Sprintf("%s-%s-%s", c.id, hex.EncodeToString(randBytes), filepath.Base(m.Destination)) +- guestDest := filepath.Join(guestSharedDir, filename) ++ var guestDest string ++ _, needCopy := safeCopyFiles[filepath.Base(m.Destination)] ++ if needCopy { ++ guestDest = filepath.Join(kataGuestStorageDir, filename) ++ } else { ++ guestDest = filepath.Join(guestSharedDir, filename) ++ } + + // copy file to contaier's rootfs if filesystem sharing is not supported, otherwise + // bind mount it in the shared directory. + caps := c.sandbox.hypervisor.capabilities() +- if !caps.IsFsSharingSupported() { ++ if !caps.IsFsSharingSupported() || needCopy { + c.Logger().Debug("filesystem sharing is not supported, files will be copied") + + fileInfo, err := os.Stat(m.Source) +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch b/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch new file mode 100644 index 0000000000000000000000000000000000000000..cf16d258aecc09e2c7d9e47adb890117265b04c2 --- /dev/null +++ b/runtime/patches/0050-runtime-add-kata-network-upate-iface-subcommand.patch @@ -0,0 +1,358 @@ +From f943e5d91f6922c599b46727e2dfd5e8e7e5bea8 Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Wed, 19 Aug 2020 22:43:54 +0800 +Subject: [PATCH 50/50] runtime: add kata-network upate-iface subcommand + +reason: add kata-network update-iface subcommand to +update the exist interface in the Kata VM. + +Signed-off-by: jiangpengfei +--- + cli/network.go | 37 +++++++++- + virtcontainers/api.go | 25 +++++-- + virtcontainers/implementation.go | 5 ++ + virtcontainers/interfaces.go | 1 + + virtcontainers/pkg/vcmock/mock.go | 8 +++ + virtcontainers/pkg/vcmock/types.go | 1 + + virtcontainers/sandbox.go | 137 +++++++++++++++++++++++++++++++++++++ + 7 files changed, 207 insertions(+), 7 deletions(-) + +diff --git a/cli/network.go b/cli/network.go +index 3dd0971e..7dce0528 100644 +--- a/cli/network.go ++++ b/cli/network.go +@@ -65,8 +65,9 @@ var addIfaceCommand = cli.Command{ + "hwAddr":"", + "linkType":"tap", + "vhostUserSocket":"" ++ "queues": + } +- device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional. ++ device,name,mtu,hwAddr are required, IPAddresses, queues and vhostUserSocket are optional. + `, + Flags: []cli.Flag{}, + Action: func(context *cli.Context) error { +@@ -104,6 +105,34 @@ var delIfaceCommand = cli.Command{ + }, + } + ++var updateIfaceCommand = cli.Command{ ++ Name: "update-iface", ++ Usage: "update the interface in container", ++ ArgsUsage: `update-iface file or - for stdin ++ file or stdin for example: ++ { ++ "device":"", ++ "name":"", ++ "IPAddresses":[{"address":"","mask":""}], ++ "mtu":0, ++ "hwAddr":"", ++ "linkType":"", ++ "vhostUserSocket":"", ++ "queues":0 ++ } ++ name is required,IPAddresses is optional,device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty. ++ `, ++ Flags: []cli.Flag{}, ++ Action: func(context *cli.Context) error { ++ ctx, err := cliContextToContext(context) ++ if err != nil { ++ return err ++ } ++ ++ return networkModifyCommand(ctx, context.Args().First(), context.Args().Get(1), interfaceType, vcTypes.NetworkOpUpdate) ++ }, ++} ++ + var listIfacesCommand = cli.Command{ + Name: "list-ifaces", + Usage: "list network interfaces in a container", +@@ -262,6 +291,12 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType + kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). + WithError(err).Error("delete interface failed") + } ++ case vcTypes.NetworkOpUpdate: ++ resultingInf, err = vci.UpdateInterface(ctx, sandboxID, inf) ++ if err != nil { ++ kataLog.WithField("resulting-interface", fmt.Sprintf("%+v", resultingInf)). ++ WithError(err).Error("update interface failed") ++ } + } + + json.NewEncoder(output).Encode(resultingInf) +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 7036df8c..ca5412a9 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -889,7 +889,7 @@ func AddDevice(ctx context.Context, sandboxID string, info deviceConfig.DeviceIn + return s.AddDevice(info) + } + +-func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, add bool) (*vcTypes.Interface, error) { ++func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface, op vcTypes.NetworkOp) (*vcTypes.Interface, error) { + if sandboxID == "" { + return nil, vcTypes.ErrNeedSandboxID + } +@@ -906,11 +906,16 @@ func toggleInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa + } + defer s.releaseStatelessSandbox() + +- if add { ++ switch op { ++ case vcTypes.NetworkOpAdd: + return s.AddInterface(inf) ++ case vcTypes.NetworkOpRemove: ++ return s.RemoveInterface(inf) ++ case vcTypes.NetworkOpUpdate: ++ return s.UpdateInterface(inf) ++ default: ++ return nil, fmt.Errorf("operation is not found") + } +- +- return s.RemoveInterface(inf) + } + + // AddInterface is the virtcontainers add interface entry point. +@@ -918,7 +923,7 @@ func AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) + span, ctx := trace(ctx, "AddInterface") + defer span.Finish() + +- return toggleInterface(ctx, sandboxID, inf, true) ++ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpAdd) + } + + // RemoveInterface is the virtcontainers remove interface entry point. +@@ -926,7 +931,15 @@ func RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interfa + span, ctx := trace(ctx, "RemoveInterface") + defer span.Finish() + +- return toggleInterface(ctx, sandboxID, inf, false) ++ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpRemove) ++} ++ ++// UpdateInterface updates interface entry point in the virtcontainers. ++func UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++ span, ctx := trace(ctx, "UpdateInterface") ++ defer span.Finish() ++ ++ return toggleInterface(ctx, sandboxID, inf, vcTypes.NetworkOpUpdate) + } + + // ListInterfaces is the virtcontainers list interfaces entry point. +diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go +index 0265d1ed..e4bc4ae6 100644 +--- a/virtcontainers/implementation.go ++++ b/virtcontainers/implementation.go +@@ -163,6 +163,11 @@ func (impl *VCImpl) ListInterfaces(ctx context.Context, sandboxID string) ([]*vc + return ListInterfaces(ctx, sandboxID) + } + ++// ListInterfaces implements the VC function of the same name. ++func (impl *VCImpl) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++ return UpdateInterface(ctx, sandboxID, inf) ++} ++ + // UpdateRoutes implements the VC function of the same name. + func (impl *VCImpl) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + return UpdateRoutes(ctx, sandboxID, routes, op) +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index d8f0be69..0fd12d84 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -51,6 +51,7 @@ type VC interface { + + AddInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + RemoveInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) ++ UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) + UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) + ListRoutes(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) +diff --git a/virtcontainers/pkg/vcmock/mock.go b/virtcontainers/pkg/vcmock/mock.go +index aa9bae9a..d3bf201f 100644 +--- a/virtcontainers/pkg/vcmock/mock.go ++++ b/virtcontainers/pkg/vcmock/mock.go +@@ -273,6 +273,14 @@ func (m *VCMock) ListInterfaces(ctx context.Context, sandboxID string) ([]*vcTyp + return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) + } + ++// UpdateInterface implements the VC function of the same name ++func (m *VCMock) UpdateInterface(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++ if m.UpdateRoutesFunc != nil { ++ return m.UpdateInterfaceFunc(ctx, sandboxID, inf) ++ } ++ return nil, fmt.Errorf("%s: %s (%+v): sandboxID: %v", mockErrorPrefix, getSelf(), m, sandboxID) ++} ++ + // UpdateRoutes implements the VC function of the same name. + func (m *VCMock) UpdateRoutes(ctx context.Context, sandboxID string, routes []*vcTypes.Route, op vcTypes.NetworkOp) ([]*vcTypes.Route, error) { + if m.UpdateRoutesFunc != nil { +diff --git a/virtcontainers/pkg/vcmock/types.go b/virtcontainers/pkg/vcmock/types.go +index 610b4602..152e66b7 100644 +--- a/virtcontainers/pkg/vcmock/types.go ++++ b/virtcontainers/pkg/vcmock/types.go +@@ -73,6 +73,7 @@ type VCMock struct { + AddInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + RemoveInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + ListInterfacesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Interface, error) ++ UpdateInterfaceFunc func(ctx context.Context, sandboxID string, inf *vcTypes.Interface) (*vcTypes.Interface, error) + UpdateRoutesFunc func(ctx context.Context, sandboxID string, routes []*vcTypes.Route) ([]*vcTypes.Route, error) + ListRoutesFunc func(ctx context.Context, sandboxID string) ([]*vcTypes.Route, error) + CleanupContainerFunc func(ctx context.Context, sandboxID, containerID string, force bool) error +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index d8ab6c1a..174e6cb6 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -1031,6 +1031,54 @@ func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, e + return nil, nil + } + ++// UpdateInterface updates the nic. ++func (s *Sandbox) UpdateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++ err := checkUpdateInterfaceInfo(inf) ++ if err != nil { ++ return nil, err ++ } ++ ++ nicInGuest := false ++ ++ endpoint := s.getEndpointByName(inf.Name) ++ // nic is not in local endpoints ++ if endpoint == nil { ++ interfaceInGuestInf := s.getInterfaceInfByName(inf.Name) ++ if interfaceInGuestInf == nil { ++ return nil, fmt.Errorf("can not find the network interface") ++ } ++ ++ nicInGuest = true ++ endpoint, err = s.generateEndpointForStore(interfaceInGuestInf) ++ if err != nil { ++ return nil, err ++ } ++ } ++ ++ // update endpoint ip ++ err = s.updateEndpointIP(endpoint, inf) ++ if err != nil { ++ return nil, err ++ } ++ ++ s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Update endpoint") ++ grpcIf, err := s.agent.updateInterface(inf) ++ if err != nil { ++ return nil, err ++ } ++ ++ if nicInGuest { ++ s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint) ++ } ++ ++ if err = s.Save(); err != nil { ++ s.Logger().WithError(err).Error("failed to store the network information") ++ return nil, err ++ } ++ ++ return grpcIf, nil ++} ++ + // ListInterfaces lists all nics and their configurations in the sandbox. + func (s *Sandbox) ListInterfaces() ([]*vcTypes.Interface, error) { + return s.agent.listInterfaces() +@@ -2431,6 +2479,95 @@ func (s *Sandbox) IsCompatOldCNI() bool { + return s.config.NetworkConfig.EnableCompatOldCNI + } + ++func checkUpdateInterfaceInfo(inf *vcTypes.Interface) error { ++ if inf.Mtu > 0 || inf.Queues > 0 || inf.Device != "" || inf.VhostUserSocket != "" || inf.LinkType != "" || inf.HwAddr != "" { ++ return fmt.Errorf("device,mtu,hwAddr,linkType,queues and vhostUserSocket should be 0 or empty") ++ } ++ ++ return nil ++} ++ ++// getEndpointByName returns the endpoint by name. ++func (s *Sandbox) getEndpointByName(name string) Endpoint { ++ if name == "" { ++ return nil ++ } ++ ++ for i, endpoint := range s.networkNS.Endpoints { ++ if endpoint.Name() == name { ++ return s.networkNS.Endpoints[i] ++ } ++ } ++ ++ return nil ++} ++ ++// getInterfaceInfByName returns the interface information. ++func (s *Sandbox) getInterfaceInfByName(name string) *vcTypes.Interface { ++ if name == "" { ++ return nil ++ } ++ ++ nics, err := s.agent.listInterfaces() ++ if err != nil { ++ return nil ++ } ++ ++ for _, i := range nics { ++ if i.Name == name { ++ return i ++ } ++ } ++ ++ return nil ++} ++ ++// generateEndpointForStore only generates endpoint information for store,it does not create a endpoint ++func (s *Sandbox) generateEndpointForStore(inf *vcTypes.Interface) (*TapEndpoint, error) { ++ netInfo, err := s.generateNetInfo(inf) ++ if err != nil { ++ return nil, err ++ } ++ ++ endpoint := &TapEndpoint{ ++ TapInterface: TapInterface{ ++ Name: inf.Name, ++ TAPIface: NetworkInterface{ ++ HardAddr: inf.HwAddr, ++ }, ++ }, ++ EndpointType: TapEndpointType, ++ } ++ ++ endpoint.SetProperties(netInfo) ++ ++ return endpoint, nil ++} ++ ++func (s *Sandbox) updateEndpointIP(endpoint Endpoint, inf *vcTypes.Interface) error { ++ if endpoint == nil { ++ return fmt.Errorf("endpoint is nil") ++ } ++ ++ netInfo := endpoint.Properties() ++ ++ var addrs []netlink.Addr ++ for _, addr := range inf.IPAddresses { ++ netlinkAddrStr := fmt.Sprintf("%s/%s", addr.Address, addr.Mask) ++ netlinkAddr, err := netlink.ParseAddr(netlinkAddrStr) ++ if err != nil { ++ return fmt.Errorf("could not parse %q: %v", netlinkAddrStr, err) ++ } ++ ++ addrs = append(addrs, *netlinkAddr) ++ } ++ ++ netInfo.Addrs = addrs ++ endpoint.SetProperties(netInfo) ++ ++ return nil ++} ++ + // updateStaticSandboxResources update sandbox's cpu and memory resource passed by + // sandbox_cpu and sandbox_mem annotations + func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { +-- +2.14.3 (Apple Git-98) + diff --git a/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch b/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch new file mode 100644 index 0000000000000000000000000000000000000000..385a37639b4ba49060ac60a78452724a825230af --- /dev/null +++ b/runtime/patches/0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch @@ -0,0 +1,78 @@ +From be0d60f5fa88267afb26125681a217ef9476e133 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Sat, 19 Sep 2020 12:47:14 +0800 +Subject: [PATCH] runtime: fix del-iface doesn't delete the tap interface in + the host problem + +reason: kata-runtime add-iface support add exist tap device in the host or +new created tap device by kata-network add-iface process. If add interface +is exist tap device, del-iface will not delete the tap device, so we need +to judge the deleted interface is exist device or new created, which is judged +by the VMFds of the tap Link and del-iface will load this info from the +network.json in 1.7.0 version. However, 1.11.1 version don't write the VMFds +into the persiste.json file, del-iface can not figure out which tap is exist +already or new created, so cause the this problem. +--- + virtcontainers/endpoint.go | 2 ++ + virtcontainers/persist/api/network.go | 5 +++++ + virtcontainers/tap_endpoint.go | 1 + + 3 files changed, 8 insertions(+) + +diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go +index 7efcf49c..3618792e 100644 +--- a/virtcontainers/endpoint.go ++++ b/virtcontainers/endpoint.go +@@ -129,6 +129,7 @@ func saveTapIf(tapif *TapInterface) *persistapi.TapInterface { + HardAddr: tapif.TAPIface.HardAddr, + Addrs: tapif.TAPIface.Addrs, + }, ++ VMFds: tapif.VMFds, + } + } + +@@ -167,6 +168,7 @@ func loadTapIf(tapif *persistapi.TapInterface) *TapInterface { + HardAddr: tapif.TAPIface.HardAddr, + Addrs: tapif.TAPIface.Addrs, + }, ++ VMFds: tapif.VMFds, + } + } + +diff --git a/virtcontainers/persist/api/network.go b/virtcontainers/persist/api/network.go +index 53c6de44..c5611767 100644 +--- a/virtcontainers/persist/api/network.go ++++ b/virtcontainers/persist/api/network.go +@@ -7,6 +7,8 @@ + package persistapi + + import ( ++ "os" ++ + "github.com/vishvananda/netlink" + ) + +@@ -45,6 +47,9 @@ type TapInterface struct { + Name string + TAPIface NetworkInterface + // remove VMFds and VhostFds ++ // add VMFds back to judge a tap interface is exist before add-iface ++ // or new created by kata-network add-iface ++ VMFds []*os.File + } + + // TuntapInterface defines a tap interface +diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go +index 5a3e7f7e..0b6002aa 100644 +--- a/virtcontainers/tap_endpoint.go ++++ b/virtcontainers/tap_endpoint.go +@@ -199,6 +199,7 @@ func unTapNetwork(endpoint *TapEndpoint) error { + return nil + } + ++ networkLogger().Debug("untap the new created tap interface") + name := endpoint.TapInterface.TAPIface.Name + netHandle, err := netlink.NewHandle() + if err != nil { +-- +2.11.0 + diff --git a/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch b/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch new file mode 100644 index 0000000000000000000000000000000000000000..3456c9fc7ba7145fac53908139d0709c3148af08 --- /dev/null +++ b/runtime/patches/0052-runtime-add-support-of-new-sandbox-StratoVirt.patch @@ -0,0 +1,1720 @@ +From 60b6a9c63a116db6d60a91b3dc875de3f33aa57c Mon Sep 17 00:00:00 2001 +From: LiangZhang +Date: Mon, 21 Sep 2020 15:33:17 +0800 +Subject: [PATCH] runtime: add support of new sandbox StratoVirt + +Signed-off-by: LiangZhang +--- + pkg/katautils/config.go | 5 +- + .../agent/protocols/grpc/agent.pb.go | 636 +++++++++++++-------- + virtcontainers/agent.go | 3 + + virtcontainers/hypervisor.go | 12 + + virtcontainers/kata_agent.go | 73 ++- + virtcontainers/noop_agent.go | 5 + + virtcontainers/stratovirt.go | 617 ++++++++++++++++++++ + 7 files changed, 1090 insertions(+), 261 deletions(-) + create mode 100644 virtcontainers/stratovirt.go + +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 3365b3f..ed6f03c 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -52,7 +52,7 @@ const ( + clhHypervisorTableType = "clh" + qemuHypervisorTableType = "qemu" + acrnHypervisorTableType = "acrn" +- ++ stratovirtHypervisorTableType = "stratovirt" + // supported proxy component types + kataProxyTableType = "kata" + +@@ -921,6 +921,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi + case clhHypervisorTableType: + config.HypervisorType = vc.ClhHypervisor + hConfig, err = newClhHypervisorConfig(hypervisor) ++ case stratovirtHypervisorTableType: ++ config.HypervisorType = vc.StratovirtHypervisor ++ hConfig, err = newQemuHypervisorConfig(hypervisor) + } + + if err != nil { +diff --git a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +index c50ecb5..181fcb6 100644 +--- a/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go ++++ b/vendor/github.com/kata-containers/agent/protocols/grpc/agent.pb.go +@@ -50,6 +50,7 @@ + Interfaces + Routes + UpdateInterfaceRequest ++ UpdateInterfaceHwAddrByNameRequest + UpdateRoutesRequest + ListInterfacesRequest + ListRoutesRequest +@@ -1491,6 +1492,24 @@ func (m *UpdateInterfaceRequest) GetInterface() *types.Interface { + return nil + } + ++type UpdateInterfaceHwAddrByNameRequest struct { ++ Interface *types.Interface `protobuf:"bytes,1,opt,name=interface" json:"interface,omitempty"` ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } ++func (m *UpdateInterfaceHwAddrByNameRequest) String() string { return proto.CompactTextString(m) } ++func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} ++func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { ++ return fileDescriptorAgent, []int{40} ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) GetInterface() *types.Interface { ++ if m != nil { ++ return m.Interface ++ } ++ return nil ++} ++ + type UpdateRoutesRequest struct { + Routes *Routes `protobuf:"bytes,1,opt,name=routes" json:"routes,omitempty"` + Increment bool `protobuf:"varint,2,opt,name=increment,proto3" json:"increment,omitempty"` +@@ -1499,7 +1518,7 @@ type UpdateRoutesRequest struct { + func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } + func (m *UpdateRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateRoutesRequest) ProtoMessage() {} +-func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{40} } ++func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } + + func (m *UpdateRoutesRequest) GetRoutes() *Routes { + if m != nil { +@@ -1521,7 +1540,7 @@ type ListInterfacesRequest struct { + func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } + func (m *ListInterfacesRequest) String() string { return proto.CompactTextString(m) } + func (*ListInterfacesRequest) ProtoMessage() {} +-func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{41} } ++func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } + + type ListRoutesRequest struct { + } +@@ -1529,7 +1548,7 @@ type ListRoutesRequest struct { + func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } + func (m *ListRoutesRequest) String() string { return proto.CompactTextString(m) } + func (*ListRoutesRequest) ProtoMessage() {} +-func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{42} } ++func (*ListRoutesRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } + + type OnlineCPUMemRequest struct { + // Wait specifies if the caller waits for the agent to online all resources. +@@ -1545,7 +1564,7 @@ type OnlineCPUMemRequest struct { + func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } + func (m *OnlineCPUMemRequest) String() string { return proto.CompactTextString(m) } + func (*OnlineCPUMemRequest) ProtoMessage() {} +-func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{43} } ++func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } + + func (m *OnlineCPUMemRequest) GetWait() bool { + if m != nil { +@@ -1576,7 +1595,7 @@ type ReseedRandomDevRequest struct { + func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } + func (m *ReseedRandomDevRequest) String() string { return proto.CompactTextString(m) } + func (*ReseedRandomDevRequest) ProtoMessage() {} +-func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{44} } ++func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } + + func (m *ReseedRandomDevRequest) GetData() []byte { + if m != nil { +@@ -1603,7 +1622,7 @@ type AgentDetails struct { + func (m *AgentDetails) Reset() { *m = AgentDetails{} } + func (m *AgentDetails) String() string { return proto.CompactTextString(m) } + func (*AgentDetails) ProtoMessage() {} +-func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{45} } ++func (*AgentDetails) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } + + func (m *AgentDetails) GetVersion() string { + if m != nil { +@@ -1654,7 +1673,7 @@ type GuestDetailsRequest struct { + func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } + func (m *GuestDetailsRequest) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsRequest) ProtoMessage() {} +-func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{46} } ++func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } + + func (m *GuestDetailsRequest) GetMemBlockSize() bool { + if m != nil { +@@ -1680,7 +1699,7 @@ type GuestDetailsResponse struct { + func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } + func (m *GuestDetailsResponse) String() string { return proto.CompactTextString(m) } + func (*GuestDetailsResponse) ProtoMessage() {} +-func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{47} } ++func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } + + func (m *GuestDetailsResponse) GetMemBlockSizeBytes() uint64 { + if m != nil { +@@ -1712,7 +1731,7 @@ type MemHotplugByProbeRequest struct { + func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } + func (m *MemHotplugByProbeRequest) String() string { return proto.CompactTextString(m) } + func (*MemHotplugByProbeRequest) ProtoMessage() {} +-func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{48} } ++func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } + + func (m *MemHotplugByProbeRequest) GetMemHotplugProbeAddr() []uint64 { + if m != nil { +@@ -1731,7 +1750,7 @@ type SetGuestDateTimeRequest struct { + func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } + func (m *SetGuestDateTimeRequest) String() string { return proto.CompactTextString(m) } + func (*SetGuestDateTimeRequest) ProtoMessage() {} +-func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{49} } ++func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } + + func (m *SetGuestDateTimeRequest) GetSec() int64 { + if m != nil { +@@ -1780,7 +1799,7 @@ type Storage struct { + func (m *Storage) Reset() { *m = Storage{} } + func (m *Storage) String() string { return proto.CompactTextString(m) } + func (*Storage) ProtoMessage() {} +-func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{50} } ++func (*Storage) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } + + func (m *Storage) GetDriver() string { + if m != nil { +@@ -1863,7 +1882,7 @@ type Device struct { + func (m *Device) Reset() { *m = Device{} } + func (m *Device) String() string { return proto.CompactTextString(m) } + func (*Device) ProtoMessage() {} +-func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{51} } ++func (*Device) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } + + func (m *Device) GetId() string { + if m != nil { +@@ -1909,7 +1928,7 @@ type StringUser struct { + func (m *StringUser) Reset() { *m = StringUser{} } + func (m *StringUser) String() string { return proto.CompactTextString(m) } + func (*StringUser) ProtoMessage() {} +-func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{52} } ++func (*StringUser) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } + + func (m *StringUser) GetUid() string { + if m != nil { +@@ -1957,7 +1976,7 @@ type CopyFileRequest struct { + func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } + func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } + func (*CopyFileRequest) ProtoMessage() {} +-func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{53} } ++func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } + + func (m *CopyFileRequest) GetPath() string { + if m != nil { +@@ -2021,7 +2040,7 @@ type StartTracingRequest struct { + func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } + func (m *StartTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StartTracingRequest) ProtoMessage() {} +-func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{54} } ++func (*StartTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } + + type StopTracingRequest struct { + } +@@ -2029,7 +2048,7 @@ type StopTracingRequest struct { + func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } + func (m *StopTracingRequest) String() string { return proto.CompactTextString(m) } + func (*StopTracingRequest) ProtoMessage() {} +-func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{55} } ++func (*StopTracingRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } + + type UpdateIPVSRequest struct { + // IPVS_req is the IPVS rule message needed to update +@@ -2039,7 +2058,7 @@ type UpdateIPVSRequest struct { + func (m *UpdateIPVSRequest) Reset() { *m = UpdateIPVSRequest{} } + func (m *UpdateIPVSRequest) String() string { return proto.CompactTextString(m) } + func (*UpdateIPVSRequest) ProtoMessage() {} +-func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{56} } ++func (*UpdateIPVSRequest) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } + + func (m *UpdateIPVSRequest) GetIPVSReq() string { + if m != nil { +@@ -2056,7 +2075,7 @@ type IPVSResponse struct { + func (m *IPVSResponse) Reset() { *m = IPVSResponse{} } + func (m *IPVSResponse) String() string { return proto.CompactTextString(m) } + func (*IPVSResponse) ProtoMessage() {} +-func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{57} } ++func (*IPVSResponse) Descriptor() ([]byte, []int) { return fileDescriptorAgent, []int{58} } + + func (m *IPVSResponse) GetIPVSRes() string { + if m != nil { +@@ -2106,6 +2125,7 @@ func init() { + proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") + proto.RegisterType((*Routes)(nil), "grpc.Routes") + proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") ++ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") + proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") + proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") + proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") +@@ -2163,6 +2183,7 @@ type AgentServiceClient interface { + TtyWinResize(ctx context.Context, in *TtyWinResizeRequest, opts ...grpc1.CallOption) (*google_protobuf2.Empty, error) + // networking + UpdateInterface(ctx context.Context, in *UpdateInterfaceRequest, opts ...grpc1.CallOption) (*types.Interface, error) ++ UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) + UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) + ListInterfaces(ctx context.Context, in *ListInterfacesRequest, opts ...grpc1.CallOption) (*Interfaces, error) + ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) +@@ -2342,6 +2363,15 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, in *UpdateInte + return out, nil + } + ++func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, in *UpdateInterfaceHwAddrByNameRequest, opts ...grpc1.CallOption) (*types.Interface, error) { ++ out := new(types.Interface) ++ err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateInterfaceHwAddrByName", in, out, c.cc, opts...) ++ if err != nil { ++ return nil, err ++ } ++ return out, nil ++} ++ + func (c *agentServiceClient) UpdateRoutes(ctx context.Context, in *UpdateRoutesRequest, opts ...grpc1.CallOption) (*Routes, error) { + out := new(Routes) + err := grpc1.Invoke(ctx, "/grpc.AgentService/UpdateRoutes", in, out, c.cc, opts...) +@@ -2497,6 +2527,7 @@ type AgentServiceServer interface { + TtyWinResize(context.Context, *TtyWinResizeRequest) (*google_protobuf2.Empty, error) + // networking + UpdateInterface(context.Context, *UpdateInterfaceRequest) (*types.Interface, error) ++ UpdateInterfaceHwAddrByName(context.Context, *UpdateInterfaceHwAddrByNameRequest) (*types.Interface, error) + UpdateRoutes(context.Context, *UpdateRoutesRequest) (*Routes, error) + ListInterfaces(context.Context, *ListInterfacesRequest) (*Interfaces, error) + ListRoutes(context.Context, *ListRoutesRequest) (*Routes, error) +@@ -2825,6 +2856,24 @@ func _AgentService_UpdateInterface_Handler(srv interface{}, ctx context.Context, + return interceptor(ctx, in, info, handler) + } + ++func _AgentService_UpdateInterfaceHwAddrByName_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { ++ in := new(UpdateInterfaceHwAddrByNameRequest) ++ if err := dec(in); err != nil { ++ return nil, err ++ } ++ if interceptor == nil { ++ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, in) ++ } ++ info := &grpc1.UnaryServerInfo{ ++ Server: srv, ++ FullMethod: "/grpc.AgentService/UpdateInterfaceHwAddrByName", ++ } ++ handler := func(ctx context.Context, req interface{}) (interface{}, error) { ++ return srv.(AgentServiceServer).UpdateInterfaceHwAddrByName(ctx, req.(*UpdateInterfaceHwAddrByNameRequest)) ++ } ++ return interceptor(ctx, in, info, handler) ++} ++ + func _AgentService_UpdateRoutes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc1.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateRoutesRequest) + if err := dec(in); err != nil { +@@ -3150,6 +3199,10 @@ var _AgentService_serviceDesc = grpc1.ServiceDesc{ + Handler: _AgentService_UpdateInterface_Handler, + }, + { ++ MethodName: "UpdateInterfaceHwAddrByName", ++ Handler: _AgentService_UpdateInterfaceHwAddrByName_Handler, ++ }, ++ { + MethodName: "UpdateRoutes", + Handler: _AgentService_UpdateRoutes_Handler, + }, +@@ -4899,6 +4952,34 @@ func (m *UpdateInterfaceRequest) MarshalTo(dAtA []byte) (int, error) { + return i, nil + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { ++ size := m.Size() ++ dAtA = make([]byte, size) ++ n, err := m.MarshalTo(dAtA) ++ if err != nil { ++ return nil, err ++ } ++ return dAtA[:n], nil ++} ++ ++func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { ++ var i int ++ _ = i ++ var l int ++ _ = l ++ if m.Interface != nil { ++ dAtA[i] = 0xa ++ i++ ++ i = encodeVarintAgent(dAtA, i, uint64(m.Interface.Size())) ++ n25, err := m.Interface.MarshalTo(dAtA[i:]) ++ if err != nil { ++ return 0, err ++ } ++ i += n25 ++ } ++ return i, nil ++} ++ + func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) +@@ -4918,11 +4999,11 @@ func (m *UpdateRoutesRequest) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0xa + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.Routes.Size())) +- n25, err := m.Routes.MarshalTo(dAtA[i:]) ++ n26, err := m.Routes.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n25 ++ i += n26 + } + if m.Increment { + dAtA[i] = 0x10 +@@ -5176,11 +5257,11 @@ func (m *GuestDetailsResponse) MarshalTo(dAtA []byte) (int, error) { + dAtA[i] = 0x12 + i++ + i = encodeVarintAgent(dAtA, i, uint64(m.AgentDetails.Size())) +- n26, err := m.AgentDetails.MarshalTo(dAtA[i:]) ++ n27, err := m.AgentDetails.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } +- i += n26 ++ i += n27 + } + if m.SupportMemHotplugProbe { + dAtA[i] = 0x18 +@@ -5211,21 +5292,21 @@ func (m *MemHotplugByProbeRequest) MarshalTo(dAtA []byte) (int, error) { + var l int + _ = l + if len(m.MemHotplugProbeAddr) > 0 { +- dAtA28 := make([]byte, len(m.MemHotplugProbeAddr)*10) +- var j27 int ++ dAtA29 := make([]byte, len(m.MemHotplugProbeAddr)*10) ++ var j28 int + for _, num := range m.MemHotplugProbeAddr { + for num >= 1<<7 { +- dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) ++ dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 +- j27++ ++ j28++ + } +- dAtA28[j27] = uint8(num) +- j27++ ++ dAtA29[j28] = uint8(num) ++ j28++ + } + dAtA[i] = 0xa + i++ +- i = encodeVarintAgent(dAtA, i, uint64(j27)) +- i += copy(dAtA[i:], dAtA28[:j27]) ++ i = encodeVarintAgent(dAtA, i, uint64(j28)) ++ i += copy(dAtA[i:], dAtA29[:j28]) + } + return i, nil + } +@@ -6333,6 +6414,16 @@ func (m *UpdateInterfaceRequest) Size() (n int) { + return n + } + ++func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { ++ var l int ++ _ = l ++ if m.Interface != nil { ++ l = m.Interface.Size() ++ n += 1 + l + sovAgent(uint64(l)) ++ } ++ return n ++} ++ + func (m *UpdateRoutesRequest) Size() (n int) { + var l int + _ = l +@@ -12114,6 +12205,89 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { + } + return nil + } ++func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { ++ l := len(dAtA) ++ iNdEx := 0 ++ for iNdEx < l { ++ preIndex := iNdEx ++ var wire uint64 ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ wire |= (uint64(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ fieldNum := int32(wire >> 3) ++ wireType := int(wire & 0x7) ++ if wireType == 4 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") ++ } ++ if fieldNum <= 0 { ++ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) ++ } ++ switch fieldNum { ++ case 1: ++ if wireType != 2 { ++ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) ++ } ++ var msglen int ++ for shift := uint(0); ; shift += 7 { ++ if shift >= 64 { ++ return ErrIntOverflowAgent ++ } ++ if iNdEx >= l { ++ return io.ErrUnexpectedEOF ++ } ++ b := dAtA[iNdEx] ++ iNdEx++ ++ msglen |= (int(b) & 0x7F) << shift ++ if b < 0x80 { ++ break ++ } ++ } ++ if msglen < 0 { ++ return ErrInvalidLengthAgent ++ } ++ postIndex := iNdEx + msglen ++ if postIndex > l { ++ return io.ErrUnexpectedEOF ++ } ++ if m.Interface == nil { ++ m.Interface = &types.Interface{} ++ } ++ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { ++ return err ++ } ++ iNdEx = postIndex ++ default: ++ iNdEx = preIndex ++ skippy, err := skipAgent(dAtA[iNdEx:]) ++ if err != nil { ++ return err ++ } ++ if skippy < 0 { ++ return ErrInvalidLengthAgent ++ } ++ if (iNdEx + skippy) > l { ++ return io.ErrUnexpectedEOF ++ } ++ iNdEx += skippy ++ } ++ } ++ ++ if iNdEx > l { ++ return io.ErrUnexpectedEOF ++ } ++ return nil ++} + func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 +@@ -14242,204 +14416,206 @@ var ( + func init() { proto.RegisterFile("agent.proto", fileDescriptorAgent) } + + var fileDescriptorAgent = []byte{ +- // 3183 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0x4b, 0x6f, 0x1c, 0xc7, +- 0x99, 0x98, 0x07, 0x39, 0x33, 0xdf, 0xbc, 0xc8, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, +- 0x99, 0x5e, 0xaf, 0x49, 0x8b, 0x36, 0xfc, 0x84, 0x57, 0x10, 0x29, 0xad, 0xc8, 0xb5, 0xb4, 0xe2, +- 0xf6, 0x88, 0xeb, 0x5d, 0x2c, 0x16, 0x8d, 0x66, 0x77, 0x71, 0x58, 0xe6, 0x74, 0x57, 0xbb, 0xaa, +- 0x9a, 0xe2, 0x78, 0x81, 0x3d, 0x26, 0xd7, 0x9c, 0x72, 0xcb, 0x1f, 0x08, 0x72, 0xcb, 0x2d, 0xb9, +- 0xe6, 0x60, 0x04, 0x39, 0x04, 0xf9, 0x01, 0x41, 0xe0, 0x9f, 0x90, 0x5f, 0x10, 0xd4, 0xab, 0x1f, +- 0x33, 0x43, 0x1a, 0x11, 0x04, 0xe4, 0xd2, 0xe8, 0xef, 0x51, 0xdf, 0xab, 0xaa, 0xbe, 0xfa, 0xbe, +- 0x2a, 0x68, 0xfb, 0x63, 0x1c, 0x8b, 0xad, 0x84, 0x51, 0x41, 0x51, 0x7d, 0xcc, 0x92, 0x60, 0xd8, +- 0xa2, 0x01, 0xd1, 0x88, 0xe1, 0xc7, 0x63, 0x22, 0x4e, 0xd3, 0xe3, 0xad, 0x80, 0x46, 0xdb, 0x67, +- 0xbe, 0xf0, 0xdf, 0x0f, 0x68, 0x2c, 0x7c, 0x12, 0x63, 0xc6, 0xb7, 0xd5, 0xc0, 0xed, 0xe4, 0x6c, +- 0xbc, 0x2d, 0xa6, 0x09, 0xe6, 0xfa, 0x6b, 0xc6, 0xdd, 0x1c, 0x53, 0x3a, 0x9e, 0xe0, 0x6d, 0x05, +- 0x1d, 0xa7, 0x27, 0xdb, 0x38, 0x4a, 0xc4, 0x54, 0x13, 0x9d, 0x5f, 0x54, 0x61, 0x63, 0x8f, 0x61, +- 0x5f, 0xe0, 0x3d, 0x2b, 0xcd, 0xc5, 0xdf, 0xa6, 0x98, 0x0b, 0xf4, 0x06, 0x74, 0x32, 0x0d, 0x1e, +- 0x09, 0x07, 0x95, 0x3b, 0x95, 0xcd, 0x96, 0xdb, 0xce, 0x70, 0x07, 0x21, 0xba, 0x0e, 0x0d, 0x7c, +- 0x81, 0x03, 0x49, 0xad, 0x2a, 0xea, 0xb2, 0x04, 0x0f, 0x42, 0x74, 0x0f, 0xda, 0x5c, 0x30, 0x12, +- 0x8f, 0xbd, 0x94, 0x63, 0x36, 0xa8, 0xdd, 0xa9, 0x6c, 0xb6, 0x77, 0x56, 0xb6, 0xa4, 0x4b, 0x5b, +- 0x23, 0x45, 0x38, 0xe2, 0x98, 0xb9, 0xc0, 0xb3, 0x7f, 0x74, 0x17, 0x1a, 0x21, 0x3e, 0x27, 0x01, +- 0xe6, 0x83, 0xfa, 0x9d, 0xda, 0x66, 0x7b, 0xa7, 0xa3, 0xd9, 0x1f, 0x2a, 0xa4, 0x6b, 0x89, 0xe8, +- 0x5d, 0x68, 0x72, 0x41, 0x99, 0x3f, 0xc6, 0x7c, 0xb0, 0xa4, 0x18, 0xbb, 0x56, 0xae, 0xc2, 0xba, +- 0x19, 0x19, 0xdd, 0x82, 0xda, 0xb3, 0xbd, 0x83, 0xc1, 0xb2, 0xd2, 0x0e, 0x86, 0x2b, 0xc1, 0x81, +- 0x2b, 0xd1, 0xe8, 0x4d, 0xe8, 0x72, 0x3f, 0x0e, 0x8f, 0xe9, 0x85, 0x97, 0x90, 0x30, 0xe6, 0x83, +- 0xc6, 0x9d, 0xca, 0x66, 0xd3, 0xed, 0x18, 0xe4, 0xa1, 0xc4, 0x39, 0x9f, 0xc3, 0xb5, 0x91, 0xf0, +- 0x99, 0x78, 0x89, 0xe8, 0x38, 0x47, 0xb0, 0xe1, 0xe2, 0x88, 0x9e, 0xbf, 0x54, 0x68, 0x07, 0xd0, +- 0x10, 0x24, 0xc2, 0x34, 0x15, 0x2a, 0xb4, 0x5d, 0xd7, 0x82, 0xce, 0xaf, 0x2a, 0x80, 0x1e, 0x5d, +- 0xe0, 0xe0, 0x90, 0xd1, 0x00, 0x73, 0xfe, 0x0f, 0x9a, 0xae, 0x77, 0xa0, 0x91, 0x68, 0x03, 0x06, +- 0x75, 0xc5, 0x6e, 0x66, 0xc1, 0x5a, 0x65, 0xa9, 0xce, 0x37, 0xb0, 0x3e, 0x22, 0xe3, 0xd8, 0x9f, +- 0xbc, 0x42, 0x7b, 0x37, 0x60, 0x99, 0x2b, 0x99, 0xca, 0xd4, 0xae, 0x6b, 0x20, 0xe7, 0x10, 0xd0, +- 0xd7, 0x3e, 0x11, 0xaf, 0x4e, 0x93, 0xf3, 0x3e, 0xac, 0x95, 0x24, 0xf2, 0x84, 0xc6, 0x1c, 0x2b, +- 0x03, 0x84, 0x2f, 0x52, 0xae, 0x84, 0x2d, 0xb9, 0x06, 0x72, 0x30, 0xac, 0x3f, 0x21, 0xdc, 0xb2, +- 0xe3, 0xbf, 0xc7, 0x84, 0x0d, 0x58, 0x3e, 0xa1, 0x2c, 0xf2, 0x85, 0xb5, 0x40, 0x43, 0x08, 0x41, +- 0xdd, 0x67, 0x63, 0x3e, 0xa8, 0xdd, 0xa9, 0x6d, 0xb6, 0x5c, 0xf5, 0x2f, 0x57, 0xe5, 0x8c, 0x1a, +- 0x63, 0xd7, 0x1b, 0xd0, 0x31, 0x71, 0xf7, 0x26, 0x84, 0x0b, 0xa5, 0xa7, 0xe3, 0xb6, 0x0d, 0x4e, +- 0x8e, 0x71, 0x28, 0x6c, 0x1c, 0x25, 0xe1, 0x4b, 0x6e, 0xf8, 0x1d, 0x68, 0x31, 0xcc, 0x69, 0xca, +- 0xe4, 0x36, 0xad, 0xaa, 0x79, 0x5f, 0xd7, 0xf3, 0xfe, 0x84, 0xc4, 0xe9, 0x85, 0x6b, 0x69, 0x6e, +- 0xce, 0x66, 0xb6, 0x90, 0xe0, 0x2f, 0xb3, 0x85, 0x3e, 0x87, 0x6b, 0x87, 0x7e, 0xca, 0x5f, 0xc6, +- 0x56, 0xe7, 0x0b, 0xb9, 0xfd, 0x78, 0x1a, 0xbd, 0xd4, 0xe0, 0x5f, 0x56, 0xa0, 0xb9, 0x97, 0xa4, +- 0x47, 0xdc, 0x1f, 0x63, 0xf4, 0x3a, 0xb4, 0x05, 0x15, 0xfe, 0xc4, 0x4b, 0x25, 0xa8, 0xd8, 0xeb, +- 0x2e, 0x28, 0x94, 0x66, 0x90, 0x61, 0xc7, 0x2c, 0x48, 0x52, 0xc3, 0x51, 0xbd, 0x53, 0xdb, 0xac, +- 0xbb, 0x6d, 0x8d, 0xd3, 0x2c, 0x5b, 0xb0, 0xa6, 0x68, 0x1e, 0x89, 0xbd, 0x33, 0xcc, 0x62, 0x3c, +- 0x89, 0x68, 0x88, 0xd5, 0xfa, 0xad, 0xbb, 0xab, 0x8a, 0x74, 0x10, 0x7f, 0x95, 0x11, 0xd0, 0x3f, +- 0xc1, 0x6a, 0xc6, 0x2f, 0x37, 0xa5, 0xe2, 0xae, 0x2b, 0xee, 0xbe, 0xe1, 0x3e, 0x32, 0x68, 0xe7, +- 0xff, 0xa1, 0xf7, 0xfc, 0x94, 0x51, 0x21, 0x26, 0x24, 0x1e, 0x3f, 0xf4, 0x85, 0x2f, 0xb3, 0x47, +- 0x82, 0x19, 0xa1, 0x21, 0x37, 0xd6, 0x5a, 0x10, 0xbd, 0x07, 0xab, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, +- 0x3c, 0x55, 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0d, 0xf3, 0xdb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, +- 0xf6, 0x76, 0x33, 0xec, 0x73, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0x7a, 0x0f, 0x5a, +- 0x79, 0x1c, 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x84, +- 0xbe, 0xc8, 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, +- 0x7c, 0x01, 0xad, 0x43, 0x12, 0x72, 0xad, 0x78, 0x00, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, +- 0x6c, 0x40, 0xb4, 0x0e, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe2, +- 0x88, 0xb2, 0xa9, 0x0a, 0xd8, 0x3a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x26, 0xb4, 0x22, 0xff, +- 0x22, 0x9b, 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x00, 0x8d, 0x13, 0x9f, 0x4c, 0x82, +- 0x58, 0x98, 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0x5d, 0x15, 0xda, 0x5a, 0xa3, 0x36, +- 0x78, 0x1d, 0x96, 0x02, 0x3f, 0x38, 0xcd, 0x54, 0x2a, 0x00, 0xdd, 0xb5, 0x86, 0x54, 0x8b, 0x49, +- 0x38, 0xb7, 0xd4, 0x9a, 0xb6, 0x0d, 0xc0, 0x5f, 0xf8, 0x89, 0xb1, 0xad, 0x76, 0x09, 0x73, 0x4b, +- 0xf2, 0x68, 0x73, 0x3f, 0x84, 0x8e, 0x5e, 0x77, 0x66, 0x48, 0xfd, 0x92, 0x21, 0x6d, 0xcd, 0xa5, +- 0x07, 0xbd, 0x09, 0xdd, 0x94, 0x63, 0xef, 0x94, 0x60, 0xe6, 0xb3, 0xe0, 0x74, 0x3a, 0x58, 0xd2, +- 0x67, 0x64, 0xca, 0xf1, 0xbe, 0xc5, 0xa1, 0x1d, 0x58, 0x92, 0xe9, 0x8f, 0x0f, 0x96, 0xd5, 0x71, +- 0x7c, 0xab, 0x28, 0x52, 0xb9, 0xba, 0xa5, 0xbe, 0x8f, 0x62, 0xc1, 0xa6, 0xae, 0x66, 0x1d, 0x7e, +- 0x0a, 0x90, 0x23, 0xd1, 0x0a, 0xd4, 0xce, 0xf0, 0xd4, 0xec, 0x43, 0xf9, 0x2b, 0x83, 0x73, 0xee, +- 0x4f, 0x52, 0x1b, 0x75, 0x0d, 0x7c, 0x5e, 0xfd, 0xb4, 0xe2, 0x04, 0xd0, 0xdf, 0x9d, 0x9c, 0x11, +- 0x5a, 0x18, 0xbe, 0x0e, 0x4b, 0x91, 0xff, 0x0d, 0x65, 0x36, 0x92, 0x0a, 0x50, 0x58, 0x12, 0x53, +- 0x66, 0x45, 0x28, 0x00, 0xf5, 0xa0, 0x4a, 0x13, 0x15, 0xaf, 0x96, 0x5b, 0xa5, 0x49, 0xae, 0xa8, +- 0x5e, 0x50, 0xe4, 0xfc, 0xb9, 0x0e, 0x90, 0x6b, 0x41, 0x2e, 0x0c, 0x09, 0xf5, 0x38, 0x66, 0xb2, +- 0x04, 0xf1, 0x8e, 0xa7, 0x02, 0x73, 0x8f, 0xe1, 0x20, 0x65, 0x9c, 0x9c, 0xcb, 0xf9, 0x93, 0x6e, +- 0x5f, 0xd3, 0x6e, 0xcf, 0xd8, 0xe6, 0x5e, 0x27, 0x74, 0xa4, 0xc7, 0xed, 0xca, 0x61, 0xae, 0x1d, +- 0x85, 0x0e, 0xe0, 0x5a, 0x2e, 0x33, 0x2c, 0x88, 0xab, 0x5e, 0x25, 0x6e, 0x2d, 0x13, 0x17, 0xe6, +- 0xa2, 0x1e, 0xc1, 0x1a, 0xa1, 0xde, 0xb7, 0x29, 0x4e, 0x4b, 0x82, 0x6a, 0x57, 0x09, 0x5a, 0x25, +- 0xf4, 0x3f, 0xd4, 0x80, 0x5c, 0xcc, 0x21, 0xdc, 0x28, 0x78, 0x29, 0xb7, 0x7b, 0x41, 0x58, 0xfd, +- 0x2a, 0x61, 0x1b, 0x99, 0x55, 0x32, 0x1f, 0xe4, 0x12, 0xff, 0x0d, 0x36, 0x08, 0xf5, 0x5e, 0xf8, +- 0x44, 0xcc, 0x8a, 0x5b, 0xfa, 0x11, 0x27, 0xe5, 0xa1, 0x5b, 0x96, 0xa5, 0x9d, 0x8c, 0x30, 0x1b, +- 0x97, 0x9c, 0x5c, 0xfe, 0x11, 0x27, 0x9f, 0xaa, 0x01, 0xb9, 0x98, 0x07, 0xb0, 0x4a, 0xe8, 0xac, +- 0x35, 0x8d, 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x0b, 0xab, 0x1c, 0x07, 0x82, 0xb2, 0xe2, +- 0x22, 0x68, 0x5e, 0x25, 0x62, 0xc5, 0xf0, 0x67, 0x32, 0x9c, 0xff, 0x81, 0xce, 0x7e, 0x3a, 0xc6, +- 0x62, 0x72, 0x9c, 0x25, 0x83, 0x57, 0x96, 0x7f, 0x9c, 0xbf, 0x56, 0xa1, 0xbd, 0x37, 0x66, 0x34, +- 0x4d, 0x4a, 0x39, 0x59, 0x6f, 0xd2, 0xd9, 0x9c, 0xac, 0x58, 0x54, 0x4e, 0xd6, 0xcc, 0x1f, 0x41, +- 0x27, 0x52, 0x5b, 0xd7, 0xf0, 0xeb, 0x3c, 0xb4, 0x3a, 0xb7, 0xa9, 0xdd, 0x76, 0x54, 0x48, 0x66, +- 0x5b, 0x00, 0x09, 0x09, 0xb9, 0x19, 0xa3, 0xd3, 0x51, 0xdf, 0x54, 0x84, 0x36, 0x45, 0xbb, 0xad, +- 0x24, 0xcb, 0xd6, 0xf7, 0xa0, 0x7d, 0x2c, 0x83, 0x64, 0x06, 0x94, 0x92, 0x51, 0x1e, 0x3d, 0x17, +- 0x8e, 0xf3, 0x4d, 0xb8, 0x0f, 0xdd, 0x53, 0x1d, 0x32, 0x33, 0x48, 0xaf, 0xa1, 0x37, 0x8d, 0x27, +- 0xb9, 0xbf, 0x5b, 0xc5, 0xc8, 0xea, 0x09, 0xe8, 0x9c, 0x16, 0x50, 0xc3, 0x11, 0xac, 0xce, 0xb1, +- 0x2c, 0xc8, 0x41, 0x9b, 0xc5, 0x1c, 0xd4, 0xde, 0x41, 0x5a, 0x51, 0x71, 0x64, 0x31, 0x2f, 0xfd, +- 0xac, 0x0a, 0xbd, 0x83, 0x58, 0x60, 0x76, 0xe2, 0x07, 0x58, 0x5b, 0x8c, 0xa0, 0x1e, 0xfb, 0x11, +- 0x36, 0x32, 0xd5, 0x3f, 0xba, 0x01, 0x4d, 0x76, 0xa1, 0x53, 0x88, 0x99, 0xd1, 0x06, 0xbb, 0x50, +- 0xa9, 0x01, 0xbd, 0x06, 0xc0, 0x2e, 0xbc, 0xc4, 0x0f, 0xce, 0xb0, 0x89, 0x61, 0xdd, 0x6d, 0xb1, +- 0x8b, 0x43, 0x8d, 0x90, 0x8b, 0x81, 0x5d, 0x78, 0x98, 0x31, 0xca, 0xb8, 0xc9, 0x56, 0x4d, 0x76, +- 0xf1, 0x48, 0xc1, 0x66, 0x6c, 0xc8, 0x68, 0x92, 0xe0, 0x50, 0x65, 0x69, 0x35, 0xf6, 0xa1, 0x46, +- 0x48, 0xad, 0xc2, 0x6a, 0x5d, 0xd6, 0x5a, 0x45, 0xae, 0x55, 0xe4, 0x5a, 0x1b, 0x7a, 0xa4, 0x28, +- 0x6a, 0x15, 0x99, 0xd6, 0xa6, 0xd6, 0x2a, 0x0a, 0x5a, 0x45, 0xae, 0xb5, 0x65, 0xc7, 0x1a, 0xad, +- 0xce, 0x6f, 0xaa, 0xd0, 0x78, 0x1e, 0xa8, 0x49, 0x41, 0x77, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, +- 0x84, 0x9f, 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, +- 0x22, 0xc3, 0xa7, 0xf1, 0x48, 0x9e, 0xe0, 0x86, 0xc4, 0x70, 0x70, 0x6e, 0xd7, 0x3a, 0x9f, 0xc6, +- 0x2e, 0x0e, 0xce, 0xa5, 0x7d, 0x27, 0x24, 0x56, 0x39, 0xe6, 0x9e, 0x8d, 0xca, 0x09, 0x89, 0x65, +- 0xfe, 0xb8, 0x57, 0x24, 0xee, 0x98, 0xa0, 0x58, 0xe2, 0x8e, 0xf2, 0x4c, 0xa6, 0x01, 0x49, 0x35, +- 0x41, 0x69, 0x4a, 0x84, 0xa4, 0xaa, 0xc3, 0x79, 0x42, 0x39, 0x36, 0x01, 0xd1, 0x80, 0xf4, 0x57, +- 0xfd, 0xe8, 0x31, 0x3a, 0x1a, 0x2d, 0x85, 0x51, 0x83, 0x6e, 0x40, 0x73, 0xe2, 0x73, 0xe1, 0xf9, +- 0xc1, 0x99, 0x09, 0x46, 0x43, 0xc2, 0x0f, 0x82, 0x33, 0x59, 0xdd, 0xcb, 0x82, 0x1c, 0xc7, 0x03, +- 0x50, 0x04, 0x03, 0xa9, 0xaa, 0x65, 0x42, 0x39, 0x89, 0xc7, 0x83, 0xb6, 0xa9, 0x5a, 0x34, 0xe8, +- 0xa4, 0xd0, 0x38, 0x0a, 0x75, 0xec, 0xf2, 0xc1, 0x95, 0xd9, 0xc1, 0x36, 0xf6, 0x26, 0x60, 0x06, +- 0x34, 0x6b, 0x45, 0x9f, 0x08, 0x26, 0x62, 0x4d, 0x76, 0xa1, 0x13, 0xbe, 0x99, 0x52, 0x43, 0xac, +- 0xdb, 0x29, 0xd5, 0x44, 0xe7, 0x0f, 0x15, 0xe8, 0xfc, 0x3b, 0x16, 0x2f, 0x28, 0x3b, 0xb3, 0xf9, +- 0x00, 0x88, 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0x5e, +- 0x87, 0x9a, 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x0d, 0xa8, +- 0x8b, 0x20, 0xf9, 0xd8, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, +- 0x26, 0x24, 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xac, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, +- 0x9f, 0x56, 0x60, 0x63, 0xb6, 0xfb, 0x30, 0xbd, 0xd2, 0x47, 0xd0, 0x09, 0x54, 0xd2, 0x28, 0x25, +- 0xc6, 0xd5, 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x04, 0xba, 0xb1, 0x0e, 0x4f, 0x29, +- 0x3f, 0x9a, 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x9a, 0x11, 0x81, +- 0x47, 0x82, 0x61, 0x3f, 0x7a, 0x15, 0x5d, 0x30, 0x82, 0xba, 0x2a, 0x99, 0x6b, 0xaa, 0xc9, 0x53, +- 0xff, 0xce, 0x3b, 0xb0, 0x56, 0xd2, 0x62, 0x7c, 0x5d, 0x81, 0xda, 0xc4, 0x2c, 0x9f, 0xae, 0x2b, +- 0x7f, 0x1d, 0x1f, 0x56, 0x5d, 0xec, 0x87, 0xaf, 0xce, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x04, +- 0x54, 0x54, 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0x3f, 0x83, 0xd5, 0x3d, 0xb9, 0x8b, 0x46, +- 0x22, 0x24, 0xf1, 0xab, 0x68, 0xdb, 0xff, 0x0f, 0xd6, 0x9e, 0x8b, 0xe9, 0xd7, 0x52, 0x18, 0x27, +- 0xdf, 0xe1, 0x57, 0xe4, 0x1f, 0xa3, 0x2f, 0xac, 0x7f, 0x8c, 0xbe, 0x90, 0xdb, 0x32, 0xa0, 0x93, +- 0x34, 0x8a, 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa1, 0xa3, 0x1b, 0xb9, 0xa7, 0x34, 0x4c, +- 0x27, 0x78, 0xe1, 0x31, 0x70, 0x1b, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, +- 0xcb, 0x2d, 0x60, 0x9c, 0x9f, 0x57, 0x61, 0x5d, 0xdf, 0xcb, 0x8d, 0xf4, 0x75, 0x94, 0x75, 0x61, +- 0x08, 0xcd, 0x53, 0xca, 0x45, 0x41, 0x60, 0x06, 0x4b, 0x13, 0xc3, 0xd8, 0x4a, 0x93, 0xbf, 0xa5, +- 0xcb, 0xb2, 0xda, 0xd5, 0x97, 0x65, 0x73, 0xd7, 0x61, 0xf5, 0xf9, 0xeb, 0x30, 0x99, 0x00, 0x2d, +- 0x13, 0xd1, 0xc7, 0x4c, 0xcb, 0x6d, 0x19, 0xcc, 0x41, 0x88, 0xee, 0x42, 0x7f, 0x2c, 0xad, 0xf4, +- 0x4e, 0x29, 0x3d, 0xf3, 0x12, 0x5f, 0x9c, 0xaa, 0xc4, 0xda, 0x72, 0xbb, 0x0a, 0xbd, 0x4f, 0xe9, +- 0xd9, 0xa1, 0x2f, 0x4e, 0xd1, 0x67, 0xd0, 0x33, 0xbd, 0x48, 0xa4, 0x42, 0xc4, 0x4d, 0x05, 0x66, +- 0x76, 0x51, 0x31, 0x7a, 0x6e, 0xf7, 0xac, 0x00, 0x71, 0xe7, 0x3a, 0x5c, 0x7b, 0x88, 0xb9, 0x60, +- 0x74, 0x5a, 0x0e, 0x8c, 0xf3, 0x2f, 0x00, 0x07, 0x79, 0xfe, 0xf9, 0xa0, 0x08, 0x99, 0xac, 0xb5, +- 0xb2, 0xa5, 0xaf, 0x45, 0x33, 0x82, 0x5b, 0xe0, 0x71, 0xb6, 0x60, 0xd9, 0xa5, 0xa9, 0x3c, 0x11, +- 0xdf, 0xb2, 0x7f, 0x66, 0x5c, 0xc7, 0x8c, 0x53, 0x48, 0xd7, 0xd0, 0x9c, 0x7d, 0x7b, 0x8f, 0x92, +- 0x8b, 0x33, 0x53, 0xb4, 0x05, 0xad, 0x2c, 0x13, 0x9a, 0xac, 0x32, 0xaf, 0x3a, 0x67, 0x71, 0xfe, +- 0x1b, 0xd6, 0xb4, 0x24, 0x2d, 0xd9, 0x8a, 0x79, 0x0b, 0x96, 0x99, 0x35, 0xa3, 0x92, 0xdf, 0x87, +- 0x1a, 0x26, 0x43, 0x43, 0xb7, 0xa4, 0xb2, 0x80, 0xe1, 0xc8, 0x1e, 0x9b, 0x4d, 0x37, 0x47, 0xc8, +- 0x68, 0x3d, 0x21, 0x5c, 0xe4, 0x6e, 0xda, 0x68, 0xad, 0xc1, 0xaa, 0x24, 0x94, 0x34, 0x3a, 0xff, +- 0x0b, 0x6b, 0xcf, 0xe2, 0x09, 0x89, 0xf1, 0xde, 0xe1, 0xd1, 0x53, 0x9c, 0x65, 0x05, 0x04, 0x75, +- 0x75, 0xde, 0x55, 0x94, 0x74, 0xf5, 0x2f, 0xb7, 0x49, 0x7c, 0xec, 0x05, 0x49, 0xca, 0xcd, 0xf5, +- 0xe4, 0x72, 0x7c, 0xbc, 0x97, 0xa4, 0x5c, 0x9e, 0x81, 0xb2, 0xd6, 0xa4, 0xf1, 0x64, 0xaa, 0xf6, +- 0x4a, 0xd3, 0x6d, 0x04, 0x49, 0xfa, 0x2c, 0x9e, 0x4c, 0x9d, 0x7f, 0x56, 0x17, 0x32, 0x18, 0x87, +- 0xae, 0x1f, 0x87, 0x34, 0x7a, 0x88, 0xcf, 0x0b, 0x1a, 0xb2, 0xe6, 0xdf, 0xe6, 0x84, 0xef, 0x2b, +- 0xd0, 0x79, 0x30, 0xc6, 0xb1, 0x78, 0x88, 0x85, 0x4f, 0x26, 0xaa, 0xc1, 0x3f, 0xc7, 0x8c, 0x13, +- 0x1a, 0x9b, 0x85, 0x6f, 0x41, 0xf4, 0x3a, 0xb4, 0x49, 0x4c, 0x84, 0x17, 0xfa, 0x38, 0xa2, 0xb1, +- 0x89, 0x02, 0x48, 0xd4, 0x43, 0x85, 0x41, 0xef, 0x40, 0x5f, 0x5f, 0x1f, 0x7b, 0xa7, 0x7e, 0x1c, +- 0x4e, 0xe4, 0x96, 0xd3, 0xd7, 0x69, 0x3d, 0x8d, 0xde, 0x37, 0x58, 0xf4, 0x2e, 0xac, 0x98, 0x0d, +- 0x91, 0x73, 0xd6, 0x15, 0x67, 0xdf, 0xe0, 0x4b, 0xac, 0x69, 0x92, 0x50, 0x26, 0xb8, 0xc7, 0x71, +- 0x10, 0xd0, 0x28, 0x31, 0xdd, 0x71, 0xdf, 0xe2, 0x47, 0x1a, 0xed, 0x8c, 0x61, 0xed, 0xb1, 0xf4, +- 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0xd3, 0x94, +- 0x89, 0xb0, 0xac, 0xbf, 0x77, 0x25, 0x72, 0x44, 0xbe, 0x53, 0x17, 0x41, 0x92, 0xeb, 0x94, 0x8a, +- 0x64, 0x92, 0x8e, 0xbd, 0x84, 0xd1, 0x63, 0x6c, 0x5c, 0xec, 0x47, 0x38, 0xda, 0xd7, 0xf8, 0x43, +- 0x89, 0x76, 0x7e, 0x5b, 0x81, 0xf5, 0xb2, 0x26, 0x93, 0x74, 0xb7, 0x61, 0xbd, 0xac, 0xca, 0xd4, +- 0x82, 0xba, 0x9e, 0x58, 0x2d, 0x2a, 0xd4, 0x55, 0xe1, 0x27, 0xd0, 0x55, 0x6f, 0x0a, 0x5e, 0xa8, +- 0x25, 0x95, 0x8f, 0xb9, 0xe2, 0xbc, 0xb8, 0x1d, 0xbf, 0x38, 0x4b, 0x9f, 0xc1, 0x0d, 0xe3, 0xbe, +- 0x37, 0x6f, 0xb6, 0x5e, 0x10, 0x1b, 0x86, 0xe1, 0xe9, 0x8c, 0xf5, 0x4f, 0x60, 0x90, 0xa3, 0x76, +- 0xa7, 0x0a, 0x69, 0x63, 0xf5, 0x01, 0xac, 0xcd, 0x38, 0xfb, 0x20, 0x0c, 0x99, 0xda, 0xa0, 0x75, +- 0x77, 0x11, 0xc9, 0xb9, 0x0f, 0xd7, 0x47, 0x58, 0xe8, 0x68, 0xf8, 0xc2, 0x34, 0xa6, 0x5a, 0xd8, +- 0x0a, 0xd4, 0x46, 0x38, 0x50, 0xce, 0xd7, 0x5c, 0xf9, 0x2b, 0x17, 0xe0, 0x11, 0xc7, 0x81, 0xf2, +- 0xb2, 0xe6, 0xaa, 0x7f, 0xe7, 0xd7, 0x15, 0x68, 0x98, 0x34, 0x29, 0x53, 0x7d, 0xc8, 0xc8, 0x39, +- 0x66, 0x66, 0xe9, 0x19, 0x08, 0xbd, 0x0d, 0x3d, 0xfd, 0xe7, 0xd1, 0x44, 0x10, 0x9a, 0x25, 0xdf, +- 0xae, 0xc6, 0x3e, 0xd3, 0x48, 0x75, 0x5d, 0xac, 0x6e, 0x43, 0xcd, 0xc5, 0x83, 0x81, 0xd4, 0x9d, +- 0x2f, 0x97, 0x99, 0x41, 0x25, 0xdb, 0x96, 0x6b, 0x20, 0xb9, 0xd4, 0xad, 0xbc, 0x25, 0x25, 0xcf, +- 0x82, 0x72, 0xa9, 0x47, 0x34, 0x8d, 0x85, 0x97, 0x50, 0x12, 0x0b, 0x93, 0x5d, 0x41, 0xa1, 0x0e, +- 0x25, 0xc6, 0xf9, 0x49, 0x05, 0x96, 0xf5, 0x93, 0x09, 0xea, 0x41, 0x35, 0x3b, 0xe3, 0xaa, 0x44, +- 0xd5, 0x0b, 0x4a, 0x97, 0x3e, 0xd7, 0xd4, 0xbf, 0xdc, 0xc7, 0xe7, 0x91, 0xce, 0xd4, 0xc6, 0xb4, +- 0xf3, 0x48, 0xa5, 0xe8, 0xb7, 0xa1, 0x97, 0x1f, 0x95, 0x8a, 0xae, 0x4d, 0xec, 0x66, 0x58, 0xc5, +- 0x76, 0xa9, 0xa5, 0xce, 0x7f, 0x01, 0xe4, 0x4f, 0x07, 0x32, 0xe4, 0x69, 0x66, 0x8c, 0xfc, 0x95, +- 0x98, 0x71, 0x76, 0xc8, 0xca, 0x5f, 0x74, 0x17, 0x7a, 0x7e, 0x18, 0x12, 0x39, 0xdc, 0x9f, 0x3c, +- 0x26, 0x61, 0xb6, 0x49, 0xcb, 0x58, 0xe7, 0xf7, 0x15, 0xe8, 0xef, 0xd1, 0x64, 0xfa, 0xaf, 0x64, +- 0x82, 0x0b, 0x19, 0x44, 0x19, 0x69, 0xce, 0x58, 0xf9, 0xaf, 0xab, 0xff, 0x09, 0xd6, 0x5b, 0x4b, +- 0xcf, 0x6c, 0x53, 0x22, 0xd4, 0xb6, 0xb2, 0xc4, 0xec, 0x16, 0xb6, 0xab, 0x89, 0x4f, 0x69, 0xa8, +- 0x9a, 0xb4, 0x90, 0x30, 0x2f, 0xbb, 0x73, 0xed, 0xba, 0x8d, 0x90, 0x30, 0x45, 0x32, 0x8e, 0x2c, +- 0xa9, 0x6b, 0xff, 0xa2, 0x23, 0xcb, 0x1a, 0x23, 0x1d, 0xd9, 0x80, 0x65, 0x7a, 0x72, 0xc2, 0xb1, +- 0x50, 0xdd, 0x43, 0xcd, 0x35, 0x50, 0x96, 0xe6, 0x9a, 0x85, 0x34, 0x77, 0x0d, 0xd6, 0xd4, 0x03, +- 0xd3, 0x73, 0xe6, 0x07, 0x24, 0x1e, 0xdb, 0x54, 0xbc, 0x0e, 0x68, 0x24, 0x68, 0x32, 0x83, 0xdd, +- 0x82, 0x55, 0x73, 0xe6, 0x1c, 0xfe, 0xe7, 0xc8, 0xba, 0x7e, 0x03, 0x9a, 0x12, 0xf4, 0x18, 0xfe, +- 0xd6, 0x26, 0x46, 0x43, 0x76, 0xde, 0x85, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, +- 0xbe, 0xf3, 0xa7, 0x15, 0x93, 0x6e, 0xcd, 0x45, 0x0e, 0x7a, 0x0c, 0xfd, 0x99, 0x87, 0x41, 0x64, +- 0x6e, 0xf6, 0x16, 0xbf, 0x17, 0x0e, 0x37, 0xb6, 0xf4, 0x43, 0xe3, 0x96, 0x7d, 0x68, 0xdc, 0x7a, +- 0x14, 0x25, 0x62, 0x8a, 0x1e, 0x41, 0xaf, 0xfc, 0x84, 0x86, 0x6e, 0xda, 0x1a, 0x64, 0xc1, 0xc3, +- 0xda, 0xa5, 0x62, 0x1e, 0x43, 0x7f, 0xe6, 0x35, 0xcd, 0xda, 0xb3, 0xf8, 0x91, 0xed, 0x52, 0x41, +- 0xf7, 0xa1, 0x5d, 0x78, 0x3e, 0x43, 0x03, 0x2d, 0x64, 0xfe, 0x45, 0xed, 0x52, 0x01, 0x7b, 0xd0, +- 0x2d, 0xbd, 0x68, 0xa1, 0xa1, 0xf1, 0x67, 0xc1, 0x33, 0xd7, 0xa5, 0x42, 0x76, 0xa1, 0x5d, 0x78, +- 0x58, 0xb2, 0x56, 0xcc, 0xbf, 0x5e, 0x0d, 0x6f, 0x2c, 0xa0, 0x98, 0xe9, 0xdc, 0x87, 0x6e, 0xe9, +- 0x19, 0xc8, 0x1a, 0xb2, 0xe8, 0x09, 0x6a, 0x78, 0x73, 0x21, 0xcd, 0x48, 0x7a, 0x0c, 0xfd, 0x99, +- 0x47, 0x21, 0x1b, 0xdc, 0xc5, 0x6f, 0x45, 0x97, 0xba, 0xf5, 0x95, 0x9a, 0xec, 0x42, 0xbb, 0x55, +- 0x98, 0xec, 0xf9, 0x27, 0xa0, 0xe1, 0xad, 0xc5, 0x44, 0x63, 0xd5, 0x23, 0xe8, 0x95, 0x5f, 0x7f, +- 0xac, 0xb0, 0x85, 0x6f, 0x42, 0x57, 0xaf, 0x9c, 0xd2, 0x43, 0x50, 0xbe, 0x72, 0x16, 0xbd, 0x0f, +- 0x5d, 0x2a, 0xe8, 0x01, 0x80, 0x69, 0xae, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0xd4, 0x65, 0x53, +- 0xb6, 0xa0, 0x11, 0xbb, 0x0f, 0xa0, 0x7b, 0xa2, 0x90, 0xa6, 0x02, 0x5d, 0xb7, 0x66, 0xcc, 0x34, +- 0x62, 0xc3, 0xc1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x65, 0x04, 0x7c, 0x09, 0x90, 0xf7, 0x5a, +- 0x56, 0xc0, 0x5c, 0xf7, 0x75, 0x45, 0x0c, 0x3a, 0xc5, 0xce, 0x0a, 0x19, 0x5f, 0x17, 0x74, 0x5b, +- 0x57, 0x88, 0xe8, 0xcf, 0x54, 0xce, 0xe5, 0xc5, 0x36, 0x5b, 0x50, 0x0f, 0xe7, 0xaa, 0x67, 0xf4, +- 0x09, 0x74, 0x8a, 0x25, 0xb3, 0xb5, 0x62, 0x41, 0x19, 0x3d, 0x2c, 0x95, 0xcd, 0xe8, 0x3e, 0xf4, +- 0xca, 0x05, 0x31, 0x2a, 0xec, 0x8b, 0xb9, 0x32, 0x79, 0xb8, 0x32, 0x73, 0xd1, 0xc1, 0xd1, 0x87, +- 0x00, 0x79, 0xe1, 0x6c, 0xc3, 0x37, 0x57, 0x4a, 0xcf, 0x68, 0xfd, 0x12, 0x7a, 0x85, 0xbc, 0x2d, +- 0x7b, 0xc2, 0xeb, 0x25, 0x87, 0xf3, 0x6c, 0x3e, 0x34, 0x15, 0x56, 0x29, 0x6d, 0x3f, 0x80, 0x4e, +- 0xf1, 0x8c, 0xb0, 0xde, 0x2e, 0x38, 0x37, 0xae, 0x4a, 0x7a, 0x85, 0xf3, 0xc4, 0xae, 0xdd, 0xf9, +- 0x23, 0xe6, 0xaa, 0xa4, 0x57, 0xea, 0x47, 0x6d, 0xae, 0x59, 0xd4, 0xa4, 0x5e, 0x75, 0x14, 0x94, +- 0x9b, 0x37, 0x1b, 0xfd, 0x85, 0x2d, 0xdd, 0x55, 0x6b, 0xb0, 0xd8, 0xa7, 0xd8, 0x78, 0x2c, 0xe8, +- 0x5d, 0x7e, 0x24, 0x27, 0x14, 0x7b, 0x91, 0x42, 0x4e, 0x58, 0xd0, 0xa2, 0x5c, 0x2a, 0x68, 0x1f, +- 0xfa, 0x8f, 0x6d, 0x99, 0x69, 0x4a, 0x60, 0x63, 0xce, 0x82, 0x92, 0x7f, 0x38, 0x5c, 0x44, 0x32, +- 0xb3, 0xfc, 0x15, 0xac, 0xce, 0x95, 0xbf, 0xe8, 0x76, 0x76, 0xef, 0xbe, 0xb0, 0x2e, 0xbe, 0xd4, +- 0xac, 0x03, 0x58, 0x99, 0xad, 0x7e, 0xd1, 0x6b, 0x66, 0xd2, 0x17, 0x57, 0xc5, 0x97, 0x8a, 0xfa, +- 0x0c, 0x9a, 0xb6, 0xda, 0x42, 0xe6, 0x7d, 0x63, 0xa6, 0xfa, 0xba, 0x6c, 0xe8, 0x6e, 0xe7, 0xfb, +- 0x1f, 0x6e, 0x57, 0xfe, 0xf8, 0xc3, 0xed, 0xca, 0x5f, 0x7e, 0xb8, 0x5d, 0x39, 0x5e, 0x56, 0xd4, +- 0x0f, 0xff, 0x16, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x16, 0x45, 0x76, 0xe7, 0x24, 0x00, 0x00, ++ // 3214 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x39, 0xcb, 0x6e, 0x1c, 0xc7, ++ 0x76, 0x98, 0x07, 0x39, 0x33, 0x67, 0x5e, 0x9c, 0x22, 0x45, 0x8d, 0x46, 0xb2, 0x2c, 0xb7, 0x6d, ++ 0x99, 0x8e, 0x63, 0xd2, 0xa2, 0x0d, 0x3f, 0xe1, 0x08, 0x22, 0xa5, 0x88, 0x8c, 0x25, 0x8b, 0x69, ++ 0x8a, 0x71, 0x8c, 0x20, 0x68, 0x34, 0xbb, 0x8b, 0xc3, 0x32, 0xa7, 0xbb, 0xda, 0x55, 0xd5, 0x14, ++ 0xc7, 0x01, 0xb2, 0x4c, 0xb6, 0x59, 0x65, 0x97, 0x1f, 0x08, 0x92, 0x55, 0x76, 0xc9, 0x36, 0x0b, ++ 0x23, 0xc8, 0xe2, 0x7e, 0xc1, 0xc5, 0x85, 0x3f, 0xe1, 0x7e, 0xc1, 0x45, 0xbd, 0xfa, 0x31, 0x33, ++ 0xa4, 0x71, 0x09, 0x01, 0x77, 0xd3, 0xe8, 0xf3, 0xa8, 0xf3, 0xaa, 0xaa, 0x53, 0xe7, 0x54, 0x41, ++ 0xdb, 0x1f, 0xe3, 0x58, 0x6c, 0x26, 0x8c, 0x0a, 0x8a, 0xea, 0x63, 0x96, 0x04, 0xa3, 0x16, 0x0d, ++ 0x88, 0x46, 0x8c, 0x3e, 0x1d, 0x13, 0x71, 0x9a, 0x1e, 0x6f, 0x06, 0x34, 0xda, 0x3a, 0xf3, 0x85, ++ 0xff, 0x61, 0x40, 0x63, 0xe1, 0x93, 0x18, 0x33, 0xbe, 0xa5, 0x06, 0x6e, 0x25, 0x67, 0xe3, 0x2d, ++ 0x31, 0x4d, 0x30, 0xd7, 0x5f, 0x33, 0xee, 0xf6, 0x98, 0xd2, 0xf1, 0x04, 0x6f, 0x29, 0xe8, 0x38, ++ 0x3d, 0xd9, 0xc2, 0x51, 0x22, 0xa6, 0x9a, 0xe8, 0xfc, 0x5b, 0x15, 0xd6, 0x77, 0x19, 0xf6, 0x05, ++ 0xde, 0xb5, 0xd2, 0x5c, 0xfc, 0x63, 0x8a, 0xb9, 0x40, 0x6f, 0x41, 0x27, 0xd3, 0xe0, 0x91, 0x70, ++ 0x58, 0xb9, 0x57, 0xd9, 0x68, 0xb9, 0xed, 0x0c, 0xb7, 0x1f, 0xa2, 0x9b, 0xd0, 0xc0, 0x17, 0x38, ++ 0x90, 0xd4, 0xaa, 0xa2, 0x2e, 0x4b, 0x70, 0x3f, 0x44, 0x0f, 0xa0, 0xcd, 0x05, 0x23, 0xf1, 0xd8, ++ 0x4b, 0x39, 0x66, 0xc3, 0xda, 0xbd, 0xca, 0x46, 0x7b, 0x7b, 0x65, 0x53, 0xba, 0xb4, 0x79, 0xa8, ++ 0x08, 0x47, 0x1c, 0x33, 0x17, 0x78, 0xf6, 0x8f, 0xee, 0x43, 0x23, 0xc4, 0xe7, 0x24, 0xc0, 0x7c, ++ 0x58, 0xbf, 0x57, 0xdb, 0x68, 0x6f, 0x77, 0x34, 0xfb, 0x63, 0x85, 0x74, 0x2d, 0x11, 0xbd, 0x0f, ++ 0x4d, 0x2e, 0x28, 0xf3, 0xc7, 0x98, 0x0f, 0x97, 0x14, 0x63, 0xd7, 0xca, 0x55, 0x58, 0x37, 0x23, ++ 0xa3, 0x3b, 0x50, 0x7b, 0xb1, 0xbb, 0x3f, 0x5c, 0x56, 0xda, 0xc1, 0x70, 0x25, 0x38, 0x70, 0x25, ++ 0x1a, 0xbd, 0x0d, 0x5d, 0xee, 0xc7, 0xe1, 0x31, 0xbd, 0xf0, 0x12, 0x12, 0xc6, 0x7c, 0xd8, 0xb8, ++ 0x57, 0xd9, 0x68, 0xba, 0x1d, 0x83, 0x3c, 0x90, 0x38, 0xe7, 0x4b, 0xb8, 0x71, 0x28, 0x7c, 0x26, ++ 0xae, 0x11, 0x1d, 0xe7, 0x08, 0xd6, 0x5d, 0x1c, 0xd1, 0xf3, 0x6b, 0x85, 0x76, 0x08, 0x0d, 0x41, ++ 0x22, 0x4c, 0x53, 0xa1, 0x42, 0xdb, 0x75, 0x2d, 0xe8, 0xfc, 0x47, 0x05, 0xd0, 0x93, 0x0b, 0x1c, ++ 0x1c, 0x30, 0x1a, 0x60, 0xce, 0xff, 0x44, 0xd3, 0xf5, 0x1e, 0x34, 0x12, 0x6d, 0xc0, 0xb0, 0xae, ++ 0xd8, 0xcd, 0x2c, 0x58, 0xab, 0x2c, 0xd5, 0xf9, 0x01, 0xd6, 0x0e, 0xc9, 0x38, 0xf6, 0x27, 0xaf, ++ 0xd1, 0xde, 0x75, 0x58, 0xe6, 0x4a, 0xa6, 0x32, 0xb5, 0xeb, 0x1a, 0xc8, 0x39, 0x00, 0xf4, 0x9d, ++ 0x4f, 0xc4, 0xeb, 0xd3, 0xe4, 0x7c, 0x08, 0xab, 0x25, 0x89, 0x3c, 0xa1, 0x31, 0xc7, 0xca, 0x00, ++ 0xe1, 0x8b, 0x94, 0x2b, 0x61, 0x4b, 0xae, 0x81, 0x1c, 0x0c, 0x6b, 0xcf, 0x08, 0xb7, 0xec, 0xf8, ++ 0x8f, 0x31, 0x61, 0x1d, 0x96, 0x4f, 0x28, 0x8b, 0x7c, 0x61, 0x2d, 0xd0, 0x10, 0x42, 0x50, 0xf7, ++ 0xd9, 0x98, 0x0f, 0x6b, 0xf7, 0x6a, 0x1b, 0x2d, 0x57, 0xfd, 0xcb, 0x55, 0x39, 0xa3, 0xc6, 0xd8, ++ 0xf5, 0x16, 0x74, 0x4c, 0xdc, 0xbd, 0x09, 0xe1, 0x42, 0xe9, 0xe9, 0xb8, 0x6d, 0x83, 0x93, 0x63, ++ 0x1c, 0x0a, 0xeb, 0x47, 0x49, 0x78, 0xcd, 0x0d, 0xbf, 0x0d, 0x2d, 0x86, 0x39, 0x4d, 0x99, 0xdc, ++ 0xa6, 0x55, 0x35, 0xef, 0x6b, 0x7a, 0xde, 0x9f, 0x91, 0x38, 0xbd, 0x70, 0x2d, 0xcd, 0xcd, 0xd9, ++ 0xcc, 0x16, 0x12, 0xfc, 0x3a, 0x5b, 0xe8, 0x4b, 0xb8, 0x71, 0xe0, 0xa7, 0xfc, 0x3a, 0xb6, 0x3a, ++ 0x5f, 0xc9, 0xed, 0xc7, 0xd3, 0xe8, 0x5a, 0x83, 0xff, 0xbd, 0x02, 0xcd, 0xdd, 0x24, 0x3d, 0xe2, ++ 0xfe, 0x18, 0xa3, 0x37, 0xa1, 0x2d, 0xa8, 0xf0, 0x27, 0x5e, 0x2a, 0x41, 0xc5, 0x5e, 0x77, 0x41, ++ 0xa1, 0x34, 0x83, 0x0c, 0x3b, 0x66, 0x41, 0x92, 0x1a, 0x8e, 0xea, 0xbd, 0xda, 0x46, 0xdd, 0x6d, ++ 0x6b, 0x9c, 0x66, 0xd9, 0x84, 0x55, 0x45, 0xf3, 0x48, 0xec, 0x9d, 0x61, 0x16, 0xe3, 0x49, 0x44, ++ 0x43, 0xac, 0xd6, 0x6f, 0xdd, 0x1d, 0x28, 0xd2, 0x7e, 0xfc, 0x4d, 0x46, 0x40, 0x7f, 0x06, 0x83, ++ 0x8c, 0x5f, 0x6e, 0x4a, 0xc5, 0x5d, 0x57, 0xdc, 0x7d, 0xc3, 0x7d, 0x64, 0xd0, 0xce, 0x3f, 0x42, ++ 0xef, 0xe5, 0x29, 0xa3, 0x42, 0x4c, 0x48, 0x3c, 0x7e, 0xec, 0x0b, 0x5f, 0x66, 0x8f, 0x04, 0x33, ++ 0x42, 0x43, 0x6e, 0xac, 0xb5, 0x20, 0xfa, 0x00, 0x06, 0x42, 0xf3, 0xe2, 0xd0, 0xb3, 0x3c, 0x55, ++ 0xc5, 0xb3, 0x92, 0x11, 0x0e, 0x0c, 0xf3, 0xbb, 0xd0, 0xcb, 0x99, 0x65, 0xfe, 0x31, 0xf6, 0x76, ++ 0x33, 0xec, 0x4b, 0x12, 0x61, 0xe7, 0x5c, 0xc5, 0x4a, 0x4d, 0x32, 0xfa, 0x00, 0x5a, 0x79, 0x1c, ++ 0x2a, 0x6a, 0x85, 0xf4, 0xf4, 0x0a, 0xb1, 0xe1, 0x74, 0x9b, 0x59, 0x50, 0xbe, 0x86, 0xbe, 0xc8, ++ 0x0c, 0xf7, 0x42, 0x5f, 0xf8, 0xe5, 0x45, 0x55, 0xf6, 0xca, 0xed, 0x89, 0x12, 0xec, 0x7c, 0x05, ++ 0xad, 0x03, 0x12, 0x72, 0xad, 0x78, 0x08, 0x8d, 0x20, 0x65, 0x0c, 0xc7, 0xc2, 0xba, 0x6c, 0x40, ++ 0xb4, 0x06, 0x4b, 0x13, 0x12, 0x11, 0x61, 0xdc, 0xd4, 0x80, 0x43, 0x01, 0x9e, 0xe3, 0x88, 0xb2, ++ 0xa9, 0x0a, 0xd8, 0x1a, 0x2c, 0x15, 0x27, 0x57, 0x03, 0xe8, 0x36, 0xb4, 0x22, 0xff, 0x22, 0x9b, ++ 0x54, 0x49, 0x69, 0x46, 0xfe, 0x85, 0x36, 0x7e, 0x08, 0x8d, 0x13, 0x9f, 0x4c, 0x82, 0x58, 0x98, ++ 0xa8, 0x58, 0x30, 0x57, 0x58, 0x2f, 0x2a, 0xfc, 0xdf, 0x2a, 0xb4, 0xb5, 0x46, 0x6d, 0xf0, 0x1a, ++ 0x2c, 0x05, 0x7e, 0x70, 0x9a, 0xa9, 0x54, 0x00, 0xba, 0x6f, 0x0d, 0xa9, 0x16, 0x93, 0x70, 0x6e, ++ 0xa9, 0x35, 0x6d, 0x0b, 0x80, 0xbf, 0xf2, 0x13, 0x63, 0x5b, 0xed, 0x12, 0xe6, 0x96, 0xe4, 0xd1, ++ 0xe6, 0x7e, 0x0c, 0x1d, 0xbd, 0xee, 0xcc, 0x90, 0xfa, 0x25, 0x43, 0xda, 0x9a, 0x4b, 0x0f, 0x7a, ++ 0x1b, 0xba, 0x29, 0xc7, 0xde, 0x29, 0xc1, 0xcc, 0x67, 0xc1, 0xe9, 0x74, 0xb8, 0xa4, 0xcf, 0xc8, ++ 0x94, 0xe3, 0x3d, 0x8b, 0x43, 0xdb, 0xb0, 0x24, 0xd3, 0x1f, 0x1f, 0x2e, 0xab, 0xe3, 0xf8, 0x4e, ++ 0x51, 0xa4, 0x72, 0x75, 0x53, 0x7d, 0x9f, 0xc4, 0x82, 0x4d, 0x5d, 0xcd, 0x3a, 0xfa, 0x1c, 0x20, ++ 0x47, 0xa2, 0x15, 0xa8, 0x9d, 0xe1, 0xa9, 0xd9, 0x87, 0xf2, 0x57, 0x06, 0xe7, 0xdc, 0x9f, 0xa4, ++ 0x36, 0xea, 0x1a, 0xf8, 0xb2, 0xfa, 0x79, 0xc5, 0x09, 0xa0, 0xbf, 0x33, 0x39, 0x23, 0xb4, 0x30, ++ 0x7c, 0x0d, 0x96, 0x22, 0xff, 0x07, 0xca, 0x6c, 0x24, 0x15, 0xa0, 0xb0, 0x24, 0xa6, 0xcc, 0x8a, ++ 0x50, 0x00, 0xea, 0x41, 0x95, 0x26, 0x2a, 0x5e, 0x2d, 0xb7, 0x4a, 0x93, 0x5c, 0x51, 0xbd, 0xa0, ++ 0xc8, 0xf9, 0x6d, 0x1d, 0x20, 0xd7, 0x82, 0x5c, 0x18, 0x11, 0xea, 0x71, 0xcc, 0x64, 0x09, 0xe2, ++ 0x1d, 0x4f, 0x05, 0xe6, 0x1e, 0xc3, 0x41, 0xca, 0x38, 0x39, 0x97, 0xf3, 0x27, 0xdd, 0xbe, 0xa1, ++ 0xdd, 0x9e, 0xb1, 0xcd, 0xbd, 0x49, 0xe8, 0xa1, 0x1e, 0xb7, 0x23, 0x87, 0xb9, 0x76, 0x14, 0xda, ++ 0x87, 0x1b, 0xb9, 0xcc, 0xb0, 0x20, 0xae, 0x7a, 0x95, 0xb8, 0xd5, 0x4c, 0x5c, 0x98, 0x8b, 0x7a, ++ 0x02, 0xab, 0x84, 0x7a, 0x3f, 0xa6, 0x38, 0x2d, 0x09, 0xaa, 0x5d, 0x25, 0x68, 0x40, 0xe8, 0x5f, ++ 0xab, 0x01, 0xb9, 0x98, 0x03, 0xb8, 0x55, 0xf0, 0x52, 0x6e, 0xf7, 0x82, 0xb0, 0xfa, 0x55, 0xc2, ++ 0xd6, 0x33, 0xab, 0x64, 0x3e, 0xc8, 0x25, 0xfe, 0x15, 0xac, 0x13, 0xea, 0xbd, 0xf2, 0x89, 0x98, ++ 0x15, 0xb7, 0xf4, 0x2b, 0x4e, 0xca, 0x43, 0xb7, 0x2c, 0x4b, 0x3b, 0x19, 0x61, 0x36, 0x2e, 0x39, ++ 0xb9, 0xfc, 0x2b, 0x4e, 0x3e, 0x57, 0x03, 0x72, 0x31, 0x8f, 0x60, 0x40, 0xe8, 0xac, 0x35, 0x8d, ++ 0xab, 0x84, 0xf4, 0x09, 0x2d, 0x5b, 0xb2, 0x03, 0x03, 0x8e, 0x03, 0x41, 0x59, 0x71, 0x11, 0x34, ++ 0xaf, 0x12, 0xb1, 0x62, 0xf8, 0x33, 0x19, 0xce, 0xdf, 0x41, 0x67, 0x2f, 0x1d, 0x63, 0x31, 0x39, ++ 0xce, 0x92, 0xc1, 0x6b, 0xcb, 0x3f, 0xce, 0xef, 0xab, 0xd0, 0xde, 0x1d, 0x33, 0x9a, 0x26, 0xa5, ++ 0x9c, 0xac, 0x37, 0xe9, 0x6c, 0x4e, 0x56, 0x2c, 0x2a, 0x27, 0x6b, 0xe6, 0x4f, 0xa0, 0x13, 0xa9, ++ 0xad, 0x6b, 0xf8, 0x75, 0x1e, 0x1a, 0xcc, 0x6d, 0x6a, 0xb7, 0x1d, 0x15, 0x92, 0xd9, 0x26, 0x40, ++ 0x42, 0x42, 0x6e, 0xc6, 0xe8, 0x74, 0xd4, 0x37, 0x15, 0xa1, 0x4d, 0xd1, 0x6e, 0x2b, 0xc9, 0xb2, ++ 0xf5, 0x03, 0x68, 0x1f, 0xcb, 0x20, 0x99, 0x01, 0xa5, 0x64, 0x94, 0x47, 0xcf, 0x85, 0xe3, 0x7c, ++ 0x13, 0xee, 0x41, 0xf7, 0x54, 0x87, 0xcc, 0x0c, 0xd2, 0x6b, 0xe8, 0x6d, 0xe3, 0x49, 0xee, 0xef, ++ 0x66, 0x31, 0xb2, 0x7a, 0x02, 0x3a, 0xa7, 0x05, 0xd4, 0xe8, 0x10, 0x06, 0x73, 0x2c, 0x0b, 0x72, ++ 0xd0, 0x46, 0x31, 0x07, 0xb5, 0xb7, 0x91, 0x56, 0x54, 0x1c, 0x59, 0xcc, 0x4b, 0xff, 0x52, 0x85, ++ 0xde, 0x7e, 0x2c, 0x30, 0x3b, 0xf1, 0x03, 0xac, 0x2d, 0x46, 0x50, 0x8f, 0xfd, 0x08, 0x1b, 0x99, ++ 0xea, 0x1f, 0xdd, 0x82, 0x26, 0xbb, 0xd0, 0x29, 0xc4, 0xcc, 0x68, 0x83, 0x5d, 0xa8, 0xd4, 0x80, ++ 0xde, 0x00, 0x60, 0x17, 0x5e, 0xe2, 0x07, 0x67, 0xd8, 0xc4, 0xb0, 0xee, 0xb6, 0xd8, 0xc5, 0x81, ++ 0x46, 0xc8, 0xc5, 0xc0, 0x2e, 0x3c, 0xcc, 0x18, 0x65, 0xdc, 0x64, 0xab, 0x26, 0xbb, 0x78, 0xa2, ++ 0x60, 0x33, 0x36, 0x64, 0x34, 0x49, 0x70, 0xa8, 0xb2, 0xb4, 0x1a, 0xfb, 0x58, 0x23, 0xa4, 0x56, ++ 0x61, 0xb5, 0x2e, 0x6b, 0xad, 0x22, 0xd7, 0x2a, 0x72, 0xad, 0x0d, 0x3d, 0x52, 0x14, 0xb5, 0x8a, ++ 0x4c, 0x6b, 0x53, 0x6b, 0x15, 0x05, 0xad, 0x22, 0xd7, 0xda, 0xb2, 0x63, 0x8d, 0x56, 0xe7, 0xbf, ++ 0xab, 0xd0, 0x78, 0x19, 0xa8, 0x49, 0x41, 0xf7, 0xa0, 0x8d, 0xb9, 0xf0, 0x8f, 0x27, 0x84, 0x9f, ++ 0xe2, 0xd0, 0x2c, 0xf3, 0x22, 0x4a, 0xda, 0xc8, 0xa7, 0xb1, 0xc7, 0xe5, 0x09, 0x6e, 0x22, 0xc3, ++ 0xa7, 0xf1, 0xa1, 0x3c, 0xc1, 0x0d, 0x89, 0xe1, 0xe0, 0xdc, 0xae, 0x75, 0x3e, 0x8d, 0x5d, 0x1c, ++ 0x9c, 0x4b, 0xfb, 0x4e, 0x48, 0xac, 0x72, 0xcc, 0x03, 0x1b, 0x95, 0x13, 0x12, 0xcb, 0xfc, 0xf1, ++ 0xa0, 0x48, 0xdc, 0x36, 0x41, 0xb1, 0xc4, 0x6d, 0xe5, 0x99, 0x4c, 0x03, 0x92, 0x6a, 0x82, 0xd2, ++ 0x94, 0x08, 0x49, 0x55, 0x87, 0xf3, 0x84, 0x72, 0x6c, 0x02, 0xa2, 0x01, 0xe9, 0xaf, 0xfa, 0xd1, ++ 0x63, 0x74, 0x34, 0x5a, 0x0a, 0xa3, 0x06, 0xdd, 0x82, 0xe6, 0xc4, 0xe7, 0xc2, 0xf3, 0x83, 0x33, ++ 0x13, 0x8c, 0x86, 0x84, 0x1f, 0x05, 0x67, 0xb2, 0xba, 0x97, 0x05, 0x39, 0x8e, 0x87, 0xa0, 0x08, ++ 0x06, 0x52, 0x55, 0xcb, 0x84, 0x72, 0x12, 0x8f, 0x87, 0x6d, 0x53, 0xb5, 0x68, 0xd0, 0x49, 0xa1, ++ 0x71, 0x14, 0xea, 0xd8, 0xe5, 0x83, 0x2b, 0xb3, 0x83, 0x6d, 0xec, 0x4d, 0xc0, 0x0c, 0x68, 0xd6, ++ 0x8a, 0x3e, 0x11, 0x4c, 0xc4, 0x9a, 0xec, 0x42, 0x27, 0x7c, 0x33, 0xa5, 0x86, 0x58, 0xb7, 0x53, ++ 0xaa, 0x89, 0xce, 0xff, 0x57, 0xa0, 0xf3, 0x2d, 0x16, 0xaf, 0x28, 0x3b, 0xb3, 0xf9, 0x00, 0x88, ++ 0x5d, 0xd6, 0xdc, 0x9c, 0x75, 0xa6, 0x3c, 0x2b, 0x2f, 0x77, 0xb7, 0xc0, 0x87, 0xde, 0x84, 0x9a, ++ 0x08, 0x12, 0xb3, 0x73, 0x4c, 0x6b, 0x68, 0x96, 0x82, 0x2b, 0x29, 0xe8, 0x2d, 0xa8, 0x8b, 0x20, ++ 0xf9, 0xd4, 0xa4, 0x8a, 0x19, 0x0e, 0x45, 0x92, 0x32, 0xd2, 0x30, 0x29, 0xb7, 0x97, 0x26, 0x24, ++ 0xae, 0xa4, 0x48, 0x19, 0x69, 0x98, 0x7c, 0xaa, 0x66, 0x76, 0x8e, 0x43, 0x91, 0x9c, 0x7f, 0xae, ++ 0xc0, 0xfa, 0x6c, 0xf7, 0x61, 0x7a, 0xa5, 0x4f, 0xa0, 0x13, 0xa8, 0xa4, 0x51, 0x4a, 0x8c, 0x83, ++ 0xb9, 0x74, 0xe2, 0xb6, 0x83, 0x42, 0x2e, 0xfd, 0x0c, 0xba, 0xb1, 0x0e, 0x4f, 0x29, 0x3f, 0x9a, ++ 0xe4, 0x50, 0x8c, 0x9c, 0xdb, 0x89, 0x0b, 0x90, 0x13, 0x02, 0xfa, 0x8e, 0x11, 0x81, 0x0f, 0x05, ++ 0xc3, 0x7e, 0xf4, 0x3a, 0xba, 0x60, 0x04, 0x75, 0x55, 0x32, 0xd7, 0x54, 0x93, 0xa7, 0xfe, 0x9d, ++ 0xf7, 0x60, 0xb5, 0xa4, 0xc5, 0xf8, 0xba, 0x02, 0xb5, 0x89, 0x59, 0x3e, 0x5d, 0x57, 0xfe, 0x3a, ++ 0x3e, 0x0c, 0x5c, 0xec, 0x87, 0xaf, 0xcf, 0x1a, 0xa3, 0xa2, 0x96, 0xab, 0xd8, 0x00, 0x54, 0x54, ++ 0x61, 0x4c, 0xb1, 0x56, 0x57, 0x0a, 0x56, 0xbf, 0x80, 0xc1, 0xae, 0xdc, 0x45, 0x87, 0x22, 0x24, ++ 0xf1, 0xeb, 0x68, 0xdb, 0xff, 0x01, 0x56, 0x5f, 0x8a, 0xe9, 0x77, 0x52, 0x18, 0x27, 0x3f, 0xe1, ++ 0xd7, 0xe4, 0x1f, 0xa3, 0xaf, 0xac, 0x7f, 0x8c, 0xbe, 0x92, 0xdb, 0x32, 0xa0, 0x93, 0x34, 0x8a, ++ 0xd5, 0x12, 0xed, 0xba, 0x06, 0x72, 0x76, 0xa0, 0xa3, 0x1b, 0xb9, 0xe7, 0x34, 0x4c, 0x27, 0x78, ++ 0xe1, 0x31, 0x70, 0x17, 0x20, 0xf1, 0x99, 0x1f, 0x61, 0x81, 0x19, 0x57, 0x25, 0x5f, 0xcb, 0x2d, ++ 0x60, 0x9c, 0x7f, 0xad, 0xc2, 0x9a, 0xbe, 0x97, 0x3b, 0xd4, 0xd7, 0x51, 0xd6, 0x85, 0x11, 0x34, ++ 0x4f, 0x29, 0x17, 0x05, 0x81, 0x19, 0x2c, 0x4d, 0x0c, 0x63, 0x2b, 0x4d, 0xfe, 0x96, 0x2e, 0xcb, ++ 0x6a, 0x57, 0x5f, 0x96, 0xcd, 0x5d, 0x87, 0xd5, 0xe7, 0xaf, 0xc3, 0x64, 0x02, 0xb4, 0x4c, 0x44, ++ 0x1f, 0x33, 0x2d, 0xb7, 0x65, 0x30, 0xfb, 0x21, 0xba, 0x0f, 0xfd, 0xb1, 0xb4, 0xd2, 0x3b, 0xa5, ++ 0xf4, 0xcc, 0x4b, 0x7c, 0x71, 0xaa, 0x12, 0x6b, 0xcb, 0xed, 0x2a, 0xf4, 0x1e, 0xa5, 0x67, 0x07, ++ 0xbe, 0x38, 0x45, 0x5f, 0x40, 0xcf, 0xf4, 0x22, 0x91, 0x0a, 0x11, 0x37, 0x15, 0x98, 0xd9, 0x45, ++ 0xc5, 0xe8, 0xb9, 0xdd, 0xb3, 0x02, 0xc4, 0x9d, 0x9b, 0x70, 0xe3, 0x31, 0xe6, 0x82, 0xd1, 0x69, ++ 0x39, 0x30, 0xce, 0x5f, 0x00, 0xec, 0xe7, 0xf9, 0xe7, 0xa3, 0x22, 0x64, 0xb2, 0xd6, 0xca, 0xa6, ++ 0xbe, 0x16, 0xcd, 0x08, 0x6e, 0x81, 0xc7, 0xd9, 0x84, 0x65, 0x97, 0xa6, 0xf2, 0x44, 0x7c, 0xc7, ++ 0xfe, 0x99, 0x71, 0x1d, 0x33, 0x4e, 0x21, 0x5d, 0x43, 0x73, 0xf6, 0xec, 0x3d, 0x4a, 0x2e, 0xce, ++ 0x4c, 0xd1, 0x26, 0xb4, 0xb2, 0x4c, 0x68, 0xb2, 0xca, 0xbc, 0xea, 0x9c, 0xc5, 0x79, 0x09, 0xce, ++ 0x8c, 0xa4, 0xbd, 0x57, 0x8f, 0xc2, 0x90, 0xed, 0x4c, 0xbf, 0xf5, 0xa3, 0x6b, 0x4b, 0xfd, 0x1e, ++ 0x56, 0xb5, 0x54, 0x6d, 0xaf, 0x15, 0xf3, 0x0e, 0x2c, 0x33, 0xeb, 0x5c, 0x25, 0xbf, 0x65, 0x35, ++ 0x4c, 0x86, 0x86, 0xee, 0x48, 0x65, 0x01, 0xc3, 0x91, 0x3d, 0x8c, 0x9b, 0x6e, 0x8e, 0x90, 0x73, ++ 0xf0, 0x8c, 0x70, 0x91, 0x07, 0xcf, 0xce, 0xc1, 0x2a, 0x0c, 0x24, 0xa1, 0xa4, 0xd1, 0xf9, 0x7b, ++ 0x58, 0x7d, 0x11, 0x4f, 0x48, 0x8c, 0x77, 0x0f, 0x8e, 0x9e, 0xe3, 0x2c, 0xd7, 0x20, 0xa8, 0xab, ++ 0x53, 0xb4, 0xa2, 0xa4, 0xab, 0x7f, 0xb9, 0xf9, 0xe2, 0x63, 0x2f, 0x48, 0x52, 0x6e, 0x2e, 0x3d, ++ 0x97, 0xe3, 0xe3, 0xdd, 0x24, 0xe5, 0xf2, 0x64, 0x95, 0x15, 0x2c, 0x8d, 0x27, 0x53, 0xb5, 0x03, ++ 0x9b, 0x6e, 0x23, 0x48, 0xd2, 0x17, 0xf1, 0x64, 0xea, 0xfc, 0xb9, 0xba, 0xe6, 0xc1, 0x38, 0x74, ++ 0xfd, 0x38, 0xa4, 0xd1, 0x63, 0x7c, 0x5e, 0xd0, 0x90, 0x5d, 0x29, 0xd8, 0x4c, 0xf3, 0x73, 0x05, ++ 0x3a, 0x8f, 0xc6, 0x38, 0x16, 0x8f, 0xb1, 0xf0, 0xc9, 0x44, 0x5d, 0x1b, 0x9c, 0x63, 0xc6, 0x09, ++ 0x8d, 0xcd, 0x76, 0xb2, 0x20, 0x7a, 0x13, 0xda, 0x24, 0x26, 0xc2, 0x0b, 0x7d, 0x1c, 0xd1, 0xd8, ++ 0x44, 0x01, 0x24, 0xea, 0xb1, 0xc2, 0xa0, 0xf7, 0xa0, 0xaf, 0x2f, 0xa5, 0xbd, 0x53, 0x3f, 0x0e, ++ 0x27, 0x72, 0x23, 0xeb, 0x4b, 0xba, 0x9e, 0x46, 0xef, 0x19, 0x2c, 0x7a, 0x1f, 0x56, 0xcc, 0x36, ++ 0xcb, 0x39, 0xeb, 0x8a, 0xb3, 0x6f, 0xf0, 0x25, 0xd6, 0x34, 0x49, 0x28, 0x13, 0xdc, 0xe3, 0x38, ++ 0x08, 0x68, 0x94, 0x98, 0x9e, 0xbb, 0x6f, 0xf1, 0x87, 0x1a, 0xed, 0x8c, 0x61, 0xf5, 0xa9, 0xf4, ++ 0xd3, 0x78, 0x92, 0x4f, 0x70, 0x2f, 0xc2, 0x91, 0x77, 0x3c, 0xa1, 0xc1, 0x99, 0x27, 0x93, 0x9f, ++ 0x89, 0xb0, 0xac, 0xea, 0x77, 0x24, 0xf2, 0x90, 0xfc, 0xa4, 0xae, 0x97, 0x24, 0xd7, 0x29, 0x15, ++ 0xc9, 0x24, 0x1d, 0x7b, 0x09, 0xa3, 0xc7, 0xd8, 0xb8, 0xd8, 0x8f, 0x70, 0xb4, 0xa7, 0xf1, 0x07, ++ 0x12, 0xed, 0xfc, 0x4f, 0x05, 0xd6, 0xca, 0x9a, 0x4c, 0x2a, 0xdf, 0x82, 0xb5, 0xb2, 0x2a, 0x53, ++ 0x61, 0xea, 0x2a, 0x65, 0x50, 0x54, 0xa8, 0x6b, 0xcd, 0xcf, 0xa0, 0xab, 0x5e, 0x2a, 0xbc, 0x50, ++ 0x4b, 0x2a, 0x1f, 0x9e, 0xc5, 0x79, 0x71, 0x3b, 0x7e, 0x71, 0x96, 0xbe, 0x80, 0x5b, 0xc6, 0x7d, ++ 0x6f, 0xde, 0x6c, 0xbd, 0x20, 0xd6, 0x0d, 0xc3, 0xf3, 0x19, 0xeb, 0x9f, 0xc1, 0x30, 0x47, 0xed, ++ 0x4c, 0x15, 0xd2, 0xc6, 0xea, 0x23, 0x58, 0x9d, 0x71, 0x56, 0xee, 0x3b, 0xb5, 0xed, 0xeb, 0xee, ++ 0x22, 0x92, 0xf3, 0x10, 0x6e, 0x1e, 0x62, 0xa1, 0xa3, 0xe1, 0x0b, 0xd3, 0xee, 0x6a, 0x61, 0x2b, ++ 0x50, 0x3b, 0xc4, 0x81, 0x72, 0xbe, 0xe6, 0xca, 0x5f, 0xb9, 0x00, 0x8f, 0x38, 0x0e, 0x94, 0x97, ++ 0x35, 0x57, 0xfd, 0x3b, 0xff, 0x55, 0x81, 0x86, 0x49, 0xbe, 0xf2, 0x00, 0x09, 0x19, 0x39, 0xc7, ++ 0xcc, 0x2c, 0x3d, 0x03, 0xa1, 0x77, 0xa1, 0xa7, 0xff, 0x3c, 0x9a, 0x08, 0x42, 0xb3, 0x94, 0xde, ++ 0xd5, 0xd8, 0x17, 0x1a, 0xa9, 0x2e, 0xa1, 0xd5, 0x1d, 0xab, 0xb9, 0xce, 0x30, 0x90, 0xba, 0x49, ++ 0xe6, 0x32, 0x33, 0xa8, 0x14, 0xde, 0x72, 0x0d, 0x24, 0x97, 0xba, 0x95, 0xb7, 0xa4, 0xe4, 0x59, ++ 0x50, 0x2e, 0xf5, 0x88, 0xa6, 0xb1, 0xf0, 0x12, 0x4a, 0x62, 0x61, 0x72, 0x36, 0x28, 0xd4, 0x81, ++ 0xc4, 0x38, 0xff, 0x54, 0x81, 0x65, 0xfd, 0x10, 0x83, 0x7a, 0x50, 0xcd, 0x4e, 0xce, 0x2a, 0x51, ++ 0x55, 0x88, 0xd2, 0xa5, 0x4f, 0x4b, 0xf5, 0x2f, 0xf7, 0xf1, 0x79, 0xa4, 0xf3, 0xbf, 0x31, 0xed, ++ 0x3c, 0x52, 0x89, 0xff, 0x5d, 0xe8, 0xe5, 0x07, 0xb0, 0xa2, 0x6b, 0x13, 0xbb, 0x19, 0x56, 0xb1, ++ 0x5d, 0x6a, 0xa9, 0xf3, 0xb7, 0x00, 0xf9, 0x83, 0x84, 0x0c, 0x79, 0x9a, 0x19, 0x23, 0x7f, 0x25, ++ 0x66, 0x9c, 0x1d, 0xdd, 0xf2, 0x17, 0xdd, 0x87, 0x9e, 0x1f, 0x86, 0x44, 0x0e, 0xf7, 0x27, 0x4f, ++ 0x49, 0x98, 0x6d, 0xd2, 0x32, 0xd6, 0xf9, 0xbf, 0x0a, 0xf4, 0x77, 0x69, 0x32, 0xfd, 0x4b, 0x32, ++ 0xc1, 0x85, 0x0c, 0xa2, 0x8c, 0x34, 0x27, 0xb7, 0xfc, 0xd7, 0x3d, 0xc5, 0x04, 0xeb, 0xad, 0xa5, ++ 0x67, 0xb6, 0x29, 0x11, 0x6a, 0x5b, 0x59, 0x62, 0x76, 0xb7, 0xdb, 0xd5, 0xc4, 0xe7, 0x34, 0x54, ++ 0xad, 0x5f, 0x48, 0x98, 0x97, 0xdd, 0xe4, 0x76, 0xdd, 0x46, 0x48, 0x98, 0x22, 0x19, 0x47, 0x96, ++ 0xd4, 0x63, 0x42, 0xd1, 0x91, 0x65, 0x8d, 0x91, 0x8e, 0xac, 0xc3, 0x32, 0x3d, 0x39, 0xe1, 0x58, ++ 0xa8, 0x9e, 0xa4, 0xe6, 0x1a, 0x28, 0x4b, 0x73, 0xcd, 0x42, 0x9a, 0xbb, 0x01, 0xab, 0xea, 0xd9, ++ 0xea, 0x25, 0xf3, 0x03, 0x12, 0x8f, 0x6d, 0x2a, 0x5e, 0x03, 0x74, 0x28, 0x68, 0x32, 0x83, 0xdd, ++ 0x84, 0x81, 0x39, 0x7f, 0x0e, 0xfe, 0xe6, 0xd0, 0xba, 0x7e, 0x0b, 0x9a, 0x12, 0xf4, 0x18, 0xfe, ++ 0xd1, 0x26, 0x46, 0x43, 0x76, 0xde, 0x87, 0x8e, 0xfe, 0x35, 0x69, 0x20, 0x67, 0xe5, 0x65, 0x56, ++ 0xbe, 0xfd, 0x9f, 0x03, 0x93, 0x6e, 0xcd, 0xf5, 0x10, 0x7a, 0x0a, 0xfd, 0x99, 0xe7, 0x46, 0x64, ++ 0xee, 0x0b, 0x17, 0xbf, 0x42, 0x8e, 0xd6, 0x37, 0xf5, 0xf3, 0xe5, 0xa6, 0x7d, 0xbe, 0xdc, 0x7c, ++ 0x12, 0x25, 0x62, 0x8a, 0x9e, 0x40, 0xaf, 0xfc, 0x30, 0x87, 0x6e, 0xdb, 0xca, 0x66, 0xc1, 0x73, ++ 0xdd, 0xa5, 0x62, 0x9e, 0x42, 0x7f, 0xe6, 0x8d, 0xce, 0xda, 0xb3, 0xf8, 0xe9, 0xee, 0x52, 0x41, ++ 0x0f, 0xa1, 0x5d, 0x78, 0x94, 0x43, 0x43, 0x2d, 0x64, 0xfe, 0x9d, 0xee, 0x52, 0x01, 0xbb, 0xd0, ++ 0x2d, 0xbd, 0x93, 0xa1, 0x91, 0xf1, 0x67, 0xc1, 0xe3, 0xd9, 0xa5, 0x42, 0x76, 0xa0, 0x5d, 0x78, ++ 0xae, 0xb2, 0x56, 0xcc, 0xbf, 0x89, 0x8d, 0x6e, 0x2d, 0xa0, 0x98, 0xe9, 0xdc, 0x83, 0x6e, 0xe9, ++ 0x71, 0xc9, 0x1a, 0xb2, 0xe8, 0x61, 0x6b, 0x74, 0x7b, 0x21, 0xcd, 0x48, 0x7a, 0x0a, 0xfd, 0x99, ++ 0xa7, 0x26, 0x1b, 0xdc, 0xc5, 0x2f, 0x50, 0x97, 0xba, 0xf5, 0x8d, 0x9a, 0xec, 0x42, 0x13, 0x57, ++ 0x98, 0xec, 0xf9, 0x87, 0xa5, 0xd1, 0x9d, 0xc5, 0x44, 0x63, 0xd5, 0x13, 0xe8, 0x95, 0xdf, 0x94, ++ 0xac, 0xb0, 0x85, 0x2f, 0x4d, 0x57, 0xaf, 0x9c, 0xd2, 0xf3, 0x52, 0xbe, 0x72, 0x16, 0xbd, 0x3a, ++ 0x5d, 0x2a, 0xe8, 0x11, 0x80, 0x69, 0xd9, 0x42, 0x12, 0x67, 0x53, 0x36, 0xd7, 0x2a, 0x66, 0x53, ++ 0xb6, 0xa0, 0xbd, 0x7b, 0x08, 0xa0, 0x3b, 0xad, 0x90, 0xa6, 0x02, 0xdd, 0xb4, 0x66, 0xcc, 0xb4, ++ 0x77, 0xa3, 0xe1, 0x3c, 0x61, 0x4e, 0x00, 0x66, 0xec, 0x3a, 0x02, 0xbe, 0x06, 0xc8, 0x3b, 0x38, ++ 0x2b, 0x60, 0xae, 0xa7, 0xbb, 0x22, 0x06, 0x9d, 0x62, 0xbf, 0x86, 0x8c, 0xaf, 0x0b, 0x7a, 0xb8, ++ 0x2b, 0x44, 0xf4, 0x67, 0xaa, 0xe8, 0xf2, 0x62, 0x9b, 0x2d, 0xd3, 0x47, 0x73, 0xd5, 0x33, 0xfa, ++ 0x1e, 0x6e, 0x5f, 0x51, 0x88, 0xa3, 0x8d, 0x85, 0xe2, 0x16, 0xd4, 0xea, 0x0b, 0x44, 0x7f, 0x06, ++ 0x9d, 0x62, 0x35, 0x6e, 0x1d, 0x5c, 0x50, 0xa1, 0x8f, 0x4a, 0x15, 0x39, 0x7a, 0x08, 0xbd, 0x72, ++ 0xad, 0x8d, 0x0a, 0x5b, 0x6e, 0xae, 0x02, 0x1f, 0xad, 0xcc, 0xdc, 0xcc, 0x70, 0xf4, 0x31, 0x40, ++ 0x5e, 0x93, 0xdb, 0x99, 0x99, 0xab, 0xd2, 0x67, 0xb4, 0x7e, 0x0d, 0xbd, 0xc2, 0x91, 0x20, 0x9b, ++ 0xd8, 0x9b, 0x25, 0xe7, 0xf3, 0x83, 0x62, 0x64, 0x8a, 0xb7, 0xd2, 0x89, 0xf0, 0x08, 0x3a, 0xc5, ++ 0xe3, 0xc7, 0x7a, 0xbb, 0xe0, 0x48, 0xba, 0x2a, 0x9f, 0x16, 0x8e, 0x2a, 0xbb, 0x2d, 0xe6, 0x4f, ++ 0xaf, 0xab, 0xf2, 0x69, 0xa9, 0x81, 0xb6, 0x69, 0x6c, 0x51, 0x57, 0x7d, 0xd5, 0x29, 0x53, 0xee, ++ 0x36, 0x6d, 0xf4, 0x17, 0xf6, 0xa0, 0x57, 0x2d, 0xef, 0x62, 0x0b, 0x64, 0xe3, 0xb1, 0xa0, 0x2d, ++ 0xfa, 0x95, 0x74, 0x53, 0x6c, 0x73, 0x0a, 0xe9, 0x66, 0x41, 0xf7, 0x73, 0xa9, 0xa0, 0x3d, 0xe8, ++ 0x3f, 0xb5, 0x15, 0xac, 0xa9, 0xae, 0x8d, 0x39, 0x0b, 0xba, 0x89, 0xd1, 0x68, 0x11, 0xc9, 0xcc, ++ 0xf2, 0x37, 0x30, 0x98, 0xab, 0xac, 0xd1, 0xdd, 0xec, 0xa1, 0x60, 0x61, 0xc9, 0x7d, 0xa9, 0x59, ++ 0xfb, 0xb0, 0x32, 0x5b, 0x58, 0xa3, 0x37, 0xcc, 0xa4, 0x2f, 0x2e, 0xb8, 0x2f, 0x15, 0xf5, 0x05, ++ 0x34, 0x6d, 0x21, 0x87, 0xcc, 0x83, 0xcc, 0x4c, 0x61, 0x77, 0xd9, 0xd0, 0x9d, 0xce, 0xcf, 0xbf, ++ 0xdc, 0xad, 0xfc, 0xe6, 0x97, 0xbb, 0x95, 0xdf, 0xfd, 0x72, 0xb7, 0x72, 0xbc, 0xac, 0xa8, 0x1f, ++ 0xff, 0x21, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x62, 0x55, 0x85, 0x98, 0x25, 0x00, 0x00, + } +diff --git a/virtcontainers/agent.go b/virtcontainers/agent.go +index 3f22d9f..5e3ace9 100644 +--- a/virtcontainers/agent.go ++++ b/virtcontainers/agent.go +@@ -231,6 +231,9 @@ type agent interface { + // updateInterface will tell the agent to update a nic for an existed Sandbox. + updateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) + ++ // updateInterfaceHwAddrByName will tell the agent to update a nic for an existed Sandbox by nic name. ++ updateInterfaceHwAddrByName(inf *vcTypes.Interface) (*vcTypes.Interface, error) ++ + // listInterfaces will tell the agent to list interfaces of an existed Sandbox + listInterfaces() ([]*vcTypes.Interface, error) + +diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go +index 60f1d19..3ca874e 100644 +--- a/virtcontainers/hypervisor.go ++++ b/virtcontainers/hypervisor.go +@@ -47,6 +47,9 @@ const ( + + // MockHypervisor is a mock hypervisor for testing purposes + MockHypervisor HypervisorType = "mock" ++ ++ // StratovirtHypervisor is stratovirt hypervisor ++ StratovirtHypervisor HypervisorType = "stratovirt" + ) + + const ( +@@ -171,6 +174,9 @@ func (hType *HypervisorType) Set(value string) error { + case "mock": + *hType = MockHypervisor + return nil ++ case "stratovirt": ++ *hType = StratovirtHypervisor ++ return nil + default: + return fmt.Errorf("Unknown hypervisor type %s", value) + } +@@ -189,6 +195,8 @@ func (hType *HypervisorType) String() string { + return string(ClhHypervisor) + case MockHypervisor: + return string(MockHypervisor) ++ case StratovirtHypervisor: ++ return string(StratovirtHypervisor) + default: + return "" + } +@@ -218,6 +226,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { + }, nil + case MockHypervisor: + return &mockHypervisor{}, nil ++ case StratovirtHypervisor: ++ return &stratovirt{ ++ store: store, ++ }, nil + default: + return nil, fmt.Errorf("Unknown hypervisor type %s", hType) + } +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 0f03c9d..f8cad39 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -102,36 +102,37 @@ const ( + ) + + const ( +- grpcCheckRequest = "grpc.CheckRequest" +- grpcExecProcessRequest = "grpc.ExecProcessRequest" +- grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" +- grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" +- grpcCreateContainerRequest = "grpc.CreateContainerRequest" +- grpcStartContainerRequest = "grpc.StartContainerRequest" +- grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" +- grpcSignalProcessRequest = "grpc.SignalProcessRequest" +- grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" +- grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" +- grpcListInterfacesRequest = "grpc.ListInterfacesRequest" +- grpcListRoutesRequest = "grpc.ListRoutesRequest" +- grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" +- grpcListProcessesRequest = "grpc.ListProcessesRequest" +- grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" +- grpcWaitProcessRequest = "grpc.WaitProcessRequest" +- grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" +- grpcWriteStreamRequest = "grpc.WriteStreamRequest" +- grpcCloseStdinRequest = "grpc.CloseStdinRequest" +- grpcStatsContainerRequest = "grpc.StatsContainerRequest" +- grpcPauseContainerRequest = "grpc.PauseContainerRequest" +- grpcResumeContainerRequest = "grpc.ResumeContainerRequest" +- grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" +- grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" +- grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" +- grpcCopyFileRequest = "grpc.CopyFileRequest" +- grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" +- grpcStartTracingRequest = "grpc.StartTracingRequest" +- grpcStopTracingRequest = "grpc.StopTracingRequest" +- grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" ++ grpcCheckRequest = "grpc.CheckRequest" ++ grpcExecProcessRequest = "grpc.ExecProcessRequest" ++ grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" ++ grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" ++ grpcCreateContainerRequest = "grpc.CreateContainerRequest" ++ grpcStartContainerRequest = "grpc.StartContainerRequest" ++ grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" ++ grpcSignalProcessRequest = "grpc.SignalProcessRequest" ++ grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" ++ grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" ++ grpcUpdateInterfaceHwAddrByNameRequest = "grpc.UpdateInterfaceHwAddrByNameRequest" ++ grpcListInterfacesRequest = "grpc.ListInterfacesRequest" ++ grpcListRoutesRequest = "grpc.ListRoutesRequest" ++ grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" ++ grpcListProcessesRequest = "grpc.ListProcessesRequest" ++ grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" ++ grpcWaitProcessRequest = "grpc.WaitProcessRequest" ++ grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" ++ grpcWriteStreamRequest = "grpc.WriteStreamRequest" ++ grpcCloseStdinRequest = "grpc.CloseStdinRequest" ++ grpcStatsContainerRequest = "grpc.StatsContainerRequest" ++ grpcPauseContainerRequest = "grpc.PauseContainerRequest" ++ grpcResumeContainerRequest = "grpc.ResumeContainerRequest" ++ grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" ++ grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" ++ grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" ++ grpcCopyFileRequest = "grpc.CopyFileRequest" ++ grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" ++ grpcStartTracingRequest = "grpc.StartTracingRequest" ++ grpcStopTracingRequest = "grpc.StopTracingRequest" ++ grpcUpdateIPVSRequest = "grpc.UpdateIPVSRequest" + ) + + // The function is declared this way for mocking in unit tests +@@ -628,6 +629,15 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface, + return nil, err + } + ++func (k *kataAgent) updateInterfaceHwAddrByName(ifc *vcTypes.Interface) (*vcTypes.Interface, error) { ++ ifcReq := &grpc.UpdateInterfaceHwAddrByNameRequest{ ++ Interface: k.convertToKataAgentInterface(ifc), ++ } ++ _, err := k.sendReq(ifcReq) ++ ++ return nil, err ++} ++ + func (k *kataAgent) updateInterfaces(interfaces []*vcTypes.Interface) error { + for _, ifc := range interfaces { + if _, err := k.updateInterface(ifc); err != nil { +@@ -2074,6 +2084,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { + k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest), opts...) + } ++ k.reqHandlers[grpcUpdateInterfaceHwAddrByNameRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { ++ return k.client.UpdateInterfaceHwAddrByName(ctx, req.(*grpc.UpdateInterfaceHwAddrByNameRequest), opts...) ++ } + k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}, opts ...golangGrpc.CallOption) (interface{}, error) { + return k.client.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest), opts...) + } +diff --git a/virtcontainers/noop_agent.go b/virtcontainers/noop_agent.go +index d174623..bac34be 100644 +--- a/virtcontainers/noop_agent.go ++++ b/virtcontainers/noop_agent.go +@@ -111,6 +111,11 @@ func (n *noopAgent) updateInterface(inf *vcTypes.Interface) (*vcTypes.Interface, + return nil, nil + } + ++// updateInterfaceHwAddrByName is the Noop agent Interface update implementation. It does nothing. ++func (n *noopAgent) updateInterfaceHwAddrByName(inf *vcTypes.Interface) (*vcTypes.Interface, error) { ++ return nil, nil ++} ++ + // listInterfaces is the Noop agent Interfaces list implementation. It does nothing. + func (n *noopAgent) listInterfaces() ([]*vcTypes.Interface, error) { + return nil, nil +diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go +new file mode 100644 +index 0000000..7c156d5 +--- /dev/null ++++ b/virtcontainers/stratovirt.go +@@ -0,0 +1,617 @@ ++package virtcontainers ++ ++import ( ++ "context" ++ "fmt" ++ "os" ++ "os/exec" ++ "path/filepath" ++ "strconv" ++ "strings" ++ "syscall" ++ "time" ++ ++ govmmQemu "github.com/intel/govmm/qemu" ++ "github.com/opentracing/opentracing-go" ++ "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" ++ ++ "github.com/kata-containers/runtime/virtcontainers/device/config" ++ persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" ++ vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types" ++ "github.com/kata-containers/runtime/virtcontainers/store" ++ "github.com/kata-containers/runtime/virtcontainers/types" ++) ++ ++const defaultDummyMac = "22:33:44:aa:bb:" ++const mmioBlkCount = 6 ++const mmioNetCount = 2 ++ ++type stratovirtDev struct { ++ dev interface{} ++ devType deviceType ++} ++ ++type stratovirt struct { ++ id string ++ ctx context.Context ++ sandbox *Sandbox ++ store persistapi.PersistDriver ++ config HypervisorConfig ++ pid int ++ socketPath string ++ consolePath string ++ qmpMonitorCh qmpChannel ++ devices []stratovirtDev ++ HotpluggedVCPUs []CPUDevice ++ mmioBlkSlots [mmioBlkCount]bool ++ mmioNetSlots [mmioNetCount]bool ++} ++ ++func (s *stratovirt) Logger() *logrus.Entry { ++ return virtLog.WithField("subsystem", "stratovirt") ++} ++ ++func (s *stratovirt) trace(name string) (opentracing.Span, context.Context) { ++ span, ctx := opentracing.StartSpanFromContext(s.ctx, name) ++ ++ span.SetTag("subsystem", "hypervisor") ++ span.SetTag("type", "stratovirt") ++ ++ return span, ctx ++} ++ ++func (s *stratovirt) getKernelCmdLine() string { ++ var params []string ++ ++ params = append(params, "pci=off") ++ params = append(params, "reboot=k") ++ params = append(params, "panic=1") ++ params = append(params, "iommu=off") ++ params = append(params, "acpi=off") ++ params = append(params, "quiet") ++ params = append(params, "agent.use_vsock="+strconv.FormatBool(s.config.UseVSock)) ++ params = append(params, SerializeParams(s.config.KernelParams, "=")...) ++ ++ return strings.Join(params, " ") ++} ++ ++func (s *stratovirt) hypervisorConfig() HypervisorConfig { ++ return s.config ++} ++ ++func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig, stateful bool) error { ++ s.ctx = ctx ++ ++ span, _ := s.trace("createSandbox") ++ defer span.Finish() ++ ++ sandbox, err := globalSandboxList.lookupSandbox(id) ++ if err != nil { ++ s.Logger().Error("Get sandbox failed") ++ return err ++ } ++ ++ s.id = id ++ s.sandbox = sandbox ++ s.config = *hypervisorConfig ++ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") ++ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") ++ s.qmpMonitorCh = qmpChannel{ ++ ctx: s.ctx, ++ path: s.socketPath, ++ } ++ ++ return err ++} ++ ++func (s *stratovirt) waitSandBoxStarted(timeout int) error { ++ timeStart := time.Now() ++ for { ++ err := s.qmpSetup() ++ if err == nil { ++ break ++ } ++ ++ if int(time.Since(timeStart).Seconds()) > timeout { ++ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err) ++ } ++ ++ time.Sleep(time.Duration(50) * time.Millisecond) ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil { ++ s.Logger().WithError(err).Error(qmpCapErrMsg) ++ return err ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) startSandbox(timeout int) error { ++ span, _ := s.trace("startSandbox") ++ defer span.Finish() ++ ++ var params []string ++ params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id)) ++ params = append(params, "-api-channel unix:"+s.socketPath) ++ ++ if kernelPath, err := s.config.KernelAssetPath(); err == nil { ++ params = append(params, "-kernel "+kernelPath) ++ } ++ ++ if initrdPath, err := s.config.InitrdAssetPath(); err == nil { ++ params = append(params, "-initrd "+initrdPath) ++ } ++ ++ params = append(params, "-append "+s.getKernelCmdLine()) ++ params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) ++ params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024)) ++ params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) ++ ++ // add devices to cmdline ++ for _, d := range s.devices { ++ switch v := d.dev.(type) { ++ case Endpoint: ++ name := v.Name() ++ mac := v.HardwareAddr() ++ tapName := v.NetworkPair().TapInterface.TAPIface.Name ++ params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) ++ case config.BlockDrive: ++ id := v.ID ++ path := v.File ++ params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path)) ++ case types.VSock: ++ v.VhostFd.Close() ++ params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID)) ++ default: ++ s.Logger().Error("Adding device type is unsupported") ++ } ++ } ++ ++ // daemonize ++ params = append(params, "-daemonize") ++ ++ // append logfile only on debug ++ if s.config.Debug { ++ dir := filepath.Join(store.RunVMStoragePath(), s.id) ++ params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) ++ } ++ ++ s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) ++ ++ dir := filepath.Join(store.RunVMStoragePath(), s.id) ++ err := os.MkdirAll(dir, store.DirMode) ++ if err != nil { ++ return err ++ } ++ defer func() { ++ if err != nil { ++ if err := os.RemoveAll(dir); err != nil { ++ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) ++ } ++ } ++ }() ++ ++ binPath, err := s.config.HypervisorAssetPath() ++ if err != nil { ++ s.Logger().WithField("Fail to get hypervisor bin path", err).Error() ++ return err ++ } ++ ++ cmd := exec.CommandContext(s.ctx, binPath, params...) ++ if err := cmd.Run(); err != nil { ++ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() ++ return err ++ } ++ ++ if err = s.waitSandBoxStarted(timeout); err != nil { ++ return err ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) stopSandbox(force bool) error { ++ span, _ := s.trace("stopSandbox") ++ defer span.Finish() ++ ++ defer func() { ++ dir := filepath.Join(store.RunVMStoragePath(), s.id) ++ link, _ := filepath.EvalSymlinks(dir) ++ ++ if err := os.RemoveAll(dir); err != nil { ++ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir) ++ } ++ ++ if link != dir && link != "" { ++ if err := os.RemoveAll(link); err != nil { ++ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link) ++ } ++ } ++ }() ++ ++ if !force { ++ err := s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) ++ if err != nil { ++ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT") ++ return err ++ } ++ } else { ++ if s.pid > 0 { ++ syscall.Kill(s.pid, syscall.SIGKILL) ++ } ++ } ++ return nil ++} ++ ++func (s *stratovirt) pauseSandbox() error { ++ return nil ++} ++ ++func (s *stratovirt) saveSandbox() error { ++ return nil ++} ++ ++func (s *stratovirt) resumeSandbox() error { ++ return nil ++} ++ ++func (s *stratovirt) addDevice(devInfo interface{}, devType deviceType) error { ++ dev := stratovirtDev{ ++ dev: devInfo, ++ devType: devType, ++ } ++ s.devices = append(s.devices, dev) ++ ++ return nil ++} ++ ++func (s *stratovirt) updateVMInterfaceHwAddr(Name string, HwAddr string) error { ++ inf := &vcTypes.Interface{ ++ Name: Name, ++ HwAddr: HwAddr, ++ IPAddresses: []*vcTypes.IPAddress{}, ++ PciAddr: "", ++ } ++ ++ if _, err := s.sandbox.agent.updateInterfaceHwAddrByName(inf); err != nil { ++ s.Logger().WithError(err).Error("Agent updateInterfaceHwAddrByName failed") ++ return err ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { ++ Name = filepath.Base(strings.ToLower(Name)) ++ ++ if strings.HasPrefix(Name, "eth") { ++ idxStr := strings.TrimPrefix(Name, "eth") ++ if idxStr == Name { ++ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ } ++ ++ idx, err := strconv.Atoi(idxStr) ++ if err != nil { ++ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) ++ } ++ ++ if !isPut && s.mmioNetSlots[idx] { ++ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) ++ } ++ s.mmioNetSlots[idx] = !isPut ++ ++ return idx, nil ++ } else if strings.HasPrefix(Name, "vd") { ++ charStr := strings.TrimPrefix(Name, "vd") ++ if charStr == Name { ++ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) ++ } ++ ++ char := []rune(charStr) ++ idx := int(char[0] - 'a') ++ ++ if !isPut && s.mmioBlkSlots[idx] { ++ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) ++ } ++ s.mmioBlkSlots[idx] = !isPut ++ ++ return idx, nil ++ } ++ ++ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) ++} ++ ++func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { ++ err = s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ var tap TapInterface ++ ++ switch endpoint.Type() { ++ case VethEndpointType: ++ drive := endpoint.(*VethEndpoint) ++ tap = drive.NetPair.TapInterface ++ case TapEndpointType: ++ drive := endpoint.(*TapEndpoint) ++ tap = drive.TapInterface ++ default: ++ return fmt.Errorf("Endpoint is not supported") ++ } ++ ++ switch op { ++ case addDevice: ++ var ( ++ VMFdNames []string ++ VhostFdNames []string ++ ) ++ for i, VMFd := range tap.VMFds { ++ fdName := fmt.Sprintf("fd%d", i) ++ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil { ++ return err ++ } ++ VMFdNames = append(VMFdNames, fdName) ++ } ++ for i, VhostFd := range tap.VhostFds { ++ fdName := fmt.Sprintf("vhostfd%d", i) ++ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil { ++ return err ++ } ++ VhostFd.Close() ++ VhostFdNames = append(VhostFdNames, fdName) ++ } ++ ++ slot, err := s.getDevSlot(endpoint.Name(), false) ++ if err != nil { ++ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) ++ } ++ ++ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ } else { ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ ++ // update vm interface ++ if err := s.updateVMInterfaceHwAddr(endpoint.Name(), endpoint.HardwareAddr()); err != nil { ++ s.getDevSlot(endpoint.Name(), true) ++ return err ++ } ++ case removeDevice: ++ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { ++ return err ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { ++ return err ++ } ++ ++ slot, err := s.getDevSlot(endpoint.Name(), true) ++ if err != nil { ++ slot = 0 ++ } ++ ++ // clear vm interface ++ if err := s.updateVMInterfaceHwAddr(endpoint.Name(), defaultDummyMac+fmt.Sprintf("%02x", slot)); err != nil { ++ s.getDevSlot(endpoint.Name(), false) ++ return err ++ } ++ default: ++ return fmt.Errorf("Operation is not supported") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { ++ err = s.qmpSetup() ++ if err != nil { ++ return err ++ } ++ ++ switch op { ++ case addDevice: ++ driver := "virtio-blk-pci" ++ slot, err := s.getDevSlot(drive.VirtPath, false) ++ if err != nil { ++ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID); err != nil { ++ s.getDevSlot(drive.VirtPath, true) ++ return err ++ } ++ ++ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { ++ s.getDevSlot(drive.VirtPath, true) ++ return err ++ } ++ case removeDevice: ++ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { ++ return err ++ } ++ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { ++ return err ++ } ++ ++ s.getDevSlot(drive.VirtPath, true) ++ default: ++ return fmt.Errorf("Operation is not supported") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) { ++ span, _ := s.trace("hotplugAddDevice") ++ defer span.Finish() ++ ++ switch devType { ++ case netDev: ++ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) ++ case blockDev: ++ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) ++ default: ++ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) ++ } ++} ++ ++func (s *stratovirt) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) { ++ span, _ := s.trace("hotplugRemoveDevice") ++ defer span.Finish() ++ ++ switch devType { ++ case netDev: ++ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) ++ case blockDev: ++ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) ++ default: ++ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) ++ } ++} ++ ++func (s *stratovirt) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { ++ return 0, memoryDevice{}, nil ++} ++ ++func (s *stratovirt) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { ++ return 0, 0, nil ++} ++ ++func (s *stratovirt) getSandboxConsole(id string) (string, error) { ++ return s.consolePath, nil ++} ++ ++func (s *stratovirt) getMemorySize() uint32 { ++ return 0 ++} ++ ++func (s *stratovirt) disconnect() { ++ span, _ := s.trace("disconnect") ++ defer span.Finish() ++ ++ s.qmpTeardown() ++} ++ ++func (s *stratovirt) capabilities() types.Capabilities { ++ span, _ := s.trace("capabilities") ++ defer span.Finish() ++ ++ var caps types.Capabilities ++ caps.SetBlockDeviceHotplugSupport() ++ ++ return caps ++} ++ ++func (s *stratovirt) qmpTeardown() { ++ if s.qmpMonitorCh.qmp != nil { ++ s.qmpMonitorCh.qmp.Shutdown() ++ <-s.qmpMonitorCh.disconn ++ s.qmpMonitorCh.qmp = nil ++ s.qmpMonitorCh.disconn = nil ++ } ++} ++ ++func (s *stratovirt) qmpSetup() error { ++ s.qmpTeardown() ++ ++ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} ++ ++ // Auto-closed by QMPStart(). ++ disconnectCh := make(chan struct{}) ++ ++ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh) ++ if err != nil { ++ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance") ++ return err ++ } ++ ++ s.qmpMonitorCh.qmp = qmp ++ s.qmpMonitorCh.disconn = disconnectCh ++ ++ return nil ++} ++ ++func (s *stratovirt) getThreadIDs() (vcpuThreadIDs, error) { ++ span, _ := s.trace("getThreadIDs") ++ defer span.Finish() ++ ++ tid := vcpuThreadIDs{} ++ if err := s.qmpSetup(); err != nil { ++ return tid, err ++ } ++ ++ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx) ++ if err != nil { ++ s.Logger().WithError(err).Error("Failed to query cpu infos") ++ return tid, err ++ } ++ ++ tid.vcpus = make(map[int]int, len(cpuInfos)) ++ for _, i := range cpuInfos { ++ if i.ThreadID > 0 { ++ tid.vcpus[i.CPU] = i.ThreadID ++ } ++ } ++ return tid, nil ++} ++ ++func (s *stratovirt) cleanup() error { ++ span, _ := s.trace("cleanup") ++ defer span.Finish() ++ ++ s.qmpTeardown() ++ ++ return nil ++} ++ ++func (s *stratovirt) getPids() []int { ++ return []int{s.pid} ++} ++ ++func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { ++ return nil ++} ++ ++func (s *stratovirt) toGrpc() ([]byte, error) { ++ return nil, nil ++} ++ ++func (s *stratovirt) check() error { ++ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil { ++ return errors.Wrapf(err, "Failed to ping StratoVirt process") ++ } ++ ++ return nil ++} ++ ++func (s *stratovirt) generateSocket(id string, useVsock bool) (interface{}, error) { ++ return generateVMSocket(id, useVsock, s.store.RunVMStoragePath()) ++} ++ ++func (s *stratovirt) save() (p persistapi.HypervisorState) { ++ p.Pid = s.pid ++ p.Type = string(StratovirtHypervisor) ++ return ++} ++ ++func (s *stratovirt) load(p persistapi.HypervisorState) { ++ s.pid = p.Pid ++ return ++} +-- +1.8.3.1 + diff --git a/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch b/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch new file mode 100644 index 0000000000000000000000000000000000000000..419d5aa5e5c8b7abbf23435716861f24aa7eb7ed --- /dev/null +++ b/runtime/patches/0053-kata-runtime-add-interface-for-host-cgroup.patch @@ -0,0 +1,199 @@ +From 31bbb64a0682e326b354ac54c6596402fc20a977 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Tue, 8 Sep 2020 20:03:41 +0800 +Subject: [PATCH 1/5] kata-runtime: add interface for host cgroup + +reason: add interface for host cgroup, including +CreateSandboxCgroup, DestroySandboxCgroup and +AddPidToSandboxCgroup, GetSandboxCgroupPath + +Signed-off-by: yangfeiyu +--- + virtcontainers/cgroups.go | 111 +++++++++++++++++++++++++++++++++++++++ + virtcontainers/implementation.go | 20 +++++++ + virtcontainers/interfaces.go | 5 ++ + 3 files changed, 136 insertions(+) + +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index 4459df5..df0ec30 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -8,13 +8,20 @@ package virtcontainers + + import ( + "bufio" ++ "context" ++ "encoding/json" + "fmt" ++ "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/containerd/cgroups" + specs "github.com/opencontainers/runtime-spec/specs-go" ++ "github.com/sirupsen/logrus" ++ ++ "github.com/kata-containers/runtime/virtcontainers/store" ++ "github.com/kata-containers/runtime/virtcontainers/types" + ) + + type cgroupPather interface { +@@ -30,6 +37,110 @@ const cgroupKataPath = "/kata/" + var cgroupsLoadFunc = cgroups.Load + var cgroupsNewFunc = cgroups.New + ++// CreateSandboxCgroup create cgroup based on the first container's cgroupPath of sandbox ++func CreateSandboxCgroup(ctx context.Context, path string) error { ++ if path == "" { ++ return fmt.Errorf("sandbox cgroupPath shouldn't be empty!") ++ } ++ ++ var cgroupPath string ++ vcpuCgroupPath := filepath.Join(path, "vcpu") ++ if filepath.IsAbs(vcpuCgroupPath) { ++ cgroupPath = filepath.Clean(vcpuCgroupPath) ++ } else { ++ cgroupPath = filepath.Join(filepath.Clean("/" + vcpuCgroupPath)) ++ } ++ ++ resources := specs.LinuxResources{} ++ if _, err := cgroupsNewFunc(cgroups.V1, cgroups.StaticPath(cgroupPath), &resources); err != nil { ++ return fmt.Errorf("Could not create cgroup for %v: %v", cgroupPath, err) ++ } ++ ++ return nil ++} ++ ++// DestroySandboxCgroup destroy the cgroup dir created for sandbox ++func DestroySandboxCgroup(ctx context.Context, cgroupPath string) error { ++ return deleteCgroup(cgroups.V1, cgroupPath) ++} ++ ++func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error { ++ if cgroupPath == "" { ++ logrus.Warn("delete cgroupPath shouldn't be empty!") ++ return nil ++ } ++ ++ cgroup, err := cgroupsLoadFunc(hierarchy, ++ cgroups.StaticPath(cgroupPath)) ++ ++ if err == cgroups.ErrCgroupDeleted { ++ // cgroup already deleted ++ return nil ++ } ++ ++ if err != nil { ++ return fmt.Errorf("Could not load cgroup %v: %v", cgroupPath, err) ++ } ++ ++ // move running process here, that way cgroup can be removed ++ parent, err := parentCgroup(hierarchy, cgroupPath) ++ if err != nil { ++ // parent cgroup doesn't exist, that means there are no process running ++ // and the container cgroup was removed. ++ logrus.Warn(err) ++ return nil ++ } ++ ++ if err := cgroup.MoveTo(parent); err != nil { ++ // Don't fail, cgroup can be deleted ++ logrus.Warnf("Could not move container process into parent cgroup: %v", err) ++ } ++ ++ if err := cgroup.Delete(); err != nil { ++ return fmt.Errorf("Could not delete cgroup %v: %v", cgroupPath, err) ++ } ++ ++ return nil ++} ++ ++// GetSandboxCgroupPath return the cgroup path of specified sandbox ++func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { ++ stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile) ++ ++ fileData, err := ioutil.ReadFile(stateFilePath) ++ if err != nil { ++ return "", err ++ } ++ ++ state := types.SandboxState{} ++ ++ if err := json.Unmarshal(fileData, &state); err != nil { ++ return "", err ++ } ++ ++ if state.CgroupPath == "" { ++ return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty") ++ } ++ ++ return state.CgroupPath, nil ++} ++ ++// AddPidToSandboxCgroup add kata-runtime create process to cgroup ++// the pid will be added to the cgroup of "/vcpu" ++func AddPidToSandboxCgroup(ctx context.Context, pid int, cgroupPath string) error { ++ cgroup, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath)) ++ if err != nil { ++ return err ++ } ++ ++ err = cgroup.Add(cgroups.Process{Pid: pid}) ++ if err != nil { ++ return err ++ } ++ ++ return nil ++} ++ + // V1Constraints returns the cgroups that are compatible with the VC architecture + // and hypervisor, constraints can be applied to these cgroups. + func V1Constraints() ([]cgroups.Subsystem, error) { +diff --git a/virtcontainers/implementation.go b/virtcontainers/implementation.go +index e4bc4ae..fedc51f 100644 +--- a/virtcontainers/implementation.go ++++ b/virtcontainers/implementation.go +@@ -188,3 +188,23 @@ func (impl *VCImpl) CleanupContainer(ctx context.Context, sandboxID, containerID + func (impl *VCImpl) UpdateIPVSRule(ctx context.Context, sandboxID string, IPVSRule *grpc.UpdateIPVSRequest) (*grpc.IPVSResponse, error) { + return UpdateIPVSRule(ctx, sandboxID, IPVSRule) + } ++ ++// CreateSandboxCgroup implements the VC function of the same name. ++func (impl *VCImpl) CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error { ++ return CreateSandboxCgroup(ctx, sandboxCgroupPath) ++} ++ ++// DestroySandboxCgroup implements the VC function of the same name. ++func (impl *VCImpl) DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error { ++ return DestroySandboxCgroup(ctx, sandboxCgroupPath) ++} ++ ++// AddPidToSandboxCgroup implements the VC function of the same name. ++func (impl *VCImpl) AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error { ++ return AddPidToSandboxCgroup(ctx, pid, sandboxCgroupPath) ++} ++ ++// GetSandboxCgroupPath implements the VC function of the same name. ++func (impl *VCImpl) GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { ++ return GetSandboxCgroupPath(ctx, sandboxID) ++} +diff --git a/virtcontainers/interfaces.go b/virtcontainers/interfaces.go +index 0fd12d8..4d166e0 100644 +--- a/virtcontainers/interfaces.go ++++ b/virtcontainers/interfaces.go +@@ -24,6 +24,11 @@ type VC interface { + SetLogger(ctx context.Context, logger *logrus.Entry) + SetFactory(ctx context.Context, factory Factory) + ++ CreateSandboxCgroup(ctx context.Context, sandboxCgroupPath string) error ++ DestroySandboxCgroup(ctx context.Context, sandboxCgroupPath string) error ++ AddPidToSandboxCgroup(ctx context.Context, pid int, sandboxCgroupPath string) error ++ GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) ++ + CreateSandbox(ctx context.Context, sandboxConfig SandboxConfig) (VCSandbox, error) + DeleteSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) + FetchSandbox(ctx context.Context, sandboxID string) (VCSandbox, error) +-- +1.8.3.1 + diff --git a/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch b/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch new file mode 100644 index 0000000000000000000000000000000000000000..1c28525be2e2e2bd47376627c43fb9bd0811bdd1 --- /dev/null +++ b/runtime/patches/0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch @@ -0,0 +1,376 @@ +From 98a3c4677261e1c0364015f36928cddfb0af253e Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Wed, 9 Sep 2020 16:45:24 +0800 +Subject: [PATCH 2/5] kata-runtime: add sandbox cgroup with vcpu and emulator + switch + +reason: add sandbox cgroup with vcpu and emulator switch, if +sandbox_cgroup_with_emulator is true, it will overload the feature +of sandbox_cgroup_only, there will be two cgroups, vcpu and emulator + +Signed-off-by: yangfeiyu +--- + cli/config/configuration-qemu.toml.in | 12 +++++++++ + cli/kata-env.go | 38 ++++++++++++++------------- + pkg/katautils/config.go | 18 +++++++------ + virtcontainers/api.go | 4 ++- + virtcontainers/container.go | 6 ++--- + virtcontainers/persist.go | 30 +++++++++++---------- + virtcontainers/persist/api/config.go | 2 ++ + virtcontainers/pkg/annotations/annotations.go | 2 ++ + virtcontainers/pkg/oci/utils.go | 13 +++++++++ + virtcontainers/sandbox.go | 23 +++++++++++----- + 10 files changed, 97 insertions(+), 51 deletions(-) + +diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in +index e57a954..fae88f9 100644 +--- a/cli/config/configuration-qemu.toml.in ++++ b/cli/config/configuration-qemu.toml.in +@@ -477,6 +477,18 @@ enable_compat_old_cni = true + # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType + sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ + ++# It is a new host cgroup solution to limit the kata resouce in the host different from the ++# community original solution.If sandbox_cgroup_with_emulator is enabled, it will override ++# the config of sandbox_cgroup_only. Each Pod corresponds to a pod level cgroup directory ++# which is named with sandboxID. In each pod level cgroup, it contains two sub cgroup ++# directory: vcpu and emulator, these two sub cgroup only valid in the CPU cgroup subsystem, ++# because we just want to distinguish the emulator main thread and vcpu thread in the CPU ++# cgroup subsystem.And with this config enabled, kata-runtime and related sub processes will ++# added into the vcpu cgroup directory with resource limited, and qemu main thread and other ++# non-vcpu threads will be moved into the emulator cgroup without resource limit, which will ++# improve the IO throughput for kata-containers. ++sandbox_cgroup_with_emulator = true ++ + # Enabled experimental feature list, format: ["a", "b"]. + # Experimental features are features not stable enough for production, + # they may break compatibility, and are prepared for a big version bump. +diff --git a/cli/kata-env.go b/cli/kata-env.go +index d8a6068..48026fe 100644 +--- a/cli/kata-env.go ++++ b/cli/kata-env.go +@@ -63,15 +63,16 @@ type RuntimeConfigInfo struct { + + // RuntimeInfo stores runtime details. + type RuntimeInfo struct { +- Version RuntimeVersionInfo +- Config RuntimeConfigInfo +- Debug bool +- Trace bool +- DisableGuestSeccomp bool +- DisableNewNetNs bool +- SandboxCgroupOnly bool +- Experimental []exp.Feature +- Path string ++ Version RuntimeVersionInfo ++ Config RuntimeConfigInfo ++ Debug bool ++ Trace bool ++ DisableGuestSeccomp bool ++ DisableNewNetNs bool ++ SandboxCgroupOnly bool ++ SandboxCgroupWithEmulator bool ++ Experimental []exp.Feature ++ Path string + } + + type VersionInfo struct { +@@ -194,15 +195,16 @@ func getRuntimeInfo(configFile string, config oci.RuntimeConfig) RuntimeInfo { + runtimePath, _ := os.Executable() + + return RuntimeInfo{ +- Debug: config.Debug, +- Trace: config.Trace, +- Version: runtimeVersion, +- Config: runtimeConfig, +- Path: runtimePath, +- DisableNewNetNs: config.DisableNewNetNs, +- SandboxCgroupOnly: config.SandboxCgroupOnly, +- Experimental: config.Experimental, +- DisableGuestSeccomp: config.DisableGuestSeccomp, ++ Debug: config.Debug, ++ Trace: config.Trace, ++ Version: runtimeVersion, ++ Config: runtimeConfig, ++ Path: runtimePath, ++ DisableNewNetNs: config.DisableNewNetNs, ++ SandboxCgroupOnly: config.SandboxCgroupOnly, ++ SandboxCgroupWithEmulator: config.SandboxCgroupWithEmulator, ++ Experimental: config.Experimental, ++ DisableGuestSeccomp: config.DisableGuestSeccomp, + } + } + +diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go +index 3365b3f..89e46f6 100644 +--- a/pkg/katautils/config.go ++++ b/pkg/katautils/config.go +@@ -139,14 +139,15 @@ type proxy struct { + } + + type runtime struct { +- Debug bool `toml:"enable_debug"` +- Tracing bool `toml:"enable_tracing"` +- DisableNewNetNs bool `toml:"disable_new_netns"` +- EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` +- DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` +- SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` +- Experimental []string `toml:"experimental"` +- InterNetworkModel string `toml:"internetworking_model"` ++ Debug bool `toml:"enable_debug"` ++ Tracing bool `toml:"enable_tracing"` ++ DisableNewNetNs bool `toml:"disable_new_netns"` ++ EnableCompatOldCNI bool `toml:"enable_compat_old_cni"` ++ DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` ++ SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` ++ SandboxCgroupWithEmulator bool `toml:"sandbox_cgroup_with_emulator"` ++ Experimental []string `toml:"experimental"` ++ InterNetworkModel string `toml:"internetworking_model"` + } + + type shim struct { +@@ -1252,6 +1253,7 @@ func LoadConfiguration(configPath string, ignoreLogging, builtIn bool, debugFlag + } + + config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly ++ config.SandboxCgroupWithEmulator = tomlConf.Runtime.SandboxCgroupWithEmulator + config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs + config.EnableCompatOldCNI = tomlConf.Runtime.EnableCompatOldCNI + for _, f := range tomlConf.Runtime.Experimental { +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index ca5412a..08bcbb5 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -103,7 +103,9 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f + }() + + // Move runtime to sandbox cgroup so all process are created there. +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupWithEmulator{ ++ // emulator ++ } else if s.config.SandboxCgroupOnly { + if err := s.setupSandboxCgroup(); err != nil { + return nil, err + } +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 4060ebb..1b70382 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -1009,7 +1009,7 @@ func (c *Container) create() (err error) { + } + } + +- if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly { ++ if !rootless.IsRootless() && !c.sandbox.config.SandboxCgroupOnly && !c.sandbox.config.SandboxCgroupWithEmulator { + if err = c.cgroupsCreate(); err != nil { + return + } +@@ -1034,7 +1034,7 @@ func (c *Container) delete() error { + } + + // If running rootless, there are no cgroups to remove +- if !c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless() { ++ if !c.sandbox.config.SandboxCgroupWithEmulator && (!c.sandbox.config.SandboxCgroupOnly || !rootless.IsRootless()) { + if err := c.cgroupsDelete(); err != nil { + return err + } +@@ -1348,7 +1348,7 @@ func (c *Container) update(resources specs.LinuxResources) error { + } + } + +- if !c.sandbox.config.SandboxCgroupOnly { ++ if !c.sandbox.config.SandboxCgroupWithEmulator && !c.sandbox.config.SandboxCgroupOnly { + if err := c.cgroupsUpdate(resources); err != nil { + return err + } +diff --git a/virtcontainers/persist.go b/virtcontainers/persist.go +index fe00bf9..efa4506 100644 +--- a/virtcontainers/persist.go ++++ b/virtcontainers/persist.go +@@ -194,13 +194,14 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { + InterworkingModel: int(sconfig.NetworkConfig.InterworkingModel), + }, + +- ShmSize: sconfig.ShmSize, +- SharePidNs: sconfig.SharePidNs, +- Stateful: sconfig.Stateful, +- SystemdCgroup: sconfig.SystemdCgroup, +- SandboxCgroupOnly: sconfig.SandboxCgroupOnly, +- DisableGuestSeccomp: sconfig.DisableGuestSeccomp, +- Cgroups: sconfig.Cgroups, ++ ShmSize: sconfig.ShmSize, ++ SharePidNs: sconfig.SharePidNs, ++ Stateful: sconfig.Stateful, ++ SystemdCgroup: sconfig.SystemdCgroup, ++ SandboxCgroupOnly: sconfig.SandboxCgroupOnly, ++ SandboxCgroupWithEmulator: sconfig.SandboxCgroupWithEmulator, ++ DisableGuestSeccomp: sconfig.DisableGuestSeccomp, ++ Cgroups: sconfig.Cgroups, + } + + for _, e := range sconfig.Experimental { +@@ -485,13 +486,14 @@ func loadSandboxConfig(id string) (*SandboxConfig, error) { + InterworkingModel: NetInterworkingModel(savedConf.NetworkConfig.InterworkingModel), + }, + +- ShmSize: savedConf.ShmSize, +- SharePidNs: savedConf.SharePidNs, +- Stateful: savedConf.Stateful, +- SystemdCgroup: savedConf.SystemdCgroup, +- SandboxCgroupOnly: savedConf.SandboxCgroupOnly, +- DisableGuestSeccomp: savedConf.DisableGuestSeccomp, +- Cgroups: savedConf.Cgroups, ++ ShmSize: savedConf.ShmSize, ++ SharePidNs: savedConf.SharePidNs, ++ Stateful: savedConf.Stateful, ++ SystemdCgroup: savedConf.SystemdCgroup, ++ SandboxCgroupOnly: savedConf.SandboxCgroupOnly, ++ SandboxCgroupWithEmulator: savedConf.SandboxCgroupWithEmulator, ++ DisableGuestSeccomp: savedConf.DisableGuestSeccomp, ++ Cgroups: savedConf.Cgroups, + } + + for _, name := range savedConf.Experimental { +diff --git a/virtcontainers/persist/api/config.go b/virtcontainers/persist/api/config.go +index 3a2df32..28204fc 100644 +--- a/virtcontainers/persist/api/config.go ++++ b/virtcontainers/persist/api/config.go +@@ -258,6 +258,8 @@ type SandboxConfig struct { + // SandboxCgroupOnly enables cgroup only at podlevel in the host + SandboxCgroupOnly bool + ++ SandboxCgroupWithEmulator bool ++ + DisableGuestSeccomp bool + + // Experimental enables experimental features +diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go +index 528dfa6..96c4ef2 100644 +--- a/virtcontainers/pkg/annotations/annotations.go ++++ b/virtcontainers/pkg/annotations/annotations.go +@@ -215,6 +215,8 @@ const ( + // SandboxCgroupOnly is a sandbox annotation that determines if kata processes are managed only in sandbox cgroup. + SandboxCgroupOnly = kataAnnotRuntimePrefix + "sandbox_cgroup_only" + ++ SandboxCgroupWithEmulator = kataAnnotRuntimePrefix + "sandbox_cgroup_with_emulator" ++ + // Experimental is a sandbox annotation that determines if experimental features enabled. + Experimental = kataAnnotRuntimePrefix + "experimental" + +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 3b2af75..91067fb 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -139,6 +139,8 @@ type RuntimeConfig struct { + //Determines kata processes are managed only in sandbox cgroup + SandboxCgroupOnly bool + ++ SandboxCgroupWithEmulator bool ++ + //Experimental features enabled + Experimental []exp.Feature + } +@@ -746,6 +748,15 @@ func addRuntimeConfigOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e + sbConfig.SandboxCgroupOnly = sandboxCgroupOnly + } + ++ if value, ok := ocispec.Annotations[vcAnnotations.SandboxCgroupWithEmulator]; ok { ++ sandboxCgroupWithEmulator, err := strconv.ParseBool(value) ++ if err != nil { ++ return fmt.Errorf("error parsing annotation for sandbox_cgroup_with_emulator : Please specify boolean value 'true|false'") ++ } ++ ++ sbConfig.SandboxCgroupWithEmulator = sandboxCgroupWithEmulator ++ } ++ + if value, ok := ocispec.Annotations[vcAnnotations.Experimental]; ok { + features := strings.Split(value, " ") + sbConfig.Experimental = []exp.Feature{} +@@ -869,6 +880,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid, c + + SandboxCgroupOnly: runtime.SandboxCgroupOnly, + ++ SandboxCgroupWithEmulator: runtime.SandboxCgroupWithEmulator, ++ + DisableGuestSeccomp: runtime.DisableGuestSeccomp, + + // Q: Is this really necessary? @weizhang555 +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 174e6cb..b479cf5 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -126,6 +126,8 @@ type SandboxConfig struct { + // SandboxCgroupOnly enables cgroup only at podlevel in the host + SandboxCgroupOnly bool + ++ SandboxCgroupWithEmulator bool ++ + DisableGuestSeccomp bool + + // Experimental features enabled +@@ -1532,8 +1534,9 @@ func (s *Sandbox) Stats() (SandboxStats, error) { + + var path string + var cgroupSubsystems cgroups.Hierarchy +- +- if s.config.SandboxCgroupOnly { ++ if !s.config.SandboxCgroupWithEmulator { ++ // vcpu and emulator ++ } else if s.config.SandboxCgroupOnly { + cgroupSubsystems = cgroups.V1 + path = s.state.CgroupPath + } else { +@@ -1793,7 +1796,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) + span, _ := s.trace("HotplugAddDevice") + defer span.Finish() + +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupWithEmulator { ++ // emulator ++ } else if s.config.SandboxCgroupOnly { + // We are about to add a device to the hypervisor, + // the device cgroup MUST be updated since the hypervisor + // will need access to such device +@@ -1849,7 +1854,9 @@ func (s *Sandbox) HotplugAddDevice(device api.Device, devType config.DeviceType) + // Sandbox implement DeviceReceiver interface from device/api/interface.go + func (s *Sandbox) HotplugRemoveDevice(device api.Device, devType config.DeviceType) error { + defer func() { +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupWithEmulator { ++ ++ } else if s.config.SandboxCgroupOnly { + // Remove device from cgroup, the hypervisor + // should not have access to such device anymore. + hdev := device.GetHostPath() +@@ -2107,7 +2114,7 @@ func (s *Sandbox) cgroupsUpdate() error { + // If Kata is configured for SandboxCgroupOnly, the VMM and its processes are already + // in the Kata sandbox cgroup (inherited). No need to move threads/processes, and we should + // rely on parent's cgroup CPU/memory values +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupWithEmulator || s.config.SandboxCgroupOnly { + return nil + } + +@@ -2154,7 +2161,9 @@ func (s *Sandbox) cgroupsDelete() error { + var path string + var cgroupSubsystems cgroups.Hierarchy + +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupWithEmulator { ++ // emulator ++ } else if s.config.SandboxCgroupOnly { + return s.cgroupMgr.Destroy() + } + +@@ -2197,7 +2206,7 @@ func (s *Sandbox) constrainHypervisor(cgroup cgroups.Cgroup) error { + // Kata/VMM into account, Kata may fail to boot due to being overconstrained. + // If !SandboxCgroupOnly, place the VMM into an unconstrained cgroup, and the vCPU threads into constrained + // cgroup +- if s.config.SandboxCgroupOnly { ++ if s.config.SandboxCgroupOnly || s.config.SandboxCgroupWithEmulator { + // Kata components were moved into the sandbox-cgroup already, so VMM + // will already land there as well. No need to take action + return nil +-- +1.8.3.1 + diff --git a/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch b/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch new file mode 100644 index 0000000000000000000000000000000000000000..6d63e830b4cd083376f2caae1466b7baff9e9259 --- /dev/null +++ b/runtime/patches/0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch @@ -0,0 +1,428 @@ +From ce7523dfe1bb60cf54254e16a103fd3fc9503618 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Thu, 17 Sep 2020 10:38:38 +0800 +Subject: [PATCH 3/5] kata_runtime: support host cgroup with emulator policy + +reason: support host cgroup with emulator policy when +sandbox_cgroup_with_emulator is set true + +Signed-off-by: yangfeiyu +--- + cli/create.go | 38 ++++++++++++ + virtcontainers/api.go | 10 ++- + virtcontainers/cgroups.go | 132 ++++++++++++++++++++++++++++++++++------ + virtcontainers/persist/fs/fs.go | 8 +++ + virtcontainers/pkg/oci/utils.go | 14 +++++ + virtcontainers/sandbox.go | 70 ++++++++++++++++++++- + 6 files changed, 250 insertions(+), 22 deletions(-) + +diff --git a/cli/create.go b/cli/create.go +index 02cb2c5..b14434b 100644 +--- a/cli/create.go ++++ b/cli/create.go +@@ -11,6 +11,7 @@ import ( + "errors" + "fmt" + "os" ++ "path/filepath" + + "github.com/kata-containers/runtime/pkg/katautils" + vc "github.com/kata-containers/runtime/virtcontainers" +@@ -134,11 +135,48 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s + var process vc.Process + switch containerType { + case vc.PodSandbox: ++ if runtimeConfig.SandboxCgroupWithEmulator { ++ // create the sandbox level cgroup ++ cgroupPath := ociSpec.Linux.CgroupsPath ++ if err = vci.CreateSandboxCgroup(ctx, cgroupPath); err != nil { ++ return err ++ } ++ ++ defer func() { ++ if err != nil { ++ _ = vci.DestroySandboxCgroup(ctx, cgroupPath) ++ } ++ }() ++ ++ // add kata-runtime create process into /vcpu cgroup ++ vcpuCgroupPath := filepath.Join(cgroupPath, "vcpu") ++ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil { ++ return err ++ } ++ } ++ + _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false) + if err != nil { + return err + } + case vc.PodContainer: ++ if runtimeConfig.SandboxCgroupWithEmulator { ++ sandboxID, err := oci.GetSandboxIDFromAnnotations(&ociSpec) ++ if err != nil { ++ return fmt.Errorf("container annotation doesn't contain sandboxID") ++ } ++ ++ sandboxCgroupPath, err := vci.GetSandboxCgroupPath(ctx, sandboxID) ++ if err != nil { ++ return err ++ } ++ ++ // add kata-runtime create process into /vcpu cgroup ++ vcpuCgroupPath := filepath.Join(sandboxCgroupPath, "vcpu") ++ if err = vci.AddPidToSandboxCgroup(ctx, os.Getpid(), vcpuCgroupPath); err != nil { ++ return err ++ } ++ } + process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false) + if err != nil { + return err +diff --git a/virtcontainers/api.go b/virtcontainers/api.go +index 08bcbb5..38c8235 100644 +--- a/virtcontainers/api.go ++++ b/virtcontainers/api.go +@@ -103,9 +103,7 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f + }() + + // Move runtime to sandbox cgroup so all process are created there. +- if s.config.SandboxCgroupWithEmulator{ +- // emulator +- } else if s.config.SandboxCgroupOnly { ++ if !s.config.SandboxCgroupWithEmulator && s.config.SandboxCgroupOnly { + if err := s.setupSandboxCgroup(); err != nil { + return nil, err + } +@@ -129,6 +127,12 @@ func createSandboxFromConfig(ctx context.Context, sandboxConfig SandboxConfig, f + return nil, err + } + ++ if s.config.SandboxCgroupWithEmulator { ++ if err := s.setupHostCgroupsWithEmulator(); err != nil { ++ return nil, err ++ } ++ } ++ + // Create Containers + if err = s.createContainers(); err != nil { + return nil, err +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index df0ec30..65d2001 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -9,19 +9,15 @@ package virtcontainers + import ( + "bufio" + "context" +- "encoding/json" + "fmt" +- "io/ioutil" + "os" + "path/filepath" ++ "strconv" + "strings" + + "github.com/containerd/cgroups" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" +- +- "github.com/kata-containers/runtime/virtcontainers/store" +- "github.com/kata-containers/runtime/virtcontainers/types" + ) + + type cgroupPather interface { +@@ -32,7 +28,11 @@ type cgroupPather interface { + // unconstrained cgroups are placed here. + // for example /sys/fs/cgroup/memory/kata/$CGPATH + // where path is defined by the containers manager +-const cgroupKataPath = "/kata/" ++const ( ++ cgroupKataPath = "/kata/" ++ vcpuCgroupName = "vcpu" ++ emulatorCgroupName = "emulator" ++) + + var cgroupsLoadFunc = cgroups.Load + var cgroupsNewFunc = cgroups.New +@@ -105,24 +105,16 @@ func deleteCgroup(hierarchy cgroups.Hierarchy, cgroupPath string) error { + + // GetSandboxCgroupPath return the cgroup path of specified sandbox + func GetSandboxCgroupPath(ctx context.Context, sandboxID string) (string, error) { +- stateFilePath := filepath.Join(store.RunStoragePath(), sandboxID, store.StateFile) +- +- fileData, err := ioutil.ReadFile(stateFilePath) ++ config, err := loadSandboxConfig(sandboxID) + if err != nil { + return "", err + } + +- state := types.SandboxState{} +- +- if err := json.Unmarshal(fileData, &state); err != nil { +- return "", err +- } +- +- if state.CgroupPath == "" { +- return "", fmt.Errorf("get sandbox cgroup path error: cgroupPath is empty") ++ if config.Cgroups == nil { ++ return "", fmt.Errorf("the cgroups of sandbox %s is nil", sandboxID) + } + +- return state.CgroupPath, nil ++ return config.Cgroups.Path, nil + } + + // AddPidToSandboxCgroup add kata-runtime create process to cgroup +@@ -276,3 +268,107 @@ func validCPUResources(cpuSpec *specs.LinuxCPU) *specs.LinuxCPU { + + return &cpu + } ++ ++// getQemuTaskWithoutVcpu filter out tasks under /proc/{qemu pid}/task, to find out the task of not VCPU, ++// VCPU task is filtered by "query-cpus" qmp command ++func getQemuTaskWithoutVcpu(sandbox *Sandbox, vmPid int) []int { ++ procPath := fmt.Sprintf("/proc/%d/task", vmPid) ++ ++ dirReader, err := os.Open(procPath) ++ if err != nil { ++ logrus.Warningf("cannot open %s: %s", procPath, err) ++ return nil ++ } ++ ++ defer dirReader.Close() ++ ++ dirs, err := dirReader.Readdirnames(0) ++ if err != nil { ++ logrus.Warningf("walking dirs in %s failed: %s", procPath, err) ++ return nil ++ } ++ ++ vcpuThreadInfo, err := sandbox.hypervisor.getThreadIDs() ++ if err != nil { ++ logrus.Warnf("get hypervisor Thread ID failed: %v", err) ++ return nil ++ } ++ ++ var vcpuThreadIDs []int ++ for _, value := range vcpuThreadInfo.vcpus { ++ vcpuThreadIDs = append(vcpuThreadIDs, value) ++ } ++ ++ var allThreadIDs []int ++ for _, dir := range dirs { ++ p, err := strconv.Atoi(dir) ++ if err != nil { ++ logrus.Warnf("can not change string dir: %s to int type", dir) ++ return nil ++ } ++ ++ allThreadIDs = append(allThreadIDs, p) ++ } ++ ++ nonVCPUThreads := diffSlice(allThreadIDs, vcpuThreadIDs) ++ ++ return nonVCPUThreads ++} ++ ++func pulloutQemuThread(sandbox *Sandbox, vmPid int, path string) error { ++ control, err := cgroups.New(cgroups.SingleSubsystem(cgroups.V1, cgroups.Cpu), ++ cgroups.StaticPath(path), ++ &specs.LinuxResources{}) ++ if err != nil { ++ return err ++ } ++ taskIds := getQemuTaskWithoutVcpu(sandbox, vmPid) ++ if len(taskIds) == 0 { ++ logrus.Warnf("no taskId id in qemu other than vcpu found of pid %d", vmPid) ++ return nil ++ } ++ for _, taskId := range taskIds { ++ if err := control.AddTask(cgroups.Process{ ++ Pid: taskId, ++ }); err != nil { ++ logrus.Errorf("failed to add task %d to cgroup of %s", taskId, path) ++ return err ++ } ++ } ++ ++ return nil ++} ++ ++// checkCgroupExist check cgroup exist or not ++func checkCgroupExist(hierarchy cgroups.Hierarchy, path string) bool { ++ subSystems, _ := hierarchy() ++ for _, s := range cgroupPathers(subSystems) { ++ if _, err := os.Lstat(s.Path(path)); err != nil { ++ if os.IsNotExist(err) { ++ return false ++ } ++ } ++ } ++ ++ return true ++} ++ ++// diffSlice return the s1 - s2 ++func diffSlice(s1, s2 []int) []int { ++ var diffSlice []int ++ for _, p := range s1 { ++ if !isInSlice(p, s2) { ++ diffSlice = append(diffSlice, p) ++ } ++ } ++ return diffSlice ++} ++ ++func isInSlice(i int, s []int) bool { ++ for _, v := range s { ++ if i == v { ++ return true ++ } ++ } ++ return false ++} +diff --git a/virtcontainers/persist/fs/fs.go b/virtcontainers/persist/fs/fs.go +index 38efdba..641d64e 100644 +--- a/virtcontainers/persist/fs/fs.go ++++ b/virtcontainers/persist/fs/fs.go +@@ -14,6 +14,8 @@ import ( + "path/filepath" + "syscall" + ++ "github.com/opencontainers/runc/libcontainer/configs" ++ + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/sirupsen/logrus" + ) +@@ -78,6 +80,12 @@ func (fs *FS) ToDisk(ss persistapi.SandboxState, cs map[string]persistapi.Contai + return fmt.Errorf("sandbox container id required") + } + ++ if ss.Config.Cgroups == nil { ++ ss.Config.Cgroups = &configs.Cgroup{ ++ Path: ss.CgroupPath, ++ } ++ } ++ + fs.sandboxState = &ss + fs.containerState = cs + +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index 91067fb..e8ef41b 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -1136,3 +1136,17 @@ func validateSandboxDNS(value string) error { + + return nil + } ++ ++func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) { ++ if s == nil { ++ return "", fmt.Errorf("spec is nil") ++ } ++ ++ for _, v := range CRISandboxNameKeyList { ++ if sandboxID, ok := s.Annotations[v]; ok { ++ return sandboxID, nil ++ } ++ } ++ ++ return "", fmt.Errorf("failed to find the sandbox ID") ++} +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index b479cf5..ca4e700 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -2162,7 +2162,9 @@ func (s *Sandbox) cgroupsDelete() error { + var cgroupSubsystems cgroups.Hierarchy + + if s.config.SandboxCgroupWithEmulator { +- // emulator ++ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil { ++ return err ++ } + } else if s.config.SandboxCgroupOnly { + return s.cgroupMgr.Destroy() + } +@@ -2381,6 +2383,68 @@ func (s *Sandbox) setupSandboxCgroup() error { + return nil + } + ++func (s *Sandbox) setupHostCgroupsWithEmulator() error { ++ if len(s.config.Containers) == 0 { ++ return nil ++ } ++ ++ sandboxContainerSpec := s.GetPatchedOCISpec() ++ if sandboxContainerSpec == nil { ++ return fmt.Errorf("sandbox container should not be empty") ++ } ++ ++ // Set sandbox's cgroup path ++ s.state.CgroupPath = sandboxContainerSpec.Linux.CgroupsPath ++ ++ if !checkCgroupExist(cgroups.V1, s.state.CgroupPath) { ++ return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath) ++ } ++ ++ // pull out qemu threads other than vcpu to the cgroup of "/emulator" ++ if s.config.HypervisorType == QemuHypervisor { ++ emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName) ++ hypervisorPids := s.hypervisor.getPids() ++ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { ++ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) ++ } ++ if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil { ++ return err ++ } ++ } ++ ++ // limit cpu to "/vcpu" ++ vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName) ++ vcpuResources := specs.LinuxResources{ ++ CPU: s.cpuResources(), ++ } ++ if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil { ++ return err ++ } ++ ++ // limit blkio resource to "" ++ ++ // limit files resource ++ ++ return nil ++} ++ ++func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) error { ++ if resources == nil { ++ return nil ++ } ++ ++ control, err := cgroupsLoadFunc(cgroups.V1, cgroups.StaticPath(cgroupPath)) ++ if err != nil { ++ return fmt.Errorf("could not load cgroup %v: %v", cgroupPath, err) ++ } ++ ++ if err = control.Update(resources); err != nil { ++ return fmt.Errorf("could not update cgroup %v: %v", cgroupPath, err) ++ } ++ ++ return nil ++} ++ + // GetPatchedOCISpec returns sandbox's OCI specification + // This OCI specification was patched when the sandbox was created + // by containerCapabilities(), SetEphemeralStorageType() and others +@@ -2452,6 +2516,10 @@ func (s *Sandbox) forceDeleteSandbox() { + c.forceDeleteContainer() + } + ++ if err := deleteCgroup(cgroups.V1, s.state.CgroupPath); err != nil { ++ s.Logger().Warnf("sandbox forceDelete cgroups failed: %v", err) ++ } ++ + globalSandboxList.removeSandbox(s.id) + + if s.monitor != nil { +-- +1.8.3.1 + diff --git a/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch b/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch new file mode 100644 index 0000000000000000000000000000000000000000..dc8be5d7381792a15d6fd08170fdfe8d44c34f4e --- /dev/null +++ b/runtime/patches/0056-kata_runtime-support-the-blkio-in-host-cgroups.patch @@ -0,0 +1,318 @@ +From f4b899b933a3a30fc378ceb4d8855778cd783d9a Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Fri, 18 Sep 2020 16:54:11 +0800 +Subject: [PATCH 4/5] kata_runtime: support the blkio in host cgroups + +reason: support the blkio in host cgroups, run with paras +--annotation io.kubernetes.docker.type=podsandbox --annotation io.katacontainers.blkio_cgroup= +'{"blkiocgroup":[{"path":"/dev/sda","limits":[{"type":"throttle_read_bps","value":400}]}]}' + +Signed-off-by: yangfeiyu +--- + vendor/github.com/containerd/cgroups/blkio.go | 6 ++ + virtcontainers/cgroups.go | 102 ++++++++++++++++++++++++++ + virtcontainers/pkg/annotations/annotations.go | 3 + + virtcontainers/pkg/oci/utils.go | 50 ++++++++++++- + virtcontainers/sandbox.go | 6 ++ + virtcontainers/utils/utils.go | 22 ++++++ + 6 files changed, 186 insertions(+), 3 deletions(-) + +diff --git a/vendor/github.com/containerd/cgroups/blkio.go b/vendor/github.com/containerd/cgroups/blkio.go +index 7c498de..485ff7b 100644 +--- a/vendor/github.com/containerd/cgroups/blkio.go ++++ b/vendor/github.com/containerd/cgroups/blkio.go +@@ -23,6 +23,7 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "reflect" + "strconv" + "strings" + +@@ -55,6 +56,11 @@ func (b *blkioController) Create(path string, resources *specs.LinuxResources) e + return nil + } + for _, t := range createBlkioSettings(resources.BlockIO) { ++ ptr := reflect.ValueOf(t.value) ++ if ptr.Kind() == reflect.Ptr && ptr.IsNil() { ++ continue ++ } ++ + if t.value != nil { + if err := ioutil.WriteFile( + filepath.Join(b.Path(path), fmt.Sprintf("blkio.%s", t.name)), +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index 65d2001..e8c5a7b 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -9,6 +9,7 @@ package virtcontainers + import ( + "bufio" + "context" ++ "encoding/json" + "fmt" + "os" + "path/filepath" +@@ -16,6 +17,8 @@ import ( + "strings" + + "github.com/containerd/cgroups" ++ "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" ++ "github.com/kata-containers/runtime/virtcontainers/utils" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + ) +@@ -32,8 +35,46 @@ const ( + cgroupKataPath = "/kata/" + vcpuCgroupName = "vcpu" + emulatorCgroupName = "emulator" ++ ++ // BlkioThrottleReadBps is the key to fetch throttle_read_bps ++ BlkioThrottleReadBps = "throttle_read_bps" ++ ++ // BlkioThrottleWriteBps is the key to fetch throttle_write_bps ++ BlkioThrottleWriteBps = "throttle_write_bps" ++ ++ // BlkioThrottleReadIOPS is the key to fetch throttle_read_iops ++ BlkioThrottleReadIOPS = "throttle_read_iops" ++ ++ // BlkioThrottleWriteIOPS is the key to fetch throttle_write_iops ++ BlkioThrottleWriteIOPS = "throttle_write_iops" ++ ++ // BlkioWeight is the key to fetch blkio_weight ++ BlkioWeight = "blkio_weight" ++ ++ // BlkioLeafWeight is the key to fetch blkio_leaf_weight ++ BlkioLeafWeight = "blkio_leaf_weight" + ) + ++// BlkioCgroup for Linux cgroup 'blkio' data exchange ++type BlkioCgroup struct { ++ // Items specifies per cgroup values ++ Items []BlockIOCgroupItem `json:"blkiocgroup,omitempty"` ++} ++ ++type BlockIOCgroupItem struct { ++ // Path represent path of blkio device ++ Path string `json:"path,omitempty"` ++ // Limits specifies the blkio type and value ++ Limits []IOLimit `json:"limits,omitempty"` ++} ++ ++type IOLimit struct { ++ // Type specifies IO type ++ Type string `json:"type,omitempty"` ++ // Value specifies rate or weight value ++ Value uint64 `json:"value,omitempty"` ++} ++ + var cgroupsLoadFunc = cgroups.Load + var cgroupsNewFunc = cgroups.New + +@@ -372,3 +413,64 @@ func isInSlice(i int, s []int) bool { + } + return false + } ++ ++func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO { ++ value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey] ++ if !ok { ++ return nil ++ } ++ ++ var blkioCgroupParse BlkioCgroup ++ var linuxBlkio specs.LinuxBlockIO ++ if err := json.Unmarshal([]byte(value), &blkioCgroupParse); err != nil { ++ s.Logger().Errorf("blkio_cgroup Unmarshal error:%v", err) ++ return nil ++ } ++ ++ for _, item := range blkioCgroupParse.Items { ++ if item.Limits == nil { ++ s.Logger().Errorf("%v:limits have none data", item) ++ return nil ++ } ++ for _, limit := range item.Limits { ++ if item.Path != "" { ++ major, minor, err := utils.GetDeviceByPath(item.Path) ++ if err != nil { ++ s.Logger().Errorf("failed to find major and minor of device %s: %v", item.Path, err) ++ return nil ++ } ++ td := specs.LinuxThrottleDevice{ ++ Rate: limit.Value, ++ } ++ td.Major = major ++ td.Minor = minor ++ switch limit.Type { ++ case BlkioThrottleReadBps: ++ linuxBlkio.ThrottleReadBpsDevice = append(linuxBlkio.ThrottleReadBpsDevice, td) ++ case BlkioThrottleWriteBps: ++ linuxBlkio.ThrottleWriteBpsDevice = append(linuxBlkio.ThrottleWriteBpsDevice, td) ++ case BlkioThrottleReadIOPS: ++ linuxBlkio.ThrottleReadIOPSDevice = append(linuxBlkio.ThrottleReadIOPSDevice, td) ++ case BlkioThrottleWriteIOPS: ++ linuxBlkio.ThrottleWriteIOPSDevice = append(linuxBlkio.ThrottleWriteIOPSDevice, td) ++ default: ++ s.Logger().Errorf("the type of throtlle device:%s is not surpport", limit.Type) ++ return nil ++ } ++ } else { ++ switch limit.Type { ++ case BlkioWeight: ++ weightUint16 := uint16(limit.Value) ++ linuxBlkio.Weight = &weightUint16 ++ case BlkioLeafWeight: ++ weightLeafUint16 := uint16(limit.Value) ++ linuxBlkio.LeafWeight = &weightLeafUint16 ++ default: ++ s.Logger().Errorf("the type:%s is not one of the supported types of blkio_weight and blkio_leaf_weight", limit.Type) ++ return nil ++ } ++ } ++ } ++ } ++ return &linuxBlkio ++} +diff --git a/virtcontainers/pkg/annotations/annotations.go b/virtcontainers/pkg/annotations/annotations.go +index 96c4ef2..c35993e 100644 +--- a/virtcontainers/pkg/annotations/annotations.go ++++ b/virtcontainers/pkg/annotations/annotations.go +@@ -21,6 +21,9 @@ const ( + ContainerTypeKey = kataAnnotationsPrefix + "pkg.oci.container_type" + + SandboxConfigPathKey = kataAnnotationsPrefix + "config_path" ++ ++ // BlkioCgroupTypeKey is the annotation key to fetch sandbox blkio cgroup values ++ BlkioCgroupTypeKey = kataAnnotationsPrefix + "blkio_cgroup" + ) + + // Annotations related to Hypervisor configuration +diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go +index e8ef41b..643753b 100644 +--- a/virtcontainers/pkg/oci/utils.go ++++ b/virtcontainers/pkg/oci/utils.go +@@ -38,9 +38,10 @@ type annotationContainerType struct { + type annotationHandler func(value string) error + + var annotationHandlerList = map[string]annotationHandler{ +- vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, +- vcAnnotations.StaticMemTypeKey: validateSandboxMem, +- vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, ++ vcAnnotations.StaticCPUTypeKey: validateSandboxCPU, ++ vcAnnotations.StaticMemTypeKey: validateSandboxMem, ++ vcAnnotations.SandboxDNSTypeKey: validateSandboxDNS, ++ vcAnnotations.BlkioCgroupTypeKey: validateBlkioCgroup, + } + + var ( +@@ -1076,6 +1077,7 @@ func validateOtherSandboxAnnotations(annotation, value string) error { + // addOtherSandboxAnnotation add self defined annotation for sandbox + func addOtherSandboxAnnotation(ocispec specs.Spec, sbConfig *vc.SandboxConfig) error { + otherSandboxAnnotationsKey := []string{ ++ vcAnnotations.BlkioCgroupTypeKey, + vcAnnotations.StaticCPUTypeKey, + vcAnnotations.StaticMemTypeKey, + } +@@ -1137,6 +1139,48 @@ func validateSandboxDNS(value string) error { + return nil + } + ++func validateBlkioCgroup(value string) error { ++ var linuxBlkioCgroup vc.BlkioCgroup ++ if err := json.Unmarshal([]byte(value), &linuxBlkioCgroup); err != nil { ++ return fmt.Errorf("blkio_cgroup Unmarshal error:%v, value is %s", err, value) ++ } ++ ++ if linuxBlkioCgroup.Items == nil { ++ return fmt.Errorf("BlkioCgroup:%v fetch none data", linuxBlkioCgroup) ++ } ++ ++ // pathNull used to judge path for blkio.weight/blkio.leaf_weight only once ++ pathNull := false ++ for _, item := range linuxBlkioCgroup.Items { ++ if pathNull { ++ return fmt.Errorf("too many paths for blkio.weight/blkio.leaf_weight") ++ } ++ if item.Path == "" && !pathNull { ++ pathNull = true ++ if len(item.Limits) > 2 || len(item.Limits) == 0 { ++ return fmt.Errorf("the format of values for blkio.weight or blkio.leaf_weight is wrong") ++ } ++ ++ for _, ioLimit := range item.Limits { ++ switch ioLimit.Type { ++ case vc.BlkioWeight: ++ // Blkio.weight, between 10 and 1000 ++ if ioLimit.Value < 10 || ioLimit.Value > 1000 { ++ return fmt.Errorf("blkio.weight:%v must be between 10 and 1000", ioLimit.Value) ++ } ++ case vc.BlkioLeafWeight: ++ return fmt.Errorf("the blkio.leaf_weight is not supported now") ++ default: ++ return fmt.Errorf("the type of blkio device:%s is not surpport", ioLimit.Type) ++ } ++ } ++ } else if _, _, err := utils.GetDeviceByPath(item.Path); err != nil { ++ return fmt.Errorf("failed to find major and minor of device %s: %v", item.Path, err) ++ } ++ } ++ return nil ++} ++ + func GetSandboxIDFromAnnotations(s *specs.Spec) (string, error) { + if s == nil { + return "", fmt.Errorf("spec is nil") +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index ca4e700..9284f99 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -2422,6 +2422,12 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { + } + + // limit blkio resource to "" ++ blkioResources := specs.LinuxResources{ ++ BlockIO: s.blockIOResource(), ++ } ++ if err := applyResourceLimit(&blkioResources, vcpuCgroupPath); err != nil { ++ return err ++ } + + // limit files resource + +diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go +index 36ac67a..d4dad40 100644 +--- a/virtcontainers/utils/utils.go ++++ b/virtcontainers/utils/utils.go +@@ -370,6 +370,28 @@ func GetPhysicalCPUNumber() int { + return cpuNum + } + ++// GetDeviceByPath returns the device No. ++func GetDeviceByPath(path string) (int64, int64, error) { ++ if path == "" { ++ return -1, -1, fmt.Errorf("Path cannot be empty") ++ } ++ ++ path, err := filepath.EvalSymlinks(path) ++ if err != nil { ++ return -1, -1, err ++ } ++ ++ stat := syscall.Stat_t{} ++ err = syscall.Stat(path, &stat) ++ if err != nil { ++ return -1, -1, err ++ } ++ ++ major := int64(stat.Rdev / 256) ++ minor := int64(stat.Rdev % 256) ++ return major, minor, nil ++} ++ + func RoundVCPUNumber(value string) (int, error) { + cpuNum, err := strconv.ParseFloat(value, 64) + if err != nil || cpuNum < minCPUs { +-- +1.8.3.1 + diff --git a/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch b/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch new file mode 100644 index 0000000000000000000000000000000000000000..ba425bab217d66b4672ee2b2bcd7ec56b4e18f96 --- /dev/null +++ b/runtime/patches/0057-kata-runtime-support-files-limit-in-host-cgroups.patch @@ -0,0 +1,5834 @@ +From 04011b8a08bb69aa01ca7045bdcb477c14d5feac Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Mon, 21 Sep 2020 19:27:11 +0800 +Subject: [PATCH 5/5] kata-runtime: support --files-limit in host cgroups + +reason: support --files-limit in host cgroups, like --files-limit 2000 + +Signed-off-by: yangfeiyu +--- + vendor/github.com/containerd/cgroups/files.go | 142 + + vendor/github.com/containerd/cgroups/metrics.pb.go | 5050 +++----------------- + vendor/github.com/containerd/cgroups/subsystem.go | 2 + + vendor/github.com/containerd/cgroups/utils.go | 2 +- + .../opencontainers/runtime-spec/specs-go/config.go | 6 + + virtcontainers/cgroups.go | 82 + + virtcontainers/fileslimit/fileslimit.c | 23 + + virtcontainers/fileslimit/fileslimit.go | 23 + + virtcontainers/fileslimit/fileslimit.h | 14 + + virtcontainers/sandbox.go | 38 +- + 10 files changed, 1012 insertions(+), 4370 deletions(-) + create mode 100644 vendor/github.com/containerd/cgroups/files.go + create mode 100644 virtcontainers/fileslimit/fileslimit.c + create mode 100644 virtcontainers/fileslimit/fileslimit.go + create mode 100644 virtcontainers/fileslimit/fileslimit.h + +diff --git a/vendor/github.com/containerd/cgroups/files.go b/vendor/github.com/containerd/cgroups/files.go +new file mode 100644 +index 0000000..c8374c1 +--- /dev/null ++++ b/vendor/github.com/containerd/cgroups/files.go +@@ -0,0 +1,142 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++SPDX-License-Identifier: Apache-2.0 ++Description: common functions ++Author: jiangpengfei ++Create: 2019-06-12 ++*/ ++ ++package cgroups ++ ++import ( ++ "fmt" ++ "io/ioutil" ++ "os" ++ "path/filepath" ++ "strconv" ++ "strings" ++ ++ specs "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++func NewFiles(root string) *filesController { ++ return &filesController{ ++ root: filepath.Join(root, string(Files)), ++ } ++} ++ ++type filesController struct { ++ root string ++} ++ ++type filesSettings struct { ++ name string ++ value *uint64 ++} ++ ++func (f *filesController) Name() Name { ++ return Files ++} ++ ++func (f *filesController) Path(path string) string { ++ return filepath.Join(f.root, path) ++} ++ ++func (f *filesController) Create(path string, resources *specs.LinuxResources) error { ++ if err := os.MkdirAll(f.Path(path), defaultDirPerm); err != nil { ++ return err ++ } ++ ++ if resources.Files == nil { ++ return nil ++ } ++ ++ settings := getFilesSettings(resources) ++ for _, t := range settings { ++ if t.value != nil { ++ // If files.limit value equal 0, which regard as no limit ++ if *(t.value) == 0 { ++ if err := ioutil.WriteFile( ++ filepath.Join(f.Path(path), fmt.Sprintf("files.%s", t.name)), ++ []byte("max"), ++ defaultFilePerm, ++ ); err != nil { ++ return err ++ } ++ } else { ++ if err := ioutil.WriteFile( ++ filepath.Join(f.Path(path), fmt.Sprintf("files.%s", t.name)), ++ []byte(strconv.FormatUint(*t.value, 10)), ++ defaultFilePerm, ++ ); err != nil { ++ return err ++ } ++ } ++ } ++ } ++ ++ return nil ++} ++ ++func (f *filesController) Update(path string, resources *specs.LinuxResources) error { ++ return f.Create(path, resources) ++} ++ ++func (f *filesController) Stat(path string, stats *Metrics) error { ++ usage, err := getCgroupParamUint(path, "files.usage") ++ if err != nil { ++ return fmt.Errorf("failed to parse files.usage - %s", err) ++ } ++ ++ maxString, err := getCgroupParamString(path, "files.limit") ++ if err != nil { ++ return fmt.Errorf("failed to parse files.limit - %s", err) ++ } ++ ++ // Default if files.limit == "max" is 0 -- which represents "no limit". ++ var max uint64 ++ if maxString != "max" { ++ max, err = parseUint(maxString, 10, 64) ++ if err != nil { ++ return fmt.Errorf("failed to parse files.limit -- unable to parse %q as a uint from Cgroup file %q", maxString, filepath.Join(path, "file.limits")) ++ } ++ } ++ ++ stats.Files.Usage = usage ++ stats.Files.Limit = max ++ return nil ++} ++ ++func getFilesSettings(resources *specs.LinuxResources) []filesSettings { ++ return []filesSettings{ ++ { ++ name: "limit", ++ value: resources.Files.Limit, ++ }, ++ } ++} ++ ++// Gets a single uint64 value from the specified cgroup file. ++func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) { ++ fileName := filepath.Join(cgroupPath, cgroupFile) ++ contents, err := ioutil.ReadFile(fileName) ++ if err != nil { ++ return 0, err ++ } ++ ++ res, err := parseUint(strings.TrimSpace(string(contents)), 10, 64) ++ if err != nil { ++ return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName) ++ } ++ return res, nil ++} ++ ++// Gets a string value from the specified cgroup file ++func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) { ++ contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile)) ++ if err != nil { ++ return "", err ++ } ++ ++ return strings.TrimSpace(string(contents)), nil ++} +diff --git a/vendor/github.com/containerd/cgroups/metrics.pb.go b/vendor/github.com/containerd/cgroups/metrics.pb.go +index 7dd7f6f..652744d 100644 +--- a/vendor/github.com/containerd/cgroups/metrics.pb.go ++++ b/vendor/github.com/containerd/cgroups/metrics.pb.go +@@ -2,37 +2,38 @@ + // source: github.com/containerd/cgroups/metrics.proto + + /* +- Package cgroups is a generated protocol buffer package. +- +- It is generated from these files: +- github.com/containerd/cgroups/metrics.proto +- +- It has these top-level messages: +- Metrics +- HugetlbStat +- PidsStat +- CPUStat +- CPUUsage +- Throttle +- MemoryStat +- MemoryEntry +- BlkIOStat +- BlkIOEntry +- RdmaStat +- RdmaEntry +- NetworkStat ++Package cgroups is a generated protocol buffer package. ++ ++It is generated from these files: ++ github.com/containerd/cgroups/metrics.proto ++ ++It has these top-level messages: ++ Metrics ++ HugetlbStat ++ PidsStat ++ CPUStat ++ CPUUsage ++ Throttle ++ MemoryStat ++ MemoryEntry ++ BlkIOStat ++ BlkIOEntry ++ RdmaStat ++ RdmaEntry ++ NetworkStat ++ FilesStat + */ + package cgroups + +-import proto "github.com/gogo/protobuf/proto" +-import fmt "fmt" +-import math "math" +-import _ "github.com/gogo/protobuf/gogoproto" ++import ( ++ fmt "fmt" + +-import strings "strings" +-import reflect "reflect" ++ proto "github.com/gogo/protobuf/proto" + +-import io "io" ++ math "math" ++ ++ _ "github.com/gogo/protobuf/gogoproto" ++) + + // Reference imports to suppress errors if they are not otherwise used. + var _ = proto.Marshal +@@ -53,12 +54,70 @@ type Metrics struct { + Blkio *BlkIOStat `protobuf:"bytes,5,opt,name=blkio" json:"blkio,omitempty"` + Rdma *RdmaStat `protobuf:"bytes,6,opt,name=rdma" json:"rdma,omitempty"` + Network []*NetworkStat `protobuf:"bytes,7,rep,name=network" json:"network,omitempty"` ++ Files *FilesStat `protobuf:"bytes,8,opt,name=files" json:"files,omitempty"` + } + + func (m *Metrics) Reset() { *m = Metrics{} } ++func (m *Metrics) String() string { return proto.CompactTextString(m) } + func (*Metrics) ProtoMessage() {} + func (*Metrics) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{0} } + ++func (m *Metrics) GetHugetlb() []*HugetlbStat { ++ if m != nil { ++ return m.Hugetlb ++ } ++ return nil ++} ++ ++func (m *Metrics) GetPids() *PidsStat { ++ if m != nil { ++ return m.Pids ++ } ++ return nil ++} ++ ++func (m *Metrics) GetCPU() *CPUStat { ++ if m != nil { ++ return m.CPU ++ } ++ return nil ++} ++ ++func (m *Metrics) GetMemory() *MemoryStat { ++ if m != nil { ++ return m.Memory ++ } ++ return nil ++} ++ ++func (m *Metrics) GetBlkio() *BlkIOStat { ++ if m != nil { ++ return m.Blkio ++ } ++ return nil ++} ++ ++func (m *Metrics) GetRdma() *RdmaStat { ++ if m != nil { ++ return m.Rdma ++ } ++ return nil ++} ++ ++func (m *Metrics) GetNetwork() []*NetworkStat { ++ if m != nil { ++ return m.Network ++ } ++ return nil ++} ++ ++func (m *Metrics) GetFiles() *FilesStat { ++ if m != nil { ++ return m.Files ++ } ++ return nil ++} ++ + type HugetlbStat struct { + Usage uint64 `protobuf:"varint,1,opt,name=usage,proto3" json:"usage,omitempty"` + Max uint64 `protobuf:"varint,2,opt,name=max,proto3" json:"max,omitempty"` +@@ -67,27 +126,86 @@ type HugetlbStat struct { + } + + func (m *HugetlbStat) Reset() { *m = HugetlbStat{} } ++func (m *HugetlbStat) String() string { return proto.CompactTextString(m) } + func (*HugetlbStat) ProtoMessage() {} + func (*HugetlbStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{1} } + ++func (m *HugetlbStat) GetUsage() uint64 { ++ if m != nil { ++ return m.Usage ++ } ++ return 0 ++} ++ ++func (m *HugetlbStat) GetMax() uint64 { ++ if m != nil { ++ return m.Max ++ } ++ return 0 ++} ++ ++func (m *HugetlbStat) GetFailcnt() uint64 { ++ if m != nil { ++ return m.Failcnt ++ } ++ return 0 ++} ++ ++func (m *HugetlbStat) GetPagesize() string { ++ if m != nil { ++ return m.Pagesize ++ } ++ return "" ++} ++ + type PidsStat struct { + Current uint64 `protobuf:"varint,1,opt,name=current,proto3" json:"current,omitempty"` + Limit uint64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + } + + func (m *PidsStat) Reset() { *m = PidsStat{} } ++func (m *PidsStat) String() string { return proto.CompactTextString(m) } + func (*PidsStat) ProtoMessage() {} + func (*PidsStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{2} } + ++func (m *PidsStat) GetCurrent() uint64 { ++ if m != nil { ++ return m.Current ++ } ++ return 0 ++} ++ ++func (m *PidsStat) GetLimit() uint64 { ++ if m != nil { ++ return m.Limit ++ } ++ return 0 ++} ++ + type CPUStat struct { + Usage *CPUUsage `protobuf:"bytes,1,opt,name=usage" json:"usage,omitempty"` + Throttling *Throttle `protobuf:"bytes,2,opt,name=throttling" json:"throttling,omitempty"` + } + + func (m *CPUStat) Reset() { *m = CPUStat{} } ++func (m *CPUStat) String() string { return proto.CompactTextString(m) } + func (*CPUStat) ProtoMessage() {} + func (*CPUStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{3} } + ++func (m *CPUStat) GetUsage() *CPUUsage { ++ if m != nil { ++ return m.Usage ++ } ++ return nil ++} ++ ++func (m *CPUStat) GetThrottling() *Throttle { ++ if m != nil { ++ return m.Throttling ++ } ++ return nil ++} ++ + type CPUUsage struct { + // values in nanoseconds + Total uint64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` +@@ -97,9 +215,38 @@ type CPUUsage struct { + } + + func (m *CPUUsage) Reset() { *m = CPUUsage{} } ++func (m *CPUUsage) String() string { return proto.CompactTextString(m) } + func (*CPUUsage) ProtoMessage() {} + func (*CPUUsage) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{4} } + ++func (m *CPUUsage) GetTotal() uint64 { ++ if m != nil { ++ return m.Total ++ } ++ return 0 ++} ++ ++func (m *CPUUsage) GetKernel() uint64 { ++ if m != nil { ++ return m.Kernel ++ } ++ return 0 ++} ++ ++func (m *CPUUsage) GetUser() uint64 { ++ if m != nil { ++ return m.User ++ } ++ return 0 ++} ++ ++func (m *CPUUsage) GetPerCPU() []uint64 { ++ if m != nil { ++ return m.PerCPU ++ } ++ return nil ++} ++ + type Throttle struct { + Periods uint64 `protobuf:"varint,1,opt,name=periods,proto3" json:"periods,omitempty"` + ThrottledPeriods uint64 `protobuf:"varint,2,opt,name=throttled_periods,json=throttledPeriods,proto3" json:"throttled_periods,omitempty"` +@@ -107,9 +254,31 @@ type Throttle struct { + } + + func (m *Throttle) Reset() { *m = Throttle{} } ++func (m *Throttle) String() string { return proto.CompactTextString(m) } + func (*Throttle) ProtoMessage() {} + func (*Throttle) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{5} } + ++func (m *Throttle) GetPeriods() uint64 { ++ if m != nil { ++ return m.Periods ++ } ++ return 0 ++} ++ ++func (m *Throttle) GetThrottledPeriods() uint64 { ++ if m != nil { ++ return m.ThrottledPeriods ++ } ++ return 0 ++} ++ ++func (m *Throttle) GetThrottledTime() uint64 { ++ if m != nil { ++ return m.ThrottledTime ++ } ++ return 0 ++} ++ + type MemoryStat struct { + Cache uint64 `protobuf:"varint,1,opt,name=cache,proto3" json:"cache,omitempty"` + RSS uint64 `protobuf:"varint,2,opt,name=rss,proto3" json:"rss,omitempty"` +@@ -150,4547 +319,698 @@ type MemoryStat struct { + } + + func (m *MemoryStat) Reset() { *m = MemoryStat{} } ++func (m *MemoryStat) String() string { return proto.CompactTextString(m) } + func (*MemoryStat) ProtoMessage() {} + func (*MemoryStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{6} } + +-type MemoryEntry struct { +- Limit uint64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` +- Usage uint64 `protobuf:"varint,2,opt,name=usage,proto3" json:"usage,omitempty"` +- Max uint64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` +- Failcnt uint64 `protobuf:"varint,4,opt,name=failcnt,proto3" json:"failcnt,omitempty"` ++func (m *MemoryStat) GetCache() uint64 { ++ if m != nil { ++ return m.Cache ++ } ++ return 0 + } + +-func (m *MemoryEntry) Reset() { *m = MemoryEntry{} } +-func (*MemoryEntry) ProtoMessage() {} +-func (*MemoryEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{7} } +- +-type BlkIOStat struct { +- IoServiceBytesRecursive []*BlkIOEntry `protobuf:"bytes,1,rep,name=io_service_bytes_recursive,json=ioServiceBytesRecursive" json:"io_service_bytes_recursive,omitempty"` +- IoServicedRecursive []*BlkIOEntry `protobuf:"bytes,2,rep,name=io_serviced_recursive,json=ioServicedRecursive" json:"io_serviced_recursive,omitempty"` +- IoQueuedRecursive []*BlkIOEntry `protobuf:"bytes,3,rep,name=io_queued_recursive,json=ioQueuedRecursive" json:"io_queued_recursive,omitempty"` +- IoServiceTimeRecursive []*BlkIOEntry `protobuf:"bytes,4,rep,name=io_service_time_recursive,json=ioServiceTimeRecursive" json:"io_service_time_recursive,omitempty"` +- IoWaitTimeRecursive []*BlkIOEntry `protobuf:"bytes,5,rep,name=io_wait_time_recursive,json=ioWaitTimeRecursive" json:"io_wait_time_recursive,omitempty"` +- IoMergedRecursive []*BlkIOEntry `protobuf:"bytes,6,rep,name=io_merged_recursive,json=ioMergedRecursive" json:"io_merged_recursive,omitempty"` +- IoTimeRecursive []*BlkIOEntry `protobuf:"bytes,7,rep,name=io_time_recursive,json=ioTimeRecursive" json:"io_time_recursive,omitempty"` +- SectorsRecursive []*BlkIOEntry `protobuf:"bytes,8,rep,name=sectors_recursive,json=sectorsRecursive" json:"sectors_recursive,omitempty"` ++func (m *MemoryStat) GetRSS() uint64 { ++ if m != nil { ++ return m.RSS ++ } ++ return 0 + } + +-func (m *BlkIOStat) Reset() { *m = BlkIOStat{} } +-func (*BlkIOStat) ProtoMessage() {} +-func (*BlkIOStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{8} } +- +-type BlkIOEntry struct { +- Op string `protobuf:"bytes,1,opt,name=op,proto3" json:"op,omitempty"` +- Device string `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` +- Major uint64 `protobuf:"varint,3,opt,name=major,proto3" json:"major,omitempty"` +- Minor uint64 `protobuf:"varint,4,opt,name=minor,proto3" json:"minor,omitempty"` +- Value uint64 `protobuf:"varint,5,opt,name=value,proto3" json:"value,omitempty"` ++func (m *MemoryStat) GetRSSHuge() uint64 { ++ if m != nil { ++ return m.RSSHuge ++ } ++ return 0 + } + +-func (m *BlkIOEntry) Reset() { *m = BlkIOEntry{} } +-func (*BlkIOEntry) ProtoMessage() {} +-func (*BlkIOEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{9} } +- +-type RdmaStat struct { +- Current []*RdmaEntry `protobuf:"bytes,1,rep,name=current" json:"current,omitempty"` +- Limit []*RdmaEntry `protobuf:"bytes,2,rep,name=limit" json:"limit,omitempty"` ++func (m *MemoryStat) GetMappedFile() uint64 { ++ if m != nil { ++ return m.MappedFile ++ } ++ return 0 + } + +-func (m *RdmaStat) Reset() { *m = RdmaStat{} } +-func (*RdmaStat) ProtoMessage() {} +-func (*RdmaStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{10} } +- +-type RdmaEntry struct { +- Device string `protobuf:"bytes,1,opt,name=device,proto3" json:"device,omitempty"` +- HcaHandles uint32 `protobuf:"varint,2,opt,name=hca_handles,json=hcaHandles,proto3" json:"hca_handles,omitempty"` +- HcaObjects uint32 `protobuf:"varint,3,opt,name=hca_objects,json=hcaObjects,proto3" json:"hca_objects,omitempty"` ++func (m *MemoryStat) GetDirty() uint64 { ++ if m != nil { ++ return m.Dirty ++ } ++ return 0 + } + +-func (m *RdmaEntry) Reset() { *m = RdmaEntry{} } +-func (*RdmaEntry) ProtoMessage() {} +-func (*RdmaEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{11} } +- +-type NetworkStat struct { +- Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +- RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` +- RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` +- RxErrors uint64 `protobuf:"varint,4,opt,name=rx_errors,json=rxErrors,proto3" json:"rx_errors,omitempty"` +- RxDropped uint64 `protobuf:"varint,5,opt,name=rx_dropped,json=rxDropped,proto3" json:"rx_dropped,omitempty"` +- TxBytes uint64 `protobuf:"varint,6,opt,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` +- TxPackets uint64 `protobuf:"varint,7,opt,name=tx_packets,json=txPackets,proto3" json:"tx_packets,omitempty"` +- TxErrors uint64 `protobuf:"varint,8,opt,name=tx_errors,json=txErrors,proto3" json:"tx_errors,omitempty"` +- TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` ++func (m *MemoryStat) GetWriteback() uint64 { ++ if m != nil { ++ return m.Writeback ++ } ++ return 0 + } + +-func (m *NetworkStat) Reset() { *m = NetworkStat{} } +-func (*NetworkStat) ProtoMessage() {} +-func (*NetworkStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{12} } +- +-func init() { +- proto.RegisterType((*Metrics)(nil), "io.containerd.cgroups.v1.Metrics") +- proto.RegisterType((*HugetlbStat)(nil), "io.containerd.cgroups.v1.HugetlbStat") +- proto.RegisterType((*PidsStat)(nil), "io.containerd.cgroups.v1.PidsStat") +- proto.RegisterType((*CPUStat)(nil), "io.containerd.cgroups.v1.CPUStat") +- proto.RegisterType((*CPUUsage)(nil), "io.containerd.cgroups.v1.CPUUsage") +- proto.RegisterType((*Throttle)(nil), "io.containerd.cgroups.v1.Throttle") +- proto.RegisterType((*MemoryStat)(nil), "io.containerd.cgroups.v1.MemoryStat") +- proto.RegisterType((*MemoryEntry)(nil), "io.containerd.cgroups.v1.MemoryEntry") +- proto.RegisterType((*BlkIOStat)(nil), "io.containerd.cgroups.v1.BlkIOStat") +- proto.RegisterType((*BlkIOEntry)(nil), "io.containerd.cgroups.v1.BlkIOEntry") +- proto.RegisterType((*RdmaStat)(nil), "io.containerd.cgroups.v1.RdmaStat") +- proto.RegisterType((*RdmaEntry)(nil), "io.containerd.cgroups.v1.RdmaEntry") +- proto.RegisterType((*NetworkStat)(nil), "io.containerd.cgroups.v1.NetworkStat") +-} +-func (m *Metrics) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetPgPgIn() uint64 { ++ if m != nil { ++ return m.PgPgIn + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *Metrics) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.Hugetlb) > 0 { +- for _, msg := range m.Hugetlb { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if m.Pids != nil { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Pids.Size())) +- n1, err := m.Pids.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n1 +- } +- if m.CPU != nil { +- dAtA[i] = 0x1a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.CPU.Size())) +- n2, err := m.CPU.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n2 ++func (m *MemoryStat) GetPgPgOut() uint64 { ++ if m != nil { ++ return m.PgPgOut + } +- if m.Memory != nil { +- dAtA[i] = 0x22 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Memory.Size())) +- n3, err := m.Memory.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n3 +- } +- if m.Blkio != nil { +- dAtA[i] = 0x2a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Blkio.Size())) +- n4, err := m.Blkio.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n4 +- } +- if m.Rdma != nil { +- dAtA[i] = 0x32 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Rdma.Size())) +- n5, err := m.Rdma.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n5 +- } +- if len(m.Network) > 0 { +- for _, msg := range m.Network { +- dAtA[i] = 0x3a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- return i, nil ++ return 0 + } + +-func (m *HugetlbStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetPgFault() uint64 { ++ if m != nil { ++ return m.PgFault + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *HugetlbStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Usage != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) +- } +- if m.Max != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) +- } +- if m.Failcnt != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) +- } +- if len(m.Pagesize) > 0 { +- dAtA[i] = 0x22 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) +- i += copy(dAtA[i:], m.Pagesize) ++func (m *MemoryStat) GetPgMajFault() uint64 { ++ if m != nil { ++ return m.PgMajFault + } +- return i, nil ++ return 0 + } + +-func (m *PidsStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetInactiveAnon() uint64 { ++ if m != nil { ++ return m.InactiveAnon + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *PidsStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Current != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) ++func (m *MemoryStat) GetActiveAnon() uint64 { ++ if m != nil { ++ return m.ActiveAnon + } +- if m.Limit != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) +- } +- return i, nil ++ return 0 + } + +-func (m *CPUStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetInactiveFile() uint64 { ++ if m != nil { ++ return m.InactiveFile + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *CPUStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Usage != nil { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) +- n6, err := m.Usage.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n6 +- } +- if m.Throttling != nil { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Throttling.Size())) +- n7, err := m.Throttling.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n7 ++func (m *MemoryStat) GetActiveFile() uint64 { ++ if m != nil { ++ return m.ActiveFile + } +- return i, nil ++ return 0 + } + +-func (m *CPUUsage) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetUnevictable() uint64 { ++ if m != nil { ++ return m.Unevictable + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *CPUUsage) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Total != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Total)) +- } +- if m.Kernel != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel)) +- } +- if m.User != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.User)) ++func (m *MemoryStat) GetHierarchicalMemoryLimit() uint64 { ++ if m != nil { ++ return m.HierarchicalMemoryLimit + } +- if len(m.PerCPU) > 0 { +- dAtA9 := make([]byte, len(m.PerCPU)*10) +- var j8 int +- for _, num := range m.PerCPU { +- for num >= 1<<7 { +- dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) +- num >>= 7 +- j8++ +- } +- dAtA9[j8] = uint8(num) +- j8++ +- } +- dAtA[i] = 0x22 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(j8)) +- i += copy(dAtA[i:], dAtA9[:j8]) +- } +- return i, nil ++ return 0 + } + +-func (m *Throttle) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetHierarchicalSwapLimit() uint64 { ++ if m != nil { ++ return m.HierarchicalSwapLimit + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *Throttle) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Periods != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Periods)) +- } +- if m.ThrottledPeriods != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledPeriods)) ++func (m *MemoryStat) GetTotalCache() uint64 { ++ if m != nil { ++ return m.TotalCache + } +- if m.ThrottledTime != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledTime)) +- } +- return i, nil ++ return 0 + } + +-func (m *MemoryStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalRSS() uint64 { ++ if m != nil { ++ return m.TotalRSS + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *MemoryStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Cache != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Cache)) +- } +- if m.RSS != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RSS)) +- } +- if m.RSSHuge != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RSSHuge)) +- } +- if m.MappedFile != 0 { +- dAtA[i] = 0x20 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.MappedFile)) +- } +- if m.Dirty != 0 { +- dAtA[i] = 0x28 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Dirty)) +- } +- if m.Writeback != 0 { +- dAtA[i] = 0x30 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Writeback)) +- } +- if m.PgPgIn != 0 { +- dAtA[i] = 0x38 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgIn)) +- } +- if m.PgPgOut != 0 { +- dAtA[i] = 0x40 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgOut)) +- } +- if m.PgFault != 0 { +- dAtA[i] = 0x48 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.PgFault)) +- } +- if m.PgMajFault != 0 { +- dAtA[i] = 0x50 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.PgMajFault)) +- } +- if m.InactiveAnon != 0 { +- dAtA[i] = 0x58 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) +- } +- if m.ActiveAnon != 0 { +- dAtA[i] = 0x60 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) +- } +- if m.InactiveFile != 0 { +- dAtA[i] = 0x68 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) +- } +- if m.ActiveFile != 0 { +- dAtA[i] = 0x70 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) +- } +- if m.Unevictable != 0 { +- dAtA[i] = 0x78 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) +- } +- if m.HierarchicalMemoryLimit != 0 { +- dAtA[i] = 0x80 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalMemoryLimit)) +- } +- if m.HierarchicalSwapLimit != 0 { +- dAtA[i] = 0x88 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalSwapLimit)) +- } +- if m.TotalCache != 0 { +- dAtA[i] = 0x90 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalCache)) +- } +- if m.TotalRSS != 0 { +- dAtA[i] = 0x98 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSS)) +- } +- if m.TotalRSSHuge != 0 { +- dAtA[i] = 0xa0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSSHuge)) +- } +- if m.TotalMappedFile != 0 { +- dAtA[i] = 0xa8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalMappedFile)) +- } +- if m.TotalDirty != 0 { +- dAtA[i] = 0xb0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalDirty)) +- } +- if m.TotalWriteback != 0 { +- dAtA[i] = 0xb8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalWriteback)) ++func (m *MemoryStat) GetTotalRSSHuge() uint64 { ++ if m != nil { ++ return m.TotalRSSHuge + } +- if m.TotalPgPgIn != 0 { +- dAtA[i] = 0xc0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgIn)) +- } +- if m.TotalPgPgOut != 0 { +- dAtA[i] = 0xc8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgOut)) +- } +- if m.TotalPgFault != 0 { +- dAtA[i] = 0xd0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgFault)) +- } +- if m.TotalPgMajFault != 0 { +- dAtA[i] = 0xd8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgMajFault)) +- } +- if m.TotalInactiveAnon != 0 { +- dAtA[i] = 0xe0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveAnon)) +- } +- if m.TotalActiveAnon != 0 { +- dAtA[i] = 0xe8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveAnon)) +- } +- if m.TotalInactiveFile != 0 { +- dAtA[i] = 0xf0 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveFile)) +- } +- if m.TotalActiveFile != 0 { +- dAtA[i] = 0xf8 +- i++ +- dAtA[i] = 0x1 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveFile)) +- } +- if m.TotalUnevictable != 0 { +- dAtA[i] = 0x80 +- i++ +- dAtA[i] = 0x2 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TotalUnevictable)) +- } +- if m.Usage != nil { +- dAtA[i] = 0x8a +- i++ +- dAtA[i] = 0x2 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) +- n10, err := m.Usage.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n10 +- } +- if m.Swap != nil { +- dAtA[i] = 0x92 +- i++ +- dAtA[i] = 0x2 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Swap.Size())) +- n11, err := m.Swap.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n11 +- } +- if m.Kernel != nil { +- dAtA[i] = 0x9a +- i++ +- dAtA[i] = 0x2 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel.Size())) +- n12, err := m.Kernel.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n12 +- } +- if m.KernelTCP != nil { +- dAtA[i] = 0xa2 +- i++ +- dAtA[i] = 0x2 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.KernelTCP.Size())) +- n13, err := m.KernelTCP.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n13 +- } +- return i, nil ++ return 0 + } + +-func (m *MemoryEntry) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalMappedFile() uint64 { ++ if m != nil { ++ return m.TotalMappedFile + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *MemoryEntry) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if m.Limit != 0 { +- dAtA[i] = 0x8 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) +- } +- if m.Usage != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) ++func (m *MemoryStat) GetTotalDirty() uint64 { ++ if m != nil { ++ return m.TotalDirty + } +- if m.Max != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) +- } +- if m.Failcnt != 0 { +- dAtA[i] = 0x20 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) +- } +- return i, nil ++ return 0 + } + +-func (m *BlkIOStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalWriteback() uint64 { ++ if m != nil { ++ return m.TotalWriteback + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *BlkIOStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.IoServiceBytesRecursive) > 0 { +- for _, msg := range m.IoServiceBytesRecursive { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.IoServicedRecursive) > 0 { +- for _, msg := range m.IoServicedRecursive { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.IoQueuedRecursive) > 0 { +- for _, msg := range m.IoQueuedRecursive { +- dAtA[i] = 0x1a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.IoServiceTimeRecursive) > 0 { +- for _, msg := range m.IoServiceTimeRecursive { +- dAtA[i] = 0x22 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.IoWaitTimeRecursive) > 0 { +- for _, msg := range m.IoWaitTimeRecursive { +- dAtA[i] = 0x2a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.IoMergedRecursive) > 0 { +- for _, msg := range m.IoMergedRecursive { +- dAtA[i] = 0x32 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } ++func (m *MemoryStat) GetTotalPgPgIn() uint64 { ++ if m != nil { ++ return m.TotalPgPgIn + } +- if len(m.IoTimeRecursive) > 0 { +- for _, msg := range m.IoTimeRecursive { +- dAtA[i] = 0x3a +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.SectorsRecursive) > 0 { +- for _, msg := range m.SectorsRecursive { +- dAtA[i] = 0x42 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- return i, nil ++ return 0 + } + +-func (m *BlkIOEntry) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalPgPgOut() uint64 { ++ if m != nil { ++ return m.TotalPgPgOut + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *BlkIOEntry) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.Op) > 0 { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Op))) +- i += copy(dAtA[i:], m.Op) +- } +- if len(m.Device) > 0 { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) +- i += copy(dAtA[i:], m.Device) +- } +- if m.Major != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) ++func (m *MemoryStat) GetTotalPgFault() uint64 { ++ if m != nil { ++ return m.TotalPgFault + } +- if m.Minor != 0 { +- dAtA[i] = 0x20 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Minor)) +- } +- if m.Value != 0 { +- dAtA[i] = 0x28 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.Value)) +- } +- return i, nil ++ return 0 + } + +-func (m *RdmaStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalPgMajFault() uint64 { ++ if m != nil { ++ return m.TotalPgMajFault + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *RdmaStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.Current) > 0 { +- for _, msg := range m.Current { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } +- } +- if len(m.Limit) > 0 { +- for _, msg := range m.Limit { +- dAtA[i] = 0x12 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) +- n, err := msg.MarshalTo(dAtA[i:]) +- if err != nil { +- return 0, err +- } +- i += n +- } ++func (m *MemoryStat) GetTotalInactiveAnon() uint64 { ++ if m != nil { ++ return m.TotalInactiveAnon + } +- return i, nil ++ return 0 + } + +-func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalActiveAnon() uint64 { ++ if m != nil { ++ return m.TotalActiveAnon + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *RdmaEntry) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.Device) > 0 { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) +- i += copy(dAtA[i:], m.Device) ++func (m *MemoryStat) GetTotalInactiveFile() uint64 { ++ if m != nil { ++ return m.TotalInactiveFile + } +- if m.HcaHandles != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) +- } +- if m.HcaObjects != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.HcaObjects)) +- } +- return i, nil ++ return 0 + } + +-func (m *NetworkStat) Marshal() (dAtA []byte, err error) { +- size := m.Size() +- dAtA = make([]byte, size) +- n, err := m.MarshalTo(dAtA) +- if err != nil { +- return nil, err ++func (m *MemoryStat) GetTotalActiveFile() uint64 { ++ if m != nil { ++ return m.TotalActiveFile + } +- return dAtA[:n], nil ++ return 0 + } + +-func (m *NetworkStat) MarshalTo(dAtA []byte) (int, error) { +- var i int +- _ = i +- var l int +- _ = l +- if len(m.Name) > 0 { +- dAtA[i] = 0xa +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(len(m.Name))) +- i += copy(dAtA[i:], m.Name) +- } +- if m.RxBytes != 0 { +- dAtA[i] = 0x10 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RxBytes)) +- } +- if m.RxPackets != 0 { +- dAtA[i] = 0x18 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RxPackets)) +- } +- if m.RxErrors != 0 { +- dAtA[i] = 0x20 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RxErrors)) +- } +- if m.RxDropped != 0 { +- dAtA[i] = 0x28 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.RxDropped)) +- } +- if m.TxBytes != 0 { +- dAtA[i] = 0x30 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TxBytes)) +- } +- if m.TxPackets != 0 { +- dAtA[i] = 0x38 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TxPackets)) ++func (m *MemoryStat) GetTotalUnevictable() uint64 { ++ if m != nil { ++ return m.TotalUnevictable + } +- if m.TxErrors != 0 { +- dAtA[i] = 0x40 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TxErrors)) +- } +- if m.TxDropped != 0 { +- dAtA[i] = 0x48 +- i++ +- i = encodeVarintMetrics(dAtA, i, uint64(m.TxDropped)) +- } +- return i, nil ++ return 0 + } + +-func encodeVarintMetrics(dAtA []byte, offset int, v uint64) int { +- for v >= 1<<7 { +- dAtA[offset] = uint8(v&0x7f | 0x80) +- v >>= 7 +- offset++ ++func (m *MemoryStat) GetUsage() *MemoryEntry { ++ if m != nil { ++ return m.Usage + } +- dAtA[offset] = uint8(v) +- return offset + 1 ++ return nil + } +-func (m *Metrics) Size() (n int) { +- var l int +- _ = l +- if len(m.Hugetlb) > 0 { +- for _, e := range m.Hugetlb { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if m.Pids != nil { +- l = m.Pids.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.CPU != nil { +- l = m.CPU.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.Memory != nil { +- l = m.Memory.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.Blkio != nil { +- l = m.Blkio.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.Rdma != nil { +- l = m.Rdma.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if len(m.Network) > 0 { +- for _, e := range m.Network { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } ++ ++func (m *MemoryStat) GetSwap() *MemoryEntry { ++ if m != nil { ++ return m.Swap + } +- return n ++ return nil + } + +-func (m *HugetlbStat) Size() (n int) { +- var l int +- _ = l +- if m.Usage != 0 { +- n += 1 + sovMetrics(uint64(m.Usage)) +- } +- if m.Max != 0 { +- n += 1 + sovMetrics(uint64(m.Max)) +- } +- if m.Failcnt != 0 { +- n += 1 + sovMetrics(uint64(m.Failcnt)) +- } +- l = len(m.Pagesize) +- if l > 0 { +- n += 1 + l + sovMetrics(uint64(l)) ++func (m *MemoryStat) GetKernel() *MemoryEntry { ++ if m != nil { ++ return m.Kernel + } +- return n ++ return nil + } + +-func (m *PidsStat) Size() (n int) { +- var l int +- _ = l +- if m.Current != 0 { +- n += 1 + sovMetrics(uint64(m.Current)) +- } +- if m.Limit != 0 { +- n += 1 + sovMetrics(uint64(m.Limit)) ++func (m *MemoryStat) GetKernelTCP() *MemoryEntry { ++ if m != nil { ++ return m.KernelTCP + } +- return n ++ return nil + } + +-func (m *CPUStat) Size() (n int) { +- var l int +- _ = l +- if m.Usage != nil { +- l = m.Usage.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.Throttling != nil { +- l = m.Throttling.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- return n ++type MemoryEntry struct { ++ Limit uint64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` ++ Usage uint64 `protobuf:"varint,2,opt,name=usage,proto3" json:"usage,omitempty"` ++ Max uint64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` ++ Failcnt uint64 `protobuf:"varint,4,opt,name=failcnt,proto3" json:"failcnt,omitempty"` + } + +-func (m *CPUUsage) Size() (n int) { +- var l int +- _ = l +- if m.Total != 0 { +- n += 1 + sovMetrics(uint64(m.Total)) +- } +- if m.Kernel != 0 { +- n += 1 + sovMetrics(uint64(m.Kernel)) +- } +- if m.User != 0 { +- n += 1 + sovMetrics(uint64(m.User)) +- } +- if len(m.PerCPU) > 0 { +- l = 0 +- for _, e := range m.PerCPU { +- l += sovMetrics(uint64(e)) +- } +- n += 1 + sovMetrics(uint64(l)) + l ++func (m *MemoryEntry) Reset() { *m = MemoryEntry{} } ++func (m *MemoryEntry) String() string { return proto.CompactTextString(m) } ++func (*MemoryEntry) ProtoMessage() {} ++func (*MemoryEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{7} } ++ ++func (m *MemoryEntry) GetLimit() uint64 { ++ if m != nil { ++ return m.Limit + } +- return n ++ return 0 + } + +-func (m *Throttle) Size() (n int) { +- var l int +- _ = l +- if m.Periods != 0 { +- n += 1 + sovMetrics(uint64(m.Periods)) +- } +- if m.ThrottledPeriods != 0 { +- n += 1 + sovMetrics(uint64(m.ThrottledPeriods)) +- } +- if m.ThrottledTime != 0 { +- n += 1 + sovMetrics(uint64(m.ThrottledTime)) ++func (m *MemoryEntry) GetUsage() uint64 { ++ if m != nil { ++ return m.Usage + } +- return n ++ return 0 + } + +-func (m *MemoryStat) Size() (n int) { +- var l int +- _ = l +- if m.Cache != 0 { +- n += 1 + sovMetrics(uint64(m.Cache)) +- } +- if m.RSS != 0 { +- n += 1 + sovMetrics(uint64(m.RSS)) +- } +- if m.RSSHuge != 0 { +- n += 1 + sovMetrics(uint64(m.RSSHuge)) +- } +- if m.MappedFile != 0 { +- n += 1 + sovMetrics(uint64(m.MappedFile)) +- } +- if m.Dirty != 0 { +- n += 1 + sovMetrics(uint64(m.Dirty)) +- } +- if m.Writeback != 0 { +- n += 1 + sovMetrics(uint64(m.Writeback)) +- } +- if m.PgPgIn != 0 { +- n += 1 + sovMetrics(uint64(m.PgPgIn)) +- } +- if m.PgPgOut != 0 { +- n += 1 + sovMetrics(uint64(m.PgPgOut)) +- } +- if m.PgFault != 0 { +- n += 1 + sovMetrics(uint64(m.PgFault)) +- } +- if m.PgMajFault != 0 { +- n += 1 + sovMetrics(uint64(m.PgMajFault)) +- } +- if m.InactiveAnon != 0 { +- n += 1 + sovMetrics(uint64(m.InactiveAnon)) +- } +- if m.ActiveAnon != 0 { +- n += 1 + sovMetrics(uint64(m.ActiveAnon)) +- } +- if m.InactiveFile != 0 { +- n += 1 + sovMetrics(uint64(m.InactiveFile)) +- } +- if m.ActiveFile != 0 { +- n += 1 + sovMetrics(uint64(m.ActiveFile)) +- } +- if m.Unevictable != 0 { +- n += 1 + sovMetrics(uint64(m.Unevictable)) +- } +- if m.HierarchicalMemoryLimit != 0 { +- n += 2 + sovMetrics(uint64(m.HierarchicalMemoryLimit)) +- } +- if m.HierarchicalSwapLimit != 0 { +- n += 2 + sovMetrics(uint64(m.HierarchicalSwapLimit)) +- } +- if m.TotalCache != 0 { +- n += 2 + sovMetrics(uint64(m.TotalCache)) ++func (m *MemoryEntry) GetMax() uint64 { ++ if m != nil { ++ return m.Max + } +- if m.TotalRSS != 0 { +- n += 2 + sovMetrics(uint64(m.TotalRSS)) +- } +- if m.TotalRSSHuge != 0 { +- n += 2 + sovMetrics(uint64(m.TotalRSSHuge)) +- } +- if m.TotalMappedFile != 0 { +- n += 2 + sovMetrics(uint64(m.TotalMappedFile)) +- } +- if m.TotalDirty != 0 { +- n += 2 + sovMetrics(uint64(m.TotalDirty)) +- } +- if m.TotalWriteback != 0 { +- n += 2 + sovMetrics(uint64(m.TotalWriteback)) +- } +- if m.TotalPgPgIn != 0 { +- n += 2 + sovMetrics(uint64(m.TotalPgPgIn)) +- } +- if m.TotalPgPgOut != 0 { +- n += 2 + sovMetrics(uint64(m.TotalPgPgOut)) +- } +- if m.TotalPgFault != 0 { +- n += 2 + sovMetrics(uint64(m.TotalPgFault)) +- } +- if m.TotalPgMajFault != 0 { +- n += 2 + sovMetrics(uint64(m.TotalPgMajFault)) +- } +- if m.TotalInactiveAnon != 0 { +- n += 2 + sovMetrics(uint64(m.TotalInactiveAnon)) +- } +- if m.TotalActiveAnon != 0 { +- n += 2 + sovMetrics(uint64(m.TotalActiveAnon)) +- } +- if m.TotalInactiveFile != 0 { +- n += 2 + sovMetrics(uint64(m.TotalInactiveFile)) +- } +- if m.TotalActiveFile != 0 { +- n += 2 + sovMetrics(uint64(m.TotalActiveFile)) +- } +- if m.TotalUnevictable != 0 { +- n += 2 + sovMetrics(uint64(m.TotalUnevictable)) +- } +- if m.Usage != nil { +- l = m.Usage.Size() +- n += 2 + l + sovMetrics(uint64(l)) +- } +- if m.Swap != nil { +- l = m.Swap.Size() +- n += 2 + l + sovMetrics(uint64(l)) +- } +- if m.Kernel != nil { +- l = m.Kernel.Size() +- n += 2 + l + sovMetrics(uint64(l)) +- } +- if m.KernelTCP != nil { +- l = m.KernelTCP.Size() +- n += 2 + l + sovMetrics(uint64(l)) +- } +- return n ++ return 0 + } + +-func (m *MemoryEntry) Size() (n int) { +- var l int +- _ = l +- if m.Limit != 0 { +- n += 1 + sovMetrics(uint64(m.Limit)) +- } +- if m.Usage != 0 { +- n += 1 + sovMetrics(uint64(m.Usage)) ++func (m *MemoryEntry) GetFailcnt() uint64 { ++ if m != nil { ++ return m.Failcnt + } +- if m.Max != 0 { +- n += 1 + sovMetrics(uint64(m.Max)) +- } +- if m.Failcnt != 0 { +- n += 1 + sovMetrics(uint64(m.Failcnt)) +- } +- return n ++ return 0 + } + +-func (m *BlkIOStat) Size() (n int) { +- var l int +- _ = l +- if len(m.IoServiceBytesRecursive) > 0 { +- for _, e := range m.IoServiceBytesRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoServicedRecursive) > 0 { +- for _, e := range m.IoServicedRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoQueuedRecursive) > 0 { +- for _, e := range m.IoQueuedRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoServiceTimeRecursive) > 0 { +- for _, e := range m.IoServiceTimeRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoWaitTimeRecursive) > 0 { +- for _, e := range m.IoWaitTimeRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoMergedRecursive) > 0 { +- for _, e := range m.IoMergedRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.IoTimeRecursive) > 0 { +- for _, e := range m.IoTimeRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- if len(m.SectorsRecursive) > 0 { +- for _, e := range m.SectorsRecursive { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- return n ++type BlkIOStat struct { ++ IoServiceBytesRecursive []*BlkIOEntry `protobuf:"bytes,1,rep,name=io_service_bytes_recursive,json=ioServiceBytesRecursive" json:"io_service_bytes_recursive,omitempty"` ++ IoServicedRecursive []*BlkIOEntry `protobuf:"bytes,2,rep,name=io_serviced_recursive,json=ioServicedRecursive" json:"io_serviced_recursive,omitempty"` ++ IoQueuedRecursive []*BlkIOEntry `protobuf:"bytes,3,rep,name=io_queued_recursive,json=ioQueuedRecursive" json:"io_queued_recursive,omitempty"` ++ IoServiceTimeRecursive []*BlkIOEntry `protobuf:"bytes,4,rep,name=io_service_time_recursive,json=ioServiceTimeRecursive" json:"io_service_time_recursive,omitempty"` ++ IoWaitTimeRecursive []*BlkIOEntry `protobuf:"bytes,5,rep,name=io_wait_time_recursive,json=ioWaitTimeRecursive" json:"io_wait_time_recursive,omitempty"` ++ IoMergedRecursive []*BlkIOEntry `protobuf:"bytes,6,rep,name=io_merged_recursive,json=ioMergedRecursive" json:"io_merged_recursive,omitempty"` ++ IoTimeRecursive []*BlkIOEntry `protobuf:"bytes,7,rep,name=io_time_recursive,json=ioTimeRecursive" json:"io_time_recursive,omitempty"` ++ SectorsRecursive []*BlkIOEntry `protobuf:"bytes,8,rep,name=sectors_recursive,json=sectorsRecursive" json:"sectors_recursive,omitempty"` + } + +-func (m *BlkIOEntry) Size() (n int) { +- var l int +- _ = l +- l = len(m.Op) +- if l > 0 { +- n += 1 + l + sovMetrics(uint64(l)) +- } +- l = len(m.Device) +- if l > 0 { +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.Major != 0 { +- n += 1 + sovMetrics(uint64(m.Major)) +- } +- if m.Minor != 0 { +- n += 1 + sovMetrics(uint64(m.Minor)) +- } +- if m.Value != 0 { +- n += 1 + sovMetrics(uint64(m.Value)) ++func (m *BlkIOStat) Reset() { *m = BlkIOStat{} } ++func (m *BlkIOStat) String() string { return proto.CompactTextString(m) } ++func (*BlkIOStat) ProtoMessage() {} ++func (*BlkIOStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{8} } ++ ++func (m *BlkIOStat) GetIoServiceBytesRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoServiceBytesRecursive + } +- return n ++ return nil + } + +-func (m *RdmaStat) Size() (n int) { +- var l int +- _ = l +- if len(m.Current) > 0 { +- for _, e := range m.Current { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } ++func (m *BlkIOStat) GetIoServicedRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoServicedRecursive + } +- if len(m.Limit) > 0 { +- for _, e := range m.Limit { +- l = e.Size() +- n += 1 + l + sovMetrics(uint64(l)) +- } +- } +- return n ++ return nil + } + +-func (m *RdmaEntry) Size() (n int) { +- var l int +- _ = l +- l = len(m.Device) +- if l > 0 { +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.HcaHandles != 0 { +- n += 1 + sovMetrics(uint64(m.HcaHandles)) ++func (m *BlkIOStat) GetIoQueuedRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoQueuedRecursive + } +- if m.HcaObjects != 0 { +- n += 1 + sovMetrics(uint64(m.HcaObjects)) +- } +- return n ++ return nil + } + +-func (m *NetworkStat) Size() (n int) { +- var l int +- _ = l +- l = len(m.Name) +- if l > 0 { +- n += 1 + l + sovMetrics(uint64(l)) +- } +- if m.RxBytes != 0 { +- n += 1 + sovMetrics(uint64(m.RxBytes)) +- } +- if m.RxPackets != 0 { +- n += 1 + sovMetrics(uint64(m.RxPackets)) +- } +- if m.RxErrors != 0 { +- n += 1 + sovMetrics(uint64(m.RxErrors)) +- } +- if m.RxDropped != 0 { +- n += 1 + sovMetrics(uint64(m.RxDropped)) +- } +- if m.TxBytes != 0 { +- n += 1 + sovMetrics(uint64(m.TxBytes)) ++func (m *BlkIOStat) GetIoServiceTimeRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoServiceTimeRecursive + } +- if m.TxPackets != 0 { +- n += 1 + sovMetrics(uint64(m.TxPackets)) +- } +- if m.TxErrors != 0 { +- n += 1 + sovMetrics(uint64(m.TxErrors)) +- } +- if m.TxDropped != 0 { +- n += 1 + sovMetrics(uint64(m.TxDropped)) +- } +- return n ++ return nil + } + +-func sovMetrics(x uint64) (n int) { +- for { +- n++ +- x >>= 7 +- if x == 0 { +- break +- } ++func (m *BlkIOStat) GetIoWaitTimeRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoWaitTimeRecursive + } +- return n +-} +-func sozMetrics(x uint64) (n int) { +- return sovMetrics(uint64((x << 1) ^ uint64((int64(x) >> 63)))) ++ return nil + } +-func (this *Metrics) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOStat) GetIoMergedRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoMergedRecursive + } +- s := strings.Join([]string{`&Metrics{`, +- `Hugetlb:` + strings.Replace(fmt.Sprintf("%v", this.Hugetlb), "HugetlbStat", "HugetlbStat", 1) + `,`, +- `Pids:` + strings.Replace(fmt.Sprintf("%v", this.Pids), "PidsStat", "PidsStat", 1) + `,`, +- `CPU:` + strings.Replace(fmt.Sprintf("%v", this.CPU), "CPUStat", "CPUStat", 1) + `,`, +- `Memory:` + strings.Replace(fmt.Sprintf("%v", this.Memory), "MemoryStat", "MemoryStat", 1) + `,`, +- `Blkio:` + strings.Replace(fmt.Sprintf("%v", this.Blkio), "BlkIOStat", "BlkIOStat", 1) + `,`, +- `Rdma:` + strings.Replace(fmt.Sprintf("%v", this.Rdma), "RdmaStat", "RdmaStat", 1) + `,`, +- `Network:` + strings.Replace(fmt.Sprintf("%v", this.Network), "NetworkStat", "NetworkStat", 1) + `,`, +- `}`, +- }, "") +- return s ++ return nil + } +-func (this *HugetlbStat) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOStat) GetIoTimeRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.IoTimeRecursive + } +- s := strings.Join([]string{`&HugetlbStat{`, +- `Usage:` + fmt.Sprintf("%v", this.Usage) + `,`, +- `Max:` + fmt.Sprintf("%v", this.Max) + `,`, +- `Failcnt:` + fmt.Sprintf("%v", this.Failcnt) + `,`, +- `Pagesize:` + fmt.Sprintf("%v", this.Pagesize) + `,`, +- `}`, +- }, "") +- return s ++ return nil + } +-func (this *PidsStat) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOStat) GetSectorsRecursive() []*BlkIOEntry { ++ if m != nil { ++ return m.SectorsRecursive + } +- s := strings.Join([]string{`&PidsStat{`, +- `Current:` + fmt.Sprintf("%v", this.Current) + `,`, +- `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, +- `}`, +- }, "") +- return s ++ return nil + } +-func (this *CPUStat) String() string { +- if this == nil { +- return "nil" +- } +- s := strings.Join([]string{`&CPUStat{`, +- `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "CPUUsage", "CPUUsage", 1) + `,`, +- `Throttling:` + strings.Replace(fmt.Sprintf("%v", this.Throttling), "Throttle", "Throttle", 1) + `,`, +- `}`, +- }, "") +- return s ++ ++type BlkIOEntry struct { ++ Op string `protobuf:"bytes,1,opt,name=op,proto3" json:"op,omitempty"` ++ Device string `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` ++ Major uint64 `protobuf:"varint,3,opt,name=major,proto3" json:"major,omitempty"` ++ Minor uint64 `protobuf:"varint,4,opt,name=minor,proto3" json:"minor,omitempty"` ++ Value uint64 `protobuf:"varint,5,opt,name=value,proto3" json:"value,omitempty"` + } +-func (this *CPUUsage) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOEntry) Reset() { *m = BlkIOEntry{} } ++func (m *BlkIOEntry) String() string { return proto.CompactTextString(m) } ++func (*BlkIOEntry) ProtoMessage() {} ++func (*BlkIOEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{9} } ++ ++func (m *BlkIOEntry) GetOp() string { ++ if m != nil { ++ return m.Op + } +- s := strings.Join([]string{`&CPUUsage{`, +- `Total:` + fmt.Sprintf("%v", this.Total) + `,`, +- `Kernel:` + fmt.Sprintf("%v", this.Kernel) + `,`, +- `User:` + fmt.Sprintf("%v", this.User) + `,`, +- `PerCPU:` + fmt.Sprintf("%v", this.PerCPU) + `,`, +- `}`, +- }, "") +- return s ++ return "" + } +-func (this *Throttle) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOEntry) GetDevice() string { ++ if m != nil { ++ return m.Device + } +- s := strings.Join([]string{`&Throttle{`, +- `Periods:` + fmt.Sprintf("%v", this.Periods) + `,`, +- `ThrottledPeriods:` + fmt.Sprintf("%v", this.ThrottledPeriods) + `,`, +- `ThrottledTime:` + fmt.Sprintf("%v", this.ThrottledTime) + `,`, +- `}`, +- }, "") +- return s ++ return "" + } +-func (this *MemoryStat) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOEntry) GetMajor() uint64 { ++ if m != nil { ++ return m.Major + } +- s := strings.Join([]string{`&MemoryStat{`, +- `Cache:` + fmt.Sprintf("%v", this.Cache) + `,`, +- `RSS:` + fmt.Sprintf("%v", this.RSS) + `,`, +- `RSSHuge:` + fmt.Sprintf("%v", this.RSSHuge) + `,`, +- `MappedFile:` + fmt.Sprintf("%v", this.MappedFile) + `,`, +- `Dirty:` + fmt.Sprintf("%v", this.Dirty) + `,`, +- `Writeback:` + fmt.Sprintf("%v", this.Writeback) + `,`, +- `PgPgIn:` + fmt.Sprintf("%v", this.PgPgIn) + `,`, +- `PgPgOut:` + fmt.Sprintf("%v", this.PgPgOut) + `,`, +- `PgFault:` + fmt.Sprintf("%v", this.PgFault) + `,`, +- `PgMajFault:` + fmt.Sprintf("%v", this.PgMajFault) + `,`, +- `InactiveAnon:` + fmt.Sprintf("%v", this.InactiveAnon) + `,`, +- `ActiveAnon:` + fmt.Sprintf("%v", this.ActiveAnon) + `,`, +- `InactiveFile:` + fmt.Sprintf("%v", this.InactiveFile) + `,`, +- `ActiveFile:` + fmt.Sprintf("%v", this.ActiveFile) + `,`, +- `Unevictable:` + fmt.Sprintf("%v", this.Unevictable) + `,`, +- `HierarchicalMemoryLimit:` + fmt.Sprintf("%v", this.HierarchicalMemoryLimit) + `,`, +- `HierarchicalSwapLimit:` + fmt.Sprintf("%v", this.HierarchicalSwapLimit) + `,`, +- `TotalCache:` + fmt.Sprintf("%v", this.TotalCache) + `,`, +- `TotalRSS:` + fmt.Sprintf("%v", this.TotalRSS) + `,`, +- `TotalRSSHuge:` + fmt.Sprintf("%v", this.TotalRSSHuge) + `,`, +- `TotalMappedFile:` + fmt.Sprintf("%v", this.TotalMappedFile) + `,`, +- `TotalDirty:` + fmt.Sprintf("%v", this.TotalDirty) + `,`, +- `TotalWriteback:` + fmt.Sprintf("%v", this.TotalWriteback) + `,`, +- `TotalPgPgIn:` + fmt.Sprintf("%v", this.TotalPgPgIn) + `,`, +- `TotalPgPgOut:` + fmt.Sprintf("%v", this.TotalPgPgOut) + `,`, +- `TotalPgFault:` + fmt.Sprintf("%v", this.TotalPgFault) + `,`, +- `TotalPgMajFault:` + fmt.Sprintf("%v", this.TotalPgMajFault) + `,`, +- `TotalInactiveAnon:` + fmt.Sprintf("%v", this.TotalInactiveAnon) + `,`, +- `TotalActiveAnon:` + fmt.Sprintf("%v", this.TotalActiveAnon) + `,`, +- `TotalInactiveFile:` + fmt.Sprintf("%v", this.TotalInactiveFile) + `,`, +- `TotalActiveFile:` + fmt.Sprintf("%v", this.TotalActiveFile) + `,`, +- `TotalUnevictable:` + fmt.Sprintf("%v", this.TotalUnevictable) + `,`, +- `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "MemoryEntry", "MemoryEntry", 1) + `,`, +- `Swap:` + strings.Replace(fmt.Sprintf("%v", this.Swap), "MemoryEntry", "MemoryEntry", 1) + `,`, +- `Kernel:` + strings.Replace(fmt.Sprintf("%v", this.Kernel), "MemoryEntry", "MemoryEntry", 1) + `,`, +- `KernelTCP:` + strings.Replace(fmt.Sprintf("%v", this.KernelTCP), "MemoryEntry", "MemoryEntry", 1) + `,`, +- `}`, +- }, "") +- return s ++ return 0 + } +-func (this *MemoryEntry) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOEntry) GetMinor() uint64 { ++ if m != nil { ++ return m.Minor + } +- s := strings.Join([]string{`&MemoryEntry{`, +- `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, +- `Usage:` + fmt.Sprintf("%v", this.Usage) + `,`, +- `Max:` + fmt.Sprintf("%v", this.Max) + `,`, +- `Failcnt:` + fmt.Sprintf("%v", this.Failcnt) + `,`, +- `}`, +- }, "") +- return s ++ return 0 + } +-func (this *BlkIOStat) String() string { +- if this == nil { +- return "nil" ++ ++func (m *BlkIOEntry) GetValue() uint64 { ++ if m != nil { ++ return m.Value + } +- s := strings.Join([]string{`&BlkIOStat{`, +- `IoServiceBytesRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceBytesRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoServicedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServicedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoQueuedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoQueuedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoServiceTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoWaitTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoWaitTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoMergedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoMergedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `IoTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `SectorsRecursive:` + strings.Replace(fmt.Sprintf("%v", this.SectorsRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, +- `}`, +- }, "") +- return s ++ return 0 + } +-func (this *BlkIOEntry) String() string { +- if this == nil { +- return "nil" +- } +- s := strings.Join([]string{`&BlkIOEntry{`, +- `Op:` + fmt.Sprintf("%v", this.Op) + `,`, +- `Device:` + fmt.Sprintf("%v", this.Device) + `,`, +- `Major:` + fmt.Sprintf("%v", this.Major) + `,`, +- `Minor:` + fmt.Sprintf("%v", this.Minor) + `,`, +- `Value:` + fmt.Sprintf("%v", this.Value) + `,`, +- `}`, +- }, "") +- return s ++ ++type RdmaStat struct { ++ Current []*RdmaEntry `protobuf:"bytes,1,rep,name=current" json:"current,omitempty"` ++ Limit []*RdmaEntry `protobuf:"bytes,2,rep,name=limit" json:"limit,omitempty"` + } +-func (this *RdmaStat) String() string { +- if this == nil { +- return "nil" ++ ++func (m *RdmaStat) Reset() { *m = RdmaStat{} } ++func (m *RdmaStat) String() string { return proto.CompactTextString(m) } ++func (*RdmaStat) ProtoMessage() {} ++func (*RdmaStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{10} } ++ ++func (m *RdmaStat) GetCurrent() []*RdmaEntry { ++ if m != nil { ++ return m.Current + } +- s := strings.Join([]string{`&RdmaStat{`, +- `Current:` + strings.Replace(fmt.Sprintf("%v", this.Current), "RdmaEntry", "RdmaEntry", 1) + `,`, +- `Limit:` + strings.Replace(fmt.Sprintf("%v", this.Limit), "RdmaEntry", "RdmaEntry", 1) + `,`, +- `}`, +- }, "") +- return s ++ return nil + } +-func (this *RdmaEntry) String() string { +- if this == nil { +- return "nil" ++ ++func (m *RdmaStat) GetLimit() []*RdmaEntry { ++ if m != nil { ++ return m.Limit + } +- s := strings.Join([]string{`&RdmaEntry{`, +- `Device:` + fmt.Sprintf("%v", this.Device) + `,`, +- `HcaHandles:` + fmt.Sprintf("%v", this.HcaHandles) + `,`, +- `HcaObjects:` + fmt.Sprintf("%v", this.HcaObjects) + `,`, +- `}`, +- }, "") +- return s ++ return nil + } +-func (this *NetworkStat) String() string { +- if this == nil { +- return "nil" +- } +- s := strings.Join([]string{`&NetworkStat{`, +- `Name:` + fmt.Sprintf("%v", this.Name) + `,`, +- `RxBytes:` + fmt.Sprintf("%v", this.RxBytes) + `,`, +- `RxPackets:` + fmt.Sprintf("%v", this.RxPackets) + `,`, +- `RxErrors:` + fmt.Sprintf("%v", this.RxErrors) + `,`, +- `RxDropped:` + fmt.Sprintf("%v", this.RxDropped) + `,`, +- `TxBytes:` + fmt.Sprintf("%v", this.TxBytes) + `,`, +- `TxPackets:` + fmt.Sprintf("%v", this.TxPackets) + `,`, +- `TxErrors:` + fmt.Sprintf("%v", this.TxErrors) + `,`, +- `TxDropped:` + fmt.Sprintf("%v", this.TxDropped) + `,`, +- `}`, +- }, "") +- return s ++ ++type RdmaEntry struct { ++ Device string `protobuf:"bytes,1,opt,name=device,proto3" json:"device,omitempty"` ++ HcaHandles uint32 `protobuf:"varint,2,opt,name=hca_handles,json=hcaHandles,proto3" json:"hca_handles,omitempty"` ++ HcaObjects uint32 `protobuf:"varint,3,opt,name=hca_objects,json=hcaObjects,proto3" json:"hca_objects,omitempty"` + } +-func valueToStringMetrics(v interface{}) string { +- rv := reflect.ValueOf(v) +- if rv.IsNil() { +- return "nil" ++ ++func (m *RdmaEntry) Reset() { *m = RdmaEntry{} } ++func (m *RdmaEntry) String() string { return proto.CompactTextString(m) } ++func (*RdmaEntry) ProtoMessage() {} ++func (*RdmaEntry) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{11} } ++ ++func (m *RdmaEntry) GetDevice() string { ++ if m != nil { ++ return m.Device + } +- pv := reflect.Indirect(rv).Interface() +- return fmt.Sprintf("*%v", pv) ++ return "" + } +-func (m *Metrics) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: Metrics: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: Metrics: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Hugetlb", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Hugetlb = append(m.Hugetlb, &HugetlbStat{}) +- if err := m.Hugetlb[len(m.Hugetlb)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Pids", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Pids == nil { +- m.Pids = &PidsStat{} +- } +- if err := m.Pids.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 3: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field CPU", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.CPU == nil { +- m.CPU = &CPUStat{} +- } +- if err := m.CPU.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 4: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Memory", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Memory == nil { +- m.Memory = &MemoryStat{} +- } +- if err := m.Memory.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 5: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Blkio", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Blkio == nil { +- m.Blkio = &BlkIOStat{} +- } +- if err := m.Blkio.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 6: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Rdma", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Rdma == nil { +- m.Rdma = &RdmaStat{} +- } +- if err := m.Rdma.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 7: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Network", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Network = append(m.Network, &NetworkStat{}) +- if err := m.Network[len(m.Network)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *RdmaEntry) GetHcaHandles() uint32 { ++ if m != nil { ++ return m.HcaHandles + } +- return nil ++ return 0 + } +-func (m *HugetlbStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: HugetlbStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: HugetlbStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) +- } +- m.Usage = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Usage |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) +- } +- m.Max = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Max |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Failcnt", wireType) +- } +- m.Failcnt = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Failcnt |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Pagesize", wireType) +- } +- var stringLen uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- stringLen |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- intStringLen := int(stringLen) +- if intStringLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + intStringLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Pagesize = string(dAtA[iNdEx:postIndex]) +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *RdmaEntry) GetHcaObjects() uint32 { ++ if m != nil { ++ return m.HcaObjects + } +- return nil ++ return 0 + } +-func (m *PidsStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: PidsStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: PidsStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) +- } +- m.Current = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Current |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) +- } +- m.Limit = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Limit |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF +- } +- return nil ++type NetworkStat struct { ++ Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` ++ RxBytes uint64 `protobuf:"varint,2,opt,name=rx_bytes,json=rxBytes,proto3" json:"rx_bytes,omitempty"` ++ RxPackets uint64 `protobuf:"varint,3,opt,name=rx_packets,json=rxPackets,proto3" json:"rx_packets,omitempty"` ++ RxErrors uint64 `protobuf:"varint,4,opt,name=rx_errors,json=rxErrors,proto3" json:"rx_errors,omitempty"` ++ RxDropped uint64 `protobuf:"varint,5,opt,name=rx_dropped,json=rxDropped,proto3" json:"rx_dropped,omitempty"` ++ TxBytes uint64 `protobuf:"varint,6,opt,name=tx_bytes,json=txBytes,proto3" json:"tx_bytes,omitempty"` ++ TxPackets uint64 `protobuf:"varint,7,opt,name=tx_packets,json=txPackets,proto3" json:"tx_packets,omitempty"` ++ TxErrors uint64 `protobuf:"varint,8,opt,name=tx_errors,json=txErrors,proto3" json:"tx_errors,omitempty"` ++ TxDropped uint64 `protobuf:"varint,9,opt,name=tx_dropped,json=txDropped,proto3" json:"tx_dropped,omitempty"` + } +-func (m *CPUStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: CPUStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: CPUStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Usage == nil { +- m.Usage = &CPUUsage{} +- } +- if err := m.Usage.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Throttling", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Throttling == nil { +- m.Throttling = &Throttle{} +- } +- if err := m.Throttling.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) Reset() { *m = NetworkStat{} } ++func (m *NetworkStat) String() string { return proto.CompactTextString(m) } ++func (*NetworkStat) ProtoMessage() {} ++func (*NetworkStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{12} } ++ ++func (m *NetworkStat) GetName() string { ++ if m != nil { ++ return m.Name + } +- return nil ++ return "" + } +-func (m *CPUUsage) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: CPUUsage: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: CPUUsage: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) +- } +- m.Total = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Total |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Kernel", wireType) +- } +- m.Kernel = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Kernel |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field User", wireType) +- } +- m.User = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.User |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType == 0 { +- var v uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- v |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- m.PerCPU = append(m.PerCPU, v) +- } else if wireType == 2 { +- var packedLen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- packedLen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if packedLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + packedLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- for iNdEx < postIndex { +- var v uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- v |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- m.PerCPU = append(m.PerCPU, v) +- } +- } else { +- return fmt.Errorf("proto: wrong wireType = %d for field PerCPU", wireType) +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetRxBytes() uint64 { ++ if m != nil { ++ return m.RxBytes + } +- return nil ++ return 0 + } +-func (m *Throttle) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: Throttle: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: Throttle: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Periods", wireType) +- } +- m.Periods = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Periods |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field ThrottledPeriods", wireType) +- } +- m.ThrottledPeriods = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.ThrottledPeriods |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field ThrottledTime", wireType) +- } +- m.ThrottledTime = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.ThrottledTime |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetRxPackets() uint64 { ++ if m != nil { ++ return m.RxPackets + } +- return nil ++ return 0 + } +-func (m *MemoryStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: MemoryStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: MemoryStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Cache", wireType) +- } +- m.Cache = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Cache |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RSS", wireType) +- } +- m.RSS = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RSS |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RSSHuge", wireType) +- } +- m.RSSHuge = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RSSHuge |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field MappedFile", wireType) +- } +- m.MappedFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.MappedFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 5: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Dirty", wireType) +- } +- m.Dirty = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Dirty |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 6: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Writeback", wireType) +- } +- m.Writeback = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Writeback |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 7: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field PgPgIn", wireType) +- } +- m.PgPgIn = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.PgPgIn |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 8: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field PgPgOut", wireType) +- } +- m.PgPgOut = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.PgPgOut |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 9: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field PgFault", wireType) +- } +- m.PgFault = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.PgFault |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 10: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field PgMajFault", wireType) +- } +- m.PgMajFault = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.PgMajFault |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 11: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field InactiveAnon", wireType) +- } +- m.InactiveAnon = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.InactiveAnon |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 12: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field ActiveAnon", wireType) +- } +- m.ActiveAnon = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.ActiveAnon |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 13: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field InactiveFile", wireType) +- } +- m.InactiveFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.InactiveFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 14: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field ActiveFile", wireType) +- } +- m.ActiveFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.ActiveFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 15: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Unevictable", wireType) +- } +- m.Unevictable = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Unevictable |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 16: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field HierarchicalMemoryLimit", wireType) +- } +- m.HierarchicalMemoryLimit = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.HierarchicalMemoryLimit |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 17: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field HierarchicalSwapLimit", wireType) +- } +- m.HierarchicalSwapLimit = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.HierarchicalSwapLimit |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 18: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalCache", wireType) +- } +- m.TotalCache = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalCache |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 19: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalRSS", wireType) +- } +- m.TotalRSS = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalRSS |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 20: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalRSSHuge", wireType) +- } +- m.TotalRSSHuge = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalRSSHuge |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 21: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalMappedFile", wireType) +- } +- m.TotalMappedFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalMappedFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 22: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalDirty", wireType) +- } +- m.TotalDirty = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalDirty |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 23: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalWriteback", wireType) +- } +- m.TotalWriteback = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalWriteback |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 24: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgPgIn", wireType) +- } +- m.TotalPgPgIn = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalPgPgIn |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 25: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgPgOut", wireType) +- } +- m.TotalPgPgOut = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalPgPgOut |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 26: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgFault", wireType) +- } +- m.TotalPgFault = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalPgFault |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 27: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalPgMajFault", wireType) +- } +- m.TotalPgMajFault = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalPgMajFault |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 28: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalInactiveAnon", wireType) +- } +- m.TotalInactiveAnon = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalInactiveAnon |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 29: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalActiveAnon", wireType) +- } +- m.TotalActiveAnon = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalActiveAnon |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 30: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalInactiveFile", wireType) +- } +- m.TotalInactiveFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalInactiveFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 31: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalActiveFile", wireType) +- } +- m.TotalActiveFile = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalActiveFile |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 32: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TotalUnevictable", wireType) +- } +- m.TotalUnevictable = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TotalUnevictable |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 33: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Usage == nil { +- m.Usage = &MemoryEntry{} +- } +- if err := m.Usage.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 34: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Swap", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Swap == nil { +- m.Swap = &MemoryEntry{} +- } +- if err := m.Swap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 35: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Kernel", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.Kernel == nil { +- m.Kernel = &MemoryEntry{} +- } +- if err := m.Kernel.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 36: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field KernelTCP", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- if m.KernelTCP == nil { +- m.KernelTCP = &MemoryEntry{} +- } +- if err := m.KernelTCP.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetRxErrors() uint64 { ++ if m != nil { ++ return m.RxErrors + } +- return nil ++ return 0 + } +-func (m *MemoryEntry) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: MemoryEntry: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: MemoryEntry: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) +- } +- m.Limit = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Limit |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Usage", wireType) +- } +- m.Usage = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Usage |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Max", wireType) +- } +- m.Max = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Max |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Failcnt", wireType) +- } +- m.Failcnt = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Failcnt |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetRxDropped() uint64 { ++ if m != nil { ++ return m.RxDropped + } +- return nil ++ return 0 + } +-func (m *BlkIOStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: BlkIOStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: BlkIOStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoServiceBytesRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoServiceBytesRecursive = append(m.IoServiceBytesRecursive, &BlkIOEntry{}) +- if err := m.IoServiceBytesRecursive[len(m.IoServiceBytesRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoServicedRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoServicedRecursive = append(m.IoServicedRecursive, &BlkIOEntry{}) +- if err := m.IoServicedRecursive[len(m.IoServicedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 3: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoQueuedRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoQueuedRecursive = append(m.IoQueuedRecursive, &BlkIOEntry{}) +- if err := m.IoQueuedRecursive[len(m.IoQueuedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 4: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoServiceTimeRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoServiceTimeRecursive = append(m.IoServiceTimeRecursive, &BlkIOEntry{}) +- if err := m.IoServiceTimeRecursive[len(m.IoServiceTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 5: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoWaitTimeRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoWaitTimeRecursive = append(m.IoWaitTimeRecursive, &BlkIOEntry{}) +- if err := m.IoWaitTimeRecursive[len(m.IoWaitTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 6: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoMergedRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoMergedRecursive = append(m.IoMergedRecursive, &BlkIOEntry{}) +- if err := m.IoMergedRecursive[len(m.IoMergedRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 7: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field IoTimeRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.IoTimeRecursive = append(m.IoTimeRecursive, &BlkIOEntry{}) +- if err := m.IoTimeRecursive[len(m.IoTimeRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 8: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field SectorsRecursive", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.SectorsRecursive = append(m.SectorsRecursive, &BlkIOEntry{}) +- if err := m.SectorsRecursive[len(m.SectorsRecursive)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetTxBytes() uint64 { ++ if m != nil { ++ return m.TxBytes + } +- return nil ++ return 0 + } +-func (m *BlkIOEntry) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: BlkIOEntry: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: BlkIOEntry: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) +- } +- var stringLen uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- stringLen |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- intStringLen := int(stringLen) +- if intStringLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + intStringLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Op = string(dAtA[iNdEx:postIndex]) +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) +- } +- var stringLen uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- stringLen |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- intStringLen := int(stringLen) +- if intStringLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + intStringLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Device = string(dAtA[iNdEx:postIndex]) +- iNdEx = postIndex +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Major", wireType) +- } +- m.Major = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Major |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Minor", wireType) +- } +- m.Minor = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Minor |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 5: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) +- } +- m.Value = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.Value |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetTxPackets() uint64 { ++ if m != nil { ++ return m.TxPackets + } +- return nil ++ return 0 + } +-func (m *RdmaStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: RdmaStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: RdmaStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Current", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Current = append(m.Current, &RdmaEntry{}) +- if err := m.Current[len(m.Current)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- case 2: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Limit", wireType) +- } +- var msglen int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- msglen |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- if msglen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + msglen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Limit = append(m.Limit, &RdmaEntry{}) +- if err := m.Limit[len(m.Limit)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { +- return err +- } +- iNdEx = postIndex +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetTxErrors() uint64 { ++ if m != nil { ++ return m.TxErrors + } +- return nil ++ return 0 + } +-func (m *RdmaEntry) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: RdmaEntry: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: RdmaEntry: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType) +- } +- var stringLen uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- stringLen |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- intStringLen := int(stringLen) +- if intStringLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + intStringLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Device = string(dAtA[iNdEx:postIndex]) +- iNdEx = postIndex +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field HcaHandles", wireType) +- } +- m.HcaHandles = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.HcaHandles |= (uint32(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field HcaObjects", wireType) +- } +- m.HcaObjects = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.HcaObjects |= (uint32(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++func (m *NetworkStat) GetTxDropped() uint64 { ++ if m != nil { ++ return m.TxDropped + } +- return nil ++ return 0 + } +-func (m *NetworkStat) Unmarshal(dAtA []byte) error { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- preIndex := iNdEx +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- fieldNum := int32(wire >> 3) +- wireType := int(wire & 0x7) +- if wireType == 4 { +- return fmt.Errorf("proto: NetworkStat: wiretype end group for non-group") +- } +- if fieldNum <= 0 { +- return fmt.Errorf("proto: NetworkStat: illegal tag %d (wire type %d)", fieldNum, wire) +- } +- switch fieldNum { +- case 1: +- if wireType != 2 { +- return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) +- } +- var stringLen uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- stringLen |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- intStringLen := int(stringLen) +- if intStringLen < 0 { +- return ErrInvalidLengthMetrics +- } +- postIndex := iNdEx + intStringLen +- if postIndex > l { +- return io.ErrUnexpectedEOF +- } +- m.Name = string(dAtA[iNdEx:postIndex]) +- iNdEx = postIndex +- case 2: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RxBytes", wireType) +- } +- m.RxBytes = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RxBytes |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 3: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RxPackets", wireType) +- } +- m.RxPackets = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RxPackets |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 4: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RxErrors", wireType) +- } +- m.RxErrors = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RxErrors |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 5: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field RxDropped", wireType) +- } +- m.RxDropped = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.RxDropped |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 6: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TxBytes", wireType) +- } +- m.TxBytes = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TxBytes |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 7: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TxPackets", wireType) +- } +- m.TxPackets = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TxPackets |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 8: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TxErrors", wireType) +- } +- m.TxErrors = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TxErrors |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- case 9: +- if wireType != 0 { +- return fmt.Errorf("proto: wrong wireType = %d for field TxDropped", wireType) +- } +- m.TxDropped = 0 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- m.TxDropped |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- default: +- iNdEx = preIndex +- skippy, err := skipMetrics(dAtA[iNdEx:]) +- if err != nil { +- return err +- } +- if skippy < 0 { +- return ErrInvalidLengthMetrics +- } +- if (iNdEx + skippy) > l { +- return io.ErrUnexpectedEOF +- } +- iNdEx += skippy +- } +- } + +- if iNdEx > l { +- return io.ErrUnexpectedEOF ++type FilesStat struct { ++ Usage uint64 `protobuf:"varint,1,opt,name=usage,proto3" json:"usage,omitempty"` ++ Limit uint64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` ++} ++ ++func (m *FilesStat) Reset() { *m = FilesStat{} } ++func (m *FilesStat) String() string { return proto.CompactTextString(m) } ++func (*FilesStat) ProtoMessage() {} ++func (*FilesStat) Descriptor() ([]byte, []int) { return fileDescriptorMetrics, []int{13} } ++ ++func (m *FilesStat) GetUsage() uint64 { ++ if m != nil { ++ return m.Usage + } +- return nil ++ return 0 + } +-func skipMetrics(dAtA []byte) (n int, err error) { +- l := len(dAtA) +- iNdEx := 0 +- for iNdEx < l { +- var wire uint64 +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return 0, ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return 0, io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- wire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- wireType := int(wire & 0x7) +- switch wireType { +- case 0: +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return 0, ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return 0, io.ErrUnexpectedEOF +- } +- iNdEx++ +- if dAtA[iNdEx-1] < 0x80 { +- break +- } +- } +- return iNdEx, nil +- case 1: +- iNdEx += 8 +- return iNdEx, nil +- case 2: +- var length int +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return 0, ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return 0, io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- length |= (int(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- iNdEx += length +- if length < 0 { +- return 0, ErrInvalidLengthMetrics +- } +- return iNdEx, nil +- case 3: +- for { +- var innerWire uint64 +- var start int = iNdEx +- for shift := uint(0); ; shift += 7 { +- if shift >= 64 { +- return 0, ErrIntOverflowMetrics +- } +- if iNdEx >= l { +- return 0, io.ErrUnexpectedEOF +- } +- b := dAtA[iNdEx] +- iNdEx++ +- innerWire |= (uint64(b) & 0x7F) << shift +- if b < 0x80 { +- break +- } +- } +- innerWireType := int(innerWire & 0x7) +- if innerWireType == 4 { +- break +- } +- next, err := skipMetrics(dAtA[start:]) +- if err != nil { +- return 0, err +- } +- iNdEx = start + next +- } +- return iNdEx, nil +- case 4: +- return iNdEx, nil +- case 5: +- iNdEx += 4 +- return iNdEx, nil +- default: +- return 0, fmt.Errorf("proto: illegal wireType %d", wireType) +- } ++ ++func (m *FilesStat) GetLimit() uint64 { ++ if m != nil { ++ return m.Limit + } +- panic("unreachable") ++ return 0 + } + +-var ( +- ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") +- ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") +-) ++func init() { ++ proto.RegisterType((*Metrics)(nil), "io.containerd.cgroups.v1.Metrics") ++ proto.RegisterType((*HugetlbStat)(nil), "io.containerd.cgroups.v1.HugetlbStat") ++ proto.RegisterType((*PidsStat)(nil), "io.containerd.cgroups.v1.PidsStat") ++ proto.RegisterType((*CPUStat)(nil), "io.containerd.cgroups.v1.CPUStat") ++ proto.RegisterType((*CPUUsage)(nil), "io.containerd.cgroups.v1.CPUUsage") ++ proto.RegisterType((*Throttle)(nil), "io.containerd.cgroups.v1.Throttle") ++ proto.RegisterType((*MemoryStat)(nil), "io.containerd.cgroups.v1.MemoryStat") ++ proto.RegisterType((*MemoryEntry)(nil), "io.containerd.cgroups.v1.MemoryEntry") ++ proto.RegisterType((*BlkIOStat)(nil), "io.containerd.cgroups.v1.BlkIOStat") ++ proto.RegisterType((*BlkIOEntry)(nil), "io.containerd.cgroups.v1.BlkIOEntry") ++ proto.RegisterType((*RdmaStat)(nil), "io.containerd.cgroups.v1.RdmaStat") ++ proto.RegisterType((*RdmaEntry)(nil), "io.containerd.cgroups.v1.RdmaEntry") ++ proto.RegisterType((*NetworkStat)(nil), "io.containerd.cgroups.v1.NetworkStat") ++ proto.RegisterType((*FilesStat)(nil), "io.containerd.cgroups.v1.FilesStat") ++} + + func init() { proto.RegisterFile("github.com/containerd/cgroups/metrics.proto", fileDescriptorMetrics) } + + var fileDescriptorMetrics = []byte{ +- // 1549 bytes of a gzipped FileDescriptorProto +- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x57, 0x4d, 0x6f, 0x1b, 0xb7, +- 0x16, 0x8d, 0x2c, 0xd9, 0xd2, 0x5c, 0xd9, 0x8e, 0x4d, 0x27, 0xce, 0xd8, 0x49, 0x2c, 0x47, 0xb6, +- 0xdf, 0xf3, 0x7b, 0x06, 0x64, 0xbc, 0x3c, 0x20, 0x68, 0xd2, 0x04, 0x45, 0xe4, 0x24, 0x48, 0xd0, +- 0xba, 0x51, 0x46, 0x36, 0xd2, 0xae, 0x06, 0xd4, 0x88, 0x19, 0xd1, 0x96, 0x86, 0x13, 0x0e, 0xc7, +- 0x96, 0xbb, 0xea, 0xa2, 0x40, 0x57, 0xfd, 0x33, 0xfd, 0x15, 0x59, 0x76, 0x53, 0xa0, 0xdd, 0x18, +- 0x8d, 0x7e, 0x49, 0x41, 0x72, 0x3e, 0xa8, 0x24, 0x8e, 0xab, 0xdd, 0x90, 0x3c, 0xe7, 0xdc, 0xcb, +- 0x3b, 0x87, 0xc3, 0x3b, 0xb0, 0xe3, 0x53, 0xd1, 0x8b, 0x3b, 0x0d, 0x8f, 0x0d, 0x76, 0x3d, 0x16, +- 0x08, 0x4c, 0x03, 0xc2, 0xbb, 0xbb, 0x9e, 0xcf, 0x59, 0x1c, 0x46, 0xbb, 0x03, 0x22, 0x38, 0xf5, +- 0xa2, 0x46, 0xc8, 0x99, 0x60, 0xc8, 0xa6, 0xac, 0x91, 0x83, 0x1a, 0x09, 0xa8, 0x71, 0xf2, 0xbf, +- 0xd5, 0x6b, 0x3e, 0xf3, 0x99, 0x02, 0xed, 0xca, 0x27, 0x8d, 0xaf, 0xff, 0x5a, 0x84, 0xf2, 0xbe, +- 0x56, 0x40, 0x5f, 0x41, 0xb9, 0x17, 0xfb, 0x44, 0xf4, 0x3b, 0x76, 0x61, 0xbd, 0xb8, 0x5d, 0xbd, +- 0xbb, 0xd5, 0xb8, 0x48, 0xad, 0xf1, 0x5c, 0x03, 0xdb, 0x02, 0x0b, 0x27, 0x65, 0xa1, 0x7b, 0x50, +- 0x0a, 0x69, 0x37, 0xb2, 0xa7, 0xd6, 0x0b, 0xdb, 0xd5, 0xbb, 0xf5, 0x8b, 0xd9, 0x2d, 0xda, 0x8d, +- 0x14, 0x55, 0xe1, 0xd1, 0x43, 0x28, 0x7a, 0x61, 0x6c, 0x17, 0x15, 0xed, 0xce, 0xc5, 0xb4, 0xbd, +- 0xd6, 0xa1, 0x64, 0x35, 0xcb, 0xa3, 0xf3, 0x5a, 0x71, 0xaf, 0x75, 0xe8, 0x48, 0x1a, 0x7a, 0x08, +- 0x33, 0x03, 0x32, 0x60, 0xfc, 0xcc, 0x2e, 0x29, 0x81, 0xcd, 0x8b, 0x05, 0xf6, 0x15, 0x4e, 0x45, +- 0x4e, 0x38, 0xe8, 0x3e, 0x4c, 0x77, 0xfa, 0xc7, 0x94, 0xd9, 0xd3, 0x8a, 0xbc, 0x71, 0x31, 0xb9, +- 0xd9, 0x3f, 0x7e, 0xf1, 0x52, 0x71, 0x35, 0x43, 0x6e, 0x97, 0x77, 0x07, 0xd8, 0x9e, 0xb9, 0x6c, +- 0xbb, 0x4e, 0x77, 0x80, 0xf5, 0x76, 0x25, 0x5e, 0xd6, 0x39, 0x20, 0xe2, 0x94, 0xf1, 0x63, 0xbb, +- 0x7c, 0x59, 0x9d, 0xbf, 0xd5, 0x40, 0x5d, 0xe7, 0x84, 0x55, 0x3f, 0x86, 0xaa, 0x51, 0x7f, 0x74, +- 0x0d, 0xa6, 0xe3, 0x08, 0xfb, 0xc4, 0x2e, 0xac, 0x17, 0xb6, 0x4b, 0x8e, 0x1e, 0xa0, 0x05, 0x28, +- 0x0e, 0xf0, 0x50, 0xbd, 0x8b, 0x92, 0x23, 0x1f, 0x91, 0x0d, 0xe5, 0x37, 0x98, 0xf6, 0xbd, 0x40, +- 0xa8, 0x52, 0x97, 0x9c, 0x74, 0x88, 0x56, 0xa1, 0x12, 0x62, 0x9f, 0x44, 0xf4, 0x07, 0xa2, 0x8a, +- 0x68, 0x39, 0xd9, 0xb8, 0xfe, 0x00, 0x2a, 0xe9, 0xeb, 0x92, 0x0a, 0x5e, 0xcc, 0x39, 0x09, 0x44, +- 0x12, 0x2b, 0x1d, 0xca, 0x1c, 0xfa, 0x74, 0x40, 0x45, 0x12, 0x4f, 0x0f, 0xea, 0x3f, 0x17, 0xa0, +- 0x9c, 0xbc, 0x34, 0xf4, 0x85, 0x99, 0xe5, 0x67, 0xcb, 0xb5, 0xd7, 0x3a, 0x3c, 0x94, 0xc8, 0x74, +- 0x27, 0x4d, 0x00, 0xd1, 0xe3, 0x4c, 0x88, 0x3e, 0x0d, 0xfc, 0xcb, 0xcd, 0x75, 0xa0, 0xb1, 0xc4, +- 0x31, 0x58, 0xf5, 0xb7, 0x50, 0x49, 0x65, 0x65, 0xae, 0x82, 0x09, 0xdc, 0x4f, 0xeb, 0xa5, 0x06, +- 0x68, 0x19, 0x66, 0x8e, 0x09, 0x0f, 0x48, 0x3f, 0xd9, 0x42, 0x32, 0x42, 0x08, 0x4a, 0x71, 0x44, +- 0x78, 0x52, 0x32, 0xf5, 0x8c, 0x36, 0xa0, 0x1c, 0x12, 0xee, 0x4a, 0xd3, 0x96, 0xd6, 0x8b, 0xdb, +- 0xa5, 0x26, 0x8c, 0xce, 0x6b, 0x33, 0x2d, 0xc2, 0xa5, 0x29, 0x67, 0x42, 0xc2, 0xf7, 0xc2, 0xb8, +- 0x3e, 0x84, 0x4a, 0x9a, 0x8a, 0x2c, 0x5c, 0x48, 0x38, 0x65, 0xdd, 0x28, 0x2d, 0x5c, 0x32, 0x44, +- 0x3b, 0xb0, 0x98, 0xa4, 0x49, 0xba, 0x6e, 0x8a, 0xd1, 0x19, 0x2c, 0x64, 0x0b, 0xad, 0x04, 0xbc, +- 0x05, 0xf3, 0x39, 0x58, 0xd0, 0x01, 0x49, 0xb2, 0x9a, 0xcb, 0x66, 0x0f, 0xe8, 0x80, 0xd4, 0xff, +- 0xac, 0x02, 0xe4, 0x56, 0x97, 0xfb, 0xf5, 0xb0, 0xd7, 0xcb, 0xfc, 0xa1, 0x06, 0x68, 0x05, 0x8a, +- 0x3c, 0x4a, 0x42, 0xe9, 0x13, 0xe5, 0xb4, 0xdb, 0x8e, 0x9c, 0x43, 0xff, 0x82, 0x0a, 0x8f, 0x22, +- 0x57, 0x1e, 0x6b, 0x1d, 0xa0, 0x59, 0x1d, 0x9d, 0xd7, 0xca, 0x4e, 0xbb, 0x2d, 0x6d, 0xe7, 0x94, +- 0x79, 0x14, 0xc9, 0x07, 0x54, 0x83, 0xea, 0x00, 0x87, 0x21, 0xe9, 0xba, 0x6f, 0x68, 0x5f, 0x3b, +- 0xa7, 0xe4, 0x80, 0x9e, 0x7a, 0x46, 0xfb, 0xaa, 0xd2, 0x5d, 0xca, 0xc5, 0x99, 0x3a, 0x5c, 0x25, +- 0x47, 0x0f, 0xd0, 0x2d, 0xb0, 0x4e, 0x39, 0x15, 0xa4, 0x83, 0xbd, 0x63, 0x75, 0x78, 0x4a, 0x4e, +- 0x3e, 0x81, 0x6c, 0xa8, 0x84, 0xbe, 0x1b, 0xfa, 0x2e, 0x0d, 0xec, 0xb2, 0x7e, 0x13, 0xa1, 0xdf, +- 0xf2, 0x5f, 0x04, 0x68, 0x15, 0x2c, 0xbd, 0xc2, 0x62, 0x61, 0x57, 0x92, 0x32, 0xfa, 0x2d, 0xff, +- 0x65, 0x2c, 0xd0, 0x8a, 0x62, 0xbd, 0xc1, 0x71, 0x5f, 0xd8, 0x56, 0xba, 0xf4, 0x4c, 0x0e, 0xd1, +- 0x3a, 0xcc, 0x86, 0xbe, 0x3b, 0xc0, 0x47, 0xc9, 0x32, 0xe8, 0x34, 0x43, 0x7f, 0x1f, 0x1f, 0x69, +- 0xc4, 0x06, 0xcc, 0xd1, 0x00, 0x7b, 0x82, 0x9e, 0x10, 0x17, 0x07, 0x2c, 0xb0, 0xab, 0x0a, 0x32, +- 0x9b, 0x4e, 0x3e, 0x0e, 0x58, 0x20, 0x37, 0x6b, 0x42, 0x66, 0xb5, 0x8a, 0x01, 0x30, 0x55, 0x54, +- 0x3d, 0xe6, 0xc6, 0x55, 0x54, 0x45, 0x72, 0x15, 0x05, 0x99, 0x37, 0x55, 0x14, 0x60, 0x1d, 0xaa, +- 0x71, 0x40, 0x4e, 0xa8, 0x27, 0x70, 0xa7, 0x4f, 0xec, 0xab, 0x0a, 0x60, 0x4e, 0xa1, 0x07, 0xb0, +- 0xd2, 0xa3, 0x84, 0x63, 0xee, 0xf5, 0xa8, 0x87, 0xfb, 0xae, 0xfe, 0x90, 0xb9, 0xfa, 0xf8, 0x2d, +- 0x28, 0xfc, 0x0d, 0x13, 0xa0, 0x9d, 0xf0, 0x8d, 0x5c, 0x46, 0xf7, 0x60, 0x6c, 0xc9, 0x8d, 0x4e, +- 0x71, 0x98, 0x30, 0x17, 0x15, 0xf3, 0xba, 0xb9, 0xdc, 0x3e, 0xc5, 0xa1, 0xe6, 0xd5, 0xa0, 0xaa, +- 0x4e, 0x89, 0xab, 0x8d, 0x84, 0x74, 0xda, 0x6a, 0x6a, 0x4f, 0xb9, 0xe9, 0x3f, 0x60, 0x69, 0x80, +- 0xf4, 0xd4, 0x92, 0xf2, 0xcc, 0xec, 0xe8, 0xbc, 0x56, 0x39, 0x90, 0x93, 0xd2, 0x58, 0x15, 0xb5, +- 0xec, 0x44, 0x11, 0xba, 0x07, 0xf3, 0x19, 0x54, 0x7b, 0xec, 0x9a, 0xc2, 0x2f, 0x8c, 0xce, 0x6b, +- 0xb3, 0x29, 0x5e, 0x19, 0x6d, 0x36, 0xe5, 0x28, 0xb7, 0xfd, 0x17, 0x16, 0x35, 0xcf, 0xf4, 0xdc, +- 0x75, 0x95, 0xc9, 0x55, 0xb5, 0xb0, 0x9f, 0x1b, 0x2f, 0xcb, 0x57, 0xdb, 0x6f, 0xd9, 0xc8, 0xf7, +- 0x89, 0xf2, 0xe0, 0xbf, 0x41, 0x73, 0xdc, 0xdc, 0x89, 0x37, 0x14, 0x48, 0xe7, 0xf6, 0x3a, 0xb3, +- 0xe3, 0x46, 0x9a, 0x6d, 0x66, 0x4a, 0x5b, 0xbf, 0x12, 0x35, 0xdb, 0xd2, 0xce, 0xdc, 0x4a, 0xd5, +- 0x72, 0x7f, 0xae, 0xe8, 0x97, 0x9f, 0xa1, 0xa4, 0x49, 0x37, 0x0d, 0x2d, 0xed, 0xc5, 0xd5, 0x31, +- 0x94, 0x76, 0xe3, 0x0e, 0xa0, 0x0c, 0x95, 0xbb, 0xf6, 0xa6, 0xb1, 0xd1, 0x56, 0x6e, 0xdd, 0x06, +- 0x2c, 0x69, 0xf0, 0xb8, 0x81, 0x6f, 0x29, 0xb4, 0xae, 0xd7, 0x0b, 0xd3, 0xc5, 0x59, 0x11, 0x4d, +- 0xf4, 0x6d, 0x43, 0xfb, 0x71, 0x8e, 0xfd, 0x58, 0x5b, 0x95, 0x7c, 0xed, 0x13, 0xda, 0xaa, 0xe8, +- 0x1f, 0x6a, 0x2b, 0x74, 0xed, 0x23, 0x6d, 0x85, 0xdd, 0x49, 0xb1, 0xa6, 0xd9, 0xd7, 0x93, 0xcf, +- 0x9e, 0x5c, 0x38, 0x34, 0x1c, 0xff, 0x65, 0x7a, 0x75, 0xdc, 0x51, 0xdf, 0xfe, 0xad, 0xcb, 0x2e, +- 0xf8, 0xa7, 0x81, 0xe0, 0x67, 0xe9, 0xed, 0x71, 0x1f, 0x4a, 0xd2, 0xe5, 0x76, 0x7d, 0x12, 0xae, +- 0xa2, 0xa0, 0x47, 0xd9, 0x95, 0xb0, 0x31, 0x09, 0x39, 0xbd, 0x39, 0xda, 0x00, 0xfa, 0xc9, 0x15, +- 0x5e, 0x68, 0x6f, 0x4e, 0x20, 0xd1, 0x9c, 0x1b, 0x9d, 0xd7, 0xac, 0xaf, 0x15, 0xf9, 0x60, 0xaf, +- 0xe5, 0x58, 0x5a, 0xe7, 0xc0, 0x0b, 0xeb, 0x04, 0xaa, 0x06, 0x30, 0xbf, 0x77, 0x0b, 0xc6, 0xbd, +- 0x9b, 0x77, 0x04, 0x53, 0x9f, 0xe8, 0x08, 0x8a, 0x9f, 0xec, 0x08, 0x4a, 0x63, 0x1d, 0x41, 0xfd, +- 0xf7, 0x69, 0xb0, 0xb2, 0x86, 0x07, 0x61, 0x58, 0xa5, 0xcc, 0x8d, 0x08, 0x3f, 0xa1, 0x1e, 0x71, +- 0x3b, 0x67, 0x82, 0x44, 0x2e, 0x27, 0x5e, 0xcc, 0x23, 0x7a, 0x42, 0x92, 0x66, 0x71, 0xf3, 0x92, +- 0xce, 0x49, 0xd7, 0xe6, 0x06, 0x65, 0x6d, 0x2d, 0xd3, 0x94, 0x2a, 0x4e, 0x2a, 0x82, 0xbe, 0x83, +- 0xeb, 0x79, 0x88, 0xae, 0xa1, 0x3e, 0x35, 0x81, 0xfa, 0x52, 0xa6, 0xde, 0xcd, 0x95, 0x0f, 0x60, +- 0x89, 0x32, 0xf7, 0x6d, 0x4c, 0xe2, 0x31, 0xdd, 0xe2, 0x04, 0xba, 0x8b, 0x94, 0xbd, 0x52, 0xfc, +- 0x5c, 0xd5, 0x85, 0x15, 0xa3, 0x24, 0xf2, 0x2e, 0x36, 0xb4, 0x4b, 0x13, 0x68, 0x2f, 0x67, 0x39, +- 0xcb, 0xbb, 0x3b, 0x0f, 0xf0, 0x3d, 0x2c, 0x53, 0xe6, 0x9e, 0x62, 0x2a, 0x3e, 0x54, 0x9f, 0x9e, +- 0xac, 0x22, 0xaf, 0x31, 0x15, 0xe3, 0xd2, 0xba, 0x22, 0x03, 0xc2, 0xfd, 0xb1, 0x8a, 0xcc, 0x4c, +- 0x56, 0x91, 0x7d, 0xc5, 0xcf, 0x55, 0x5b, 0xb0, 0x48, 0xd9, 0x87, 0xb9, 0x96, 0x27, 0xd0, 0xbc, +- 0x4a, 0xd9, 0x78, 0x9e, 0xaf, 0x60, 0x31, 0x22, 0x9e, 0x60, 0xdc, 0x74, 0x5b, 0x65, 0x02, 0xc5, +- 0x85, 0x84, 0x9e, 0x49, 0xd6, 0x4f, 0x00, 0xf2, 0x75, 0x34, 0x0f, 0x53, 0x2c, 0x54, 0x47, 0xc7, +- 0x72, 0xa6, 0x58, 0x28, 0x7b, 0xc0, 0xae, 0xfc, 0xec, 0xe8, 0x83, 0x63, 0x39, 0xc9, 0x48, 0x9e, +- 0xa7, 0x01, 0x3e, 0x62, 0x69, 0x13, 0xa8, 0x07, 0x6a, 0x96, 0x06, 0x8c, 0x27, 0x67, 0x47, 0x0f, +- 0xe4, 0xec, 0x09, 0xee, 0xc7, 0x24, 0xed, 0x79, 0xd4, 0xa0, 0xfe, 0x53, 0x01, 0x2a, 0xe9, 0x6f, +- 0x00, 0x7a, 0x64, 0xb6, 0xd1, 0xc5, 0xcf, 0xff, 0x75, 0x48, 0x92, 0xde, 0x4c, 0xd6, 0x6b, 0xdf, +- 0xcf, 0x7b, 0xed, 0x7f, 0x4c, 0x4e, 0x1a, 0x72, 0x02, 0x56, 0x36, 0x67, 0xec, 0xb6, 0x30, 0xb6, +- 0xdb, 0x1a, 0x54, 0x7b, 0x1e, 0x76, 0x7b, 0x38, 0xe8, 0xf6, 0x89, 0xee, 0x10, 0xe7, 0x1c, 0xe8, +- 0x79, 0xf8, 0xb9, 0x9e, 0x49, 0x01, 0xac, 0x73, 0x44, 0x3c, 0x11, 0xa9, 0xa2, 0x68, 0xc0, 0x4b, +- 0x3d, 0x53, 0xff, 0x65, 0x0a, 0xaa, 0xc6, 0x9f, 0x8b, 0xec, 0xa1, 0x03, 0x3c, 0x48, 0xe3, 0xa8, +- 0x67, 0xd9, 0xb1, 0xf1, 0xa1, 0xfe, 0x96, 0x24, 0x9f, 0xa9, 0x32, 0x1f, 0xaa, 0x8f, 0x02, 0xba, +- 0x0d, 0xc0, 0x87, 0x6e, 0x88, 0xbd, 0x63, 0x92, 0xc8, 0x97, 0x1c, 0x8b, 0x0f, 0x5b, 0x7a, 0x02, +- 0xdd, 0x04, 0x8b, 0x0f, 0x5d, 0xc2, 0x39, 0xe3, 0x51, 0x52, 0xfb, 0x0a, 0x1f, 0x3e, 0x55, 0xe3, +- 0x84, 0xdb, 0xe5, 0x4c, 0xf6, 0x02, 0xc9, 0x3b, 0xb0, 0xf8, 0xf0, 0x89, 0x9e, 0x90, 0x51, 0x45, +- 0x1a, 0x55, 0xb7, 0x9e, 0x65, 0x91, 0x47, 0x15, 0x79, 0x54, 0xdd, 0x7a, 0x5a, 0xc2, 0x8c, 0x2a, +- 0xb2, 0xa8, 0xba, 0xfb, 0xac, 0x08, 0x23, 0xaa, 0xc8, 0xa3, 0x5a, 0x29, 0x37, 0x89, 0xda, 0xb4, +- 0xdf, 0xbd, 0x5f, 0xbb, 0xf2, 0xc7, 0xfb, 0xb5, 0x2b, 0x3f, 0x8e, 0xd6, 0x0a, 0xef, 0x46, 0x6b, +- 0x85, 0xdf, 0x46, 0x6b, 0x85, 0xbf, 0x46, 0x6b, 0x85, 0xce, 0x8c, 0xfa, 0x0d, 0xff, 0xff, 0xdf, +- 0x01, 0x00, 0x00, 0xff, 0xff, 0x19, 0x9d, 0xe2, 0xd3, 0xe5, 0x0f, 0x00, 0x00, ++ // 1542 bytes of a gzipped FileDescriptorProto ++ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x57, 0x5f, 0x53, 0xdb, 0xc6, ++ 0x17, 0x1d, 0x63, 0x83, 0xad, 0x6b, 0x20, 0xb0, 0x24, 0x44, 0x90, 0xe4, 0x67, 0x62, 0xe0, 0x57, ++ 0x5a, 0x66, 0xcc, 0x34, 0x9d, 0x49, 0x9b, 0x34, 0x99, 0x4e, 0x20, 0xc9, 0x24, 0xd3, 0xd2, 0x38, ++ 0x32, 0x4c, 0xda, 0x27, 0xcd, 0x5a, 0xde, 0xc8, 0x0b, 0xb6, 0x56, 0x59, 0xad, 0xc0, 0xf4, 0xb9, ++ 0x33, 0x7d, 0xea, 0xc7, 0xeb, 0x43, 0x5f, 0x78, 0xe0, 0xbd, 0xdf, 0xa1, 0xb3, 0x77, 0xf5, 0xcf, ++ 0x09, 0x84, 0xfa, 0x4d, 0xbb, 0x7b, 0xce, 0xb9, 0x77, 0xef, 0x9e, 0x95, 0xae, 0x60, 0xdb, 0xe7, ++ 0xaa, 0x1f, 0x77, 0x5b, 0x9e, 0x18, 0xee, 0x78, 0x22, 0x50, 0x94, 0x07, 0x4c, 0xf6, 0x76, 0x3c, ++ 0x5f, 0x8a, 0x38, 0x8c, 0x76, 0x86, 0x4c, 0x49, 0xee, 0x45, 0xad, 0x50, 0x0a, 0x25, 0x88, 0xcd, ++ 0x45, 0x2b, 0x07, 0xb5, 0x12, 0x50, 0xeb, 0xe4, 0xeb, 0xd5, 0x9b, 0xbe, 0xf0, 0x05, 0x82, 0x76, ++ 0xf4, 0x93, 0xc1, 0x37, 0xff, 0x29, 0x43, 0x75, 0xdf, 0x28, 0x90, 0x1f, 0xa0, 0xda, 0x8f, 0x7d, ++ 0xa6, 0x06, 0x5d, 0xbb, 0xb4, 0x56, 0xde, 0xaa, 0x3f, 0xd8, 0x6c, 0x5d, 0xa5, 0xd6, 0x7a, 0x65, ++ 0x80, 0x1d, 0x45, 0x95, 0x93, 0xb2, 0xc8, 0x43, 0xa8, 0x84, 0xbc, 0x17, 0xd9, 0x53, 0x6b, 0xa5, ++ 0xad, 0xfa, 0x83, 0xe6, 0xd5, 0xec, 0x36, 0xef, 0x45, 0x48, 0x45, 0x3c, 0x79, 0x02, 0x65, 0x2f, ++ 0x8c, 0xed, 0x32, 0xd2, 0xee, 0x5f, 0x4d, 0xdb, 0x6b, 0x1f, 0x6a, 0xd6, 0x6e, 0xf5, 0xe2, 0xbc, ++ 0x51, 0xde, 0x6b, 0x1f, 0x3a, 0x9a, 0x46, 0x9e, 0xc0, 0xcc, 0x90, 0x0d, 0x85, 0x3c, 0xb3, 0x2b, ++ 0x28, 0xb0, 0x71, 0xb5, 0xc0, 0x3e, 0xe2, 0x30, 0x72, 0xc2, 0x21, 0x8f, 0x60, 0xba, 0x3b, 0x38, ++ 0xe6, 0xc2, 0x9e, 0x46, 0xf2, 0xfa, 0xd5, 0xe4, 0xdd, 0xc1, 0xf1, 0xeb, 0x37, 0xc8, 0x35, 0x0c, ++ 0xbd, 0x5d, 0xd9, 0x1b, 0x52, 0x7b, 0xe6, 0xba, 0xed, 0x3a, 0xbd, 0x21, 0x35, 0xdb, 0xd5, 0x78, ++ 0x5d, 0xe7, 0x80, 0xa9, 0x53, 0x21, 0x8f, 0xed, 0xea, 0x75, 0x75, 0xfe, 0xd9, 0x00, 0x4d, 0x9d, ++ 0x13, 0x96, 0xce, 0xf9, 0x3d, 0x1f, 0xb0, 0xc8, 0xae, 0x5d, 0x97, 0xf3, 0x4b, 0x0d, 0x33, 0x39, ++ 0x23, 0xa3, 0x79, 0x0c, 0xf5, 0xc2, 0xd1, 0x91, 0x9b, 0x30, 0x1d, 0x47, 0xd4, 0x67, 0x76, 0x69, ++ 0xad, 0xb4, 0x55, 0x71, 0xcc, 0x80, 0x2c, 0x40, 0x79, 0x48, 0x47, 0x78, 0x8c, 0x15, 0x47, 0x3f, ++ 0x12, 0x1b, 0xaa, 0xef, 0x29, 0x1f, 0x78, 0x81, 0xc2, 0x53, 0xaa, 0x38, 0xe9, 0x90, 0xac, 0x42, ++ 0x2d, 0xa4, 0x3e, 0x8b, 0xf8, 0x6f, 0x0c, 0xeb, 0x6f, 0x39, 0xd9, 0xb8, 0xf9, 0x18, 0x6a, 0xe9, ++ 0x49, 0x6b, 0x05, 0x2f, 0x96, 0x92, 0x05, 0x2a, 0x89, 0x95, 0x0e, 0x75, 0x0e, 0x03, 0x3e, 0xe4, ++ 0x2a, 0x89, 0x67, 0x06, 0xcd, 0x3f, 0x4a, 0x50, 0x4d, 0xce, 0x9b, 0x7c, 0x57, 0xcc, 0xf2, 0xb3, ++ 0x95, 0xde, 0x6b, 0x1f, 0x1e, 0x6a, 0x64, 0xba, 0x93, 0x5d, 0x00, 0xd5, 0x97, 0x42, 0xa9, 0x01, ++ 0x0f, 0xfc, 0xeb, 0x7d, 0x79, 0x60, 0xb0, 0xcc, 0x29, 0xb0, 0x9a, 0x1f, 0xa0, 0x96, 0xca, 0xea, ++ 0x5c, 0x95, 0x50, 0x74, 0x90, 0xd6, 0x0b, 0x07, 0x64, 0x19, 0x66, 0x8e, 0x99, 0x0c, 0xd8, 0x20, ++ 0xd9, 0x42, 0x32, 0x22, 0x04, 0x2a, 0x71, 0xc4, 0x64, 0x52, 0x32, 0x7c, 0x26, 0xeb, 0x50, 0x0d, ++ 0x99, 0x74, 0xb5, 0xdf, 0x2b, 0x6b, 0xe5, 0xad, 0xca, 0x2e, 0x5c, 0x9c, 0x37, 0x66, 0xda, 0x4c, ++ 0x6a, 0x3f, 0xcf, 0x84, 0x4c, 0xee, 0x85, 0x71, 0x73, 0x04, 0xb5, 0x34, 0x15, 0x5d, 0xb8, 0x90, ++ 0x49, 0x2e, 0x7a, 0x51, 0x5a, 0xb8, 0x64, 0x48, 0xb6, 0x61, 0x31, 0x49, 0x93, 0xf5, 0xdc, 0x14, ++ 0x63, 0x32, 0x58, 0xc8, 0x16, 0xda, 0x09, 0x78, 0x13, 0xe6, 0x73, 0xb0, 0xe2, 0x43, 0x96, 0x64, ++ 0x35, 0x97, 0xcd, 0x1e, 0xf0, 0x21, 0x6b, 0xfe, 0x5d, 0x07, 0xc8, 0x6f, 0x89, 0xde, 0xaf, 0x47, ++ 0xbd, 0x7e, 0xe6, 0x0f, 0x1c, 0x90, 0x15, 0x28, 0xcb, 0x28, 0x09, 0x65, 0x2e, 0xa3, 0xd3, 0xe9, ++ 0x38, 0x7a, 0x8e, 0xfc, 0x1f, 0x6a, 0x32, 0x8a, 0x5c, 0xfd, 0x46, 0x30, 0x01, 0x76, 0xeb, 0x17, ++ 0xe7, 0x8d, 0xaa, 0xd3, 0xe9, 0x68, 0xdb, 0x39, 0x55, 0x19, 0x45, 0xfa, 0x81, 0x34, 0xa0, 0x3e, ++ 0xa4, 0x61, 0xc8, 0x7a, 0xae, 0xf6, 0x25, 0x3a, 0xa7, 0xe2, 0x80, 0x99, 0xd2, 0xa6, 0xd5, 0x91, ++ 0x7b, 0x5c, 0xaa, 0x33, 0xbc, 0x97, 0x15, 0xc7, 0x0c, 0xc8, 0x5d, 0xb0, 0x4e, 0x25, 0x57, 0xac, ++ 0x4b, 0xbd, 0x63, 0xbc, 0x77, 0x15, 0x27, 0x9f, 0x20, 0x36, 0xd4, 0x42, 0xdf, 0x0d, 0x7d, 0x97, ++ 0x07, 0x76, 0xd5, 0x9c, 0x44, 0xe8, 0xb7, 0xfd, 0xd7, 0x01, 0x59, 0x05, 0xcb, 0xac, 0x88, 0x58, ++ 0xe1, 0xad, 0xd1, 0x65, 0xf4, 0xdb, 0xfe, 0x9b, 0x58, 0x91, 0x15, 0x64, 0xbd, 0xa7, 0xf1, 0x40, ++ 0xd9, 0x56, 0xba, 0xf4, 0x52, 0x0f, 0xc9, 0x1a, 0xcc, 0x86, 0xbe, 0x3b, 0xa4, 0x47, 0xc9, 0x32, ++ 0x98, 0x34, 0x43, 0x7f, 0x9f, 0x1e, 0x19, 0xc4, 0x3a, 0xcc, 0xf1, 0x80, 0x7a, 0x8a, 0x9f, 0x30, ++ 0x97, 0x06, 0x22, 0xb0, 0xeb, 0x08, 0x99, 0x4d, 0x27, 0x9f, 0x05, 0x22, 0xd0, 0x9b, 0x2d, 0x42, ++ 0x66, 0x8d, 0x4a, 0x01, 0x50, 0x54, 0xc1, 0x7a, 0xcc, 0x8d, 0xab, 0x60, 0x45, 0x72, 0x15, 0x84, ++ 0xcc, 0x17, 0x55, 0x10, 0xb0, 0x06, 0xf5, 0x38, 0x60, 0x27, 0xdc, 0x53, 0xb4, 0x3b, 0x60, 0xf6, ++ 0x0d, 0x04, 0x14, 0xa7, 0xc8, 0x63, 0x58, 0xe9, 0x73, 0x26, 0xa9, 0xf4, 0xfa, 0xdc, 0xa3, 0x03, ++ 0xd7, 0xbc, 0x03, 0x5d, 0x73, 0xfd, 0x16, 0x10, 0x7f, 0xbb, 0x08, 0x30, 0x4e, 0xf8, 0x49, 0x2f, ++ 0x93, 0x87, 0x30, 0xb6, 0xe4, 0x46, 0xa7, 0x34, 0x4c, 0x98, 0x8b, 0xc8, 0xbc, 0x55, 0x5c, 0xee, ++ 0x9c, 0xd2, 0xd0, 0xf0, 0x1a, 0x50, 0xc7, 0x5b, 0xe2, 0x1a, 0x23, 0x11, 0x93, 0x36, 0x4e, 0xed, ++ 0xa1, 0x9b, 0xbe, 0x04, 0xcb, 0x00, 0xb4, 0xa7, 0x96, 0xd0, 0x33, 0xb3, 0x17, 0xe7, 0x8d, 0xda, ++ 0x81, 0x9e, 0xd4, 0xc6, 0xaa, 0xe1, 0xb2, 0x13, 0x45, 0xe4, 0x21, 0xcc, 0x67, 0x50, 0xe3, 0xb1, ++ 0x9b, 0x88, 0x5f, 0xb8, 0x38, 0x6f, 0xcc, 0xa6, 0x78, 0x34, 0xda, 0x6c, 0xca, 0x41, 0xb7, 0x7d, ++ 0x05, 0x8b, 0x86, 0x57, 0xf4, 0xdc, 0x2d, 0xcc, 0xe4, 0x06, 0x2e, 0xec, 0xe7, 0xc6, 0xcb, 0xf2, ++ 0x35, 0xf6, 0x5b, 0x2e, 0xe4, 0xfb, 0x1c, 0x3d, 0xf8, 0x05, 0x18, 0x8e, 0x9b, 0x3b, 0xf1, 0x36, ++ 0x82, 0x4c, 0x6e, 0xef, 0x32, 0x3b, 0xae, 0xa7, 0xd9, 0x66, 0xa6, 0xb4, 0xcd, 0x91, 0xe0, 0x6c, ++ 0xdb, 0x38, 0x73, 0x33, 0x55, 0xcb, 0xfd, 0xb9, 0x62, 0x0e, 0x3f, 0x43, 0x69, 0x93, 0x6e, 0x14, ++ 0xb4, 0x8c, 0x17, 0x57, 0xc7, 0x50, 0xc6, 0x8d, 0xdb, 0x40, 0x32, 0x54, 0xee, 0xda, 0x3b, 0x85, ++ 0x8d, 0xb6, 0x73, 0xeb, 0xb6, 0x60, 0xc9, 0x80, 0xc7, 0x0d, 0x7c, 0x17, 0xd1, 0xa6, 0x5e, 0xaf, ++ 0x8b, 0x2e, 0xce, 0x8a, 0x58, 0x44, 0xdf, 0x2b, 0x68, 0x3f, 0xcb, 0xb1, 0x9f, 0x6a, 0x63, 0xc9, ++ 0xff, 0x77, 0x89, 0x36, 0x16, 0xfd, 0x63, 0x6d, 0x44, 0x37, 0x3e, 0xd1, 0x46, 0xec, 0x76, 0x8a, ++ 0x2d, 0x9a, 0x7d, 0x2d, 0x79, 0xed, 0xe9, 0x85, 0xc3, 0x82, 0xe3, 0xbf, 0x4f, 0x3f, 0x1d, 0xf7, ++ 0xf1, 0xdd, 0xbf, 0x79, 0x5d, 0x6f, 0xf0, 0x22, 0x50, 0xf2, 0x2c, 0xfd, 0x7a, 0x3c, 0x82, 0x8a, ++ 0x76, 0xb9, 0xdd, 0x9c, 0x84, 0x8b, 0x14, 0xf2, 0x34, 0xfb, 0x24, 0xac, 0x4f, 0x42, 0x4e, 0xbf, ++ 0x1c, 0x1d, 0x00, 0xf3, 0xe4, 0x2a, 0x2f, 0xb4, 0x37, 0x26, 0x90, 0xd8, 0x9d, 0xbb, 0x38, 0x6f, ++ 0x58, 0x3f, 0x22, 0xf9, 0x60, 0xaf, 0xed, 0x58, 0x46, 0xe7, 0xc0, 0x0b, 0x9b, 0x0c, 0xea, 0x05, ++ 0x60, 0xfe, 0xdd, 0x2d, 0x15, 0xbe, 0xbb, 0x79, 0x47, 0x30, 0x75, 0x49, 0x47, 0x50, 0xbe, 0xb4, ++ 0x23, 0xa8, 0x8c, 0x75, 0x04, 0xcd, 0xbf, 0xa6, 0xc1, 0xca, 0x7a, 0x25, 0x42, 0x61, 0x95, 0x0b, ++ 0x37, 0x62, 0xf2, 0x84, 0x7b, 0xcc, 0xed, 0x9e, 0x29, 0x16, 0xb9, 0x92, 0x79, 0xb1, 0x8c, 0xf8, ++ 0x09, 0x4b, 0xfa, 0xcc, 0x8d, 0x6b, 0x9a, 0x2e, 0x53, 0x9b, 0xdb, 0x5c, 0x74, 0x8c, 0xcc, 0xae, ++ 0x56, 0x71, 0x52, 0x11, 0xf2, 0x0b, 0xdc, 0xca, 0x43, 0xf4, 0x0a, 0xea, 0x53, 0x13, 0xa8, 0x2f, ++ 0x65, 0xea, 0xbd, 0x5c, 0xf9, 0x00, 0x96, 0xb8, 0x70, 0x3f, 0xc4, 0x2c, 0x1e, 0xd3, 0x2d, 0x4f, ++ 0xa0, 0xbb, 0xc8, 0xc5, 0x5b, 0xe4, 0xe7, 0xaa, 0x2e, 0xac, 0x14, 0x4a, 0xa2, 0xbf, 0xc5, 0x05, ++ 0xed, 0xca, 0x04, 0xda, 0xcb, 0x59, 0xce, 0xfa, 0xdb, 0x9d, 0x07, 0xf8, 0x15, 0x96, 0xb9, 0x70, ++ 0x4f, 0x29, 0x57, 0x1f, 0xab, 0x4f, 0x4f, 0x56, 0x91, 0x77, 0x94, 0xab, 0x71, 0x69, 0x53, 0x91, ++ 0x21, 0x93, 0xfe, 0x58, 0x45, 0x66, 0x26, 0xab, 0xc8, 0x3e, 0xf2, 0x73, 0xd5, 0x36, 0x2c, 0x72, ++ 0xf1, 0x71, 0xae, 0xd5, 0x09, 0x34, 0x6f, 0x70, 0x31, 0x9e, 0xe7, 0x5b, 0x58, 0x8c, 0x98, 0xa7, ++ 0x84, 0x2c, 0xba, 0xad, 0x36, 0x81, 0xe2, 0x42, 0x42, 0xcf, 0x24, 0x9b, 0x27, 0x00, 0xf9, 0x3a, ++ 0x99, 0x87, 0x29, 0x11, 0xe2, 0xd5, 0xb1, 0x9c, 0x29, 0x11, 0xea, 0x1e, 0xb0, 0xa7, 0x5f, 0x3b, ++ 0xe6, 0xe2, 0x58, 0x4e, 0x32, 0xd2, 0xf7, 0x69, 0x48, 0x8f, 0x44, 0xda, 0x04, 0x9a, 0x01, 0xce, ++ 0xf2, 0x40, 0xc8, 0xe4, 0xee, 0x98, 0x81, 0x9e, 0x3d, 0xa1, 0x83, 0x98, 0xa5, 0x3d, 0x0f, 0x0e, ++ 0x9a, 0xbf, 0x97, 0xa0, 0x96, 0xfe, 0x41, 0x90, 0xa7, 0xc5, 0x36, 0xba, 0xfc, 0xf9, 0xe6, 0x5f, ++ 0x93, 0xcc, 0x66, 0xb2, 0x5e, 0xfb, 0x51, 0xde, 0x6b, 0xff, 0x67, 0x72, 0xd2, 0x90, 0x33, 0xb0, ++ 0xb2, 0xb9, 0xc2, 0x6e, 0x4b, 0x63, 0xbb, 0x6d, 0x40, 0xbd, 0xef, 0x51, 0xb7, 0x4f, 0x83, 0x9e, ++ 0xfe, 0x3f, 0xd1, 0xa5, 0x98, 0x73, 0xa0, 0xef, 0xd1, 0x57, 0x66, 0x26, 0x05, 0x88, 0xee, 0x11, ++ 0xf3, 0x54, 0x84, 0x45, 0x31, 0x80, 0x37, 0x66, 0xa6, 0xf9, 0xe7, 0x14, 0xd4, 0x0b, 0x3f, 0x3d, ++ 0xba, 0x87, 0x0e, 0xe8, 0x30, 0x8d, 0x83, 0xcf, 0xba, 0x63, 0x93, 0x23, 0xf3, 0x2e, 0x49, 0x5e, ++ 0x53, 0x55, 0x39, 0xc2, 0x97, 0x02, 0xb9, 0x07, 0x20, 0x47, 0x6e, 0x48, 0xbd, 0x63, 0x96, 0xc8, ++ 0x57, 0x1c, 0x4b, 0x8e, 0xda, 0x66, 0x82, 0xdc, 0x01, 0x4b, 0x8e, 0x5c, 0x26, 0xa5, 0x90, 0x51, ++ 0x52, 0xfb, 0x9a, 0x1c, 0xbd, 0xc0, 0x71, 0xc2, 0xed, 0x49, 0xa1, 0x7b, 0x81, 0xe4, 0x0c, 0x2c, ++ 0x39, 0x7a, 0x6e, 0x26, 0x74, 0x54, 0x95, 0x46, 0x35, 0xad, 0x67, 0x55, 0xe5, 0x51, 0x55, 0x1e, ++ 0xd5, 0xb4, 0x9e, 0x96, 0x2a, 0x46, 0x55, 0x59, 0x54, 0xd3, 0x7d, 0xd6, 0x54, 0x21, 0xaa, 0xca, ++ 0xa3, 0x5a, 0x29, 0x37, 0x89, 0xda, 0xfc, 0x16, 0xac, 0xec, 0x27, 0xee, 0x8a, 0xdf, 0xb5, 0x4b, ++ 0x7f, 0xa0, 0xba, 0x33, 0xf8, 0x83, 0xff, 0xcd, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x8a, 0x90, ++ 0xfa, 0xea, 0x3f, 0x10, 0x00, 0x00, + } +diff --git a/vendor/github.com/containerd/cgroups/subsystem.go b/vendor/github.com/containerd/cgroups/subsystem.go +index 23de04d..237fc4c 100644 +--- a/vendor/github.com/containerd/cgroups/subsystem.go ++++ b/vendor/github.com/containerd/cgroups/subsystem.go +@@ -39,6 +39,7 @@ const ( + Memory Name = "memory" + Blkio Name = "blkio" + Rdma Name = "rdma" ++ Files Name = "files" + ) + + // Subsystems returns a complete list of the default cgroups +@@ -57,6 +58,7 @@ func Subsystems() []Name { + Memory, + Blkio, + Rdma, ++ Files, + } + if !isUserNS { + n = append(n, Devices) +diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go +index 82dbe2d..820b516 100644 +--- a/vendor/github.com/containerd/cgroups/utils.go ++++ b/vendor/github.com/containerd/cgroups/utils.go +@@ -69,7 +69,6 @@ func defaults(root string) ([]Subsystem, error) { + return nil, err + } + s := []Subsystem{ +- NewNamed(root, "systemd"), + NewFreezer(root), + NewPids(root), + NewNetCls(root), +@@ -81,6 +80,7 @@ func defaults(root string) ([]Subsystem, error) { + NewMemory(root), + NewBlkio(root), + NewRdma(root), ++ NewFiles(root), + } + // only add the devices cgroup if we are not in a user namespace + // because modifications are not allowed +diff --git a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +index 27268f9..7a1acf2 100644 +--- a/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go ++++ b/vendor/github.com/opencontainers/runtime-spec/specs-go/config.go +@@ -332,6 +332,10 @@ type LinuxRdma struct { + HcaObjects *uint32 `json:"hcaObjects,omitempty"` + } + ++type Files struct { ++ Limit *uint64 `json:"limit,omitempty"` ++} ++ + // LinuxResources has container runtime resource constraints + type LinuxResources struct { + // Devices configures the device whitelist. +@@ -352,6 +356,8 @@ type LinuxResources struct { + // Limits are a set of key value pairs that define RDMA resource limits, + // where the key is device name and value is resource limits. + Rdma map[string]LinuxRdma `json:"rdma,omitempty"` ++ // Files resource restriction configuration. ++ Files *Files `json:"files,omitempty"` + } + + // LinuxDevice represents the mknod information for a Linux special device file +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index e8c5a7b..1b6b04d 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -11,6 +11,7 @@ import ( + "context" + "encoding/json" + "fmt" ++ "io/ioutil" + "os" + "path/filepath" + "strconv" +@@ -35,6 +36,12 @@ const ( + cgroupKataPath = "/kata/" + vcpuCgroupName = "vcpu" + emulatorCgroupName = "emulator" ++ fileCgroupName = "files" ++ ++ defaultMinFilesLimit uint64 = 1024 ++ defaultMaxContainers uint64 = 200 ++ ++ procFileMaxPath = "/proc/sys/fs/file-max" + + // BlkioThrottleReadBps is the key to fetch throttle_read_bps + BlkioThrottleReadBps = "throttle_read_bps" +@@ -414,6 +421,81 @@ func isInSlice(i int, s []int) bool { + return false + } + ++// supportFileCgroup check current host OS support files cgroup or not ++func supportFileCgroup() bool { ++ root, err := cgroupV1MountPoint() ++ if err != nil { ++ return false ++ } ++ ++ filesCgroupPath := root + "/" + fileCgroupName ++ _, err = os.Stat(filesCgroupPath) ++ if err != nil { ++ return false ++ } ++ ++ return true ++} ++ ++func (s *Sandbox) filesResource(fileMaxPath string) *specs.Files { ++ var filesLimit uint64 = 0 ++ var err error ++ ++ // get fileslimit from sandbox container spec ++ sandboxContainerSpec := s.GetPatchedOCISpec() ++ if sandboxContainerSpec == nil || sandboxContainerSpec.Linux == nil { ++ return nil ++ } ++ ++ var configFl uint64 = 0 ++ sandboxResources := sandboxContainerSpec.Linux.Resources ++ if sandboxResources != nil && sandboxResources.Files != nil && sandboxResources.Files.Limit != nil { ++ configFl = *sandboxResources.Files.Limit ++ } ++ ++ if configFl > 0 { ++ if configFl < defaultMinFilesLimit { ++ logrus.Warnf("At least %d fds are needed to support vm running", defaultMinFilesLimit) ++ filesLimit = defaultMinFilesLimit ++ } else { ++ filesLimit = configFl ++ } ++ } else { ++ filesLimit, err = getDefFilesLimitFromHost(fileMaxPath) ++ if err != nil { ++ // do nothing but print the warning log ++ logrus.Errorf("get default files limit failed : %#v", err) ++ return nil ++ } ++ } ++ ++ return &specs.Files{ ++ Limit: &filesLimit, ++ } ++} ++ ++// getDefFilesLimitFromHost read default file-max files limit value from host ++func getDefFilesLimitFromHost(fileMaxPath string) (uint64, error) { ++ data, err := ioutil.ReadFile(fileMaxPath) ++ if err != nil { ++ return 0, fmt.Errorf("read file-max failed : %#v", err) ++ } ++ ++ tmpFilesLimit, err := strconv.Atoi(strings.Replace(string(data), "\n", "", -1)) ++ if err != nil { ++ return 0, fmt.Errorf("convert file-max failed : %#v", err) ++ } ++ ++ var filesLimit uint64 ++ // Distribute the maximum value of each container evenly, and take 50% as the default value ++ filesLimit = (uint64(tmpFilesLimit) / defaultMaxContainers) / 2 ++ if filesLimit < defaultMinFilesLimit { ++ return 0, fmt.Errorf("file describe resource is shortage, require : %d, get : %d ", defaultMinFilesLimit, filesLimit) ++ } ++ ++ return filesLimit, nil ++} ++ + func (s *Sandbox) blockIOResource() *specs.LinuxBlockIO { + value, ok := s.config.Annotations[annotations.BlkioCgroupTypeKey] + if !ok { +diff --git a/virtcontainers/fileslimit/fileslimit.c b/virtcontainers/fileslimit/fileslimit.c +new file mode 100644 +index 0000000..0e9ae72 +--- /dev/null ++++ b/virtcontainers/fileslimit/fileslimit.c +@@ -0,0 +1,23 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: jiangpengfei ++// Create: 2019-06-12 ++ ++#define _GNU_SOURCE ++#include ++#include ++#include ++#include "fileslimit.h" ++ ++int setProcessFilesLimit(int pid, int soft, int hard){ ++ struct rlimit new; ++ new.rlim_cur = soft; ++ new.rlim_max = hard; ++ ++ if (prlimit(pid, RLIMIT_NOFILE, &new, NULL) == -1) { ++ return errno; ++ } ++ ++ return 0; ++} +diff --git a/virtcontainers/fileslimit/fileslimit.go b/virtcontainers/fileslimit/fileslimit.go +new file mode 100644 +index 0000000..a0d2e2a +--- /dev/null ++++ b/virtcontainers/fileslimit/fileslimit.go +@@ -0,0 +1,23 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: jiangpengfei ++// Create: 2019-06-12 ++ ++package fileslimit ++ ++/* ++#cgo CFLAGS: -Wall ++extern int setProcessFilesLimit(int pid, int soft, int hard); ++*/ ++import "C" ++import "fmt" ++ ++func SetProcessFilesLimit(pid, soft, hard int) error { ++ ret := C.setProcessFilesLimit(C.int(pid), C.int(soft), C.int(hard)) ++ if ret != 0 { ++ return fmt.Errorf("failed to set fileslimit of vm process ,errno:%d", ret) ++ } ++ ++ return nil ++} +diff --git a/virtcontainers/fileslimit/fileslimit.h b/virtcontainers/fileslimit/fileslimit.h +new file mode 100644 +index 0000000..4249d7a +--- /dev/null ++++ b/virtcontainers/fileslimit/fileslimit.h +@@ -0,0 +1,14 @@ ++// Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved. ++// SPDX-License-Identifier: Apache-2.0 ++// Description: common functions ++// Author: jiangpengfei ++// Create: 2019-06-12 ++ ++#ifndef FILES_LIMIT_H ++#define FILES_LIMIT_H ++ ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ ++#endif /* FILES_LIMIT_H */ +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 9284f99..de8652f 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -27,6 +27,7 @@ import ( + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + deviceManager "github.com/kata-containers/runtime/virtcontainers/device/manager" + exp "github.com/kata-containers/runtime/virtcontainers/experimental" ++ "github.com/kata-containers/runtime/virtcontainers/fileslimit" + "github.com/kata-containers/runtime/virtcontainers/persist" + persistapi "github.com/kata-containers/runtime/virtcontainers/persist/api" + "github.com/kata-containers/runtime/virtcontainers/pkg/annotations" +@@ -2400,13 +2401,14 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { + return fmt.Errorf("sandbox's cgroup %s doesn't exist", s.state.CgroupPath) + } + ++ hypervisorPids := s.hypervisor.getPids() ++ if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { ++ return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) ++ } ++ + // pull out qemu threads other than vcpu to the cgroup of "/emulator" + if s.config.HypervisorType == QemuHypervisor { + emulatorCgroupPath := filepath.Join(s.state.CgroupPath, emulatorCgroupName) +- hypervisorPids := s.hypervisor.getPids() +- if len(hypervisorPids) == 0 || hypervisorPids[0] == 0 { +- return fmt.Errorf("hypervisor pid: %v invalid", hypervisorPids) +- } + if err := pulloutQemuThread(s, hypervisorPids[0], emulatorCgroupPath); err != nil { + return err + } +@@ -2430,6 +2432,18 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { + } + + // limit files resource ++ filesResources := specs.LinuxResources{ ++ Files: s.filesResource(procFileMaxPath), ++ } ++ if supportFileCgroup() { ++ if err := applyResourceLimit(&filesResources, vcpuCgroupPath); err != nil { ++ return err ++ } ++ } else { ++ if err := applyFilelimit(hypervisorPids[0], &filesResources); err != nil { ++ return err ++ } ++ } + + return nil + } +@@ -2451,6 +2465,22 @@ func applyResourceLimit(resources *specs.LinuxResources, cgroupPath string) erro + return nil + } + ++// apply files limit by pr_limit ++func applyFilelimit(vmPid int, resources *specs.LinuxResources) error { ++ if resources == nil { ++ return nil ++ } ++ ++ filesLimit := *(resources.Files.Limit) ++ ++ err := fileslimit.SetProcessFilesLimit(vmPid, int(filesLimit), int(filesLimit)) ++ if err != nil { ++ return fmt.Errorf("set vm fileslimit failed.vmPid:%d, limit:%d, err:%v", vmPid, filesLimit, err.Error()) ++ } ++ ++ return nil ++} ++ + // GetPatchedOCISpec returns sandbox's OCI specification + // This OCI specification was patched when the sandbox was created + // by containerCapabilities(), SetEphemeralStorageType() and others +-- +1.8.3.1 + diff --git a/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch b/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..74f9bc66b240305bb900aba0c3917f9fb3020685 --- /dev/null +++ b/runtime/patches/0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch @@ -0,0 +1,69 @@ +From faffb26c307556e1d84399060d7aef1753e9f41a Mon Sep 17 00:00:00 2001 +From: jiangpengfei +Date: Mon, 21 Sep 2020 21:52:48 -0400 +Subject: [PATCH] runtime: fix sandboxRuntimeRootPath left problem + +reason: If pod_sandbox type container deleted before pod_container +type containers in the sandbox, which leads to kata-runtime delete +pod_container process create a new sandboxRuntimeRootPath in the +/run/vc/sbs/ dir. + +So, we fix this problem by check sandboxRuntimeRootPath is exist +before fetchSandbox function is called. + +Signed-off-by: jiangpengfei +--- + cli/delete.go | 2 +- + virtcontainers/sandbox.go | 18 ++++++++++++++++++ + 2 files changed, 19 insertions(+), 1 deletion(-) + +diff --git a/cli/delete.go b/cli/delete.go +index 871ac40d..09552b9a 100644 +--- a/cli/delete.go ++++ b/cli/delete.go +@@ -87,7 +87,7 @@ func delete(ctx context.Context, containerID string, force bool) error { + // container is deleted before pod_container type container, just return nil + // and let containerd delete container operations continue + if strings.Contains(err.Error(), "no such file or directory") { +- kataLog.Warnf("pod_sandbox deleted before pod_container: %v", err) ++ kataLog.Warnf("pod_sandbox container deleted before pod_container in the sandbox: %v", err) + return nil + } + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index de8652fc..9f5d3c08 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -729,6 +729,13 @@ func fetchSandbox(ctx context.Context, sandboxID string) (sandbox *Sandbox, err + + var config SandboxConfig + ++ // check sandbox runtime root path(for example: /run/vc/sbs/) ++ // exist or not before fetch sandbox config file ++ if !checkSandboxRuntimeRootPathExist(sandboxID) { ++ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID) ++ return nil, fmt.Errorf("sandbox %s does not exist, %s no such file or directory", sandboxID,sandboxRuntimeRootPath) ++ } ++ + // Try to load sandbox config from old store at first. + c, ctx, err := loadSandboxConfigFromOldStore(ctx, sandboxID) + if err != nil { +@@ -2707,3 +2714,14 @@ func updateStaticSandboxResources(sandboxConfig *SandboxConfig) error { + + return nil + } ++ ++// checkSandboxRuntimeRootPathExist check /run/vc/sbs// dir exist or not ++func checkSandboxRuntimeRootPathExist(sandboxID string) bool { ++ sandboxRuntimeRootPath := store.SandboxRuntimeRootPath(sandboxID) ++ _, err := os.Stat(sandboxRuntimeRootPath) ++ if os.IsNotExist(err) { ++ return false ++ } ++ ++ return true ++} +\ No newline at end of file +-- +2.11.0 + diff --git a/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch b/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch new file mode 100644 index 0000000000000000000000000000000000000000..e969f7d4aeb6844a1cf7d79809dad43f8e907bb7 --- /dev/null +++ b/runtime/patches/0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch @@ -0,0 +1,42 @@ +From e10fe6ecf2d895fe009a080bb34334ff63739f3e Mon Sep 17 00:00:00 2001 +From: LiangZhang +Date: Thu, 24 Sep 2020 14:38:04 +0800 +Subject: [PATCH] runtime: fix invalid cmdline when start sandbox stratovirt + +reason: + 1. MemorySize is in type of uint32, value larger than 4096 will trigger overflow. + 2. cannot disable seccomp now. + +Signed-off-by: LiangZhang +--- + virtcontainers/stratovirt.go | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go +index 7c156d5..6db8dc0 100644 +--- a/virtcontainers/stratovirt.go ++++ b/virtcontainers/stratovirt.go +@@ -146,7 +146,7 @@ func (s *stratovirt) startSandbox(timeout int) error { + + params = append(params, "-append "+s.getKernelCmdLine()) + params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) +- params = append(params, fmt.Sprintf("-m %d", s.config.MemorySize*1024*1024)) ++ params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024)) + params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) + + // add devices to cmdline +@@ -178,6 +178,11 @@ func (s *stratovirt) startSandbox(timeout int) error { + params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) + } + ++ // disable Seccomp ++ if s.sandbox.config.DisableGuestSeccomp { ++ params = append(params, "-disable-seccomp") ++ } ++ + s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) + + dir := filepath.Join(store.RunVMStoragePath(), s.id) +-- +1.8.3.1 + diff --git a/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch b/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch new file mode 100644 index 0000000000000000000000000000000000000000..89d473f198e231eee54c4135c6870f71e8083507 --- /dev/null +++ b/runtime/patches/0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch @@ -0,0 +1,94 @@ +From b2b11cc8fb2144c13f4f9138b4420248ea200fb6 Mon Sep 17 00:00:00 2001 +From: LiangZhang +Date: Sun, 27 Sep 2020 18:10:18 +0800 +Subject: [PATCH] runtime: fix cmd params of direct use stratovirt binary + +reason: when directly use stratovirt binary, not through bash, it will cause problem. + +Signed-off-by: LiangZhang +--- + virtcontainers/stratovirt.go | 28 ++++++++++++++-------------- + 1 file changed, 14 insertions(+), 14 deletions(-) + +diff --git a/virtcontainers/stratovirt.go b/virtcontainers/stratovirt.go +index 6db8dc0..020135e 100644 +--- a/virtcontainers/stratovirt.go ++++ b/virtcontainers/stratovirt.go +@@ -133,21 +133,21 @@ func (s *stratovirt) startSandbox(timeout int) error { + defer span.Finish() + + var params []string +- params = append(params, "-name "+fmt.Sprintf("sandbox-%s", s.id)) +- params = append(params, "-api-channel unix:"+s.socketPath) ++ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) ++ params = append(params, "-api-channel", fmt.Sprintf("unix:%s", s.socketPath)) + + if kernelPath, err := s.config.KernelAssetPath(); err == nil { +- params = append(params, "-kernel "+kernelPath) ++ params = append(params, "-kernel", kernelPath) + } + + if initrdPath, err := s.config.InitrdAssetPath(); err == nil { +- params = append(params, "-initrd "+initrdPath) ++ params = append(params, "-initrd", initrdPath) + } + +- params = append(params, "-append "+s.getKernelCmdLine()) +- params = append(params, fmt.Sprintf("-smp %d", s.config.NumVCPUs)) +- params = append(params, fmt.Sprintf("-m %d", uint64(s.config.MemorySize)*1024*1024)) +- params = append(params, fmt.Sprintf("-chardev id=charconsole0,path=%s", s.consolePath)) ++ params = append(params, "-append", s.getKernelCmdLine()) ++ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) ++ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize)*1024*1024)) ++ params = append(params, "-chardev", fmt.Sprintf("id=charconsole0,path=%s", s.consolePath)) + + // add devices to cmdline + for _, d := range s.devices { +@@ -156,14 +156,14 @@ func (s *stratovirt) startSandbox(timeout int) error { + name := v.Name() + mac := v.HardwareAddr() + tapName := v.NetworkPair().TapInterface.TAPIface.Name +- params = append(params, fmt.Sprintf("-net id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) ++ params = append(params, "-net", fmt.Sprintf("id=%s,mac=%s,host_dev_name=%s", name, mac, tapName)) + case config.BlockDrive: + id := v.ID + path := v.File +- params = append(params, fmt.Sprintf("-drive id=%s,file=%s", id, path)) ++ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) + case types.VSock: + v.VhostFd.Close() +- params = append(params, fmt.Sprintf("-device vsock,id=vsock-id,guest-cid=%d", v.ContextID)) ++ params = append(params, "-device", fmt.Sprintf("vsock,id=vsock-id,guest-cid=%d", v.ContextID)) + default: + s.Logger().Error("Adding device type is unsupported") + } +@@ -175,7 +175,7 @@ func (s *stratovirt) startSandbox(timeout int) error { + // append logfile only on debug + if s.config.Debug { + dir := filepath.Join(store.RunVMStoragePath(), s.id) +- params = append(params, fmt.Sprintf("-D %s/stratovirt.log", dir)) ++ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) + } + + // disable Seccomp +@@ -183,8 +183,6 @@ func (s *stratovirt) startSandbox(timeout int) error { + params = append(params, "-disable-seccomp") + } + +- s.Logger().Info("StratoVirt start with params: ", strings.Join(params, " ")) +- + dir := filepath.Join(store.RunVMStoragePath(), s.id) + err := os.MkdirAll(dir, store.DirMode) + if err != nil { +@@ -205,6 +203,8 @@ func (s *stratovirt) startSandbox(timeout int) error { + } + + cmd := exec.CommandContext(s.ctx, binPath, params...) ++ s.Logger().Info("StratoVirt start with params: ", cmd) ++ + if err := cmd.Run(); err != nil { + s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() + return err +-- +1.8.3.1 + diff --git a/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch b/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch new file mode 100644 index 0000000000000000000000000000000000000000..f61d224021ea605073f88bca20aed6a765c59948 --- /dev/null +++ b/runtime/patches/0061-kata-runtime-retry-inserting-of-CNI-interface.patch @@ -0,0 +1,223 @@ +From babe7b3028d601a9b00aedda673e6a472f29ad07 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Mon, 28 Sep 2020 21:01:00 +0800 +Subject: [PATCH] kata-runtime: retry inserting of CNI interface + +reason: when netmon is enable, a interface is inserted into the +netns created by the sandbox, it should wait for the generating of +IP of the new interface, and get the ipv4 address + +Signed-off-by: yangfeiyu +--- + netmon/netmon.go | 128 +++++++++++++++++++++++++++++++++++++++++------ + 1 file changed, 112 insertions(+), 16 deletions(-) + +diff --git a/netmon/netmon.go b/netmon/netmon.go +index 94305ce2..57beacfb 100644 +--- a/netmon/netmon.go ++++ b/netmon/netmon.go +@@ -52,7 +52,7 @@ var ( + version = "unknown" + + // For simplicity the code will only focus on IPv4 addresses for now. +- netlinkFamily = netlink.FAMILY_ALL ++ netlinkFamily = netlink.FAMILY_V4 + + storageParentPath = "/var/run/kata-containers/netmon/sbs" + ) +@@ -70,7 +70,9 @@ type netmon struct { + storagePath string + sharedFile string + +- netIfaces map[int]vcTypes.Interface ++ netIfaces map[int]vcTypes.Interface ++ plugedIfaces map[string]vcTypes.Interface ++ pendingRoutes map[string][]vcTypes.Route + + linkUpdateCh chan netlink.LinkUpdate + linkDoneCh chan struct{} +@@ -148,15 +150,17 @@ func newNetmon(params netmonParams) (*netmon, error) { + } + + n := &netmon{ +- netmonParams: params, +- storagePath: filepath.Join(storageParentPath, params.sandboxID), +- sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), +- netIfaces: make(map[int]vcTypes.Interface), +- linkUpdateCh: make(chan netlink.LinkUpdate), +- linkDoneCh: make(chan struct{}), +- rtUpdateCh: make(chan netlink.RouteUpdate), +- rtDoneCh: make(chan struct{}), +- netHandler: handler, ++ netmonParams: params, ++ storagePath: filepath.Join(storageParentPath, params.sandboxID), ++ sharedFile: filepath.Join(storageParentPath, params.sandboxID, sharedFile), ++ netIfaces: make(map[int]vcTypes.Interface), ++ plugedIfaces: make(map[string]vcTypes.Interface), ++ pendingRoutes: make(map[string][]vcTypes.Route), ++ linkUpdateCh: make(chan netlink.LinkUpdate), ++ linkDoneCh: make(chan struct{}), ++ rtUpdateCh: make(chan netlink.RouteUpdate), ++ rtDoneCh: make(chan struct{}), ++ netHandler: handler, + } + + if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil { +@@ -266,7 +270,6 @@ func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []net + } + + var ipAddrs []*vcTypes.IPAddress +- + for _, addr := range addrs { + if addr.IPNet == nil { + continue +@@ -450,10 +453,24 @@ func (n *netmon) updateRoutes() error { + if err != nil { + return err + } ++ if len(netlinkRoutes) == 0 { ++ n.logger().Debug("get 0 routes") ++ return nil ++ } + + // Translate them into Route structures. + routes := convertRoutes(netlinkRoutes) + ++ // if the device of the routes have not be hotplug to guest, ++ // the update operation will be failed, pending them. ++ // For all routes are belong the same device, so we just need ++ // judge the device of the first route ++ if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged { ++ n.pendingRoutes[routes[0].Device] = routes ++ n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes) ++ return nil ++ } ++ + // Update the routes through the Kata CLI. + return n.updateRoutesCLI(routes) + } +@@ -489,6 +506,9 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { + return nil + } + ++ // the link is usually not ready, and `sleep 3` is an empirical value. ++ time.Sleep(3 * time.Second) ++ + // Check if the interface exist in the internal list. + if _, exist := n.netIfaces[int(ev.Index)]; exist { + n.logger().Debugf("Ignoring interface %s because already exist", +@@ -504,10 +524,26 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { + return nil + } + +- // Get the list of IP addresses associated with this interface. +- addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily) +- if err != nil { +- return err ++ // In some scenarios, the ip have not prepared,so we should do some retries. ++ var addrs []netlink.Addr ++ var err error ++ for i := 0; i < 5; i++ { ++ // Get the list of IP addresses associated with this interface. ++ addrs, err = n.netHandler.AddrList(ev.Link, netlinkFamily) ++ if err != nil { ++ return err ++ } ++ if len(addrs) > 0 { ++ break ++ } ++ time.Sleep(500 * time.Millisecond) ++ } ++ ++ // In some scenarios, the link reported by event can not found, do extras check here. ++ if n.checkLinkByHw(linkAttrs.HardwareAddr.String()) != true { ++ n.logger().Infof("Ignore %v because can not find link by HW %s", ++ linkAttrs.Name, linkAttrs.HardwareAddr.String()) ++ return nil + } + + // Convert the interfaces in the appropriate structure format. +@@ -520,6 +556,17 @@ func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error { + + // Add the interface to the internal list. + n.netIfaces[linkAttrs.Index] = iface ++ n.plugedIfaces[iface.Name] = iface ++ ++ // The pending routes is preferentially selected. ++ if routes, ok := n.pendingRoutes[iface.Name]; ok { ++ n.logger().Infof("dev %s find pending routes:%v", iface.Name, routes) ++ err = n.updateRoutesCLI(routes) ++ if err != nil { ++ return err ++ } ++ delete(n.pendingRoutes, iface.Name) ++ } + + // Complete by updating the routes. + return n.updateRoutes() +@@ -556,6 +603,8 @@ func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error { + + // Delete the interface from the internal list. + delete(n.netIfaces, linkAttrs.Index) ++ delete(n.plugedIfaces, iface.Name) ++ delete(n.pendingRoutes, iface.Name) + + // Complete by updating the routes. + return n.updateRoutes() +@@ -640,6 +689,53 @@ func (n *netmon) handleEvents() (err error) { + } + } + ++func linkByHwAddr(netHandle *netlink.Handle, hwAddr string) (netlink.Link, error) { ++ if netHandle == nil { ++ return nil, fmt.Errorf("no handler ") ++ } ++ ++ links, err := netHandle.LinkList() ++ if err != nil { ++ return nil, err ++ } ++ ++ for _, link := range links { ++ if link == nil { ++ continue ++ } ++ ++ lAttrs := link.Attrs() ++ if lAttrs == nil { ++ continue ++ } ++ ++ if lAttrs.HardwareAddr == nil { ++ continue ++ } ++ ++ if lAttrs.HardwareAddr.String() == hwAddr { ++ return link, nil ++ } ++ } ++ ++ return nil, fmt.Errorf("could not find the link corresponding to HwAddr %s", hwAddr) ++} ++ ++func (n *netmon) checkLinkByHw(hw string) bool { ++ netHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE) ++ if err != nil { ++ return false ++ } ++ defer netHandle.Delete() ++ ++ link, err := linkByHwAddr(netHandle, hw) ++ if err != nil || link == nil { ++ return false ++ } ++ ++ return true ++} ++ + func main() { + // Parse parameters. + params := parseOptions() +-- +2.23.0 + diff --git a/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch b/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch new file mode 100644 index 0000000000000000000000000000000000000000..d62395cabe46cd0af60fd4b01b355a84fcd70031 --- /dev/null +++ b/runtime/patches/0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch @@ -0,0 +1,43 @@ +From f0d2f8a19956045b4b53ac5f2c4b59940016ca41 Mon Sep 17 00:00:00 2001 +From: yangfeiyu +Date: Fri, 9 Oct 2020 16:02:27 +0800 +Subject: [PATCH] kata-runtime: support using CNI plugin to insert mutiple + network interfaces at the same time + +reason: support using CNI plugin to insert mutiple network interfaces at the same time + +Signed-off-by: yangfeiyu +--- + netmon/netmon.go | 16 +++++++++++----- + 1 file changed, 11 insertions(+), 5 deletions(-) + +diff --git a/netmon/netmon.go b/netmon/netmon.go +index 57beacfb..a519e5ba 100644 +--- a/netmon/netmon.go ++++ b/netmon/netmon.go +@@ -463,11 +463,17 @@ func (n *netmon) updateRoutes() error { + + // if the device of the routes have not be hotplug to guest, + // the update operation will be failed, pending them. +- // For all routes are belong the same device, so we just need +- // judge the device of the first route +- if _, pluged := n.plugedIfaces[routes[0].Device]; !pluged { +- n.pendingRoutes[routes[0].Device] = routes +- n.logger().Infof("dev %s have not been added, pending:%v", routes[0].Device, routes) ++ var pendingFlag bool ++ for _, route := range routes{ ++ if _, pluged := n.plugedIfaces[route.Device]; !pluged { ++ pendingFlag = true ++ n.pendingRoutes[route.Device] = append(n.pendingRoutes[route.Device],route) ++ n.logger().Infof("dev %s have not been added, pending:%v", route.Device, n.pendingRoutes[route.Device]) ++ } ++ } ++ ++ // find pending route ++ if pendingFlag { + return nil + } + +-- +2.23.0 + diff --git a/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch b/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch new file mode 100644 index 0000000000000000000000000000000000000000..bb2dfe03c44c0a81a6ab39f0cddd0355281f548c --- /dev/null +++ b/runtime/patches/0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch @@ -0,0 +1,90 @@ +From bac206ee5b4ccb90fd8c06c0b6244ba163ac82b9 Mon Sep 17 00:00:00 2001 +From: holyfei +Date: Sun, 15 Nov 2020 21:31:49 +0800 +Subject: [PATCH] kata-runtime: fix get sandbox cpu resources problem + +reason: If sandox_cgroup_with_emulator config is enabled, +kata-runtime should get cpu resources by cpuResourcesWithEmulator +func instead of the original cpuResource func. + +Signed-off-by: holyfei +--- + virtcontainers/cgroups.go | 2 ++ + virtcontainers/sandbox.go | 34 +++++++++++++++++++++++++++++++++- + 2 files changed, 35 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/cgroups.go b/virtcontainers/cgroups.go +index 1b6b04da..21708ebf 100644 +--- a/virtcontainers/cgroups.go ++++ b/virtcontainers/cgroups.go +@@ -40,6 +40,8 @@ const ( + + defaultMinFilesLimit uint64 = 1024 + defaultMaxContainers uint64 = 200 ++ defaultCPUPeriod uint64 = 1000000 ++ defaultCPUShares uint64 = 1024 + + procFileMaxPath = "/proc/sys/fs/file-max" + +diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go +index 9f5d3c08..d3c64b4f 100644 +--- a/virtcontainers/sandbox.go ++++ b/virtcontainers/sandbox.go +@@ -14,6 +14,7 @@ import ( + "net" + "os" + "path/filepath" ++ "strconv" + "strings" + "sync" + "syscall" +@@ -2351,6 +2352,37 @@ func (s *Sandbox) cpuResources() *specs.LinuxCPU { + return validCPUResources(cpu) + } + ++func (s *Sandbox) cpuResourcesWithEmulator() *specs.LinuxCPU { ++ // Use default period and quota if they are not specified. ++ // Container will inherit the constraints from its parent. ++ quota := int64(0) ++ period := uint64(0) ++ shares := uint64(0) ++ ++ cpu := &specs.LinuxCPU{ ++ Quota: "a, ++ Period: &period, ++ Shares: &shares, ++ } ++ ++ period = defaultCPUPeriod ++ ++ cpuValue, exist := s.config.Annotations[annotations.StaticCPUTypeKey] ++ if exist { ++ // If sandbox_cpu is set, we have validate at first, so we don't need to check it again ++ cpuNum, _ := strconv.ParseFloat(cpuValue, 64) ++ quota = int64(cpuNum * float64(period)) ++ shares = uint64(cpuNum * float64(defaultCPUShares)) ++ } else { ++ // If sandbox_cpu is not set, we use the hypervisor's cpu number as cpu limit ++ hypervisorConfig := s.hypervisor.hypervisorConfig() ++ quota = int64(uint64(hypervisorConfig.NumVCPUs) * period) ++ shares = uint64(hypervisorConfig.NumVCPUs) * defaultCPUShares ++ } ++ ++ return cpu ++} ++ + // setupSandboxCgroup creates and joins sandbox cgroups for the sandbox config + func (s *Sandbox) setupSandboxCgroup() error { + var err error +@@ -2424,7 +2456,7 @@ func (s *Sandbox) setupHostCgroupsWithEmulator() error { + // limit cpu to "/vcpu" + vcpuCgroupPath := filepath.Join(s.state.CgroupPath, vcpuCgroupName) + vcpuResources := specs.LinuxResources{ +- CPU: s.cpuResources(), ++ CPU: s.cpuResourcesWithEmulator(), + } + if err := applyResourceLimit(&vcpuResources, vcpuCgroupPath); err != nil { + return err +-- +2.11.0 + diff --git a/runtime/runtime-1.11.1.tar.gz b/runtime/runtime-1.11.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..b28623804379e1546efff89b603827662cd883a3 Binary files /dev/null and b/runtime/runtime-1.11.1.tar.gz differ diff --git a/runtime/series.conf b/runtime/series.conf new file mode 100644 index 0000000000000000000000000000000000000000..37b1b5f0cf1dcc29c0ca685509da28f8c3e662ec --- /dev/null +++ b/runtime/series.conf @@ -0,0 +1,63 @@ +0001-qmp-fix-kata-runtime-hungs-when-qemu-process-is-D-T-.patch +0002-kata-runtime-fix-kata-runtime-skip-read-lines-in-pro.patch +0003-kata-runtime-fix-kata-proxy-process-left-problem.patch +0004-kata-runtime-keep-the-process-name-of-qemu-same-as-c.patch +0005-cgroups-increase-delete-cgroup-retry-times.patch +0006-kata-runtime-fix-umount-container-rootfs-dir-return-.patch +0007-kata-runtime-enhance-reliability-when-kata-related-p.patch +0008-kata-runtime-fix-kata-runtime-resource-left-problem.patch +0009-kata-runtime-add-kata-runtime-global-flag-debug.patch +0010-kata-runtime-fix-kata-shim-pid-reused-problem.patch +0011-kata-runtime-check-the-process-info-before-send-SIGK.patch +0012-kata-runtime-truncate-the-log.json-file-before-kata-.patch +0013-kata-runtime-get-container-info-by-containerID-prefi.patch +0014-kata-runtime-add-self-defined-annotations-framework.patch +0015-kata-runtime-add-reuse-hypervisor-cpu-and-memory-fea.patch +0016-virtcontainers-fix-hotplug-huge-size-memory-cause-ag.patch +0017-kata-runtime-validate-sandbox-cpu-and-memory-size.patch +0018-sandbox-Stop-and-clean-up-containers-that-fail-to-cr.patch +0019-virtcontainers-fix-delete-sandbox-failed-problem.patch +0020-virtcontainers-add-enable_cpu_memory_hotplug-config-.patch +0021-kata-runtime-add-sandbox_cpu-and-sandbox_mem-annotat.patch +0022-kata-runtime-skip-go-version-check-and-do-not-build-.patch +0023-kata-runtime-set-PCIBridgeMaxCapacity-limit-to-25.patch +0024-kata-runtime-support-hotplug-tap-interface-into-kata.patch +0025-network-keep-list-ifaces-result-compatible-with-cni.patch +0026-network-add-enable_compat_old_cni-config.patch +0027-network-add-more-strict-check-for-input-network-inte.patch +0028-network-add-kata-network-add-route-subcommand.patch +0029-network-add-kata-network-del-route-subcommand.patch +0030-network-kata-network-list-routes-support-display-com.patch +0031-device_mangaer-check-VFIO-when-create-device.patch +0032-network-add-more-detail-usage-for-update-routes-subc.patch +0033-network-do-not-delete-the-exist-tap-device-in-the-ho.patch +0034-kata-runtime-add-kata-ipvs-command.patch +0035-device-mount-blockdevices-in-the-guest-VM.patch +0036-mount-limit-the-maximum-number-of-virtio-scsi-bus-sl.patch +0037-runtime-add-IPVS-test.patch +0038-pcie-using-pcie-root-port-driver-to-hotplug-device-i.patch +0039-storage-add-storage-common-functions-and-structs.patch +0040-storage-add-go-tests-for-storage.patch +0041-storage-mount-nfs-and-gpath-with-given-annotation.patch +0042-kata-runtime-do-not-ignore-updateInterface-return-er.patch +0043-kata-runtime-support-add-hypervisor-global-parameter.patch +0044-network-support-dpdk-vhost_user-net-device.patch +0045-network-support-set-dns.patch +0046-network-support-multiqueue-when-inserting-interface-.patch +0047-network-add-support-to-get-network-stats-of-vm-throu.patch +0048-console-fix-the-file-resource-leak.patch +0049-container-fix-the-write-operation-transparently-tran.patch +0050-runtime-add-kata-network-upate-iface-subcommand.patch +0051-network-fix-del-iface-doesn-t-delete-the-tap-interfa.patch +0052-runtime-add-support-of-new-sandbox-StratoVirt.patch +0053-kata-runtime-add-interface-for-host-cgroup.patch +0054-kata-runtime-add-sandbox-cgroup-with-vcpu-and-emulat.patch +0055-kata_runtime-support-host-cgroup-with-emulator-polic.patch +0056-kata_runtime-support-the-blkio-in-host-cgroups.patch +0057-kata-runtime-support-files-limit-in-host-cgroups.patch +0058-runtime-fix-sandboxRuntimeRootPath-left-problem.patch +0059-runtime-fix-invalid-cmdline-when-start-sandbox-stratovirt.patch +0060-runtime-fix-cmd-params-of-direct-use-stratovirt-bina.patch +0061-kata-runtime-retry-inserting-of-CNI-interface.patch +0062-kata-runtime-support-using-CNI-plugin-to-insert-muti.patch +0063-kata-runtime-fix-get-sandbox-cpu-resources-problem.patch diff --git a/shim/apply-patches b/shim/apply-patches new file mode 100755 index 0000000000000000000000000000000000000000..5ac8bba4acea6fec8d4747d8f2c28cec69455ab5 --- /dev/null +++ b/shim/apply-patches @@ -0,0 +1,21 @@ +#!/bin/bash + +if [[ -f ./patch_flag ]];then + echo "shim patched!" + exit 0 +fi + +tar -zxvf shim-*.tar.gz +cp -fr ./shim-*/* ./ +rm -rf ./shim-* +cat ./series.conf | while read line +do + if [[ $line == '' || $line =~ ^\s*# ]]; then + continue + fi + echo "====patch $line======" + pwd + patch -p1 -F1 -s < ./patches/$line +done + +touch ./patch_flag diff --git a/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch b/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch new file mode 100644 index 0000000000000000000000000000000000000000..b75e0645a792a29a7fc3644f2b7d356715cf2b06 --- /dev/null +++ b/shim/patches/0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch @@ -0,0 +1,38 @@ +From f5d031ef7d3437a05fe41f80aaf0e59ac54a575b Mon Sep 17 00:00:00 2001 +From: jiangpengfei9 +Date: Mon, 1 Apr 2019 12:35:59 -0400 +Subject: [PATCH 2/2] kata-shim: fix kata-shim process wait long time + to exit problem + +reason: If containerd-shim process is killed and call kata-runtime delete -f command +to delete container, it will wait 10s for kata-shim process exit in the waitForShim +function, however 10s wait time is too long in the case of containerd-shim and containerd +process is killed.So when containerd-shim process is killed, kata-shim process will +receive a SIGHUP signal, we can handle this signal as exit signal which makes kata-shim +process exit. + +Signed-off-by: jiangpengfei9 +--- + shim.go | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/shim.go b/shim.go +index a93a00d..90172fe 100644 +--- a/shim.go ++++ b/shim.go +@@ -134,6 +134,12 @@ func (s *shim) handleSignals(ctx context.Context, tty *os.File) chan os.Signal { + backtrace() + } + ++ // if containerd-shim process is killed, kata-shim will receive SIGHUP signal ++ if sysSig == syscall.SIGHUP { ++ logger().WithField("signal", sig).Warn("received SIGHUP signal") ++ os.Exit(1) ++ } ++ + // forward this signal to container + _, err := s.agent.SignalProcess(s.ctx, &pb.SignalProcessRequest{ + ContainerId: s.containerID, +-- +1.8.3.1 + diff --git a/shim/series.conf b/shim/series.conf new file mode 100644 index 0000000000000000000000000000000000000000..ce1ab400e6d85580734751282687de0cbfbb2b41 --- /dev/null +++ b/shim/series.conf @@ -0,0 +1 @@ +0001-kata-shim-fix-kata-shim-process-wait-long-tim.patch diff --git a/shim/shim-1.11.1.tar.gz b/shim/shim-1.11.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..8a6d21bc55fe41fb8ebe8bbc1c39419040134a10 Binary files /dev/null and b/shim/shim-1.11.1.tar.gz differ