diff --git a/cadvisor-0.57.0-crio-add-retry-to-containerinfo-call-b0ad403.patch b/cadvisor-0.57.0-crio-add-retry-to-containerinfo-call-b0ad403.patch new file mode 100644 index 0000000000000000000000000000000000000000..925ddb7088bf7e7d1ce5b2a9d327c189b13d9eaa --- /dev/null +++ b/cadvisor-0.57.0-crio-add-retry-to-containerinfo-call-b0ad403.patch @@ -0,0 +1,120 @@ +From b0ad403d2f3c6e1e8e498725a5c705cb5048112b Mon Sep 17 00:00:00 2001 +From: Chandan Maurya +Date: Tue, 23 Jun 2026 11:52:41 +0530 +Subject: [PATCH] crio: add retry to ContainerInfo call + +When cadvisor detects a new container cgroup via inotify, the container +may not yet be fully registered in CRI-O. This race condition causes +ContainerInfo to return HTTP 404, which cadvisor treats as a fatal error +and logs "Failed to process watch event". + +This is the same race condition that the containerd handler already +handles with a retry+backoff loop around TaskPid (see +lib/container/containerd/handler.go). + +Apply the same pattern for CRI-O: +- Add an ErrContainerNotFound sentinel error in client.go +- Wrap HTTP 404 responses from CRI-O with the sentinel +- Retry ContainerInfo up to 5 times with 100ms exponential backoff + in handler.go + +This is particularly impactful in environments using GPG-signed +container images, where image signature verification delays container +registration in CRI-O, widening the race window. + +Supersedes #3842 +--- + container/crio/client.go | 9 +++++++++ + container/crio/handler.go | 28 +++++++++++++++++++++++++--- + 2 files changed, 34 insertions(+), 3 deletions(-) +diff --git a/container/crio/client.go b/container/crio/client.go +index 8e271d1..c1a7299 100644 +--- a/container/crio/client.go ++++ b/container/crio/client.go +@@ -17,6 +17,7 @@ package crio + import ( + "context" + "encoding/json" ++ "errors" + "flag" + "fmt" + "io" +@@ -29,6 +30,11 @@ import ( + + var crioClientTimeout = flag.Duration("crio_client_timeout", time.Duration(0), "CRI-O client timeout. Default is no timeout.") + ++// ErrContainerNotFound indicates that CRI-O does not yet know about the ++// requested container. This is expected during a short window after the ++// container cgroup is created but before CRI-O finishes registration. ++var ErrContainerNotFound = errors.New("container not found") ++ + const ( + CrioSocket = "/var/run/crio/crio.sock" + maxUnixSocketPathSize = len(syscall.RawSockaddrUnix{}.Path) +@@ -152,6 +158,9 @@ func (c *crioClientImpl) ContainerInfo(id string) (*ContainerInfo, error) { + if err != nil { + return nil, fmt.Errorf("error finding container %s: status %d", id, resp.StatusCode) + } ++ if resp.StatusCode == http.StatusNotFound { ++ return nil, fmt.Errorf("error finding container %s: %s: %w", id, string(respBody), ErrContainerNotFound) ++ } + return nil, fmt.Errorf("error finding container %s: status %d returned error %s", id, resp.StatusCode, string(respBody)) + } + +diff --git a/container/crio/handler.go b/container/crio/handler.go +index 31349a0..47dcad7 100644 +--- a/container/crio/handler.go ++++ b/container/crio/handler.go +@@ -18,10 +18,12 @@ + package crio + + import ( ++ "errors" + "fmt" + "path/filepath" + "strconv" + "strings" ++ "time" + + "github.com/opencontainers/cgroups" + +@@ -30,6 +32,8 @@ import ( + containerlibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" ++ ++ "k8s.io/klog/v2" + ) + + type crioContainerHandler struct { +@@ -108,9 +112,27 @@ func newCrioContainerHandler( + id := ContainerNameToCrioId(name) + pidKnown := true + +- cInfo, err := client.ContainerInfo(id) +- if err != nil { +- return nil, err ++ // Cgroup is created during container setup. When cadvisor sees the cgroup ++ // via inotify, the container may not be fully registered in CRI-O yet. ++ // Use retry+backoff to tolerate the race condition, mirroring the ++ // containerd handler's approach for the same issue. ++ var cInfo *ContainerInfo ++ backoff := 100 * time.Millisecond ++ retry := 5 ++ for { ++ cInfo, err = client.ContainerInfo(id) ++ if err == nil && cInfo != nil { ++ break ++ } ++ ++ if !errors.Is(err, ErrContainerNotFound) || retry == 0 { ++ return nil, err ++ } ++ ++ klog.V(4).Infof("Container %s not yet registered in CRI-O, retrying (%d retries left): %v", id, retry, err) ++ retry-- ++ time.Sleep(backoff) ++ backoff *= 2 + } + if cInfo.Pid == 0 { + // If pid is not known yet, network related stats can not be retrieved by the diff --git a/cadvisor.spec b/cadvisor.spec index 3aa15244d384825877aed0676784ea514d753dce..85d363a91559d00995a8fedcd05d4685bb6d7097 100644 --- a/cadvisor.spec +++ b/cadvisor.spec @@ -3,7 +3,7 @@ Summary: Analyzes resource usage and performance characteristics of running containers Name: cadvisor Version: 0.57.0 -Release: 2%{?dist} +Release: 3%{?dist} License: Apache-2.0 URL: https://github.com/google/cadvisor Source0: https://github.com/google/cadvisor/archive/refs/tags/v%{version}.tar.gz #/%{name}-%{version}.tar.gz @@ -12,6 +12,7 @@ Source0: https://github.com/google/cadvisor/archive/refs/tags/v%{version} Source1: mod.tar.gz Source2: cadvisor Source3: cadvisor.service +Patch0001: cadvisor-0.57.0-crio-add-retry-to-containerinfo-call-b0ad403.patch Patch5000: use-local-vendor.patch BuildRequires: golang systemd glibc-static git @@ -62,6 +63,10 @@ install -D -m 0755 %{SOURCE3} %{buildroot}%{_unitdir}/%{name}.service %changelog +* Mon Jul 20 2026 PkgAgent Robot - 0.57.0-3 +- [Type] security +- [DESC] Fix CRI-O ContainerInfo race condition with retry+backoff on container not found + * Thu Jun 11 2026 hudson zhu - 0.57.0-2 - [Type] bugfix - [DESC] fix version unknown