cri, sandbox: pass sandbox resource details if available, applicable

egernst · egernst · commit 20419feaac19 · 2021-12-13T08:41:41.000-08:00
CRI API has been updated to include a an optional `resources` field in the
LinuxPodSandboxConfig field, as part of the RunPodSandbox request.

Having sandbox level resource details at sandbox creation time will have
large benefits for sandboxed runtimes. In the case of Kata Containers,
for example, this'll allow for better support of SW/HW architectures
which don't allow for CPU/memory hotplug, and it'll allow for better
queue sizing for virtio devices associated with the sandbox (in the VM
case).

If this sandbox resource information is provided as part of the run
sandbox request, let's introduce a pattern where we will update the
pause container's runtiem spec to include this information in the
annotations field.

Signed-off-by: Eric Ernst &lt;eric_ernst@apple.com&gt;
diff --git a/pkg/cri/annotations/annotations.go b/pkg/cri/annotations/annotations.go
@@ -32,6 +32,16 @@ const (
 	// SandboxID is the sandbox ID annotation
 	SandboxID = "io.kubernetes.cri.sandbox-id"
 
+	// SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the
+	// sum of container CPU resources, optionally provided by Kubelet (introduced  in 1.23) as part of the PodSandboxConfig
+	SandboxCPUPeriod = "io.kubernetes.cri.sandbox-cpu-period"
+	SandboxCPUQuota  = "io.kubernetes.cri.sandbox-cpu-quota"
+	SandboxCPUShares = "io.kubernetes.cri.sandbox-cpu-shares"
+
+	// SandboxMemory is the initial amount of memory associated with this sandbox. This is calculated as the sum
+	// of container memory, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig.
+	SandboxMem = "io.kubernetes.cri.sandbox-memory"
+
 	// SandboxLogDir is the pod log directory annotation.
 	// If the sandbox needs to generate any log, it will put it into this directory.
 	// Kubelet will be responsible for:
diff --git a/pkg/cri/server/sandbox_run_linux.go b/pkg/cri/server/sandbox_run_linux.go
@@ -19,6 +19,7 @@ package server
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/containerd/containerd"
@@ -155,6 +156,15 @@ func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxC
 	if !c.config.DisableCgroup {
 		specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
 	}
+
+	if res := config.GetLinux().GetResources(); res != nil {
+		specOpts = append(specOpts,
+			customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)),
+			customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)),
+			customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)),
+			customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10)))
+	}
+
 	specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
 
 	for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
diff --git a/pkg/cri/server/sandbox_run_linux_test.go b/pkg/cri/server/sandbox_run_linux_test.go
@@ -19,6 +19,7 @@ package server
 import (
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 
 	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -27,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+	v1 "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"github.com/containerd/containerd/pkg/cri/annotations"
 	"github.com/containerd/containerd/pkg/cri/opts"
@@ -173,6 +175,65 @@ func TestLinuxSandboxContainerSpec(t *testing.T) {
 				assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "1 1000")
 			},
 		},
+		"sandbox sizing annotations should be set if LinuxContainerResources were provided": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Linux.Resources = &v1.LinuxContainerResources{
+					CpuPeriod:          100,
+					CpuQuota:           200,
+					CpuShares:          5000,
+					MemoryLimitInBytes: 1024,
+				}
+			},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				value, ok := spec.Annotations[annotations.SandboxCPUPeriod]
+				assert.True(t, ok)
+				assert.EqualValues(t, strconv.FormatInt(100, 10), value)
+				assert.EqualValues(t, "100", value)
+
+				value, ok = spec.Annotations[annotations.SandboxCPUQuota]
+				assert.True(t, ok)
+				assert.EqualValues(t, "200", value)
+
+				value, ok = spec.Annotations[annotations.SandboxCPUShares]
+				assert.True(t, ok)
+				assert.EqualValues(t, "5000", value)
+
+				value, ok = spec.Annotations[annotations.SandboxMem]
+				assert.True(t, ok)
+				assert.EqualValues(t, "1024", value)
+			},
+		},
+		"sandbox sizing annotations should not be set if LinuxContainerResources were not provided": {
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				_, ok := spec.Annotations[annotations.SandboxCPUPeriod]
+				assert.False(t, ok)
+				_, ok = spec.Annotations[annotations.SandboxCPUQuota]
+				assert.False(t, ok)
+				_, ok = spec.Annotations[annotations.SandboxCPUShares]
+				assert.False(t, ok)
+				_, ok = spec.Annotations[annotations.SandboxMem]
+				assert.False(t, ok)
+			},
+		},
+		"sandbox sizing annotations are zero if the resources are set to 0": {
+			configChange: func(c *runtime.PodSandboxConfig) {
+				c.Linux.Resources = &v1.LinuxContainerResources{}
+			},
+			specCheck: func(t *testing.T, spec *runtimespec.Spec) {
+				value, ok := spec.Annotations[annotations.SandboxCPUPeriod]
+				assert.True(t, ok)
+				assert.EqualValues(t, "0", value)
+				value, ok = spec.Annotations[annotations.SandboxCPUQuota]
+				assert.True(t, ok)
+				assert.EqualValues(t, "0", value)
+				value, ok = spec.Annotations[annotations.SandboxCPUShares]
+				assert.True(t, ok)
+				assert.EqualValues(t, "0", value)
+				value, ok = spec.Annotations[annotations.SandboxMem]
+				assert.True(t, ok)
+				assert.EqualValues(t, "0", value)
+			},
+		},
 	} {
 		t.Logf("TestCase %q", desc)
 		c := newTestCRIService()
