koordlet: unset cfs_quota for LSR/LSE pods (#662)

Signed-off-by: saintube <saintube@foxmail.com>

pkg/koordlet/metricsadvisor/collector.go

@@ -300,7 +300,7 @@ func (c *collector) collectContainerResUsed(meta *statesinformer.PodMeta) {
 				klog.V(6).Infof("failed to collect non-running container usage for %s/%s/%s, "+
 					"CPU err: %s, Memory err: %s", pod.Namespace, pod.Name, containerStat.Name, err0, err1)
 			} else {
-				klog.Warningf("failed to collect container usage for %s/%s/%s, CPU err: %s, Memory err: %s",
+				klog.V(4).Infof("failed to collect container usage for %s/%s/%s, CPU err: %s, Memory err: %s",
 					pod.Namespace, pod.Name, containerStat.Name, err0, err1)
 			}
 			continue

pkg/koordlet/resmanager/cgroup_reconcile.go

@@ -193,8 +193,8 @@ func (m *CgroupResourcesReconcile) calculatePodAndContainerResources(podMeta *st
 	for _, container := range pod.Spec.Containers {
 		_, containerStatus, err := util.FindContainerIdAndStatusByName(&pod.Status, container.Name)
 		if err != nil {
-			klog.Warningf("failed to find containerStatus, pod %s, container %s, err: %v", util.GetPodKey(pod),
-				container.Name, err)
+			klog.V(4).Infof("failed to find containerStatus, pod %s, container %s, err: %v",
+				util.GetPodKey(pod), container.Name, err)
 			continue
 		}
 		containerDir, err := util.GetContainerCgroupPathWithKube(podMeta.CgroupDir, containerStatus)

pkg/koordlet/resmanager/cpu_burst.go

@@ -194,9 +194,8 @@ func (b *CPUBurst) start() {
 			klog.Warningf("podMeta is illegal, detail %v", podMeta)
 			continue
 		}
-		podQOS := apiext.GetPodQoSClass(podMeta.Pod)
-		if podQOS == apiext.QoSLSR || podQOS == apiext.QoSBE {
-			// ignore LSR and BE pod
+		if !util.IsPodCPUBurstable(podMeta.Pod) {
+			// ignore non-burstable pod, e.g. LSR, BE pods
 			continue
 		}
 		// merge burst config from pod and node

pkg/koordlet/runtimehooks/hooks/cpuset/cpuset.go

@@ -23,13 +23,13 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 
-	ext "github.com/koordinator-sh/koordinator/apis/extension"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/hooks"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/protocol"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/reconciler"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/runtimehooks/rule"
 	"github.com/koordinator-sh/koordinator/pkg/koordlet/statesinformer"
 	rmconfig "github.com/koordinator-sh/koordinator/pkg/runtimeproxy/config"
+	"github.com/koordinator-sh/koordinator/pkg/util"
 	sysutil "github.com/koordinator-sh/koordinator/pkg/util/system"
 )
 
@@ -47,11 +47,18 @@ func (p *cpusetPlugin) Register() {
 	klog.V(5).Infof("register hook %v", name)
 	hooks.Register(rmconfig.PreCreateContainer, name, description, p.SetContainerCPUSet)
 	hooks.Register(rmconfig.PreUpdateContainerResources, name, description, p.SetContainerCPUSet)
+	hooks.Register(rmconfig.PreRunPodSandbox, name, "unset pod cpu quota", UnsetPodCPUQuota)
+	hooks.Register(rmconfig.PreCreateContainer, name, "unset container cpu quota", UnsetContainerCPUQuota)
+	hooks.Register(rmconfig.PreUpdateContainerResources, name, "unset container cpu quota", UnsetContainerCPUQuota)
 	rule.Register(name, description,
 		rule.WithParseFunc(statesinformer.RegisterTypeNodeTopology, p.parseRule),
 		rule.WithUpdateCallback(p.ruleUpdateCb))
 	reconciler.RegisterCgroupReconciler(reconciler.ContainerLevel, sysutil.CPUSet, p.SetContainerCPUSet,
 		"set container cpuset")
+	reconciler.RegisterCgroupReconciler(reconciler.PodLevel, sysutil.CPUCFSQuota, UnsetPodCPUQuota,
+		"unset pod cpu quota")
+	reconciler.RegisterCgroupReconciler(reconciler.ContainerLevel, sysutil.CPUCFSQuota, UnsetContainerCPUQuota,
+		"unset container cpu quota")
 }
 
 var singleton *cpusetPlugin
@@ -72,7 +79,7 @@ func (p *cpusetPlugin) SetContainerCPUSet(proto protocol.HooksProtocol) error {
 	containerReq := containerCtx.Request
 
 	// cpuset from pod annotation (LSE, LSR)
-	if cpusetVal, err := getCPUSetFromPod(containerReq.PodAnnotations); err != nil {
+	if cpusetVal, err := util.GetCPUSetFromPod(containerReq.PodAnnotations); err != nil {
 		return err
 	} else if cpusetVal != "" {
 		containerCtx.Response.Resources.CPUSet = pointer.StringPtr(cpusetVal)
@@ -93,10 +100,44 @@ func (p *cpusetPlugin) SetContainerCPUSet(proto protocol.HooksProtocol) error {
 	return nil
 }
 
-func getCPUSetFromPod(podAnnotations map[string]string) (string, error) {
-	podAlloc, err := ext.GetResourceStatus(podAnnotations)
-	if err != nil {
-		return "", err
+func UnsetPodCPUQuota(proto protocol.HooksProtocol) error {
+	podCtx := proto.(*protocol.PodContext)
+	if podCtx == nil {
+		return fmt.Errorf("pod protocol is nil for plugin %v", name)
+	}
+	req := podCtx.Request
+
+	// cpuset from pod annotation (LSE, LSR)
+	// NOTE: unset cfs quota for cpuset pods to avoid unexpected throttles.
+	// https://github.com/koordinator-sh/koordinator/issues/489
+	if needUnset, err := util.IsPodCfsQuotaNeedUnset(req.Annotations); err != nil {
+		return err
+	} else if needUnset {
+		podCtx.Response.Resources.CFSQuota = pointer.Int64Ptr(-1)
+		return nil
+	}
+
+	// do nothing for cpushare pod
+	return nil
+}
+
+func UnsetContainerCPUQuota(proto protocol.HooksProtocol) error {
+	containerCtx := proto.(*protocol.ContainerContext)
+	if containerCtx == nil {
+		return fmt.Errorf("container protocol is nil for plugin %v", name)
 	}
-	return podAlloc.CPUSet, nil
+	containerReq := containerCtx.Request
+
+	// cpuset from pod annotation (LSE, LSR)
+	// NOTE: unset cfs quota for cpuset pods to avoid unexpected throttles.
+	// https://github.com/koordinator-sh/koordinator/issues/489
+	if needUnset, err := util.IsPodCfsQuotaNeedUnset(containerReq.PodAnnotations); err != nil {
+		return err
+	} else if needUnset {
+		containerCtx.Response.Resources.CFSQuota = pointer.Int64Ptr(-1)
+		return nil
+	}
+
+	// do nothing for cpushare pod
+	return nil
 }

pkg/koordlet/runtimehooks/hooks/cpuset/cpuset_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cpuset
 
 import (
+	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -36,6 +37,14 @@ func getCPUSet(dirWithKube string, helper *system.FileTestUtil) string {
 	return helper.ReadCgroupFileContents(dirWithKube, system.CPUSet)
 }
 
+func initCPUQuota(dirWithKube string, value string, helper *system.FileTestUtil) {
+	helper.WriteCgroupFileContents(dirWithKube, system.CPUCFSQuota, value)
+}
+
+func getCPUQuota(dirWithKube string, helper *system.FileTestUtil) string {
+	return helper.ReadCgroupFileContents(dirWithKube, system.CPUCFSQuota)
+}
+
 func Test_cpusetPlugin_SetContainerCPUSet(t *testing.T) {
 	type fields struct {
 		rule *cpusetRule
@@ -225,42 +234,250 @@ func Test_cpusetPlugin_SetContainerCPUSet(t *testing.T) {
 	}
 }
 
-func Test_getCPUSetFromPod(t *testing.T) {
+func TestUnsetPodCPUQuota(t *testing.T) {
 	type args struct {
-		podAnnotations map[string]string
-		podAlloc       *ext.ResourceStatus
+		podAlloc *ext.ResourceStatus
+		proto    protocol.HooksProtocol
 	}
 	tests := []struct {
-		name    string
-		args    args
-		want    string
-		wantErr bool
+		name         string
+		args         args
+		wantErr      bool
+		wantCPUQuota *int64
 	}{
 		{
-			name: "get cpuset from annotation",
+			name: "not change cfs quota with nil protocol",
+			args: args{
+				proto: nil,
+			},
+			wantErr:      true,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "not change cfs quota by bad pod allocated format",
+			args: args{
+				proto: &protocol.PodContext{
+					Request: protocol.PodRequest{
+						Labels: map[string]string{
+							ext.LabelPodQoS: string(ext.QoSLS),
+						},
+						Annotations: map[string]string{
+							ext.AnnotationResourceStatus: "bad-format",
+						},
+						CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+					},
+					Response: protocol.PodResponse{},
+				},
+			},
+			wantErr:      true,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "set cfs quota by pod allocated",
 			args: args{
-				podAnnotations: map[string]string{},
 				podAlloc: &ext.ResourceStatus{
 					CPUSet: "2-4",
 				},
+				proto: &protocol.PodContext{
+					Request: protocol.PodRequest{
+						Labels: map[string]string{
+							ext.LabelPodQoS: string(ext.QoSLS),
+						},
+						CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+					},
+					Response: protocol.PodResponse{},
+				},
+			},
+			wantErr:      false,
+			wantCPUQuota: pointer.Int64Ptr(-1),
+		},
+		{
+			name: "not change cfs quota by pod allocated share pool",
+			args: args{
+				podAlloc: &ext.ResourceStatus{
+					CPUSharedPools: []ext.CPUSharedPool{
+						{
+							Socket: 0,
+							Node:   0,
+						},
+					},
+				},
+				proto: &protocol.PodContext{
+					Request: protocol.PodRequest{
+						Labels: map[string]string{
+							ext.LabelPodQoS: string(ext.QoSLS),
+						},
+						CgroupParent: "kubepods/pod-guaranteed-test-uid/",
+					},
+					Response: protocol.PodResponse{},
+				},
 			},
-			want:    "2-4",
-			wantErr: false,
+			wantErr:      false,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "not change cfs quota for origin besteffort pod",
+			args: args{
+				proto: &protocol.PodContext{
+					Request: protocol.PodRequest{
+						CgroupParent: "kubepods/besteffort/pod-besteffort-test-uid/",
+					},
+					Response: protocol.PodResponse{},
+				},
+			},
+			wantErr:      false,
+			wantCPUQuota: nil,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if tt.args.podAlloc != nil {
-				podAllocJson := util.DumpJSON(tt.args.podAlloc)
-				tt.args.podAnnotations[ext.AnnotationResourceStatus] = podAllocJson
+			testHelper := system.NewFileTestUtil(t)
+			var podCtx *protocol.PodContext
+
+			if tt.args.proto != nil {
+				podCtx = tt.args.proto.(*protocol.PodContext)
+				initCPUQuota(podCtx.Request.CgroupParent, "", testHelper)
+				if tt.args.podAlloc != nil {
+					podAllocJson := util.DumpJSON(tt.args.podAlloc)
+					podCtx.Request.Annotations = map[string]string{
+						ext.AnnotationResourceStatus: podAllocJson,
+					}
+				}
 			}
-			got, err := getCPUSetFromPod(tt.args.podAnnotations)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("getCPUSetFromPod() error = %v, wantErr %v", err, tt.wantErr)
+
+			err := UnsetPodCPUQuota(podCtx)
+			assert.Equal(t, err != nil, tt.wantErr)
+
+			if podCtx == nil {
+				return
+			}
+			if tt.wantCPUQuota == nil {
+				assert.Nil(t, podCtx.Response.Resources.CFSQuota, "cfs quota value should be nil")
+			} else {
+				podCtx.ReconcilerDone()
+				assert.Equal(t, *tt.wantCPUQuota, *podCtx.Response.Resources.CFSQuota, "pod cfs quota should be equal")
+				gotCPUQuota := getCPUQuota(podCtx.Request.CgroupParent, testHelper)
+				gotCPUQuotaStr, err := strconv.ParseInt(gotCPUQuota, 10, 64)
+				assert.NoError(t, err)
+				assert.Equal(t, *tt.wantCPUQuota, gotCPUQuotaStr, "pod cfs quota should be equal")
+			}
+		})
+	}
+}
+
+func TestUnsetContainerCPUQuota(t *testing.T) {
+	type args struct {
+		podAlloc *ext.ResourceStatus
+		proto    protocol.HooksProtocol
+	}
+	tests := []struct {
+		name         string
+		args         args
+		wantErr      bool
+		wantCPUQuota *int64
+	}{
+		{
+			name: "not change cfs quota with nil protocol",
+			args: args{
+				proto: nil,
+			},
+			wantErr:      true,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "not change cfs quota by bad pod allocated format",
+			args: args{
+				proto: &protocol.ContainerContext{
+					Request: protocol.ContainerRequest{
+						CgroupParent: "kubepods/test-pod/test-container/",
+						PodAnnotations: map[string]string{
+							ext.AnnotationResourceStatus: "bad-format",
+						},
+					},
+				},
+			},
+			wantErr:      true,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "set cfs quota by pod allocated",
+			args: args{
+				podAlloc: &ext.ResourceStatus{
+					CPUSet: "2-4",
+				},
+				proto: &protocol.ContainerContext{
+					Request: protocol.ContainerRequest{
+						CgroupParent: "kubepods/test-pod/test-container/",
+					},
+				},
+			},
+			wantErr:      false,
+			wantCPUQuota: pointer.Int64Ptr(-1),
+		},
+		{
+			name: "not change cfs quota by pod allocated share pool",
+			args: args{
+				podAlloc: &ext.ResourceStatus{
+					CPUSharedPools: []ext.CPUSharedPool{
+						{
+							Socket: 0,
+							Node:   0,
+						},
+					},
+				},
+				proto: &protocol.ContainerContext{
+					Request: protocol.ContainerRequest{
+						CgroupParent: "kubepods/test-pod/test-container/",
+					},
+				},
+			},
+			wantErr:      false,
+			wantCPUQuota: nil,
+		},
+		{
+			name: "not change cfs quota for origin besteffort pod",
+			args: args{
+				proto: &protocol.ContainerContext{
+					Request: protocol.ContainerRequest{
+						CgroupParent: "kubepods/besteffort/test-pod/test-container/",
+					},
+				},
+			},
+			wantErr:      false,
+			wantCPUQuota: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testHelper := system.NewFileTestUtil(t)
+			var containerCtx *protocol.ContainerContext
+
+			if tt.args.proto != nil {
+				containerCtx = tt.args.proto.(*protocol.ContainerContext)
+				initCPUQuota(containerCtx.Request.CgroupParent, "", testHelper)
+				if tt.args.podAlloc != nil {
+					podAllocJson := util.DumpJSON(tt.args.podAlloc)
+					containerCtx.Request.PodAnnotations = map[string]string{
+						ext.AnnotationResourceStatus: podAllocJson,
+					}
+				}
+			}
+
+			err := UnsetContainerCPUQuota(containerCtx)
+			assert.Equal(t, err != nil, tt.wantErr)
+
+			if containerCtx == nil {
 				return
 			}
-			if got != tt.want {
-				t.Errorf("getCPUSetFromPod() got = %v, want %v", got, tt.want)
+			if tt.wantCPUQuota == nil {
+				assert.Nil(t, containerCtx.Response.Resources.CFSQuota, "cfs quota value should be nil")
+			} else {
+				containerCtx.ReconcilerDone()
+				assert.Equal(t, *tt.wantCPUQuota, *containerCtx.Response.Resources.CFSQuota, "container cfs quota should be equal")
+				gotCPUQuota := getCPUQuota(containerCtx.Request.CgroupParent, testHelper)
+				gotCPUQuotaStr, err := strconv.ParseInt(gotCPUQuota, 10, 64)
+				assert.NoError(t, err)
+				assert.Equal(t, *tt.wantCPUQuota, gotCPUQuotaStr, "container cfs quota should be equal")
 			}
 		})
 	}

pkg/koordlet/runtimehooks/protocol/container_context.go

@@ -121,7 +121,7 @@ func (c *ContainerContext) injectForOrigin() {
 	if c.Response.Resources.CPUSet != nil {
 		if err := injectCPUSet(c.Request.CgroupParent, *c.Response.Resources.CPUSet); err != nil {
 			klog.Infof("set container %v/%v/%v cpuset %v on cgroup parent %v failed, error %v", c.Request.PodMeta.Namespace,
-				c.Request.PodMeta.Name, *c.Response.Resources.CPUSet, c.Request.CgroupParent, err)
+				c.Request.PodMeta.Name, c.Request.ContainerMeta.Name, *c.Response.Resources.CPUSet, c.Request.CgroupParent, err)
 		} else {
 			klog.V(5).Infof("set container %v/%v/%v cpuset %v on cgroup parent %v",
 				c.Request.PodMeta.Namespace, c.Request.PodMeta.Name, c.Request.ContainerMeta.Name,
@@ -130,6 +130,18 @@ func (c *ContainerContext) injectForOrigin() {
 				"set container cpuset to %v", *c.Response.Resources.CPUSet).Do()
 		}
 	}
+	if c.Response.Resources.CFSQuota != nil {
+		if err := injectCPUQuota(c.Request.CgroupParent, *c.Response.Resources.CFSQuota); err != nil {
+			klog.Infof("set container %v/%v/%v cfs quota %v on cgroup parent %v failed, error %v", c.Request.PodMeta.Namespace,
+				c.Request.PodMeta.Name, c.Request.ContainerMeta.Name, *c.Response.Resources.CFSQuota, c.Request.CgroupParent, err)
+		} else {
+			klog.V(5).Infof("set container %v/%v/%v cfs quota %v on cgroup parent %v",
+				c.Request.PodMeta.Namespace, c.Request.PodMeta.Name, c.Request.ContainerMeta.Name,
+				*c.Response.Resources.CFSQuota, c.Request.CgroupParent)
+			audit.V(2).Container(c.Request.ContainerMeta.ID).Reason("runtime-hooks").Message(
+				"set container cfs quota to %v", *c.Response.Resources.CFSQuota).Do()
+		}
+	}
 	// TODO other fields
 }
 

pkg/koordlet/runtimehooks/protocol/pod_context.go

@@ -124,4 +124,16 @@ func (p *PodContext) injectForExt() {
 				"set pod bvt to %v", *p.Response.Resources.CPUBvt).Do()
 		}
 	}
+	// pod-level cfs_quota is manually updated since pod-stage hooks do not support it
+	if p.Response.Resources.CFSQuota != nil {
+		if err := injectCPUQuota(p.Request.CgroupParent, *p.Response.Resources.CFSQuota); err != nil {
+			klog.Infof("set pod %v/%v cfs quota %v on cgroup parent %v failed, error %v", p.Request.PodMeta.Namespace,
+				p.Request.PodMeta.Name, *p.Response.Resources.CFSQuota, p.Request.CgroupParent, err)
+		} else {
+			klog.V(5).Infof("set pod %v/%v/%v cfs quota %v on cgroup parent %v",
+				p.Request.PodMeta.Namespace, p.Request.PodMeta.Name, *p.Response.Resources.CFSQuota, p.Request.CgroupParent)
+			audit.V(2).Pod(p.Request.PodMeta.Namespace, p.Request.PodMeta.Name).Reason("runtime-hooks").Message(
+				"set pod cfs quota to %v", *p.Response.Resources.CFSQuota).Do()
+		}
+	}
 }

pkg/koordlet/runtimehooks/protocol/protocol.go

@@ -74,6 +74,14 @@ func injectCPUSet(cgroupParent string, cpuset string) error {
 	return nil
 }
 
+func injectCPUQuota(cgroupParent string, cpuQuota int64) error {
+	cpuQuotaStr := strconv.FormatInt(cpuQuota, 10)
+	if err := sysutil.CgroupFileWrite(cgroupParent, sysutil.CPUCFSQuota, cpuQuotaStr); err != nil {
+		return err
+	}
+	return nil
+}
+
 func injectCPUBvt(cgroupParent string, bvtValue int64) error {
 	bvtValueStr := strconv.FormatInt(bvtValue, 10)
 	if err := sysutil.CgroupFileWrite(cgroupParent, sysutil.CPUBVTWarpNs, bvtValueStr); err != nil {

pkg/runtimeproxy/resexecutor/cri/utils.go

@@ -74,7 +74,7 @@ func updateResource(a, b *v1alpha1.LinuxContainerResources) *v1alpha1.LinuxConta
 	if b.CpuPeriod > 0 {
 		a.CpuPeriod = b.CpuPeriod
 	}
-	if b.CpuQuota > 0 {
+	if b.CpuQuota != 0 { // -1 is valid
 		a.CpuQuota = b.CpuQuota
 	}
 	if b.CpuShares > 0 {
@@ -109,7 +109,7 @@ func updateResourceByUpdateContainerResourceRequest(a, b *v1alpha1.LinuxContaine
 	if b.CpuPeriod > 0 {
 		a.CpuPeriod = b.CpuPeriod
 	}
-	if b.CpuQuota > 0 {
+	if b.CpuQuota != 0 { // -1 is valid
 		a.CpuQuota = b.CpuQuota
 	}
 	if b.CpuShares > 0 {

pkg/util/pod.go

@@ -27,6 +27,7 @@ import (
 	quotav1 "k8s.io/apiserver/pkg/quota/v1"
 	"k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
 
+	apiext "github.com/koordinator-sh/koordinator/apis/extension"
 	slov1alpha1 "github.com/koordinator-sh/koordinator/apis/slo/v1alpha1"
 	"github.com/koordinator-sh/koordinator/pkg/util/system"
 )
@@ -276,3 +277,14 @@ func GetPodRequest(pod *corev1.Pod, resourceNames ...corev1.ResourceName) corev1
 func IsPodTerminated(pod *corev1.Pod) bool {
 	return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed
 }
+
+func GetCPUSetFromPod(podAnnotations map[string]string) (string, error) {
+	if podAnnotations == nil {
+		return "", nil
+	}
+	podAlloc, err := apiext.GetResourceStatus(podAnnotations)
+	if err != nil {
+		return "", err
+	}
+	return podAlloc.CPUSet, nil
+}

pkg/util/pod_qos_utils.go

+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	corev1 "k8s.io/api/core/v1"
+
+	apiext "github.com/koordinator-sh/koordinator/apis/extension"
+)
+
+// NOTE: functions in this file can be overwrite for extension
+
+// IsPodCfsQuotaNeedUnset checks if the pod-level and container-level cfs_quota should be unset to avoid unnecessary
+// throttles.
+// https://github.com/koordinator-sh/koordinator/issues/489
+func IsPodCfsQuotaNeedUnset(annotations map[string]string) (bool, error) {
+	cpusetVal, err := GetCPUSetFromPod(annotations)
+	if err != nil {
+		return false, err
+	}
+	return cpusetVal != "", nil
+}
+
+// IsPodCPUBurstable checks if cpu burst is allowed for the pod.
+func IsPodCPUBurstable(pod *corev1.Pod) bool {
+	qos := apiext.GetPodQoSClass(pod)
+	return qos != apiext.QoSLSR && qos != apiext.QoSLSE && qos != apiext.QoSBE
+}

pkg/util/pod_qos_utils_test.go

+/*
+Copyright 2022 The Koordinator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	apiext "github.com/koordinator-sh/koordinator/apis/extension"
+)
+
+func Test_IsPodCfsQuotaNeedUnset(t *testing.T) {
+	type args struct {
+		podAnnotations map[string]string
+		podAlloc       *apiext.ResourceStatus
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    bool
+		wantErr bool
+	}{
+		{
+			name: "need unset for cpuset pod",
+			args: args{
+				podAnnotations: map[string]string{},
+				podAlloc: &apiext.ResourceStatus{
+					CPUSet: "2-4",
+				},
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "no need to unset for cpushare pod",
+			args: args{
+				podAnnotations: map[string]string{},
+				podAlloc:       &apiext.ResourceStatus{},
+			},
+			want:    false,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.args.podAlloc != nil {
+				podAllocJson := DumpJSON(tt.args.podAlloc)
+				tt.args.podAnnotations[apiext.AnnotationResourceStatus] = podAllocJson
+			}
+			got, err := IsPodCfsQuotaNeedUnset(tt.args.podAnnotations)
+			assert.Equal(t, tt.wantErr, err != nil)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/util/pod_test.go

@@ -617,3 +617,39 @@ func Test_GetPodBEMemoryRequest(t *testing.T) {
 		})
 	}
 }
+
+func Test_GetCPUSetFromPod(t *testing.T) {
+	type args struct {
+		podAnnotations map[string]string
+		podAlloc       *apiext.ResourceStatus
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "get cpuset from annotation",
+			args: args{
+				podAnnotations: map[string]string{},
+				podAlloc: &apiext.ResourceStatus{
+					CPUSet: "2-4",
+				},
+			},
+			want:    "2-4",
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.args.podAlloc != nil {
+				podAllocJson := DumpJSON(tt.args.podAlloc)
+				tt.args.podAnnotations[apiext.AnnotationResourceStatus] = podAllocJson
+			}
+			got, err := GetCPUSetFromPod(tt.args.podAnnotations)
+			assert.Equal(t, tt.wantErr, err != nil)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}