diff --git a/README.md b/README.md index d95b56618..48158cb13 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ This extension controller supports the following Kubernetes versions: | Version | Support | Conformance test results | | --------------- | ----------- | ------------------------ | +| Kubernetes 1.31 | 1.31.0+ | N/A | | Kubernetes 1.30 | 1.30.0+ | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20AWS) | | Kubernetes 1.29 | 1.29.0+ | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20AWS) | | Kubernetes 1.28 | 1.28.0+ | [![Gardener v1.28 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.28%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.28%20AWS) | diff --git a/imagevector/images.yaml b/imagevector/images.yaml index b70004a0d..e18e79705 100644 --- a/imagevector/images.yaml +++ b/imagevector/images.yaml @@ -30,7 +30,7 @@ images: - name: cloud-controller-manager sourceRepository: github.com/kubernetes/cloud-provider-aws repository: registry.k8s.io/provider-aws/cloud-controller-manager - tag: "v1.26.11" + tag: "v1.26.12" targetVersion: "1.26.x" labels: - name: 'gardener.cloud/cve-categorisation' @@ -44,7 +44,7 @@ images: - name: cloud-controller-manager sourceRepository: github.com/kubernetes/cloud-provider-aws repository: registry.k8s.io/provider-aws/cloud-controller-manager - tag: "v1.27.6" + tag: "v1.27.9" targetVersion: "1.27.x" labels: - name: 'gardener.cloud/cve-categorisation' @@ -58,7 +58,7 @@ images: - name: cloud-controller-manager sourceRepository: github.com/kubernetes/cloud-provider-aws repository: registry.k8s.io/provider-aws/cloud-controller-manager - tag: "v1.28.5" + tag: "v1.28.9" targetVersion: "1.28.x" labels: - name: 'gardener.cloud/cve-categorisation' @@ -72,7 +72,7 @@ images: - name: cloud-controller-manager sourceRepository: github.com/kubernetes/cloud-provider-aws repository: registry.k8s.io/provider-aws/cloud-controller-manager - tag: "v1.29.2" + tag: "v1.29.6" targetVersion: "1.29.x" labels: - name: 'gardener.cloud/cve-categorisation' @@ -86,8 +86,22 @@ images: - name: cloud-controller-manager sourceRepository: github.com/kubernetes/cloud-provider-aws repository: registry.k8s.io/provider-aws/cloud-controller-manager - tag: "v1.30.0" - targetVersion: ">= 1.30" + tag: "v1.30.3" + targetVersion: "1.30.x" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'protected' + authentication_enforced: false + user_interaction: 'gardener-operator' + confidentiality_requirement: 'high' + integrity_requirement: 'high' + availability_requirement: 'low' +- name: cloud-controller-manager + sourceRepository: github.com/kubernetes/cloud-provider-aws + repository: registry.k8s.io/provider-aws/cloud-controller-manager + tag: "v1.31.1" + targetVersion: ">= 1.31" labels: - name: 'gardener.cloud/cve-categorisation' value: @@ -276,6 +290,35 @@ images: sourceRepository: github.com/gardener/ecr-credential-provider repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider tag: "v1.29.0" + targetVersion: "< 1.30" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'protected' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'high' + integrity_requirement: 'high' + availability_requirement: 'low' +- name: ecr-credential-provider + sourceRepository: github.com/gardener/ecr-credential-provider + repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider + tag: "v1.30.3" + targetVersion: "1.30.x" + labels: + - name: 'gardener.cloud/cve-categorisation' + value: + network_exposure: 'protected' + authentication_enforced: false + user_interaction: 'end-user' + confidentiality_requirement: 'high' + integrity_requirement: 'high' + availability_requirement: 'low' +- name: ecr-credential-provider + sourceRepository: github.com/gardener/ecr-credential-provider + repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider + tag: "v1.31.0" + targetVersion: ">= 1.31" labels: - name: 'gardener.cloud/cve-categorisation' value: diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index 1d8c2b0bf..4d8b9dd89 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -20,12 +20,14 @@ import ( extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" "github.com/gardener/gardener/pkg/component/nodemanagement/machinecontrollermanager" gutil "github.com/gardener/gardener/pkg/utils/gardener" + imagevectorutils "github.com/gardener/gardener/pkg/utils/imagevector" versionutils "github.com/gardener/gardener/pkg/utils/version" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" kubeletconfigv1 "k8s.io/kubelet/config/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" @@ -42,6 +44,20 @@ const ( ecrCredentialBinLocation = "/opt/bin/" ) +var ( + // constraintK8sLess131 is a version constraint for versions < 1.31. + // + // TODO(ialidzhikov): Replace with versionutils.ConstraintK8sLess131 when vendoring a gardener/gardener version + // that contains https://github.com/gardener/gardener/pull/10472. + constraintK8sLess131 *semver.Constraints +) + +func init() { + var err error + constraintK8sLess131, err = semver.NewConstraint("< 1.31-0") + utilruntime.Must(err) +} + // NewEnsurer creates a new controlplane ensurer. func NewEnsurer(logger logr.Logger, client client.Client) genericmutator.Ensurer { return &ensurer{ @@ -198,15 +214,19 @@ func ensureKubeAPIServerCommandLineArgs(c *corev1.Container, k8sVersion *semver. c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", "CSIMigrationAWS=true", ",") } + if constraintK8sLess131.Check(k8sVersion) { + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "InTreePluginAWSUnregister=true", ",") + } - c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", - "InTreePluginAWSUnregister=true", ",") c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-provider=") c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-config=") - c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=", - "PersistentVolumeLabel", ",") - c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=", - "PersistentVolumeLabel", ",") + if constraintK8sLess131.Check(k8sVersion) { + c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=", + "PersistentVolumeLabel", ",") + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=", + "PersistentVolumeLabel", ",") + } } func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container, k8sVersion *semver.Version) { @@ -218,9 +238,11 @@ func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container, k8sVersion c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", "CSIMigrationAWS=true", ",") } + if constraintK8sLess131.Check(k8sVersion) { + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "InTreePluginAWSUnregister=true", ",") + } - c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", - "InTreePluginAWSUnregister=true", ",") c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-config=") c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--external-cloud-volume-plugin=") } @@ -232,9 +254,10 @@ func ensureKubeSchedulerCommandLineArgs(c *corev1.Container, k8sVersion *semver. c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", "CSIMigrationAWS=true", ",") } - - c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", - "InTreePluginAWSUnregister=true", ",") + if constraintK8sLess131.Check(k8sVersion) { + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "InTreePluginAWSUnregister=true", ",") + } } func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, k8sVersion *semver.Version) { @@ -244,9 +267,10 @@ func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, k8sVersion *sem c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", "CSIMigrationAWS=true", ",") } - - c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", - "InTreePluginAWSUnregister=true", ",") + if constraintK8sLess131.Check(k8sVersion) { + c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=", + "InTreePluginAWSUnregister=true", ",") + } } func ensureKubeControllerManagerLabels(t *corev1.PodTemplateSpec) { @@ -386,21 +410,27 @@ func ensureKubeletECRProviderCommandLineArgs(command []string) []string { // EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements. func (e *ensurer) EnsureKubeletConfiguration(_ context.Context, _ gcontext.GardenContext, kubeletVersion *semver.Version, newObj, _ *kubeletconfigv1beta1.KubeletConfiguration) error { - if newObj.FeatureGates == nil { - newObj.FeatureGates = make(map[string]bool) - } - if versionutils.ConstraintK8sLess127.Check(kubeletVersion) { - newObj.FeatureGates["CSIMigration"] = true - newObj.FeatureGates["CSIMigrationAWS"] = true + setKubeletConfigurationFeatureGate(newObj, "CSIMigration", true) + setKubeletConfigurationFeatureGate(newObj, "CSIMigrationAWS", true) + } + if constraintK8sLess131.Check(kubeletVersion) { + setKubeletConfigurationFeatureGate(newObj, "InTreePluginAWSUnregister", true) } - newObj.FeatureGates["InTreePluginAWSUnregister"] = true newObj.EnableControllerAttachDetach = ptr.To(true) return nil } +func setKubeletConfigurationFeatureGate(kubeletConfiguration *kubeletconfigv1beta1.KubeletConfiguration, featureGate string, value bool) { + if kubeletConfiguration.FeatureGates == nil { + kubeletConfiguration.FeatureGates = make(map[string]bool) + } + + kubeletConfiguration.FeatureGates[featureGate] = value +} + var regexFindProperty = regexp.MustCompile("net.ipv4.neigh.default.gc_thresh1[[:space:]]*=[[:space:]]*([[:alnum:]]+)") // EnsureKubernetesGeneralConfiguration ensures that the kubernetes general configuration conforms to the provider requirements. @@ -451,8 +481,8 @@ ExecStart=/opt/bin/mtu-customizer.sh return nil } -func (e *ensurer) credentialProviderBinaryFile() (*extensionsv1alpha1.File, error) { - image, err := imagevector.ImageVector().FindImage(aws.ECRCredentialProviderImageName) +func (e *ensurer) credentialProviderBinaryFile(k8sVersion string) (*extensionsv1alpha1.File, error) { + image, err := imagevector.ImageVector().FindImage(aws.ECRCredentialProviderImageName, imagevectorutils.TargetVersion(k8sVersion)) if err != nil { return nil, err } @@ -560,7 +590,8 @@ func (e *ensurer) EnsureAdditionalFiles(ctx context.Context, gctx gcontext.Garde return err } - k8sGreaterEqual127, err := versionutils.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, ">=", "1.27") + k8sVersion := cluster.Shoot.Spec.Kubernetes.Version + k8sGreaterEqual127, err := versionutils.CompareVersions(k8sVersion, ">=", "1.27") if err != nil { return err } @@ -584,7 +615,7 @@ func (e *ensurer) EnsureAdditionalFiles(ctx context.Context, gctx gcontext.Garde } if ptr.Deref(infraConfig.EnableECRAccess, true) { - binConfig, err := e.credentialProviderBinaryFile() + binConfig, err := e.credentialProviderBinaryFile(k8sVersion) if err != nil { return err } diff --git a/pkg/webhook/controlplane/ensurer_test.go b/pkg/webhook/controlplane/ensurer_test.go index 48c51fa14..f023dc979 100644 --- a/pkg/webhook/controlplane/ensurer_test.go +++ b/pkg/webhook/controlplane/ensurer_test.go @@ -83,6 +83,20 @@ var _ = Describe("Ensurer", func() { }, }, ) + eContextK8s131 = gcontext.NewInternalGardenContext( + &extensionscontroller.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shoot--project--foo", + }, + Shoot: &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Kubernetes: gardencorev1beta1.Kubernetes{ + Version: "1.31.1", + }, + }, + }, + }, + ) infraConfig *v1alpha1.InfrastructureConfig infrastructure *extensionsv1alpha1.Infrastructure ) @@ -149,13 +163,20 @@ var _ = Describe("Ensurer", func() { checkKubeAPIServerDeployment(dep, "1.26.5") }) - It("should add missing elements to kube-apiserver deployment (k8s >= 1.27)", func() { + It("should add missing elements to kube-apiserver deployment (k8s >= 1.27, < 1.31)", func() { err := ensurer.EnsureKubeAPIServerDeployment(ctx, eContextK8s127, dep, nil) Expect(err).To(Not(HaveOccurred())) checkKubeAPIServerDeployment(dep, "1.27.1") }) + It("should add missing elements to kube-apiserver deployment (k8s >= 1.31)", func() { + err := ensurer.EnsureKubeAPIServerDeployment(ctx, eContextK8s131, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkKubeAPIServerDeployment(dep, "1.31.1") + }) + It("should modify existing elements of kube-apiserver deployment", func() { dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: v1beta1constants.DeploymentNameKubeAPIServer}, @@ -232,13 +253,20 @@ var _ = Describe("Ensurer", func() { checkKubeControllerManagerDeployment(dep, "1.26.5") }) - It("should add missing elements to kube-controller-manager deployment (k8s >= 1.27 w/ CSI annotation)", func() { + It("should add missing elements to kube-controller-manager deployment (k8s >= 1.27, < 1.31)", func() { err := ensurer.EnsureKubeControllerManagerDeployment(ctx, eContextK8s127, dep, nil) Expect(err).To(Not(HaveOccurred())) checkKubeControllerManagerDeployment(dep, "1.27.1") }) + It("should add missing elements to kube-controller-manager deployment (k8s >= 1.31)", func() { + err := ensurer.EnsureKubeControllerManagerDeployment(ctx, eContextK8s131, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkKubeControllerManagerDeployment(dep, "1.31.1") + }) + It("should modify existing elements of kube-controller-manager deployment", func() { var ( dep = &appsv1.Deployment{ @@ -310,12 +338,19 @@ var _ = Describe("Ensurer", func() { checkKubeSchedulerDeployment(dep, "1.26.5") }) - It("should add missing elements to kube-scheduler deployment (k8s >= 1.27)", func() { + It("should add missing elements to kube-scheduler deployment (k8s >= 1.27, < 1.31)", func() { err := ensurer.EnsureKubeSchedulerDeployment(ctx, eContextK8s127, dep, nil) Expect(err).To(Not(HaveOccurred())) checkKubeSchedulerDeployment(dep, "1.27.1") }) + + It("should add missing elements to kube-scheduler deployment (k8s >= 1.31)", func() { + err := ensurer.EnsureKubeSchedulerDeployment(ctx, eContextK8s131, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkKubeSchedulerDeployment(dep, "1.31.1") + }) }) Describe("#EnsureClusterAutoscalerDeployment", func() { @@ -343,17 +378,25 @@ var _ = Describe("Ensurer", func() { ensurer = NewEnsurer(logger, c) }) - It("should add missing elements to cluster-autoscaler deployment (>= 1.27)", func() { + It("should add missing elements to cluster-autoscaler deployment (< 1.27)", func() { + err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s126, dep, nil) + Expect(err).To(Not(HaveOccurred())) + + checkClusterAutoscalerDeployment(dep, "1.26.5") + }) + + It("should add missing elements to cluster-autoscaler deployment (>= 1.27, < 1.31)", func() { err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s127, dep, nil) Expect(err).To(Not(HaveOccurred())) checkClusterAutoscalerDeployment(dep, "1.27.5") }) - It("should add missing elements to cluster-autoscaler deployment (< 1.27)", func() { - err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s126, dep, nil) + + It("should add missing elements to cluster-autoscaler deployment (>= 1.31)", func() { + err := ensurer.EnsureClusterAutoscalerDeployment(ctx, eContextK8s131, dep, nil) Expect(err).To(Not(HaveOccurred())) - checkClusterAutoscalerDeployment(dep, "1.26.5") + checkClusterAutoscalerDeployment(dep, "1.31.1") }) }) @@ -672,7 +715,7 @@ done }) DescribeTable("should modify existing elements of kubelet configuration", - func(kubeletVersion *semver.Version) { + func(kubeletVersion *semver.Version, expectedFeatureGates map[string]bool) { newKubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{ FeatureGates: map[string]bool{ "Foo": true, @@ -681,19 +724,18 @@ done } kubeletConfig := *oldKubeletConfig - if version.ConstraintK8sLess127.Check(kubeletVersion) { - newKubeletConfig.FeatureGates["CSIMigration"] = true - newKubeletConfig.FeatureGates["CSIMigrationAWS"] = true + for featureGate, value := range expectedFeatureGates { + newKubeletConfig.FeatureGates[featureGate] = value } - newKubeletConfig.FeatureGates["InTreePluginAWSUnregister"] = true err := ensurer.EnsureKubeletConfiguration(ctx, nil, kubeletVersion, &kubeletConfig, nil) Expect(err).To(Not(HaveOccurred())) Expect(&kubeletConfig).To(Equal(newKubeletConfig)) }, - Entry("kubelet < 1.26", semver.MustParse("1.26.0")), - Entry("kubelet >= 1.27", semver.MustParse("1.27.1")), + Entry("kubelet < 1.27", semver.MustParse("1.26.0"), map[string]bool{"CSIMigration": true, "CSIMigrationAWS": true, "InTreePluginAWSUnregister": true}), + Entry("kubelet >= 1.27, < 1.31", semver.MustParse("1.27.1"), map[string]bool{"InTreePluginAWSUnregister": true}), + Entry("kubelet >= 1.31", semver.MustParse("1.31.1"), map[string]bool{}), ) }) @@ -800,21 +842,26 @@ done func checkKubeAPIServerDeployment(dep *appsv1.Deployment, k8sVersion string) { k8sVersionAtLeast127, _ := version.CompareVersions(k8sVersion, ">=", "1.27") + k8sVersionAtLeast131, _ := version.CompareVersions(k8sVersion, ">=", "1.31") // Check that the kube-apiserver container still exists and contains all needed command line args, // env vars, and volume mounts c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-apiserver") Expect(c).To(Not(BeNil())) - if k8sVersionAtLeast127 { + if k8sVersionAtLeast131 { + Expect(c.Command).NotTo(ContainElement(HavePrefix("--feature-gates"))) + } else if k8sVersionAtLeast127 { Expect(c.Command).To(ContainElement("--feature-gates=InTreePluginAWSUnregister=true")) - } else { + } else { // < 1.27 Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationAWS=true,InTreePluginAWSUnregister=true")) } Expect(c.Command).NotTo(ContainElement("--cloud-provider=aws")) Expect(c.Command).NotTo(ContainElement("--cloud-config=/etc/kubernetes/cloudprovider/cloudprovider.conf")) - Expect(c.Command).NotTo(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ",")) - Expect(c.Command).To(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ",")) + if !k8sVersionAtLeast131 { + Expect(c.Command).NotTo(test.ContainElementWithPrefixContaining("--enable-admission-plugins=", "PersistentVolumeLabel", ",")) + Expect(c.Command).To(test.ContainElementWithPrefixContaining("--disable-admission-plugins=", "PersistentVolumeLabel", ",")) + } Expect(c.Env).NotTo(ContainElement(accessKeyIDEnvVar)) Expect(c.Env).NotTo(ContainElement(secretAccessKeyEnvVar)) Expect(dep.Spec.Template.Annotations).To(BeNil()) @@ -824,15 +871,18 @@ func checkKubeAPIServerDeployment(dep *appsv1.Deployment, k8sVersion string) { func checkKubeControllerManagerDeployment(dep *appsv1.Deployment, k8sVersion string) { k8sVersionAtLeast127, _ := version.CompareVersions(k8sVersion, ">=", "1.27") + k8sVersionAtLeast131, _ := version.CompareVersions(k8sVersion, ">=", "1.31") // Check that the kube-controller-manager container still exists and contains all needed command line args, // env vars, and volume mounts c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-controller-manager") Expect(c).To(Not(BeNil())) - if k8sVersionAtLeast127 { + if k8sVersionAtLeast131 { + Expect(c.Command).NotTo(ContainElement(HavePrefix("--feature-gates"))) + } else if k8sVersionAtLeast127 { Expect(c.Command).To(ContainElement("--feature-gates=InTreePluginAWSUnregister=true")) - } else { + } else { // < 1.27 Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationAWS=true,InTreePluginAWSUnregister=true")) } Expect(c.Command).To(ContainElement("--cloud-provider=external")) @@ -850,28 +900,34 @@ func checkKubeControllerManagerDeployment(dep *appsv1.Deployment, k8sVersion str func checkKubeSchedulerDeployment(dep *appsv1.Deployment, k8sVersion string) { k8sVersionAtLeast127, _ := version.CompareVersions(k8sVersion, ">=", "1.27") + k8sVersionAtLeast131, _ := version.CompareVersions(k8sVersion, ">=", "1.31") // Check that the kube-scheduler container still exists and contains all needed command line args. c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "kube-scheduler") Expect(c).To(Not(BeNil())) - if k8sVersionAtLeast127 { + if k8sVersionAtLeast131 { + Expect(c.Command).NotTo(ContainElement(HavePrefix("--feature-gates"))) + } else if k8sVersionAtLeast127 { Expect(c.Command).To(ContainElement("--feature-gates=InTreePluginAWSUnregister=true")) - } else { + } else { // < 1.27 Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationAWS=true,InTreePluginAWSUnregister=true")) } } func checkClusterAutoscalerDeployment(dep *appsv1.Deployment, k8sVersion string) { k8sVersionAtLeast127, _ := version.CompareVersions(k8sVersion, ">=", "1.27") + k8sVersionAtLeast131, _ := version.CompareVersions(k8sVersion, ">=", "1.31") // Check that the cluster-autoscaler container still exists and contains all needed command line args. c := extensionswebhook.ContainerWithName(dep.Spec.Template.Spec.Containers, "cluster-autoscaler") Expect(c).To(Not(BeNil())) - if k8sVersionAtLeast127 { + if k8sVersionAtLeast131 { + Expect(c.Command).NotTo(ContainElement(HavePrefix("--feature-gates"))) + } else if k8sVersionAtLeast127 { Expect(c.Command).To(ContainElement("--feature-gates=InTreePluginAWSUnregister=true")) - } else { + } else { // < 1.27 Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationAWS=true,InTreePluginAWSUnregister=true")) } }