Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support for Kubernetes v1.31 #1055

Merged
merged 5 commits into from
Oct 15, 2024
+186 −55
Merged

Support for Kubernetes v1.31 #1055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
91531ed
Add Kubernetes 1.31 support to README.md
ialidzhikov Sep 11, 2024
e93ff56
Update cloud-controller-manager images
ialidzhikov Sep 11, 2024
9a3aa75
Update ecr-credential-provider images
ialidzhikov Sep 11, 2024
249a153
Adapt to the removal of the `InTreePluginAWSUnregister` feature gate
ialidzhikov Sep 25, 2024
8cf0fda
Adapt to the removal of the `PersistentVolumeLabel` admission plugin
ialidzhikov Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ This extension controller supports the following Kubernetes versions:

| Version | Support | Conformance test results |
| --------------- | ----------- | ------------------------ |
| Kubernetes 1.31 | 1.31.0+ | N/A |
| Kubernetes 1.30 | 1.30.0+ | [![Gardener v1.30 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.30%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.30%20AWS) |
| Kubernetes 1.29 | 1.29.0+ | [![Gardener v1.29 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.29%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.29%20AWS) |
| Kubernetes 1.28 | 1.28.0+ | [![Gardener v1.28 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.28%20AWS/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.28%20AWS) |
Expand Down
55 changes: 49 additions & 6 deletions imagevector/images.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ images:
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.26.11"
tag: "v1.26.12"
targetVersion: "1.26.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
Expand All @@ -44,7 +44,7 @@ images:
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.27.6"
tag: "v1.27.9"
targetVersion: "1.27.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
Expand All @@ -58,7 +58,7 @@ images:
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.28.5"
tag: "v1.28.9"
targetVersion: "1.28.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
Expand All @@ -72,7 +72,7 @@ images:
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.29.2"
tag: "v1.29.6"
targetVersion: "1.29.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
Expand All @@ -86,8 +86,22 @@ images:
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.30.0"
targetVersion: ">= 1.30"
tag: "v1.30.3"
targetVersion: "1.30.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
value:
network_exposure: 'protected'
authentication_enforced: false
user_interaction: 'gardener-operator'
confidentiality_requirement: 'high'
integrity_requirement: 'high'
availability_requirement: 'low'
- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/cloud-provider-aws
repository: registry.k8s.io/provider-aws/cloud-controller-manager
tag: "v1.31.1"
targetVersion: ">= 1.31"
labels:
- name: 'gardener.cloud/cve-categorisation'
value:
Expand Down Expand Up @@ -276,6 +290,35 @@ images:
sourceRepository: github.com/gardener/ecr-credential-provider
repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider
tag: "v1.29.0"
targetVersion: "< 1.30"
labels:
- name: 'gardener.cloud/cve-categorisation'
value:
network_exposure: 'protected'
authentication_enforced: false
user_interaction: 'end-user'
confidentiality_requirement: 'high'
integrity_requirement: 'high'
availability_requirement: 'low'
- name: ecr-credential-provider
sourceRepository: github.com/gardener/ecr-credential-provider
repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider
tag: "v1.30.3"
targetVersion: "1.30.x"
labels:
- name: 'gardener.cloud/cve-categorisation'
value:
network_exposure: 'protected'
authentication_enforced: false
user_interaction: 'end-user'
confidentiality_requirement: 'high'
integrity_requirement: 'high'
availability_requirement: 'low'
- name: ecr-credential-provider
sourceRepository: github.com/gardener/ecr-credential-provider
repository: europe-docker.pkg.dev/gardener-project/releases/gardener/extensions/ecr-credential-provider
tag: "v1.31.0"
targetVersion: ">= 1.31"
labels:
- name: 'gardener.cloud/cve-categorisation'
value:
Expand Down
81 changes: 56 additions & 25 deletions pkg/webhook/controlplane/ensurer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import (
extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
"github.com/gardener/gardener/pkg/component/nodemanagement/machinecontrollermanager"
gutil "github.com/gardener/gardener/pkg/utils/gardener"
imagevectorutils "github.com/gardener/gardener/pkg/utils/imagevector"
versionutils "github.com/gardener/gardener/pkg/utils/version"
"github.com/go-logr/logr"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
vpaautoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
kubeletconfigv1 "k8s.io/kubelet/config/v1"
kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
Expand All @@ -42,6 +44,20 @@ const (
ecrCredentialBinLocation = "/opt/bin/"
)

var (
// constraintK8sLess131 is a version constraint for versions < 1.31.
//
// TODO(ialidzhikov): Replace with versionutils.ConstraintK8sLess131 when vendoring a gardener/gardener version
// that contains https://github.com/gardener/gardener/pull/10472.
constraintK8sLess131 *semver.Constraints
)

func init() {
var err error
constraintK8sLess131, err = semver.NewConstraint("< 1.31-0")
utilruntime.Must(err)
}

// NewEnsurer creates a new controlplane ensurer.
func NewEnsurer(logger logr.Logger, client client.Client) genericmutator.Ensurer {
return &ensurer{
Expand Down Expand Up @@ -198,15 +214,19 @@ func ensureKubeAPIServerCommandLineArgs(c *corev1.Container, k8sVersion *semver.
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"CSIMigrationAWS=true", ",")
}
if constraintK8sLess131.Check(k8sVersion) {
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
}

c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-provider=")
c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-config=")
c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=",
"PersistentVolumeLabel", ",")
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=",
"PersistentVolumeLabel", ",")
if constraintK8sLess131.Check(k8sVersion) {
c.Command = extensionswebhook.EnsureNoStringWithPrefixContains(c.Command, "--enable-admission-plugins=",
"PersistentVolumeLabel", ",")
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--disable-admission-plugins=",
"PersistentVolumeLabel", ",")
}
}

func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container, k8sVersion *semver.Version) {
Expand All @@ -218,9 +238,11 @@ func ensureKubeControllerManagerCommandLineArgs(c *corev1.Container, k8sVersion
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"CSIMigrationAWS=true", ",")
}
if constraintK8sLess131.Check(k8sVersion) {
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
}

c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--cloud-config=")
c.Command = extensionswebhook.EnsureNoStringWithPrefix(c.Command, "--external-cloud-volume-plugin=")
}
Expand All @@ -232,9 +254,10 @@ func ensureKubeSchedulerCommandLineArgs(c *corev1.Container, k8sVersion *semver.
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"CSIMigrationAWS=true", ",")
}

c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
if constraintK8sLess131.Check(k8sVersion) {
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
}
}

func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, k8sVersion *semver.Version) {
Expand All @@ -244,9 +267,10 @@ func ensureClusterAutoscalerCommandLineArgs(c *corev1.Container, k8sVersion *sem
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"CSIMigrationAWS=true", ",")
}

c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
if constraintK8sLess131.Check(k8sVersion) {
c.Command = extensionswebhook.EnsureStringWithPrefixContains(c.Command, "--feature-gates=",
"InTreePluginAWSUnregister=true", ",")
}
}

func ensureKubeControllerManagerLabels(t *corev1.PodTemplateSpec) {
Expand Down Expand Up @@ -386,21 +410,27 @@ func ensureKubeletECRProviderCommandLineArgs(command []string) []string {

// EnsureKubeletConfiguration ensures that the kubelet configuration conforms to the provider requirements.
func (e *ensurer) EnsureKubeletConfiguration(_ context.Context, _ gcontext.GardenContext, kubeletVersion *semver.Version, newObj, _ *kubeletconfigv1beta1.KubeletConfiguration) error {
if newObj.FeatureGates == nil {
newObj.FeatureGates = make(map[string]bool)
}

if versionutils.ConstraintK8sLess127.Check(kubeletVersion) {
newObj.FeatureGates["CSIMigration"] = true
newObj.FeatureGates["CSIMigrationAWS"] = true
setKubeletConfigurationFeatureGate(newObj, "CSIMigration", true)
setKubeletConfigurationFeatureGate(newObj, "CSIMigrationAWS", true)
}
if constraintK8sLess131.Check(kubeletVersion) {
setKubeletConfigurationFeatureGate(newObj, "InTreePluginAWSUnregister", true)
}

newObj.FeatureGates["InTreePluginAWSUnregister"] = true
newObj.EnableControllerAttachDetach = ptr.To(true)

return nil
}

func setKubeletConfigurationFeatureGate(kubeletConfiguration *kubeletconfigv1beta1.KubeletConfiguration, featureGate string, value bool) {
if kubeletConfiguration.FeatureGates == nil {
kubeletConfiguration.FeatureGates = make(map[string]bool)
}

kubeletConfiguration.FeatureGates[featureGate] = value
}

var regexFindProperty = regexp.MustCompile("net.ipv4.neigh.default.gc_thresh1[[:space:]]*=[[:space:]]*([[:alnum:]]+)")

// EnsureKubernetesGeneralConfiguration ensures that the kubernetes general configuration conforms to the provider requirements.
Expand Down Expand Up @@ -451,8 +481,8 @@ ExecStart=/opt/bin/mtu-customizer.sh
return nil
}

func (e *ensurer) credentialProviderBinaryFile() (*extensionsv1alpha1.File, error) {
image, err := imagevector.ImageVector().FindImage(aws.ECRCredentialProviderImageName)
func (e *ensurer) credentialProviderBinaryFile(k8sVersion string) (*extensionsv1alpha1.File, error) {
image, err := imagevector.ImageVector().FindImage(aws.ECRCredentialProviderImageName, imagevectorutils.TargetVersion(k8sVersion))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -560,7 +590,8 @@ func (e *ensurer) EnsureAdditionalFiles(ctx context.Context, gctx gcontext.Garde
return err
}

k8sGreaterEqual127, err := versionutils.CompareVersions(cluster.Shoot.Spec.Kubernetes.Version, ">=", "1.27")
k8sVersion := cluster.Shoot.Spec.Kubernetes.Version
k8sGreaterEqual127, err := versionutils.CompareVersions(k8sVersion, ">=", "1.27")
if err != nil {
return err
}
Expand All @@ -584,7 +615,7 @@ func (e *ensurer) EnsureAdditionalFiles(ctx context.Context, gctx gcontext.Garde
}

if ptr.Deref(infraConfig.EnableECRAccess, true) {
binConfig, err := e.credentialProviderBinaryFile()
binConfig, err := e.credentialProviderBinaryFile(k8sVersion)
if err != nil {
return err
}
Expand Down
Loading
Loading